summaryrefslogtreecommitdiff
path: root/py/makeqstrdata.py
diff options
context:
space:
mode:
Diffstat (limited to 'py/makeqstrdata.py')
-rw-r--r--py/makeqstrdata.py213
1 files changed, 128 insertions, 85 deletions
diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py
index b4f4f1b03..0ad749005 100644
--- a/py/makeqstrdata.py
+++ b/py/makeqstrdata.py
@@ -17,9 +17,9 @@ import collections
import gettext
import os.path
-if hasattr(sys.stdout, 'reconfigure'):
- sys.stdout.reconfigure(encoding='utf-8')
- sys.stderr.reconfigure(errors='backslashreplace')
+if hasattr(sys.stdout, "reconfigure"):
+ sys.stdout.reconfigure(encoding="utf-8")
+ sys.stderr.reconfigure(errors="backslashreplace")
py = os.path.dirname(sys.argv[0])
top = os.path.dirname(py)
@@ -32,43 +32,44 @@ import huffman
# - iterating through bytes is different
# - codepoint2name lives in a different module
import platform
-if platform.python_version_tuple()[0] == '2':
+
+if platform.python_version_tuple()[0] == "2":
bytes_cons = lambda val, enc=None: bytearray(val)
from htmlentitydefs import codepoint2name
-elif platform.python_version_tuple()[0] == '3':
+elif platform.python_version_tuple()[0] == "3":
bytes_cons = bytes
from html.entities import codepoint2name
# end compatibility code
-codepoint2name[ord('-')] = 'hyphen';
+codepoint2name[ord("-")] = "hyphen"
# add some custom names to map characters that aren't in HTML
-codepoint2name[ord(' ')] = 'space'
-codepoint2name[ord('\'')] = 'squot'
-codepoint2name[ord(',')] = 'comma'
-codepoint2name[ord('.')] = 'dot'
-codepoint2name[ord(':')] = 'colon'
-codepoint2name[ord(';')] = 'semicolon'
-codepoint2name[ord('/')] = 'slash'
-codepoint2name[ord('%')] = 'percent'
-codepoint2name[ord('#')] = 'hash'
-codepoint2name[ord('(')] = 'paren_open'
-codepoint2name[ord(')')] = 'paren_close'
-codepoint2name[ord('[')] = 'bracket_open'
-codepoint2name[ord(']')] = 'bracket_close'
-codepoint2name[ord('{')] = 'brace_open'
-codepoint2name[ord('}')] = 'brace_close'
-codepoint2name[ord('*')] = 'star'
-codepoint2name[ord('!')] = 'bang'
-codepoint2name[ord('\\')] = 'backslash'
-codepoint2name[ord('+')] = 'plus'
-codepoint2name[ord('$')] = 'dollar'
-codepoint2name[ord('=')] = 'equals'
-codepoint2name[ord('?')] = 'question'
-codepoint2name[ord('@')] = 'at_sign'
-codepoint2name[ord('^')] = 'caret'
-codepoint2name[ord('|')] = 'pipe'
-codepoint2name[ord('~')] = 'tilde'
+codepoint2name[ord(" ")] = "space"
+codepoint2name[ord("'")] = "squot"
+codepoint2name[ord(",")] = "comma"
+codepoint2name[ord(".")] = "dot"
+codepoint2name[ord(":")] = "colon"
+codepoint2name[ord(";")] = "semicolon"
+codepoint2name[ord("/")] = "slash"
+codepoint2name[ord("%")] = "percent"
+codepoint2name[ord("#")] = "hash"
+codepoint2name[ord("(")] = "paren_open"
+codepoint2name[ord(")")] = "paren_close"
+codepoint2name[ord("[")] = "bracket_open"
+codepoint2name[ord("]")] = "bracket_close"
+codepoint2name[ord("{")] = "brace_open"
+codepoint2name[ord("}")] = "brace_close"
+codepoint2name[ord("*")] = "star"
+codepoint2name[ord("!")] = "bang"
+codepoint2name[ord("\\")] = "backslash"
+codepoint2name[ord("+")] = "plus"
+codepoint2name[ord("$")] = "dollar"
+codepoint2name[ord("=")] = "equals"
+codepoint2name[ord("?")] = "question"
+codepoint2name[ord("@")] = "at_sign"
+codepoint2name[ord("^")] = "caret"
+codepoint2name[ord("|")] = "pipe"
+codepoint2name[ord("~")] = "tilde"
C_ESCAPES = {
"\a": "\\a",
@@ -78,8 +79,8 @@ C_ESCAPES = {
"\r": "\\r",
"\t": "\\t",
"\v": "\\v",
- "\'": "\\'",
- "\"": "\\\""
+ "'": "\\'",
+ '"': '\\"',
}
# this must match the equivalent function in qstr.c
@@ -90,6 +91,7 @@ def compute_hash(qstr, bytes_hash):
# Make sure that valid hash is never zero, zero means "hash not computed"
return (hash & ((1 << (8 * bytes_hash)) - 1)) or 1
+
def translate(translation_file, i18ns):
with open(translation_file, "rb") as f:
table = gettext.GNUTranslations(f)
@@ -105,6 +107,7 @@ def translate(translation_file, i18ns):
translations.append((original, translation))
return translations
+
class TextSplitter:
def __init__(self, words):
words.sort(key=lambda x: len(x), reverse=True)
@@ -134,6 +137,7 @@ class TextSplitter:
for m in self.pat.finditer(text):
yield m.group(0)
+
def iter_substrings(s, minlen, maxlen):
len_s = len(s)
maxlen = min(len_s, maxlen)
@@ -141,18 +145,19 @@ def iter_substrings(s, minlen, maxlen):
for begin in range(0, len_s - n + 1):
yield s[begin : begin + n]
+
def compute_huffman_coding(translations, compression_filename):
texts = [t[1] for t in translations]
words = []
start_unused = 0x80
- end_unused = 0xff
+ end_unused = 0xFF
max_ord = 0
for text in texts:
for c in text:
ord_c = ord(c)
max_ord = max(ord_c, max_ord)
- if 0x80 <= ord_c < 0xff:
+ if 0x80 <= ord_c < 0xFF:
end_unused = min(ord_c, end_unused)
max_words = end_unused - 0x80
@@ -180,10 +185,7 @@ def compute_huffman_coding(translations, compression_filename):
# Score the candidates we found. This is an empirical formula only,
# chosen for its effectiveness.
scores = sorted(
- (
- (s, (len(s) - 1) ** log(max(occ - 2, 1)), occ)
- for (s, occ) in counter.items()
- ),
+ ((s, (len(s) - 1) ** log(max(occ - 2, 1)), occ) for (s, occ) in counter.items()),
key=lambda x: x[1],
reverse=True,
)
@@ -234,8 +236,8 @@ def compute_huffman_coding(translations, compression_filename):
length_count[length] = 0
length_count[length] += 1
if last_length:
- renumbered <<= (length - last_length)
- canonical[atom] = '{0:0{width}b}'.format(renumbered, width=length)
+ renumbered <<= length - last_length
+ canonical[atom] = "{0:0{width}b}".format(renumbered, width=length)
# print(f"atom={repr(atom)} code={code}", file=sys.stderr)
if len(atom) > 1:
o = words.index(atom) + 0x80
@@ -257,7 +259,8 @@ def compute_huffman_coding(translations, compression_filename):
values = [(atom if len(atom) == 1 else chr(0x80 + words.index(atom))) for atom in values]
print("//", values, lengths)
max_translation_encoded_length = max(
- len(translation.encode("utf-8")) for (original, translation) in translations)
+ len(translation.encode("utf-8")) for (original, translation) in translations
+ )
wends = list(len(w) - 2 for w in words)
for i in range(1, len(wends)):
@@ -265,15 +268,28 @@ def compute_huffman_coding(translations, compression_filename):
with open(compression_filename, "w") as f:
f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths))))
- f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values)))
- f.write("#define compress_max_length_bits ({})\n".format(max_translation_encoded_length.bit_length()))
- f.write("const {} words[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(c)) for w in words for c in w)))
+ f.write(
+ "const {} values[] = {{ {} }};\n".format(
+ values_type, ", ".join(str(ord(u)) for u in values)
+ )
+ )
+ f.write(
+ "#define compress_max_length_bits ({})\n".format(
+ max_translation_encoded_length.bit_length()
+ )
+ )
+ f.write(
+ "const {} words[] = {{ {} }};\n".format(
+ values_type, ", ".join(str(ord(c)) for w in words for c in w)
+ )
+ )
f.write("const uint8_t wends[] = {{ {} }};\n".format(", ".join(str(p) for p in wends)))
f.write("#define word_start {}\n".format(word_start))
f.write("#define word_end {}\n".format(word_end))
return (values, lengths, words, canonical, extractor)
+
def decompress(encoding_table, encoded, encoded_length_bits):
(values, lengths, words, _, _) = encoding_table
dec = []
@@ -317,7 +333,7 @@ def decompress(encoding_table, encoded, encoded_length_bits):
else:
this_bit -= 1
if max_code > 0 and bits < max_code:
- #print('{0:0{width}b}'.format(bits, width=bit_length))
+ # print('{0:0{width}b}'.format(bits, width=bit_length))
break
max_code = (max_code << 1) + lengths[bit_length]
searched_length += lengths[bit_length]
@@ -325,9 +341,10 @@ def decompress(encoding_table, encoded, encoded_length_bits):
v = values[searched_length + bits - max_code]
if v >= chr(0x80) and v < chr(0x80 + len(words)):
v = words[ord(v) - 0x80]
- i += len(v.encode('utf-8'))
+ i += len(v.encode("utf-8"))
dec.append(v)
- return ''.join(dec)
+ return "".join(dec)
+
def compress(encoding_table, decompressed, encoded_length_bits, len_translation_encoded):
if not isinstance(decompressed, str):
@@ -362,15 +379,18 @@ def compress(encoding_table, decompressed, encoded_length_bits, len_translation_
current_byte += 1
return enc[:current_byte]
+
def qstr_escape(qst):
def esc_char(m):
c = ord(m.group(0))
try:
name = codepoint2name[c]
except KeyError:
- name = '0x%02x' % c
- return "_" + name + '_'
- return re.sub(r'[^A-Za-z0-9_]', esc_char, qst)
+ name = "0x%02x" % c
+ return "_" + name + "_"
+
+ return re.sub(r"[^A-Za-z0-9_]", esc_char, qst)
+
def parse_input_headers(infiles):
# read the qstrs in from the input files
@@ -378,28 +398,27 @@ def parse_input_headers(infiles):
qstrs = {}
i18ns = set()
for infile in infiles:
- with open(infile, 'rt') as f:
+ with open(infile, "rt") as f:
for line in f:
line = line.strip()
# is this a config line?
- match = re.match(r'^QCFG\((.+), (.+)\)', line)
+ match = re.match(r"^QCFG\((.+), (.+)\)", line)
if match:
value = match.group(2)
- if value[0] == '(' and value[-1] == ')':
+ if value[0] == "(" and value[-1] == ")":
# strip parenthesis from config value
value = value[1:-1]
qcfgs[match.group(1)] = value
continue
-
match = re.match(r'^TRANSLATE\("(.*)"\)$', line)
if match:
i18ns.add(match.group(1))
continue
# is this a QSTR line?
- match = re.match(r'^Q\((.*)\)$', line)
+ match = re.match(r"^Q\((.*)\)$", line)
if not match:
continue
@@ -407,8 +426,8 @@ def parse_input_headers(infiles):
qstr = match.group(1)
# special case to specify control characters
- if qstr == '\\n':
- qstr = '\n'
+ if qstr == "\\n":
+ qstr = "\n"
# work out the corresponding qstr name
ident = qstr_escape(qstr)
@@ -437,56 +456,73 @@ def parse_input_headers(infiles):
return qcfgs, qstrs, i18ns
+
def make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr):
- qbytes = bytes_cons(qstr, 'utf8')
+ qbytes = bytes_cons(qstr, "utf8")
qlen = len(qbytes)
qhash = compute_hash(qbytes, cfg_bytes_hash)
- if all(32 <= ord(c) <= 126 and c != '\\' and c != '"' for c in qstr):
+ if all(32 <= ord(c) <= 126 and c != "\\" and c != '"' for c in qstr):
# qstr is all printable ASCII so render it as-is (for easier debugging)
qdata = qstr
else:
# qstr contains non-printable codes so render entire thing as hex pairs
- qdata = ''.join(('\\x%02x' % b) for b in qbytes)
+ qdata = "".join(("\\x%02x" % b) for b in qbytes)
if qlen >= (1 << (8 * cfg_bytes_len)):
- print('qstr is too long:', qstr)
+ print("qstr is too long:", qstr)
assert False
- qlen_str = ('\\x%02x' * cfg_bytes_len) % tuple(((qlen >> (8 * i)) & 0xff) for i in range(cfg_bytes_len))
- qhash_str = ('\\x%02x' * cfg_bytes_hash) % tuple(((qhash >> (8 * i)) & 0xff) for i in range(cfg_bytes_hash))
+ qlen_str = ("\\x%02x" * cfg_bytes_len) % tuple(
+ ((qlen >> (8 * i)) & 0xFF) for i in range(cfg_bytes_len)
+ )
+ qhash_str = ("\\x%02x" * cfg_bytes_hash) % tuple(
+ ((qhash >> (8 * i)) & 0xFF) for i in range(cfg_bytes_hash)
+ )
return '(const byte*)"%s%s" "%s"' % (qhash_str, qlen_str, qdata)
+
def print_qstr_data(encoding_table, qcfgs, qstrs, i18ns):
# get config variables
- cfg_bytes_len = int(qcfgs['BYTES_IN_LEN'])
- cfg_bytes_hash = int(qcfgs['BYTES_IN_HASH'])
+ cfg_bytes_len = int(qcfgs["BYTES_IN_LEN"])
+ cfg_bytes_hash = int(qcfgs["BYTES_IN_HASH"])
# print out the starter of the generated C header file
- print('// This file was automatically generated by makeqstrdata.py')
- print('')
+ print("// This file was automatically generated by makeqstrdata.py")
+ print("")
# add NULL qstr with no hash or data
- print('QDEF(MP_QSTR_NULL, (const byte*)"%s%s" "")' % ('\\x00' * cfg_bytes_hash, '\\x00' * cfg_bytes_len))
+ print(
+ 'QDEF(MP_QSTR_NULL, (const byte*)"%s%s" "")'
+ % ("\\x00" * cfg_bytes_hash, "\\x00" * cfg_bytes_len)
+ )
total_qstr_size = 0
total_qstr_compressed_size = 0
# go through each qstr and print it out
for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):
qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr)
- print('QDEF(MP_QSTR_%s, %s)' % (ident, qbytes))
+ print("QDEF(MP_QSTR_%s, %s)" % (ident, qbytes))
total_qstr_size += len(qstr)
total_text_size = 0
total_text_compressed_size = 0
- max_translation_encoded_length = max(len(translation.encode("utf-8")) for original, translation in i18ns)
+ max_translation_encoded_length = max(
+ len(translation.encode("utf-8")) for original, translation in i18ns
+ )
encoded_length_bits = max_translation_encoded_length.bit_length()
for original, translation in i18ns:
translation_encoded = translation.encode("utf-8")
- compressed = compress(encoding_table, translation, encoded_length_bits, len(translation_encoded))
+ compressed = compress(
+ encoding_table, translation, encoded_length_bits, len(translation_encoded)
+ )
total_text_compressed_size += len(compressed)
decompressed = decompress(encoding_table, compressed, encoded_length_bits)
assert decompressed == translation
for c in C_ESCAPES:
decompressed = decompressed.replace(c, C_ESCAPES[c])
- print("TRANSLATION(\"{}\", {}) // {}".format(original, ", ".join(["{:d}".format(x) for x in compressed]), decompressed))
+ print(
+ 'TRANSLATION("{}", {}) // {}'.format(
+ original, ", ".join(["{:d}".format(x) for x in compressed]), decompressed
+ )
+ )
total_text_size += len(translation.encode("utf-8"))
print()
@@ -495,28 +531,35 @@ def print_qstr_data(encoding_table, qcfgs, qstrs, i18ns):
print("// {} bytes worth of translations compressed".format(total_text_compressed_size))
print("// {} bytes saved".format(total_text_size - total_text_compressed_size))
+
def print_qstr_enums(qstrs):
# print out the starter of the generated C header file
- print('// This file was automatically generated by makeqstrdata.py')
- print('')
+ print("// This file was automatically generated by makeqstrdata.py")
+ print("")
# add NULL qstr with no hash or data
- print('QENUM(MP_QSTR_NULL)')
+ print("QENUM(MP_QSTR_NULL)")
# go through each qstr and print it out
for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):
- print('QENUM(MP_QSTR_%s)' % (ident,))
+ print("QENUM(MP_QSTR_%s)" % (ident,))
+
if __name__ == "__main__":
import argparse
- parser = argparse.ArgumentParser(description='Process QSTR definitions into headers for compilation')
- parser.add_argument('infiles', metavar='N', type=str, nargs='+',
- help='an integer for the accumulator')
- parser.add_argument('--translation', default=None, type=str,
- help='translations for i18n() items')
- parser.add_argument('--compression_filename', default=None, type=str,
- help='header for compression info')
+ parser = argparse.ArgumentParser(
+ description="Process QSTR definitions into headers for compilation"
+ )
+ parser.add_argument(
+ "infiles", metavar="N", type=str, nargs="+", help="an integer for the accumulator"
+ )
+ parser.add_argument(
+ "--translation", default=None, type=str, help="translations for i18n() items"
+ )
+ parser.add_argument(
+ "--compression_filename", default=None, type=str, help="header for compression info"
+ )
args = parser.parse_args()
generated by cgit v1.2.3 (git 2.39.1) at 2026-08-08 18:29:19 +0000