From 40ab5c6b21ff93ad11ca51dc66d3613dcd77e5ef Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 12 Sep 2020 10:10:18 -0500 Subject: compression: Implement ciscorn's dictionary approach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Massive savings. Thanks so much @ciscorn for providing the initial code for choosing the dictionary. This adds a bit of time to the build, both to find the dictionary but also because (for reasons I don't fully understand), the binary search in the compress() function no longer worked and had to be replaced with a linear search. I think this is because the intended invariant is that for codebook entries that encode to the same number of bits, the entries are ordered in ascending value. However, I mis-placed the transition from "words" to "byte/char values" so the codebook entries for words are in word-order rather than their code order. Because this price is only paid at build time, I didn't care to determine exactly where the correct fix was. I also commented out a line to produce the "estimated total memory size" -- at least on the unix build with TRANSLATION=ja, this led to a build time KeyError trying to compute the codebook size for all the strings. I think this occurs because some single unicode code point ('ァ') is no longer present as itself in the compressed strings, due to always being replaced by a word. As promised, this seems to save hundreds of bytes in the German translation on the trinket m0. Testing performed: - built trinket_m0 in several languages - built and ran unix port in several languages (en, de_DE, ja) and ran simple error-producing codes like ./micropython -c '1/0' --- py/makeqstrdata.py | 201 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 139 insertions(+), 62 deletions(-) (limited to 'py/makeqstrdata.py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 721fa8320..57635807d 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -100,77 +100,153 @@ def translate(translation_file, i18ns): translations.append((original, translation)) return translations -def frequent_ngrams(corpus, sz, n): - return collections.Counter(corpus[i:i+sz] for i in range(len(corpus)-sz)).most_common(n) +class TextSplitter: + def __init__(self, words): + words.sort(key=lambda x: len(x), reverse=True) + self.words = set(words) + self.pat = re.compile("|".join(re.escape(w) for w in words) + "|.", flags=re.DOTALL) + + def iter_words(self, text): + s = [] + for m in self.pat.finditer(text): + t = m.group(0) + if t in self.words: + if s: + yield (False, "".join(s)) + s = [] + yield (True, t) + else: + s.append(t) + if s: + yield (False, "".join(s)) + + def iter(self, text): + s = [] + for m in self.pat.finditer(text): + yield m.group(0) + +def iter_substrings(s, minlen, maxlen): + maxlen = min(len(s), maxlen) + for n in range(minlen, maxlen + 1): + for begin in range(0, len(s) - n + 1): + yield s[begin : begin + n] + +def compute_huffman_coding(translations, compression_filename): + texts = [t[1] for t in translations] + all_strings_concat = "".join(texts) + words = [] + max_ord = 0 + begin_unused = 128 + end_unused = 256 + for text in texts: + for c in text: + ord_c = ord(c) + max_ord = max(max_ord, ord_c) + if 128 <= ord_c < 256: + end_unused = min(ord_c, end_unused) + max_words = end_unused - begin_unused + char_size = 1 if max_ord < 256 else 2 + + sum_word_len = 0 + while True: + extractor = TextSplitter(words) + counter = collections.Counter() + for t in texts: + for (found, word) in extractor.iter_words(t): + if not found: + for substr in iter_substrings(word, minlen=2, maxlen=9): + counter[substr] += 1 + + scores = sorted( + ( + # I don't know why this works good. This could be better. + (s, (len(s) - 1) ** ((max(occ - 2, 1) + 0.5) ** 0.8), occ) + for (s, occ) in counter.items() + ), + key=lambda x: x[1], + reverse=True, + ) + + w = None + for (s, score, occ) in scores: + if score < 0: + break + if len(s) > 1: + w = s + break + + if not w: + break + if len(w) + sum_word_len > 256: + break + if len(words) == max_words: + break + words.append(w) + sum_word_len += len(w) + + extractor = TextSplitter(words) + counter = collections.Counter() + for t in texts: + for atom in extractor.iter(t): + counter[atom] += 1 + cb = huffman.codebook(counter.items()) + + word_start = begin_unused + word_end = word_start + len(words) - 1 + print("// # words", len(words)) + print("// words", words) -def encode_ngrams(translation, ngrams): - if len(ngrams) > 32: - start = 0xe000 - else: - start = 0x80 - for i, g in enumerate(ngrams): - translation = translation.replace(g, chr(start + i)) - return translation - -def decode_ngrams(compressed, ngrams): - if len(ngrams) > 32: - start, end = 0xe000, 0xf8ff - else: - start, end = 0x80, 0x9f - return "".join(ngrams[ord(c) - start] if (start <= ord(c) <= end) else c for c in compressed) - -def compute_huffman_coding(translations, qstrs, compression_filename): - all_strings = [x[1] for x in translations] - all_strings_concat = "".join(all_strings) - ngrams = [i[0] for i in frequent_ngrams(all_strings_concat, 2, 32)] - all_strings_concat = encode_ngrams(all_strings_concat, ngrams) - counts = collections.Counter(all_strings_concat) - cb = huffman.codebook(counts.items()) values = [] length_count = {} renumbered = 0 last_l = None canonical = {} - for ch, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])): - values.append(ch) + for atom, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])): + values.append(atom) l = len(code) if l not in length_count: length_count[l] = 0 length_count[l] += 1 if last_l: renumbered <<= (l - last_l) - canonical[ch] = '{0:0{width}b}'.format(renumbered, width=l) - s = C_ESCAPES.get(ch, ch) - print("//", ord(ch), s, counts[ch], canonical[ch], renumbered) + canonical[atom] = '{0:0{width}b}'.format(renumbered, width=l) + #print(f"atom={repr(atom)} code={code}", file=sys.stderr) + if len(atom) > 1: + o = words.index(atom) + 0x80 + s = "".join(C_ESCAPES.get(ch1, ch1) for ch1 in atom) + else: + s = C_ESCAPES.get(atom, atom) + o = ord(atom) + print("//", o, s, counter[atom], canonical[atom], renumbered) renumbered += 1 last_l = l lengths = bytearray() print("// length count", length_count) - print("// bigrams", ngrams) + for i in range(1, max(length_count) + 2): lengths.append(length_count.get(i, 0)) print("// values", values, "lengths", len(lengths), lengths) - ngramdata = [ord(ni) for i in ngrams for ni in i] - print("// estimated total memory size", len(lengths) + 2*len(values) + 2 * len(ngramdata) + sum((len(cb[u]) + 7)//8 for u in all_strings_concat)) + maxord = max(ord(u) for u in values if len(u) == 1) + values_type = "uint16_t" if maxord > 255 else "uint8_t" + ch_size = 1 if maxord > 255 else 2 + print("//", values, lengths) + values = [(atom if len(atom) == 1 else chr(0x80 + words.index(atom))) for atom in values] print("//", values, lengths) - values_type = "uint16_t" if max(ord(u) for u in values) > 255 else "uint8_t" max_translation_encoded_length = max(len(translation.encode("utf-8")) for original,translation in translations) with open(compression_filename, "w") as f: f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths)))) f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values))) f.write("#define compress_max_length_bits ({})\n".format(max_translation_encoded_length.bit_length())) - f.write("const {} bigrams[] = {{ {} }};\n".format(values_type, ", ".join(str(u) for u in ngramdata))) - if len(ngrams) > 32: - bigram_start = 0xe000 - else: - bigram_start = 0x80 - bigram_end = bigram_start + len(ngrams) - 1 # End is inclusive - f.write("#define bigram_start {}\n".format(bigram_start)) - f.write("#define bigram_end {}\n".format(bigram_end)) - return values, lengths, ngrams + f.write("const {} words[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(c)) for w in words for c in w))) + f.write("const uint8_t wlen[] = {{ {} }};\n".format(", ".join(str(len(w)) for w in words))) + f.write("#define word_start {}\n".format(word_start)) + f.write("#define word_end {}\n".format(word_end)) + + extractor = TextSplitter(words) + return values, lengths, words, extractor def decompress(encoding_table, encoded, encoded_length_bits): - values, lengths, ngrams = encoding_table + values, lengths, words, extractor = encoding_table dec = [] this_byte = 0 this_bit = 7 @@ -218,7 +294,8 @@ def decompress(encoding_table, encoded, encoded_length_bits): searched_length += lengths[bit_length] v = values[searched_length + bits - max_code] - v = decode_ngrams(v, ngrams) + if v >= chr(0x80) and v < chr(0x80 + len(words)): + v = words[ord(v) - 0x80] i += len(v.encode('utf-8')) dec.append(v) return ''.join(dec) @@ -226,8 +303,8 @@ def decompress(encoding_table, encoded, encoded_length_bits): def compress(encoding_table, decompressed, encoded_length_bits, len_translation_encoded): if not isinstance(decompressed, str): raise TypeError() - values, lengths, ngrams = encoding_table - decompressed = encode_ngrams(decompressed, ngrams) + values, lengths, words, extractor = encoding_table + enc = bytearray(len(decompressed) * 3) #print(decompressed) #print(lengths) @@ -246,9 +323,15 @@ def compress(encoding_table, decompressed, encoded_length_bits, len_translation_ else: current_bit -= 1 - for c in decompressed: - #print() - #print("char", c, values.index(c)) + #print("values = ", values, file=sys.stderr) + for atom in extractor.iter(decompressed): + #print("", file=sys.stderr) + if len(atom) > 1: + c = chr(0x80 + words.index(atom)) + else: + c = atom + assert c in values + start = 0 end = lengths[0] bits = 1 @@ -258,18 +341,12 @@ def compress(encoding_table, decompressed, encoded_length_bits, len_translation_ s = start e = end #print("{0:0{width}b}".format(code, width=bits)) - # Binary search! - while e > s: - midpoint = (s + e) // 2 - #print(s, e, midpoint) - if values[midpoint] == c: - compressed = code + (midpoint - start) - #print("found {0:0{width}b}".format(compressed, width=bits)) + # Linear search! + for i in range(s, e): + if values[i] == c: + compressed = code + (i - start) + #print("found {0:0{width}b}".format(compressed, width=bits), file=sys.stderr) break - elif c < values[midpoint]: - e = midpoint - else: - s = midpoint + 1 code += end - start code <<= 1 start = end @@ -452,7 +529,7 @@ if __name__ == "__main__": if args.translation: i18ns = sorted(i18ns) translations = translate(args.translation, i18ns) - encoding_table = compute_huffman_coding(translations, qstrs, args.compression_filename) + encoding_table = compute_huffman_coding(translations, args.compression_filename) print_qstr_data(encoding_table, qcfgs, qstrs, translations) else: print_qstr_enums(qstrs) -- cgit v1.2.3 From 15964a4750d9eb1a34317c166d9bde5f40c9878d Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 12 Sep 2020 19:39:35 -0500 Subject: makeqstrdata: Avoid encoding problems Most users and the CI system are running in configurations where Python configures stdout and stderr in UTF-8 mode. However, Windows is different, setting values like CP1252. This led to a build failure on Windows, because makeqstrdata printed Unicode strings to its stdout, expecting them to be encoded as UTF-8. This script is writing (stdout) to a compiler input file and potentially printing messages (stderr) to a log or console. Explicitly configure stdout to use utf-8 to get consistent behavior on all platforms, and configure stderr so that if any log/diagnostic messages are printed that cannot be displayed correctly, they are still displayed instead of creating an error while trying to print the diagnostic information. I considered setting the encodings both to ascii, but this would just be occasionally inconvenient to developers like me who want to show diagnostic info on stderr and in comments while working with the compression code. Closes: #3408 --- py/makeqstrdata.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'py/makeqstrdata.py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 57635807d..e30f120e6 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -16,6 +16,9 @@ import collections import gettext import os.path +sys.stdout.reconfigure(encoding='utf-8') +sys.stderr.reconfigure(errors='backslashreplace') + py = os.path.dirname(sys.argv[0]) top = os.path.dirname(py) -- cgit v1.2.3 From d18d79ac4709364c2371e2eaefd8f96bb20802e7 Mon Sep 17 00:00:00 2001 From: Taku Fukada Date: Mon, 14 Sep 2020 01:25:13 +0900 Subject: Small improvements to the dictionary compression --- py/makeqstrdata.py | 135 +++++++++++++++++------------------------- supervisor/shared/translate.c | 12 ++-- 2 files changed, 60 insertions(+), 87 deletions(-) (limited to 'py/makeqstrdata.py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index e30f120e6..006021791 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -12,6 +12,7 @@ from __future__ import print_function import re import sys +from math import log import collections import gettext import os.path @@ -111,9 +112,10 @@ class TextSplitter: def iter_words(self, text): s = [] + words = self.words for m in self.pat.finditer(text): t = m.group(0) - if t in self.words: + if t in words: if s: yield (False, "".join(s)) s = [] @@ -124,33 +126,35 @@ class TextSplitter: yield (False, "".join(s)) def iter(self, text): - s = [] for m in self.pat.finditer(text): yield m.group(0) def iter_substrings(s, minlen, maxlen): - maxlen = min(len(s), maxlen) + len_s = len(s) + maxlen = min(len_s, maxlen) for n in range(minlen, maxlen + 1): - for begin in range(0, len(s) - n + 1): + for begin in range(0, len_s - n + 1): yield s[begin : begin + n] def compute_huffman_coding(translations, compression_filename): texts = [t[1] for t in translations] - all_strings_concat = "".join(texts) words = [] + + start_unused = 0x80 + end_unused = 0xff max_ord = 0 - begin_unused = 128 - end_unused = 256 for text in texts: for c in text: ord_c = ord(c) - max_ord = max(max_ord, ord_c) - if 128 <= ord_c < 256: + max_ord = max(ord_c, max_ord) + if 0x80 <= ord_c < 0xff: end_unused = min(ord_c, end_unused) - max_words = end_unused - begin_unused - char_size = 1 if max_ord < 256 else 2 + max_words = end_unused - 0x80 + + values_type = "uint16_t" if max_ord > 255 else "uint8_t" + max_words_len = 160 if max_ord > 255 else 255 - sum_word_len = 0 + sum_len = 0 while True: extractor = TextSplitter(words) counter = collections.Counter() @@ -162,30 +166,30 @@ def compute_huffman_coding(translations, compression_filename): scores = sorted( ( - # I don't know why this works good. This could be better. - (s, (len(s) - 1) ** ((max(occ - 2, 1) + 0.5) ** 0.8), occ) + (s, (len(s) - 1) ** log(max(occ - 2, 1)), occ) for (s, occ) in counter.items() ), key=lambda x: x[1], reverse=True, ) - w = None + word = None for (s, score, occ) in scores: - if score < 0: - break - if len(s) > 1: - w = s + if occ < 5: + continue + if score < 5: break + word = s + break - if not w: + if not word: break - if len(w) + sum_word_len > 256: + if sum_len + len(word) - 2 > max_words_len: break if len(words) == max_words: break - words.append(w) - sum_word_len += len(w) + words.append(word) + sum_len += len(word) - 2 extractor = TextSplitter(words) counter = collections.Counter() @@ -194,7 +198,7 @@ def compute_huffman_coding(translations, compression_filename): counter[atom] += 1 cb = huffman.codebook(counter.items()) - word_start = begin_unused + word_start = start_unused word_end = word_start + len(words) - 1 print("// # words", len(words)) print("// words", words) @@ -202,18 +206,18 @@ def compute_huffman_coding(translations, compression_filename): values = [] length_count = {} renumbered = 0 - last_l = None + last_length = None canonical = {} for atom, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])): values.append(atom) - l = len(code) - if l not in length_count: - length_count[l] = 0 - length_count[l] += 1 - if last_l: - renumbered <<= (l - last_l) - canonical[atom] = '{0:0{width}b}'.format(renumbered, width=l) - #print(f"atom={repr(atom)} code={code}", file=sys.stderr) + length = len(code) + if length not in length_count: + length_count[length] = 0 + length_count[length] += 1 + if last_length: + renumbered <<= (length - last_length) + canonical[atom] = '{0:0{width}b}'.format(renumbered, width=length) + # print(f"atom={repr(atom)} code={code}", file=sys.stderr) if len(atom) > 1: o = words.index(atom) + 0x80 s = "".join(C_ESCAPES.get(ch1, ch1) for ch1 in atom) @@ -222,34 +226,37 @@ def compute_huffman_coding(translations, compression_filename): o = ord(atom) print("//", o, s, counter[atom], canonical[atom], renumbered) renumbered += 1 - last_l = l + last_length = length lengths = bytearray() print("// length count", length_count) for i in range(1, max(length_count) + 2): lengths.append(length_count.get(i, 0)) print("// values", values, "lengths", len(lengths), lengths) - maxord = max(ord(u) for u in values if len(u) == 1) - values_type = "uint16_t" if maxord > 255 else "uint8_t" - ch_size = 1 if maxord > 255 else 2 + print("//", values, lengths) values = [(atom if len(atom) == 1 else chr(0x80 + words.index(atom))) for atom in values] print("//", values, lengths) - max_translation_encoded_length = max(len(translation.encode("utf-8")) for original,translation in translations) + max_translation_encoded_length = max( + len(translation.encode("utf-8")) for (original, translation) in translations) + + wends = list(len(w) - 2 for w in words) + for i in range(1, len(wends)): + wends[i] += wends[i - 1] + with open(compression_filename, "w") as f: f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths)))) f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values))) f.write("#define compress_max_length_bits ({})\n".format(max_translation_encoded_length.bit_length())) f.write("const {} words[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(c)) for w in words for c in w))) - f.write("const uint8_t wlen[] = {{ {} }};\n".format(", ".join(str(len(w)) for w in words))) + f.write("const uint8_t wends[] = {{ {} }};\n".format(", ".join(str(p) for p in wends))) f.write("#define word_start {}\n".format(word_start)) f.write("#define word_end {}\n".format(word_end)) - extractor = TextSplitter(words) - return values, lengths, words, extractor + return (values, lengths, words, canonical, extractor) def decompress(encoding_table, encoded, encoded_length_bits): - values, lengths, words, extractor = encoding_table + (values, lengths, words, _, _) = encoding_table dec = [] this_byte = 0 this_bit = 7 @@ -306,66 +313,32 @@ def decompress(encoding_table, encoded, encoded_length_bits): def compress(encoding_table, decompressed, encoded_length_bits, len_translation_encoded): if not isinstance(decompressed, str): raise TypeError() - values, lengths, words, extractor = encoding_table + (_, _, _, canonical, extractor) = encoding_table enc = bytearray(len(decompressed) * 3) - #print(decompressed) - #print(lengths) current_bit = 7 current_byte = 0 - code = len_translation_encoded - bits = encoded_length_bits+1 + bits = encoded_length_bits + 1 for i in range(bits - 1, 0, -1): if len_translation_encoded & (1 << (i - 1)): enc[current_byte] |= 1 << current_bit if current_bit == 0: current_bit = 7 - #print("packed {0:0{width}b}".format(enc[current_byte], width=8)) current_byte += 1 else: current_bit -= 1 - #print("values = ", values, file=sys.stderr) for atom in extractor.iter(decompressed): - #print("", file=sys.stderr) - if len(atom) > 1: - c = chr(0x80 + words.index(atom)) - else: - c = atom - assert c in values - - start = 0 - end = lengths[0] - bits = 1 - compressed = None - code = 0 - while compressed is None: - s = start - e = end - #print("{0:0{width}b}".format(code, width=bits)) - # Linear search! - for i in range(s, e): - if values[i] == c: - compressed = code + (i - start) - #print("found {0:0{width}b}".format(compressed, width=bits), file=sys.stderr) - break - code += end - start - code <<= 1 - start = end - end += lengths[bits] - bits += 1 - #print("next bit", bits) - - for i in range(bits - 1, 0, -1): - if compressed & (1 << (i - 1)): + for b in canonical[atom]: + if b == "1": enc[current_byte] |= 1 << current_bit if current_bit == 0: current_bit = 7 - #print("packed {0:0{width}b}".format(enc[current_byte], width=8)) current_byte += 1 else: current_bit -= 1 + if current_bit != 7: current_byte += 1 return enc[:current_byte] diff --git a/supervisor/shared/translate.c b/supervisor/shared/translate.c index cc0de7f61..44544c98d 100644 --- a/supervisor/shared/translate.c +++ b/supervisor/shared/translate.c @@ -48,17 +48,17 @@ STATIC int put_utf8(char *buf, int u) { *buf = u; return 1; } else if(word_start <= u && u <= word_end) { - int n = (u - 0x80); - size_t off = 0; - for(int i=0; i 0) { + pos = wends[n - 1] + (n * 2); } int ret = 0; // note that at present, entries in the words table are // guaranteed not to represent words themselves, so this adds // at most 1 level of recursive call - for(int i=0; i Date: Wed, 16 Sep 2020 07:58:55 -0500 Subject: makeqstrdata: comment my understanding of @ciscorn's code --- py/makeqstrdata.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'py/makeqstrdata.py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 006021791..39d4a6840 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -156,6 +156,14 @@ def compute_huffman_coding(translations, compression_filename): sum_len = 0 while True: + # Until the dictionary is filled to capacity, use a heuristic to find + # the best "word" (2- to 9-gram) to add to it. + # + # The TextSplitter allows us to avoid considering parts of the text + # that are already covered by a previously chosen word, for example + # if "the" is in words then not only will "the" not be considered + # again, neither will "there" or "wither", since they have "the" + # as substrings. extractor = TextSplitter(words) counter = collections.Counter() for t in texts: @@ -164,6 +172,8 @@ def compute_huffman_coding(translations, compression_filename): for substr in iter_substrings(word, minlen=2, maxlen=9): counter[substr] += 1 + # Score the candidates we found. This is an empirical formula only, + # chosen for its effectiveness. scores = sorted( ( (s, (len(s) - 1) ** log(max(occ - 2, 1)), occ) @@ -173,6 +183,8 @@ def compute_huffman_coding(translations, compression_filename): reverse=True, ) + # Do we have a "word" that occurred 5 times and got a score of at least + # 5? Horray. Pick the one with the highest score. word = None for (s, score, occ) in scores: if occ < 5: @@ -182,6 +194,8 @@ def compute_huffman_coding(translations, compression_filename): word = s break + # If we can successfully add it to the dictionary, do so. Otherwise, + # we've filled the dictionary to capacity and are done. if not word: break if sum_len + len(word) - 2 > max_words_len: -- cgit v1.2.3 From bfbbbd6c5ce58ba1bcb57323566209902e1000b0 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 19 Sep 2020 10:16:13 -0500 Subject: makeqstrdata: Work with older Python This construct (which I added without sufficient testing, apparently) is only supported in Python 3.7 and newer. Make it optional so that this script works on other Python versions. This means that if you have a system with non-UTF-8 encoding you will need to use Python 3.7. In particular, this affects a problem building circuitpython in github's ubuntu-18.04 virtual environment when Python 3.7 is not explicitly installed. cookie-cuttered libraries call for Python 3.6: ``` - name: Set up Python 3.6 uses: actions/setup-python@v1 with: python-version: 3.6 ``` Since CircuitPython's own build calls for 3.8, this problem was not detected. This problem was also encountered by discord user mdroberts1243. The failure I encountered was here: https://github.com/jepler/Jepler_CircuitPython_udecimal/runs/1138045020?check_suite_focus=true .. while my step of "clone and build circuitpython unix port" is unusual, I think the same problem would have affected "build assets" if that step had been reached. --- py/makeqstrdata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'py/makeqstrdata.py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 39d4a6840..96e3956b4 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -17,8 +17,9 @@ import collections import gettext import os.path -sys.stdout.reconfigure(encoding='utf-8') -sys.stderr.reconfigure(errors='backslashreplace') +if hasattr(sys.stdout, 'reconfigure'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(errors='backslashreplace') py = os.path.dirname(sys.argv[0]) top = os.path.dirname(py) -- cgit v1.2.3 From 0318eb359fbeb91c6b37ed2050e57711ec2740bc Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Mon, 21 Sep 2020 10:02:27 -0500 Subject: makeqstrdata: Work around python3.6 compatibility problem Discord user Folknology encountered a problem building with Python 3.6.9, `TypeError: ord() expected a character, but string of length 0 found`. I was able to reproduce the problem using Python3.5*, and discovered that the meaning of the regular expression `"|."` had changed in 3.7. Before, ``` >>> [m.group(0) for m in re.finditer("|.", "hello")] ['', '', '', '', '', ''] ``` After: ``` >>> [m.group(0) for m in re.finditer("|.", "hello")] ['', 'h', '', 'e', '', 'l', '', 'l', '', 'o', ''] ``` Check if `words` is empty and if so use `"."` as the regular expression instead. This gives the same result on both versions: ``` ['h', 'e', 'l', 'l', 'o'] ``` and fixes the generation of the huffman dictionary. Folknology verified that this fix worked for them. * I could easily install 3.5 but not 3.6. 3.5 reproduced the same problem --- py/makeqstrdata.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'py/makeqstrdata.py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 96e3956b4..b4f4f1b03 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -109,7 +109,11 @@ class TextSplitter: def __init__(self, words): words.sort(key=lambda x: len(x), reverse=True) self.words = set(words) - self.pat = re.compile("|".join(re.escape(w) for w in words) + "|.", flags=re.DOTALL) + if words: + pat = "|".join(re.escape(w) for w in words) + "|." + else: + pat = "." + self.pat = re.compile(pat, flags=re.DOTALL) def iter_words(self, text): s = [] -- cgit v1.2.3