From 08ed09acc6c73bae5475735e6e39b10ccd951d36 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 16 Aug 2020 20:38:05 -0500 Subject: makeqstrdata: don't print "compression incrased length" messages This check as implemented is misleading, because it compares the compressed size in bytes (including the length indication) with the source string length in Unicode code points. For English this is approximately fair, but for Japanese this is quite unfair and produces an excess of "increased length" messages. This message might have existed for one of two reasons: * to alert to an improperly function huffman compression * to call attention to a need for a "string is stored uncompressed" case We know by now that the huffman compression is functioning as designed and effective in general. Just to be on the safe side, I did some back-of-the-envelope estimates. I considered these three replacements for "the true source string size, in bytes": + decompressed_len_utf8 = len(decompressed.encode('utf-8')) + decompressed_len_utf16 = len(decompressed.encode('utf-16be')) + decompressed_len_bitsize = ((1+len(decompressed)) * math.ceil(math.log(1+len(values), 2)) + 7) // 8 The third counts how many bits each character requires (fewer than 128 characters in the source character set = 7, fewer than 256 = 8, fewer than 512 = 9, etc, adding a string-terminating value) and is in some way representative of the best way we would be able to store "uncompressed strings". The Japanese translation (largest as of writing) has just a few strings which increase by this metric. However, the amount of loss due to expansion in those cases is outweighed by the cost of adding 1 bit per string to indicate whether it's compressed or not. For instance, in the BOARD=trinket_m0 TRANSLATION=ja build the loss is 47 bytes over 300 strings. Adding 1 bit to each of 300 strings will cost about 37 bytes, leaving just 5 Thumb instructions to implement the code to check and decode "uncompressed" strings in order to break even. --- py/makeqstrdata.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'py/makeqstrdata.py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index df2c687e5..6132298a5 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -259,8 +259,6 @@ def compress(encoding_table, decompressed, encoded_length_bits, len_translation_ current_bit -= 1 if current_bit != 7: current_byte += 1 - if current_byte > len(decompressed): - print("Note: compression increased length", repr(decompressed), len(decompressed), current_byte, file=sys.stderr) return enc[:current_byte] def qstr_escape(qst): -- cgit v1.2.3 From 79a3796b1cd74b376e091e63c1b3a2426ec3187c Mon Sep 17 00:00:00 2001 From: Taku Fukada Date: Tue, 18 Aug 2020 23:21:14 +0900 Subject: Calculate the Huffman codebook without MP_QSTRs --- py/makeqstrdata.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'py/makeqstrdata.py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 6132298a5..04c893876 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -102,10 +102,6 @@ def translate(translation_file, i18ns): def compute_huffman_coding(translations, qstrs, compression_filename): all_strings = [x[1] for x in translations] - - # go through each qstr and print it out - for _, _, qstr in qstrs.values(): - all_strings.append(qstr) all_strings_concat = "".join(all_strings) counts = collections.Counter(all_strings_concat) cb = huffman.codebook(counts.items()) -- cgit v1.2.3