From e06a3bbceb548432969bfcfa1076dc4363478151 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Mon, 2 Dec 2019 08:41:03 -0600 Subject: translation: Compress as unicode, not bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By treating each unicode code-point as a single entity for huffman compression, the overall compression rate can be somewhat improved without changing the algorithm. On the decompression side, when compressed values above 127 are encountered, they need to be converted from a 16-bit Unicode code point into a UTF-8 byte sequence. Doing this returns approximately 1.5kB of flash storage with the zh_Latn_pinyin translation. (292 -> 1768 bytes remaining in my build of trinket_m0) Other "more ASCII" translations benefit less, and in fact zh_Latn_pinyin is no longer the most constrained translation! (de_DE 1156 -> 1384 bytes free in flash, I didn't check others before pushing for CI) English is slightly pessimized, 2840 -> 2788 bytes, probably mostly because the "values" array was changed from uint8_t to uint16_t, which is strictly not required for an all-ASCII translation. This could probably be avoided in this case, but as English is not the most constrained translation it doesn't really matter. Testing performed: built for feather nRF52840 express and trinket m0 in English and zh_Latn_pinyin; ran and verified the localized messages such as Àn xià rènhé jiàn jìnrù REPL. Shǐyòng CTRL-D chóngxīn jiāzài. and Press any key to enter the REPL. Use CTRL-D to reload. were properly displayed. --- supervisor/shared/translate.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) (limited to 'supervisor') diff --git a/supervisor/shared/translate.c b/supervisor/shared/translate.c index aa5e4517c..187d5ff8a 100644 --- a/supervisor/shared/translate.c +++ b/supervisor/shared/translate.c @@ -42,12 +42,28 @@ void serial_write_compressed(const compressed_string_t* compressed) { serial_write(decompressed); } +STATIC int put_utf8(char *buf, int u) { + if(u <= 0x7f) { + *buf = u; + return 1; + } else if(u <= 0x07ff) { + *buf++ = 0b11000000 | (u >> 6); + *buf = 0b10000000 | (u & 0b00111111); + return 2; + } else { // u <= 0xffff) + *buf++ = 0b11000000 | (u >> 12); + *buf = 0b10000000 | ((u >> 6) & 0b00111111); + *buf = 0b10000000 | (u & 0b00111111); + return 3; + } +} + char* decompress(const compressed_string_t* compressed, char* decompressed) { uint8_t this_byte = 0; uint8_t this_bit = 7; uint8_t b = compressed->data[this_byte]; // Stop one early because the last byte is always NULL. - for (uint16_t i = 0; i < compressed->length - 1; i++) { + for (uint16_t i = 0; i < compressed->length - 1;) { uint32_t bits = 0; uint8_t bit_length = 0; uint32_t max_code = lengths[0]; @@ -72,7 +88,7 @@ char* decompress(const compressed_string_t* compressed, char* decompressed) { max_code = (max_code << 1) + lengths[bit_length]; searched_length += lengths[bit_length]; } - decompressed[i] = values[searched_length + bits - max_code]; + i += put_utf8(decompressed + i, values[searched_length + bits - max_code]); } decompressed[compressed->length-1] = '\0'; -- cgit v1.2.3