Merge pull request #2407 from dhalbert/4.1.x-increase-cpx-stack-update-frozen4.1.2

4.1.x increase cpx stack update frozen

19 files changed, 77 insertions, 37 deletions

diff --git a/frozen/Adafruit_CircuitPython_BusDevice b/frozen/Adafruit_CircuitPython_BusDevice
-Subproject b9280af5142fc41639229544678e23b5cca07c3
+Subproject 2000ae3a7c5d60b850c9546a16425aee279e2a3
diff --git a/frozen/Adafruit_CircuitPython_CircuitPlayground b/frozen/Adafruit_CircuitPython_CircuitPlayground
-Subproject 154b74de020764597ba49f0d1e8cc18d55b3643
+Subproject 82ba9e40dfff41fdc0541636afde4936c930d86
diff --git a/frozen/Adafruit_CircuitPython_Crickit b/frozen/Adafruit_CircuitPython_Crickit
-Subproject 617bb0787f2c61283d248632a62b27be80f64b2
+Subproject 5534662902a223ac8562e6f999d6359e4c17dab
diff --git a/frozen/Adafruit_CircuitPython_DotStar b/frozen/Adafruit_CircuitPython_DotStar
-Subproject 409e90902ac49720c4add985e8e1a1660bbe63a
+Subproject 01e89a8437c78b62d4d655c745ded57e26dc747
diff --git a/frozen/Adafruit_CircuitPython_HID b/frozen/Adafruit_CircuitPython_HID
-Subproject 89faee0eb08a6855e14f117c514fecf2dd90769
+Subproject a23b80569f23ef109667dd8c595d319e8a30d62
diff --git a/frozen/Adafruit_CircuitPython_IRRemote b/frozen/Adafruit_CircuitPython_IRRemote
-Subproject 70865ac6e09f821b26ec727e2df300a6d9ebf6b
+Subproject 8b7611a2cc076a2ac1b368c70227519f69f1e3e
diff --git a/frozen/Adafruit_CircuitPython_LIS3DH b/frozen/Adafruit_CircuitPython_LIS3DH
-Subproject bd7ddc67dc86f7ad0115f58ab80d5605739c648
+Subproject 53146ab2e82c318c3c37bd76bac34035a597b31
diff --git a/frozen/Adafruit_CircuitPython_Motor b/frozen/Adafruit_CircuitPython_Motor
-Subproject ddc74844983b35b027bd45091c7b8bb3c8d7a2d
+Subproject f69fc9b47fa25ba1414eb3d5c82f05013280c0d
diff --git a/frozen/Adafruit_CircuitPython_NeoPixel b/frozen/Adafruit_CircuitPython_NeoPixel
-Subproject c0bdd8b10383725ee9293f5d88fb8d47eb1272b
+Subproject ff99d55115f81899902c2c4a84fdfbea9ae8382
diff --git a/frozen/Adafruit_CircuitPython_Thermistor b/frozen/Adafruit_CircuitPython_Thermistor
-Subproject 893c5ec6a9aeef38284985074c2058e87754ad3
+Subproject 2e5aedf18eb417a4120d4998ac1f387a4f60073
diff --git a/frozen/Adafruit_CircuitPython_seesaw b/frozen/Adafruit_CircuitPython_seesaw
-Subproject f1171f94083ba64d153ff3f90eeb07500331d6e
+Subproject ea5e445edd4441cacd207aa2d2bfd724b813a25
diff --git a/frozen/circuitpython-stage b/frozen/circuitpython-stage
-Subproject 6d1ae72916cf240ea86185c45f844d59f56d8ec
+Subproject c1d8e1d645cbc83d857e12cf4ba67549b988a4e
diff --git a/ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.h b/ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.h
index e6a7e0676..9ec1ea607 100644
--- a/ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.h
+++ b/ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.h
@@ -33,6 +33,9 @@
 // Explanation of how a user got into safe mode.
 #define BOARD_USER_SAFE_MODE_ACTION "pressing both buttons at start up"
 
+// Increase stack size slightly due to CPX library import nesting
+#define CIRCUITPY_DEFAULT_STACK_SIZE  (4632) // Must be divisible by 8 or will HardFault.
+
 #define DEFAULT_I2C_BUS_SCL (&pin_PB03)
 #define DEFAULT_I2C_BUS_SDA (&pin_PB02)
 
diff --git a/ports/atmel-samd/boards/circuitplayground_express_crickit/mpconfigboard.h b/ports/atmel-samd/boards/circuitplayground_express_crickit/mpconfigboard.h
index ca6f9b773..e4dcc40d9 100644
--- a/ports/atmel-samd/boards/circuitplayground_express_crickit/mpconfigboard.h
+++ b/ports/atmel-samd/boards/circuitplayground_express_crickit/mpconfigboard.h
@@ -33,6 +33,9 @@
 // Explanation of how a user got into safe mode.
 #define BOARD_USER_SAFE_MODE_ACTION "pressing both buttons at start up"
 
+// Increase stack size slightly due to CPX library import nesting
+#define CIRCUITPY_DEFAULT_STACK_SIZE  (4632) // Must be divisible by 8 or will HardFault.
+
 #define DEFAULT_I2C_BUS_SCL (&pin_PB03)
 #define DEFAULT_I2C_BUS_SDA (&pin_PB02)
 
diff --git a/ports/atmel-samd/mpconfigport.h b/ports/atmel-samd/mpconfigport.h
index 4e15a6c30..d416b2612 100644
--- a/ports/atmel-samd/mpconfigport.h
+++ b/ports/atmel-samd/mpconfigport.h
@@ -34,7 +34,6 @@
 #define CIRCUITPY_MCU_FAMILY                        samd21
 #define MICROPY_PY_SYS_PLATFORM                     "Atmel SAMD21"
 #define SPI_FLASH_MAX_BAUDRATE 8000000
-#define CIRCUITPY_DEFAULT_STACK_SIZE                4096
 #define MICROPY_PY_BUILTINS_NOTIMPLEMENTED          (0)
 #define MICROPY_PY_COLLECTIONS_ORDEREDDICT          (0)
 #define MICROPY_PY_FUNCTION_ATTRS                   (0)
@@ -69,6 +68,7 @@
 #define MICROPY_PY_UJSON                            (1)
 #define MICROPY_PY_REVERSE_SPECIAL_METHODS          (1)
 //      MICROPY_PY_UERRNO_LIST - Use the default
+
 #endif
 
 // Turning off audioio, audiobusio, and touchio as necessary
@@ -78,6 +78,19 @@
 
 #include "py/circuitpy_mpconfig.h"
 
+
+#ifdef SAMD21
+#ifndef CIRCUITPY_DEFAULT_STACK_SIZE
+#define CIRCUITPY_DEFAULT_STACK_SIZE                4096
+#endif
+#endif
+
+#ifdef SAMD51
+#ifndef CIRCUITPY_DEFAULT_STACK_SIZE
+#define CIRCUITPY_DEFAULT_STACK_SIZE                (24*1024)
+#endif
+#endif
+
 #define MICROPY_PORT_ROOT_POINTERS \
     CIRCUITPY_COMMON_ROOT_POINTERS \
     mp_obj_t playing_audio[AUDIO_DMA_CHANNEL_COUNT];
diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py
index 5b5ec1c37..0d667959d 100644
--- a/py/makeqstrdata.py
+++ b/py/makeqstrdata.py
@@ -103,14 +103,10 @@ def compute_huffman_coding(translations, qstrs, compression_filename):
     # go through each qstr and print it out
     for _, _, qstr in qstrs.values():
         all_strings.append(qstr)
-    all_strings_concat = "".join(all_strings).encode("utf-8")
+    all_strings_concat = "".join(all_strings)
     counts = collections.Counter(all_strings_concat)
-    # add other values
-    for i in range(256):
-        if i not in counts:
-            counts[i] = 0
     cb = huffman.codebook(counts.items())
-    values = bytearray()
+    values = []
     length_count = {}
     renumbered = 0
     last_l = None
@@ -124,26 +120,27 @@ def compute_huffman_coding(translations, qstrs, compression_filename):
         if last_l:
             renumbered <<= (l - last_l)
         canonical[ch] = '{0:0{width}b}'.format(renumbered, width=l)
-        if chr(ch) in C_ESCAPES:
-            s = C_ESCAPES[chr(ch)]
-        else:
-            s = chr(ch)
-        print("//", ch, s, counts[ch], canonical[ch], renumbered)
+        s = C_ESCAPES.get(ch, ch)
+        print("//", ord(ch), s, counts[ch], canonical[ch], renumbered)
         renumbered += 1
         last_l = l
     lengths = bytearray()
-    for i in range(1, max(length_count) + 1):
+    print("// length count", length_count)
+    for i in range(1, max(length_count) + 2):
         lengths.append(length_count.get(i, 0))
+    print("// values", values, "lengths", len(lengths), lengths)
+    print("// estimated total memory size", len(lengths) + 2*len(values) + sum(len(cb[u]) for u in all_strings_concat))
     print("//", values, lengths)
+    values_type = "uint16_t" if max(ord(u) for u in values) > 255 else "uint8_t"
     with open(compression_filename, "w") as f:
         f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths))))
-        f.write("const uint8_t values[256] = {{ {} }};\n".format(", ".join(map(str, values))))
+        f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values)))
     return values, lengths
 
 def decompress(encoding_table, length, encoded):
     values, lengths = encoding_table
     #print(l, encoded)
-    dec = bytearray(length)
+    dec = []
     this_byte = 0
     this_bit = 7
     b = encoded[this_byte]
@@ -173,14 +170,14 @@ def decompress(encoding_table, length, encoded):
             searched_length += lengths[bit_length]
 
         v = values[searched_length + bits - max_code]
-        dec[i] = v
-    return dec
+        dec.append(v)
+    return ''.join(dec)
 
 def compress(encoding_table, decompressed):
-    if not isinstance(decompressed, bytes):
+    if not isinstance(decompressed, str):
         raise TypeError()
     values, lengths = encoding_table
-    enc = bytearray(len(decompressed))
+    enc = bytearray(len(decompressed) * 3)
     #print(decompressed)
     #print(lengths)
     current_bit = 7
@@ -227,6 +224,8 @@ def compress(encoding_table, decompressed):
                 current_bit -= 1
     if current_bit != 7:
         current_byte += 1
+    if current_byte > len(decompressed):
+        print("Note: compression increased length", repr(decompressed), len(decompressed), current_byte, file=sys.stderr)
     return enc[:current_byte]
 
 def qstr_escape(qst):
@@ -345,9 +344,9 @@ def print_qstr_data(encoding_table, qcfgs, qstrs, i18ns):
     total_text_compressed_size = 0
     for original, translation in i18ns:
         translation_encoded = translation.encode("utf-8")
-        compressed = compress(encoding_table, translation_encoded)
+        compressed = compress(encoding_table, translation)
         total_text_compressed_size += len(compressed)
-        decompressed = decompress(encoding_table, len(translation_encoded), compressed).decode("utf-8")
+        decompressed = decompress(encoding_table, len(translation_encoded), compressed)
         for c in C_ESCAPES:
             decompressed = decompressed.replace(c, C_ESCAPES[c])
         print("TRANSLATION(\"{}\", {}, {{ {} }}) // {}".format(original, len(translation_encoded)+1, ", ".join(["0x{:02x}".format(x) for x in compressed]), decompressed))
diff --git a/supervisor/shared/rgb_led_status.c b/supervisor/shared/rgb_led_status.c
index 0e69cd2e0..0e9c41394 100644
--- a/supervisor/shared/rgb_led_status.c
+++ b/supervisor/shared/rgb_led_status.c
@@ -39,7 +39,10 @@ static digitalio_digitalinout_obj_t status_neopixel;
 
 #if defined(MICROPY_HW_APA102_MOSI) && defined(MICROPY_HW_APA102_SCK)
 uint8_t rgb_status_brightness = 255;
-static uint8_t status_apa102_color[12] = {0, 0, 0, 0, 0xff, 0, 0, 0};
+
+#define APA102_BUFFER_LENGTH 12
+static uint8_t status_apa102_color[APA102_BUFFER_LENGTH] = {0, 0, 0, 0, 0xff, 0, 0, 0, 0xff, 0xff, 0xff, 0xff};
+
 #ifdef CIRCUITPY_BITBANG_APA102
 #include "shared-bindings/bitbangio/SPI.h"
 #include "shared-module/bitbangio/types.h"
@@ -104,10 +107,12 @@ void rgb_led_status_init() {
         apa102_sck_in_use = false;
         #ifdef CIRCUITPY_BITBANG_APA102
         shared_module_bitbangio_spi_try_lock(&status_apa102);
-        shared_module_bitbangio_spi_configure(&status_apa102, 100000, 0, 1, 8);
+        // Use 1MHz for clock rate. Some APA102's are spec'd 800kHz-1200kHz,
+        // though many can run much faster. bitbang will probably run slower.
+        shared_module_bitbangio_spi_configure(&status_apa102, 1000000, 0, 0, 8);
         #else
         common_hal_busio_spi_try_lock(&status_apa102);
-        common_hal_busio_spi_configure(&status_apa102, 100000, 0, 1, 8);
+        common_hal_busio_spi_configure(&status_apa102, 1000000, 0, 0, 8);
         #endif
     #endif
 
@@ -120,7 +125,7 @@ void rgb_led_status_init() {
             common_hal_pulseio_pwmout_never_reset(&rgb_status_r);
         }
     }
-    
+
     if (common_hal_mcu_pin_is_free(CP_RGB_STATUS_G)) {
         pwmout_result_t green_result = common_hal_pulseio_pwmout_construct(&rgb_status_g, CP_RGB_STATUS_G, 0, 50000, false);
 
@@ -186,9 +191,9 @@ void new_status_color(uint32_t rgb) {
         status_apa102_color[7] = (rgb_adjusted >> 16) & 0xff;
 
         #ifdef CIRCUITPY_BITBANG_APA102
-        shared_module_bitbangio_spi_write(&status_apa102, status_apa102_color, 8);
+        shared_module_bitbangio_spi_write(&status_apa102, status_apa102_color, APA102_BUFFER_LENGTH);
         #else
-        common_hal_busio_spi_write(&status_apa102, status_apa102_color, 8);
+        common_hal_busio_spi_write(&status_apa102, status_apa102_color, APA102_BUFFER_LENGTH);
         #endif
     #endif
 
@@ -229,20 +234,20 @@ void temp_status_color(uint32_t rgb) {
         if (apa102_mosi_in_use || apa102_sck_in_use) {
             return;
         }
-        uint8_t colors[12] = {0, 0, 0, 0, 0xff, rgb_adjusted & 0xff, (rgb_adjusted >> 8) & 0xff, (rgb_adjusted >> 16) & 0xff, 0x0, 0x0, 0x0, 0x0};
+        uint8_t colors[APA102_BUFFER_LENGTH] = {0, 0, 0, 0, 0xff, rgb_adjusted & 0xff, (rgb_adjusted >> 8) & 0xff, (rgb_adjusted >> 16) & 0xff, 0xff, 0xff, 0xff, 0xff};
         #ifdef CIRCUITPY_BITBANG_APA102
-        shared_module_bitbangio_spi_write(&status_apa102, colors, 12);
+        shared_module_bitbangio_spi_write(&status_apa102, colors, APA102_BUFFER_LENGTH);
         #else
-        common_hal_busio_spi_write(&status_apa102, colors, 12);
+        common_hal_busio_spi_write(&status_apa102, colors, APA102_BUFFER_LENGTH);
         #endif
     #endif
     #if defined(CP_RGB_STATUS_LED)
         uint8_t red_u8 = (rgb_adjusted >> 16) & 0xFF;
         uint8_t green_u8 = (rgb_adjusted >> 8) & 0xFF;
         uint8_t blue_u8 = rgb_adjusted & 0xFF;
- 
+
         uint16_t temp_status_color_rgb[3] = {0};
-		
+
 	#if defined(CP_RGB_STATUS_INVERTED_PWM)
         temp_status_color_rgb[0] = (1 << 16) - 1 - ((uint16_t) (red_u8 << 8) + red_u8);
         temp_status_color_rgb[1] = (1 << 16) - 1 - ((uint16_t) (green_u8 << 8) + green_u8);
@@ -265,9 +270,9 @@ void clear_temp_status() {
     #endif
     #if defined(MICROPY_HW_APA102_MOSI) && defined(MICROPY_HW_APA102_SCK)
         #ifdef CIRCUITPY_BITBANG_APA102
-        shared_module_bitbangio_spi_write(&status_apa102, status_apa102_color, 8);
+        shared_module_bitbangio_spi_write(&status_apa102, status_apa102_color, APA102_BUFFER_LENGTH);
         #else
-        common_hal_busio_spi_write(&status_apa102, status_apa102_color, 8);
+        common_hal_busio_spi_write(&status_apa102, status_apa102_color, APA102_BUFFER_LENGTH);
         #endif
     #endif
     #if defined(CP_RGB_STATUS_LED)
diff --git a/supervisor/shared/translate.c b/supervisor/shared/translate.c
index aa5e4517c..187d5ff8a 100644
--- a/supervisor/shared/translate.c
+++ b/supervisor/shared/translate.c
@@ -42,12 +42,28 @@ void serial_write_compressed(const compressed_string_t* compressed) {
     serial_write(decompressed);
 }
 
+STATIC int put_utf8(char *buf, int u) {
+    if(u <= 0x7f) {
+        *buf = u;
+        return 1;
+    } else if(u <= 0x07ff) {
+        *buf++ = 0b11000000 | (u >> 6);
+        *buf   = 0b10000000 | (u & 0b00111111);
+        return 2;
+    } else { // u <= 0xffff)
+        *buf++ = 0b11000000 | (u >> 12);
+        *buf   = 0b10000000 | ((u >> 6) & 0b00111111);
+        *buf   = 0b10000000 | (u & 0b00111111);
+        return 3;
+    }
+}
+
 char* decompress(const compressed_string_t* compressed, char* decompressed) {
     uint8_t this_byte = 0;
     uint8_t this_bit = 7;
     uint8_t b = compressed->data[this_byte];
     // Stop one early because the last byte is always NULL.
-    for (uint16_t i = 0; i < compressed->length - 1; i++) {
+    for (uint16_t i = 0; i < compressed->length - 1;) {
         uint32_t bits = 0;
         uint8_t bit_length = 0;
         uint32_t max_code = lengths[0];
@@ -72,7 +88,7 @@ char* decompress(const compressed_string_t* compressed, char* decompressed) {
             max_code = (max_code << 1) + lengths[bit_length];
             searched_length += lengths[bit_length];
         }
-        decompressed[i] = values[searched_length + bits - max_code];
+        i += put_utf8(decompressed + i, values[searched_length + bits - max_code]);
     }
 
     decompressed[compressed->length-1] = '\0';
diff --git a/tools/build_memory_info.py b/tools/build_memory_info.py
index a8f84bbb6..f3242e2aa 100644
--- a/tools/build_memory_info.py
+++ b/tools/build_memory_info.py
@@ -61,10 +61,11 @@ for region in regions:
     space = M_PATTERN.sub(M_REPLACE, space)
     regions[region] = eval(space)
 
+ram_region = regions["RAM"]
 free_flash = regions["FLASH"] - text - data
 free_ram = regions["RAM"] - data - bss
 print(free_flash, "bytes free in flash out of", regions["FLASH"], "bytes (", regions["FLASH"] / 1024, "kb ).")
-print(free_ram, "bytes free in ram for stack out of", regions["RAM"], "bytes (", regions["RAM"] / 1024, "kb ).")
+print("{} bytes free in ram for stack and heap out of {} bytes ({}kB).".format(free_ram, ram_region, ram_region / 1024))
 print()
 
 # Check that we have free flash space. GCC doesn't fail when the text + data