Merge pull request #1 from adafruit/main

merge from adafruit

11 files changed, 195 insertions, 105 deletions

diff --git a/supervisor/shared/background_callback.c b/supervisor/shared/background_callback.c
index 8e12dd362..ef686cbab 100644
--- a/supervisor/shared/background_callback.c
+++ b/supervisor/shared/background_callback.c
@@ -29,6 +29,7 @@
 #include "py/gc.h"
 #include "py/mpconfig.h"
 #include "supervisor/background_callback.h"
+#include "supervisor/linker.h"
 #include "supervisor/shared/tick.h"
 #include "shared-bindings/microcontroller/__init__.h"
 
@@ -63,7 +64,7 @@ void background_callback_add(background_callback_t *cb, background_callback_fun
 }
 
 static bool in_background_callback;
-void background_callback_run_all() {
+void PLACE_IN_ITCM(background_callback_run_all)() {
     if (!callback_head) {
         return;
     }
diff --git a/supervisor/shared/filesystem.c b/supervisor/shared/filesystem.c
index f6b94e38b..88603be0c 100644
--- a/supervisor/shared/filesystem.c
+++ b/supervisor/shared/filesystem.c
@@ -105,13 +105,19 @@ void filesystem_init(bool create_allowed, bool force_create) {
 
         // set label
 #ifdef CIRCUITPY_DRIVE_LABEL
-        f_setlabel(&vfs_fat->fatfs, CIRCUITPY_DRIVE_LABEL);
+        res = f_setlabel(&vfs_fat->fatfs, CIRCUITPY_DRIVE_LABEL);
 #else
-        f_setlabel(&vfs_fat->fatfs, "CIRCUITPY");
+        res = f_setlabel(&vfs_fat->fatfs, "CIRCUITPY");
 #endif
+        if (res != FR_OK) {
+            return;
+        }
 
         // inhibit file indexing on MacOS
-        f_mkdir(&vfs_fat->fatfs, "/.fseventsd");
+        res = f_mkdir(&vfs_fat->fatfs, "/.fseventsd");
+        if (res != FR_OK) {
+            return;
+        }
         make_empty_file(&vfs_fat->fatfs, "/.metadata_never_index");
         make_empty_file(&vfs_fat->fatfs, "/.Trashes");
         make_empty_file(&vfs_fat->fatfs, "/.fseventsd/no_log");
@@ -119,7 +125,10 @@ void filesystem_init(bool create_allowed, bool force_create) {
         make_sample_code_file(&vfs_fat->fatfs);
 
         // create empty lib directory
-        f_mkdir(&vfs_fat->fatfs, "/lib");
+        res = f_mkdir(&vfs_fat->fatfs, "/lib");
+        if (res != FR_OK) {
+            return;
+        }
 
         // and ensure everything is flushed
         supervisor_flash_flush();
diff --git a/supervisor/shared/flash.c b/supervisor/shared/flash.c
index 9f52ddbc4..1e09fe14b 100644
--- a/supervisor/shared/flash.c
+++ b/supervisor/shared/flash.c
@@ -88,9 +88,6 @@ static void build_partition(uint8_t *buf, int boot, int type, uint32_t start_blo
 
 mp_uint_t flash_read_blocks(uint8_t *dest, uint32_t block_num, uint32_t num_blocks) {
     if (block_num == 0) {
-        if (num_blocks > 1) {
-            return 1; // error
-        }
         // fake the MBR so we can decide on our own partition table
 
         for (int i = 0; i < 446; i++) {
@@ -104,9 +101,13 @@ mp_uint_t flash_read_blocks(uint8_t *dest, uint32_t block_num, uint32_t num_bloc
 
         dest[510] = 0x55;
         dest[511] = 0xaa;
-
-        return 0; // ok
-
+        if (num_blocks > 1) {
+            dest += 512;
+            num_blocks -= 1;
+            // Fall through and do a read from flash.
+        } else {
+            return 0; // Done and ok.
+        }
     }
     return supervisor_flash_read_blocks(dest, block_num - PART1_START_BLOCK, num_blocks);
 }
@@ -159,16 +160,37 @@ STATIC mp_obj_t supervisor_flash_obj_writeblocks(mp_obj_t self, mp_obj_t block_n
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_3(supervisor_flash_obj_writeblocks_obj, supervisor_flash_obj_writeblocks);
 
+bool flash_ioctl(size_t cmd, mp_int_t* out_value) {
+    *out_value = 0;
+    switch (cmd) {
+        case BP_IOCTL_INIT:
+            supervisor_flash_init();
+            break;
+        case BP_IOCTL_DEINIT:
+            supervisor_flash_flush();
+            break; // TODO properly
+        case BP_IOCTL_SYNC:
+            supervisor_flash_flush();
+            break;
+        case BP_IOCTL_SEC_COUNT:
+            *out_value = flash_get_block_count();
+            break;
+        case BP_IOCTL_SEC_SIZE:
+            *out_value = supervisor_flash_get_block_size();
+            break;
+        default:
+            return false;
+    }
+    return true;
+}
+
 STATIC mp_obj_t supervisor_flash_obj_ioctl(mp_obj_t self, mp_obj_t cmd_in, mp_obj_t arg_in) {
     mp_int_t cmd = mp_obj_get_int(cmd_in);
-    switch (cmd) {
-        case BP_IOCTL_INIT: supervisor_flash_init(); return MP_OBJ_NEW_SMALL_INT(0);
-        case BP_IOCTL_DEINIT: supervisor_flash_flush(); return MP_OBJ_NEW_SMALL_INT(0); // TODO properly
-        case BP_IOCTL_SYNC: supervisor_flash_flush(); return MP_OBJ_NEW_SMALL_INT(0);
-        case BP_IOCTL_SEC_COUNT: return MP_OBJ_NEW_SMALL_INT(flash_get_block_count());
-        case BP_IOCTL_SEC_SIZE: return MP_OBJ_NEW_SMALL_INT(supervisor_flash_get_block_size());
-        default: return mp_const_none;
+    mp_int_t out_value;
+    if (flash_ioctl(cmd, &out_value)) {
+        return MP_OBJ_NEW_SMALL_INT(out_value);
     }
+    return mp_const_none;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_3(supervisor_flash_obj_ioctl_obj, supervisor_flash_obj_ioctl);
 
@@ -200,4 +222,5 @@ void supervisor_flash_init_vfs(fs_user_mount_t *vfs) {
     vfs->writeblocks[2] = (mp_obj_t)flash_write_blocks; // native version
     vfs->u.ioctl[0] = (mp_obj_t)&supervisor_flash_obj_ioctl_obj;
     vfs->u.ioctl[1] = (mp_obj_t)&supervisor_flash_obj;
+    vfs->u.ioctl[2] = (mp_obj_t)flash_ioctl; // native version
 }
diff --git a/supervisor/shared/memory.c b/supervisor/shared/memory.c
index 8ae8a1699..0f96ae273 100755
--- a/supervisor/shared/memory.c
+++ b/supervisor/shared/memory.c
@@ -33,6 +33,14 @@
 
 #define CIRCUITPY_SUPERVISOR_ALLOC_COUNT (12)
 
+// Using a zero length to mark an unused allocation makes the code a bit shorter (but makes it
+// impossible to support zero-length allocations).
+#define FREE 0
+
+// The lowest two bits of a valid length are always zero, so we can use them to mark an allocation
+// as freed by the client but not yet reclaimed into the FREE middle.
+#define HOLE 1
+
 static supervisor_allocation allocations[CIRCUITPY_SUPERVISOR_ALLOC_COUNT];
 // We use uint32_t* to ensure word (4 byte) alignment.
 uint32_t* low_address;
@@ -61,25 +69,31 @@ void free_memory(supervisor_allocation* allocation) {
     }
     if (allocation->ptr == high_address) {
         high_address += allocation->length / 4;
+        allocation->length = FREE;
         for (index++; index < CIRCUITPY_SUPERVISOR_ALLOC_COUNT; index++) {
-            if (allocations[index].ptr != NULL) {
+            if (!(allocations[index].length & HOLE)) {
                 break;
             }
+            // Division automatically shifts out the HOLE bit.
             high_address += allocations[index].length / 4;
+            allocations[index].length = FREE;
         }
     } else if (allocation->ptr + allocation->length / 4 == low_address) {
         low_address = allocation->ptr;
+        allocation->length = FREE;
         for (index--; index >= 0; index--) {
-            if (allocations[index].ptr != NULL) {
+            if (!(allocations[index].length & HOLE)) {
                 break;
             }
             low_address -= allocations[index].length / 4;
+            allocations[index].length = FREE;
         }
     } else {
         // Freed memory isn't in the middle so skip updating bounds. The memory will be added to the
-        // middle when the memory to the inside is freed.
+        // middle when the memory to the inside is freed. We still need its length, but setting
+        // only the lowest bit is nondestructive.
+        allocation->length |= HOLE;
     }
-    allocation->ptr = NULL;
 }
 
 supervisor_allocation* allocation_from_ptr(void *ptr) {
@@ -99,7 +113,7 @@ supervisor_allocation* allocate_remaining_memory(void) {
 }
 
 supervisor_allocation* allocate_memory(uint32_t length, bool high) {
-    if ((high_address - low_address) * 4 < (int32_t) length || length % 4 != 0) {
+    if (length == 0 || length % 4 != 0) {
         return NULL;
     }
     uint8_t index = 0;
@@ -108,15 +122,21 @@ supervisor_allocation* allocate_memory(uint32_t length, bool high) {
         index = CIRCUITPY_SUPERVISOR_ALLOC_COUNT - 1;
         direction = -1;
     }
+    supervisor_allocation* alloc;
     for (; index < CIRCUITPY_SUPERVISOR_ALLOC_COUNT; index += direction) {
-        if (allocations[index].ptr == NULL) {
+        alloc = &allocations[index];
+        if (alloc->length == FREE && (high_address - low_address) * 4 >= (int32_t) length) {
             break;
         }
+        // If a hole matches in length exactly, we can reuse it.
+        if (alloc->length == (length | HOLE)) {
+            alloc->length = length;
+            return alloc;
+        }
     }
     if (index >= CIRCUITPY_SUPERVISOR_ALLOC_COUNT) {
         return NULL;
     }
-    supervisor_allocation* alloc = &allocations[index];
     if (high) {
         high_address -= length / 4;
         alloc->ptr = high_address;
diff --git a/supervisor/shared/safe_mode.c b/supervisor/shared/safe_mode.c
index 29a0a6a4f..d59e754ed 100644
--- a/supervisor/shared/safe_mode.c
+++ b/supervisor/shared/safe_mode.c
@@ -60,6 +60,11 @@ safe_mode_t wait_for_safe_mode_reset(void) {
     common_hal_digitalio_digitalinout_construct(&status_led, MICROPY_HW_LED_STATUS);
     common_hal_digitalio_digitalinout_switch_to_output(&status_led, true, DRIVE_MODE_PUSH_PULL);
     #endif
+    #ifdef CIRCUITPY_BOOT_BUTTON
+    digitalio_digitalinout_obj_t boot_button;
+    common_hal_digitalio_digitalinout_construct(&boot_button, CIRCUITPY_BOOT_BUTTON);
+    common_hal_digitalio_digitalinout_switch_to_input(&boot_button, PULL_UP);
+    #endif
     uint64_t start_ticks = supervisor_ticks_ms64();
     uint64_t diff = 0;
     while (diff < 700) {
@@ -67,6 +72,11 @@ safe_mode_t wait_for_safe_mode_reset(void) {
         // Blink on for 100, off for 100, on for 100, off for 100 and on for 200
         common_hal_digitalio_digitalinout_set_value(&status_led, diff > 100 && diff / 100 != 2 && diff / 100 != 4);
         #endif
+        #ifdef CIRCUITPY_BOOT_BUTTON
+        if (!common_hal_digitalio_digitalinout_get_value(&boot_button)) {
+           return USER_SAFE_MODE;
+        }
+        #endif
         diff = supervisor_ticks_ms64() - start_ticks;
     }
     #ifdef MICROPY_HW_LED_STATUS
@@ -103,77 +113,79 @@ void print_safe_mode_message(safe_mode_t reason) {
         return;
     }
     serial_write("\n");
-    // Output a user safe mode string if it's set.
-    #ifdef BOARD_USER_SAFE_MODE
-    if (reason == USER_SAFE_MODE) {
-        serial_write_compressed(translate("You requested starting safe mode by "));
-        serial_write(BOARD_USER_SAFE_MODE_ACTION);
-        serial_write_compressed(translate("\nTo exit, please reset the board without "));
-        serial_write(BOARD_USER_SAFE_MODE_ACTION);
-        serial_write("\n");
-    } else
-    #endif
-        switch (reason) {
-            case MANUAL_SAFE_MODE:
-                serial_write_compressed(translate("CircuitPython is in safe mode because you pressed the reset button during boot. Press again to exit safe mode.\n"));
-                return;
-            case PROGRAMMATIC_SAFE_MODE:
-                serial_write_compressed(translate("The `microcontroller` module was used to boot into safe mode. Press reset to exit safe mode.\n"));
-                return;
-            default:
-                break;
-        }
 
-        serial_write_compressed(translate("You are in safe mode: something unanticipated happened.\n"));
-        switch (reason) {
-            case BROWNOUT:
-                serial_write_compressed(translate("The microcontroller's power dipped. Make sure your power supply provides\nenough power for the whole circuit and press reset (after ejecting CIRCUITPY).\n"));
-                return;
-            case HEAP_OVERWRITTEN:
-                serial_write_compressed(translate("The CircuitPython heap was corrupted because the stack was too small.\nPlease increase the stack size if you know how, or if not:"));
-                serial_write_compressed(FILE_AN_ISSUE);
-                return;
-            case NO_HEAP:
-                serial_write_compressed(translate("CircuitPython was unable to allocate the heap.\n"));
-                serial_write_compressed(FILE_AN_ISSUE);
-                return;
-            default:
+    switch (reason) {
+        case USER_SAFE_MODE:
+            #ifdef BOARD_USER_SAFE_MODE_ACTION
+                // Output a user safe mode string if it's set.
+                serial_write_compressed(translate("You requested starting safe mode by "));
+                serial_write_compressed(BOARD_USER_SAFE_MODE_ACTION);
+                serial_write_compressed(translate("To exit, please reset the board without "));
+                serial_write_compressed(BOARD_USER_SAFE_MODE_ACTION);
+            #else
                 break;
-        }
+            #endif
+            return;
+        case MANUAL_SAFE_MODE:
+            serial_write_compressed(translate("CircuitPython is in safe mode because you pressed the reset button during boot. Press again to exit safe mode.\n"));
+            return;
+        case PROGRAMMATIC_SAFE_MODE:
+            serial_write_compressed(translate("The `microcontroller` module was used to boot into safe mode. Press reset to exit safe mode.\n"));
+            return;
+        default:
+            break;
+    }
 
-        serial_write_compressed(translate("CircuitPython core code crashed hard. Whoops!\n"));
-        switch (reason) {
-            case HARD_CRASH:
-                serial_write_compressed(translate("Crash into the HardFault_Handler."));
-                return;
-            case MICROPY_NLR_JUMP_FAIL:
-                serial_write_compressed(translate("MicroPython NLR jump failed. Likely memory corruption."));
-                return;
-            case MICROPY_FATAL_ERROR:
-                serial_write_compressed(translate("MicroPython fatal error."));
-                break;
-            case GC_ALLOC_OUTSIDE_VM:
-                serial_write_compressed(translate("Attempted heap allocation when MicroPython VM not running."));
-                break;
-#ifdef SOFTDEVICE_PRESENT
-            // defined in ports/nrf/bluetooth/bluetooth_common.mk
-            // will print "Unknown reason" if somehow encountered on other ports
-            case NORDIC_SOFT_DEVICE_ASSERT:
-                serial_write_compressed(translate("Nordic Soft Device failure assertion."));
-                break;
-#endif
-            case FLASH_WRITE_FAIL:
-                serial_write_compressed(translate("Failed to write internal flash."));
-                break;
-            case MEM_MANAGE:
-                serial_write_compressed(translate("Invalid memory access."));
-                break;
-            case WATCHDOG_RESET:
-                serial_write_compressed(translate("Watchdog timer expired."));
-                break;
-            default:
-                serial_write_compressed(translate("Unknown reason."));
-                break;
-        }
-        serial_write_compressed(FILE_AN_ISSUE);
+    serial_write_compressed(translate("You are in safe mode: something unanticipated happened.\n"));
+    switch (reason) {
+        case BROWNOUT:
+            serial_write_compressed(translate("The microcontroller's power dipped. Make sure your power supply provides\nenough power for the whole circuit and press reset (after ejecting CIRCUITPY).\n"));
+            return;
+        case HEAP_OVERWRITTEN:
+            serial_write_compressed(translate("The CircuitPython heap was corrupted because the stack was too small.\nPlease increase the stack size if you know how, or if not:"));
+            serial_write_compressed(FILE_AN_ISSUE);
+            return;
+        case NO_HEAP:
+            serial_write_compressed(translate("CircuitPython was unable to allocate the heap.\n"));
+            serial_write_compressed(FILE_AN_ISSUE);
+            return;
+        default:
+            break;
+    }
+
+    serial_write_compressed(translate("CircuitPython core code crashed hard. Whoops!\n"));
+    switch (reason) {
+        case HARD_CRASH:
+            serial_write_compressed(translate("Crash into the HardFault_Handler."));
+            return;
+        case MICROPY_NLR_JUMP_FAIL:
+            serial_write_compressed(translate("MicroPython NLR jump failed. Likely memory corruption."));
+            return;
+        case MICROPY_FATAL_ERROR:
+            serial_write_compressed(translate("MicroPython fatal error."));
+            break;
+        case GC_ALLOC_OUTSIDE_VM:
+            serial_write_compressed(translate("Attempted heap allocation when MicroPython VM not running."));
+            break;
+        #ifdef SOFTDEVICE_PRESENT
+        // defined in ports/nrf/bluetooth/bluetooth_common.mk
+        // will print "Unknown reason" if somehow encountered on other ports
+        case NORDIC_SOFT_DEVICE_ASSERT:
+            serial_write_compressed(translate("Nordic Soft Device failure assertion."));
+            break;
+        #endif
+        case FLASH_WRITE_FAIL:
+            serial_write_compressed(translate("Failed to write internal flash."));
+            break;
+        case MEM_MANAGE:
+            serial_write_compressed(translate("Invalid memory access."));
+            break;
+        case WATCHDOG_RESET:
+            serial_write_compressed(translate("Watchdog timer expired."));
+            break;
+        default:
+            serial_write_compressed(translate("Unknown reason."));
+            break;
+    }
+    serial_write_compressed(FILE_AN_ISSUE);
 }
diff --git a/supervisor/shared/safe_mode.h b/supervisor/shared/safe_mode.h
index 7d3cd63b5..34fc3c8ae 100644
--- a/supervisor/shared/safe_mode.h
+++ b/supervisor/shared/safe_mode.h
@@ -27,6 +27,8 @@
 #ifndef MICROPY_INCLUDED_SUPERVISOR_SAFE_MODE_H
 #define MICROPY_INCLUDED_SUPERVISOR_SAFE_MODE_H
 
+#include "py/mpconfig.h"
+
 typedef enum {
   NO_SAFE_MODE = 0,
   BROWNOUT,
@@ -48,7 +50,7 @@ typedef enum {
 safe_mode_t wait_for_safe_mode_reset(void);
 
 void safe_mode_on_next_reset(safe_mode_t reason);
-void reset_into_safe_mode(safe_mode_t reason);
+void reset_into_safe_mode(safe_mode_t reason) NORETURN;
 
 void print_safe_mode_message(safe_mode_t reason);
 
diff --git a/supervisor/shared/serial.c b/supervisor/shared/serial.c
index 303f89e75..7383cc228 100644
--- a/supervisor/shared/serial.c
+++ b/supervisor/shared/serial.c
@@ -69,7 +69,9 @@ bool serial_connected(void) {
 #if defined(DEBUG_UART_TX) && defined(DEBUG_UART_RX)
     return true;
 #else
-    return tud_cdc_connected();
+    // True if DTR is asserted, and the USB connection is up.
+    // tud_cdc_get_line_state(): bit 0 is DTR, bit 1 is RTS
+    return (tud_cdc_get_line_state() & 1) && tud_ready();
 #endif
 }
 
diff --git a/supervisor/shared/tick.c b/supervisor/shared/tick.c
index 91ffa307d..a2855a570 100644
--- a/supervisor/shared/tick.c
+++ b/supervisor/shared/tick.c
@@ -125,9 +125,7 @@ void supervisor_tick(void) {
 
 uint64_t supervisor_ticks_ms64() {
     uint64_t result;
-    common_hal_mcu_disable_interrupts();
     result = port_get_raw_ticks(NULL);
-    common_hal_mcu_enable_interrupts();
     result = result * 1000 / 1024;
     return result;
 }
diff --git a/supervisor/shared/translate.c b/supervisor/shared/translate.c
index 5cd7b8dd8..44544c98d 100644
--- a/supervisor/shared/translate.c
+++ b/supervisor/shared/translate.c
@@ -47,13 +47,22 @@ STATIC int put_utf8(char *buf, int u) {
     if(u <= 0x7f) {
         *buf = u;
         return 1;
-    } else if(bigram_start <= u && u <= bigram_end) {
-        int n = (u - 0x80) * 2;
-        // (note that at present, entries in the bigrams table are
-        // guaranteed not to represent bigrams themselves, so this adds
+    } else if(word_start <= u && u <= word_end) {
+        uint n = (u - word_start);
+        size_t pos = 0;
+        if (n > 0) {
+            pos = wends[n - 1] + (n * 2);
+        }
+        int ret = 0;
+        // note that at present, entries in the words table are
+        // guaranteed not to represent words themselves, so this adds
         // at most 1 level of recursive call
-        int ret = put_utf8(buf, bigrams[n]);
-        return ret + put_utf8(buf + ret, bigrams[n+1]);
+        for(; pos < wends[n] + (n + 1) * 2; pos++) {
+            int len = put_utf8(buf, words[pos]);
+            buf += len;
+            ret += len;
+        }
+        return ret;
     } else if(u <= 0x07ff) {
         *buf++ = 0b11000000 | (u >> 6);
         *buf   = 0b10000000 | (u & 0b00111111);
diff --git a/supervisor/shared/translate.h b/supervisor/shared/translate.h
index 731b26d12..16296a416 100644
--- a/supervisor/shared/translate.h
+++ b/supervisor/shared/translate.h
@@ -43,6 +43,19 @@
 //   (building the huffman encoding on UTF-16 code points gave better
 //   compression than building it on UTF-8 bytes)
 //
+// - code points starting at 128 (word_start) and potentially extending
+//   to 255 (word_end) (but never interfering with the target
+//   language's used code points) stand for dictionary entries in a
+//   dictionary with size up to 256 code points.  The dictionary entries
+//   are computed with a heuristic based on frequent substrings of 2 to
+//   9 code points.  These are called "words" but are not, grammatically
+//   speaking, words.  They're just spans of code points that frequently
+//   occur together.
+//
+// - dictionary entries are non-overlapping, and the _ending_ index of each
+//   entry is stored in an array.  Since the index given is the ending
+//   index, the array is called "wends".
+//
 // The "data" / "tail" construct is so that the struct's last member is a
 // "flexible array".  However, the _only_ member is not permitted to be
 // a flexible member, so we have to declare the first byte as a separte
diff --git a/supervisor/shared/usb/usb.c b/supervisor/shared/usb/usb.c
index e8541669a..93d3436e9 100644
--- a/supervisor/shared/usb/usb.c
+++ b/supervisor/shared/usb/usb.c
@@ -29,6 +29,7 @@
 #include "shared-module/usb_midi/__init__.h"
 #include "supervisor/background_callback.h"
 #include "supervisor/port.h"
+#include "supervisor/serial.h"
 #include "supervisor/usb.h"
 #include "lib/utils/interrupt_char.h"
 #include "lib/mp-readline/readline.h"