Merge branch 'main' into stm32-timer-allocator

11 files changed, 322 insertions, 23 deletions

diff --git a/supervisor/background_callback.h b/supervisor/background_callback.h
new file mode 100644
index 000000000..535dd656b
--- /dev/null
+++ b/supervisor/background_callback.h
@@ -0,0 +1,87 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 Jeff Epler for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef CIRCUITPY_INCLUDED_SUPERVISOR_BACKGROUND_CALLBACK_H
+#define CIRCUITPY_INCLUDED_SUPERVISOR_BACKGROUND_CALLBACK_H
+
+/** Background callbacks are a linked list of tasks to call in the background.
+ *
+ * Include a member of type `background_callback_t` inside an object
+ * which needs to queue up background work, and zero-initialize it.
+ *
+ * To schedule the work, use background_callback_add, with fun as the
+ * function to call and data pointing to the object itself.
+ *
+ * Next time run_background_tasks_if_tick is called, the callback will
+ * be run and removed from the linked list.
+ *
+ * Queueing a task that is already queued does nothing.  Unconditionally
+ * re-queueing it from its own background task will cause it to run during the
+ * very next background-tasks invocation, leading to a CircuitPython freeze, so
+ * don't do that.
+ *
+ * background_callback_add can be called from interrupt context.
+ */
+typedef void (*background_callback_fun)(void *data);
+typedef struct background_callback {
+    background_callback_fun fun;
+    void *data;
+    struct background_callback *next;
+    struct background_callback *prev;
+} background_callback_t;
+
+/* Add a background callback for which 'fun' and 'data' were previously set */
+void background_callback_add_core(background_callback_t *cb);
+
+/* Add a background callback to the given function with the given data.  When
+ * the callback involves an object on the GC heap, the 'data' must be a pointer
+ * to that object itself, not an internal pointer.  Otherwise, it can be the
+ * case that no other references to the object itself survive, and the object
+ * becomes garbage collected while an outstanding background callback still
+ * exists.
+ */
+void background_callback_add(background_callback_t *cb, background_callback_fun fun, void *data);
+
+/* Run all background callbacks.  Normally, this is done by the supervisor
+ * whenever the list is non-empty */
+void background_callback_run_all(void);
+
+/* During soft reset, remove all pending callbacks and clear the critical section flag */
+void background_callback_reset(void);
+
+/* Sometimes background callbacks must be blocked.  Use these functions to
+ * bracket the section of code where this is the case.  These calls nest, and
+ * begins must be balanced with ends.
+ */
+void background_callback_begin_critical_section(void);
+void background_callback_end_critical_section(void);
+
+/*
+ * Background callbacks may stop objects from being collected
+ */
+void background_callback_gc_collect(void);
+
+#endif
diff --git a/supervisor/flash.h b/supervisor/flash.h
index a8a77bf04..cd69cbfa9 100644
--- a/supervisor/flash.h
+++ b/supervisor/flash.h
@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
diff --git a/supervisor/port.h b/supervisor/port.h
index 8a12d34c8..ad5b3cf32 100644
--- a/supervisor/port.h
+++ b/supervisor/port.h
@@ -91,4 +91,12 @@ void port_interrupt_after_ticks(uint32_t ticks);
 // Sleep the CPU until an interrupt is received.
 void port_sleep_until_interrupt(void);
 
+// Execute port specific actions during background tasks.
+void port_background_task(void);
+
+// Take port specific actions at the beginning and end of background tasks.
+// This is used e.g., to set a monitoring pin for debug purposes.  "Actual
+// work" should be done in port_background_task() instead.
+void port_start_background_task(void);
+void port_finish_background_task(void);
 #endif  // MICROPY_INCLUDED_SUPERVISOR_PORT_H
diff --git a/supervisor/shared/background_callback.c b/supervisor/shared/background_callback.c
new file mode 100644
index 000000000..8e12dd362
--- /dev/null
+++ b/supervisor/shared/background_callback.c
@@ -0,0 +1,138 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 Jeff Epler for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+
+#include "py/gc.h"
+#include "py/mpconfig.h"
+#include "supervisor/background_callback.h"
+#include "supervisor/shared/tick.h"
+#include "shared-bindings/microcontroller/__init__.h"
+
+STATIC volatile background_callback_t *callback_head, *callback_tail;
+
+#define CALLBACK_CRITICAL_BEGIN (common_hal_mcu_disable_interrupts())
+#define CALLBACK_CRITICAL_END (common_hal_mcu_enable_interrupts())
+
+void background_callback_add_core(background_callback_t *cb) {
+    CALLBACK_CRITICAL_BEGIN;
+    if (cb->prev || callback_head == cb) {
+        CALLBACK_CRITICAL_END;
+        return;
+    }
+    cb->next = 0;
+    cb->prev = (background_callback_t*)callback_tail;
+    if (callback_tail) {
+        callback_tail->next = cb;
+        cb->prev = (background_callback_t*)callback_tail;
+    }
+    if (!callback_head) {
+        callback_head = cb;
+    }
+    callback_tail = cb;
+    CALLBACK_CRITICAL_END;
+}
+
+void background_callback_add(background_callback_t *cb, background_callback_fun fun, void *data) {
+    cb->fun = fun;
+    cb->data = data;
+    background_callback_add_core(cb);
+}
+
+static bool in_background_callback;
+void background_callback_run_all() {
+    if (!callback_head) {
+        return;
+    }
+    CALLBACK_CRITICAL_BEGIN;
+    if (in_background_callback) {
+        CALLBACK_CRITICAL_END;
+        return;
+    }
+    in_background_callback = true;
+    background_callback_t *cb = (background_callback_t*)callback_head;
+    callback_head = NULL;
+    callback_tail = NULL;
+    while (cb) {
+        background_callback_t *next = cb->next;
+        cb->next = cb->prev = NULL;
+        background_callback_fun fun = cb->fun;
+        void *data = cb->data;
+        CALLBACK_CRITICAL_END;
+        // Leave the critical section in order to run the callback function
+        if (fun) {
+            fun(data);
+        }
+        CALLBACK_CRITICAL_BEGIN;
+        cb = next;
+    }
+    in_background_callback = false;
+    CALLBACK_CRITICAL_END;
+}
+
+void background_callback_begin_critical_section() {
+    CALLBACK_CRITICAL_BEGIN;
+}
+
+void background_callback_end_critical_section() {
+    CALLBACK_CRITICAL_END;
+}
+
+void background_callback_reset() {
+    CALLBACK_CRITICAL_BEGIN;
+    background_callback_t *cb = (background_callback_t*)callback_head;
+    while(cb) {
+        background_callback_t *next = cb->next;
+        memset(cb, 0, sizeof(*cb));
+        cb = next;
+    }
+    callback_head = NULL;
+    callback_tail = NULL;
+    in_background_callback = false;
+    CALLBACK_CRITICAL_END;
+}
+
+void background_callback_gc_collect(void) {
+    // We don't enter the callback critical section here.  We rely on
+    // gc_collect_ptr _NOT_ entering background callbacks, so it is not
+    // possible for the list to be cleared.
+    //
+    // However, it is possible for the list to be extended.  We make the
+    // minor assumption that no newly added callback is for a
+    // collectable object.  That is, we only plug the hole where an
+    // object becomes collectable AFTER it is added but before the
+    // callback is run, not the hole where an object was ALREADY
+    // collectable but adds a background task for itself.
+    //
+    // It's necessary to traverse the whole list here, as the callbacks
+    // themselves can be in non-gc memory, and some of the cb->data
+    // objects themselves might be in non-gc memory.
+    background_callback_t *cb = (background_callback_t*)callback_head;
+    while(cb) {
+        gc_collect_ptr(cb->data);
+        cb = cb->next;
+    }
+}
diff --git a/supervisor/shared/external_flash/common_commands.h b/supervisor/shared/external_flash/common_commands.h
index 2eaa84833..37efd8ceb 100644
--- a/supervisor/shared/external_flash/common_commands.h
+++ b/supervisor/shared/external_flash/common_commands.h
@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
diff --git a/supervisor/shared/tick.c b/supervisor/shared/tick.c
index dd7dba8f3..4af59f78e 100644
--- a/supervisor/shared/tick.c
+++ b/supervisor/shared/tick.c
@@ -29,10 +29,19 @@
 #include "py/mpstate.h"
 #include "supervisor/linker.h"
 #include "supervisor/filesystem.h"
+#include "supervisor/background_callback.h"
 #include "supervisor/port.h"
 #include "supervisor/shared/autoreload.h"
+#include "supervisor/shared/stack.h"
 
-static volatile uint64_t PLACE_IN_DTCM_BSS(background_ticks);
+#if CIRCUITPY_BLEIO
+#include "supervisor/shared/bluetooth.h"
+#include "common-hal/_bleio/bonding.h"
+#endif
+
+#if CIRCUITPY_DISPLAYIO
+#include "shared-module/displayio/__init__.h"
+#endif
 
 #if CIRCUITPY_GAMEPAD
 #include "shared-module/gamepad/__init__.h"
@@ -42,6 +51,10 @@ static volatile uint64_t PLACE_IN_DTCM_BSS(background_ticks);
 #include "shared-module/gamepadshift/__init__.h"
 #endif
 
+#if CIRCUITPY_NETWORK
+#include "shared-module/network/__init__.h"
+#endif
+
 #include "shared-bindings/microcontroller/__init__.h"
 
 #if CIRCUITPY_WATCHDOG
@@ -51,6 +64,44 @@ static volatile uint64_t PLACE_IN_DTCM_BSS(background_ticks);
 #define WATCHDOG_EXCEPTION_CHECK() 0
 #endif
 
+static volatile uint64_t PLACE_IN_DTCM_BSS(background_ticks);
+
+static background_callback_t tick_callback;
+
+volatile uint64_t last_finished_tick = 0;
+
+void supervisor_background_tasks(void *unused) {
+    port_start_background_task();
+
+    assert_heap_ok();
+
+    #if CIRCUITPY_DISPLAYIO
+    displayio_background();
+    #endif
+
+    #if CIRCUITPY_NETWORK
+    network_module_background();
+    #endif
+    filesystem_background();
+
+    #if CIRCUITPY_BLEIO
+    supervisor_bluetooth_background();
+    bonding_background();
+    #endif
+
+    port_background_task();
+
+    assert_heap_ok();
+
+    last_finished_tick = port_get_raw_ticks(NULL);
+
+    port_finish_background_task();
+}
+
+bool supervisor_background_tasks_ok(void) {
+    return port_get_raw_ticks(NULL) - last_finished_tick < 1024;
+}
+
 void supervisor_tick(void) {
 #if CIRCUITPY_FILESYSTEM_FLUSH_INTERVAL_MS > 0
     filesystem_tick();
@@ -68,6 +119,7 @@ void supervisor_tick(void) {
         #endif
     }
 #endif
+    background_callback_add(&tick_callback, supervisor_background_tasks, NULL);
 }
 
 uint64_t supervisor_ticks_ms64() {
@@ -83,14 +135,9 @@ uint32_t supervisor_ticks_ms32() {
     return supervisor_ticks_ms64();
 }
 
-extern void run_background_tasks(void);
 
 void PLACE_IN_ITCM(supervisor_run_background_tasks_if_tick)() {
-    // TODO: Add a global that can be set by anyone to indicate we should run background tasks. That
-    // way we can short circuit the background tasks early. We used to do it based on time but it
-    // breaks cases where we wake up for a short period and then sleep. If we skipped the last
-    // background task or more before sleeping we may end up starving a task like USB.
-    run_background_tasks();
+    background_callback_run_all();
 }
 
 void mp_hal_delay_ms(mp_uint_t delay) {
@@ -104,13 +151,13 @@ void mp_hal_delay_ms(mp_uint_t delay) {
         // Check to see if we've been CTRL-Ced by autoreload or the user.
         if(MP_STATE_VM(mp_pending_exception) == MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_kbd_exception)))
            {
-           // clear exception and generate stacktrace
+            // clear exception and generate stacktrace
             MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL;
             nlr_raise(&MP_STATE_VM(mp_kbd_exception));
           }
         if( MP_STATE_VM(mp_pending_exception) == MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_reload_exception)) ||
            WATCHDOG_EXCEPTION_CHECK()) {
-           // stop sleeping immediately
+            // stop sleeping immediately
             break;
         }
         remaining = end_tick - port_get_raw_ticks(NULL);
diff --git a/supervisor/shared/tick.h b/supervisor/shared/tick.h
index e7e808058..3a01bd622 100644
--- a/supervisor/shared/tick.h
+++ b/supervisor/shared/tick.h
@@ -28,6 +28,7 @@
 #define __INCLUDED_SUPERVISOR_TICK_H
 
 #include <stdint.h>
+#include <stdbool.h>
 
 /** @brief To be called once every ms
  *
@@ -36,13 +37,6 @@
  * interrupt context.
  */
 extern void supervisor_tick(void);
-/** @brief Cause background tasks to be called soon
- *
- * Normally, background tasks are only run once per tick.  For other cases where
- * an event noticed from an interrupt context needs to be completed by a background
- * task activity, the interrupt can call supervisor_fake_tick.
- */
-extern void supervisor_fake_tick(void);
 /** @brief Get the lower 32 bits of the time in milliseconds
  *
  * This can be more efficient than supervisor_ticks_ms64, for sites where a wraparound
@@ -67,4 +61,12 @@ extern void supervisor_run_background_if_tick(void);
 extern void supervisor_enable_tick(void);
 extern void supervisor_disable_tick(void);
 
+/**
+ * @brief Return true if tick-based background tasks ran within the last 1s
+ *
+ * Note that when ticks are not enabled, this function can return false; this is
+ * intended.
+ */
+extern bool supervisor_background_tasks_ok(void);
+
 #endif
diff --git a/supervisor/shared/usb/usb.c b/supervisor/shared/usb/usb.c
index edf810118..36b5ec05d 100644
--- a/supervisor/shared/usb/usb.c
+++ b/supervisor/shared/usb/usb.c
@@ -27,6 +27,7 @@
 #include "py/objstr.h"
 #include "shared-bindings/microcontroller/Processor.h"
 #include "shared-module/usb_midi/__init__.h"
+#include "supervisor/background_callback.h"
 #include "supervisor/port.h"
 #include "supervisor/usb.h"
 #include "lib/utils/interrupt_char.h"
@@ -63,8 +64,8 @@ void usb_init(void) {
     tusb_init();
 
 #if MICROPY_KBD_EXCEPTION
-    // Set Ctrl+C as wanted char, tud_cdc_rx_wanted_cb() callback will be invoked when Ctrl+C is received
-    // This callback always got invoked regardless of mp_interrupt_char value since we only set it once here
+    // Set Ctrl+C as wanted char, tud_cdc_rx_wanted_cb() usb_callback will be invoked when Ctrl+C is received
+    // This usb_callback always got invoked regardless of mp_interrupt_char value since we only set it once here
     tud_cdc_set_wanted_char(CHAR_CTRL_C);
 #endif
 
@@ -82,6 +83,16 @@ void usb_background(void) {
     }
 }
 
+static background_callback_t usb_callback;
+static void usb_background_do(void* unused) {
+    usb_background();
+}
+
+void usb_irq_handler(void) {
+    tud_int_handler(0);
+    background_callback_add(&usb_callback, usb_background_do, NULL);
+}
+
 //--------------------------------------------------------------------+
 // tinyusb callbacks
 //--------------------------------------------------------------------+
diff --git a/supervisor/stub/internal_flash.c b/supervisor/stub/internal_flash.c
index 5a82f81f7..3a4ba935d 100644
--- a/supervisor/stub/internal_flash.c
+++ b/supervisor/stub/internal_flash.c
@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * SPDX-FileCopyrightText: Copyright (c) 2013, 2014 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
diff --git a/supervisor/supervisor.mk b/supervisor/supervisor.mk
index 21803ae0a..188536686 100644
--- a/supervisor/supervisor.mk
+++ b/supervisor/supervisor.mk
@@ -2,6 +2,7 @@ SRC_SUPERVISOR = \
 	main.c \
 	supervisor/port.c \
 	supervisor/shared/autoreload.c \
+	supervisor/shared/background_callback.c \
 	supervisor/shared/board.c \
 	supervisor/shared/filesystem.c \
 	supervisor/shared/flash.c \
diff --git a/supervisor/usb.h b/supervisor/usb.h
index 29280c725..2a447c368 100644
--- a/supervisor/usb.h
+++ b/supervisor/usb.h
@@ -29,10 +29,15 @@
 
 #include <stdbool.h>
 
-// Ports must call this as frequently as they can in order to keep the USB connection
-// alive and responsive.
+// Ports must call this as frequently as they can in order to keep the USB
+// connection alive and responsive.  Normally this is called from background
+// tasks after the USB IRQ handler is executed, but in specific circumstances
+// it may be necessary to call it directly.
 void usb_background(void);
 
+// Ports must call this from their particular USB IRQ handler
+void usb_irq_handler(void);
+
 // Only inits the USB peripheral clocks and pins. The peripheral will be initialized by
 // TinyUSB.
 void init_usb_hardware(void);