rgbmatrix: recover gracefully from allocation errors

e.g., allocating a 192x32x6bpp matrix would be enough to trigger this
reliably on a Metro M4 Express using the "memory hogging" layout.
Allocating 64x32x6bpp could trigger it, but somewhat unreliably.

There are several things going on here:
 * we make the failing call with interrupts off
 * we were throwing an exception with interrupts off
 * protomatter failed badly in _PM_free when it was partially-initialized

Incorporate the fix from protomatter, switch to a non-throwing malloc
variant, and ensure that interrupts get turned back on.

This decreases the quality of the MemoryError (it cannot report the size
of the failed allocation) but allows CircuitPython to survive, rather
than faulting.

2 files changed, 12 insertions, 7 deletions

diff --git a/shared-module/rgbmatrix/RGBMatrix.c b/shared-module/rgbmatrix/RGBMatrix.c
index 6dad91679..a020ef089 100644
--- a/shared-module/rgbmatrix/RGBMatrix.c
+++ b/shared-module/rgbmatrix/RGBMatrix.c
@@ -89,6 +89,7 @@ void common_hal_rgbmatrix_rgbmatrix_reconstruct(rgbmatrix_rgbmatrix_obj_t* self,
         self->bufinfo.typecode = 'H' | MP_OBJ_ARRAY_TYPECODE_FLAG_RW;
     }
 
+    memset(&self->core, 0, sizeof(self->core));
     ProtomatterStatus stat = _PM_init(&self->core,
         self->width, self->bit_depth,
         self->rgb_count/6, self->rgb_pins,
@@ -101,14 +102,17 @@ void common_hal_rgbmatrix_rgbmatrix_reconstruct(rgbmatrix_rgbmatrix_obj_t* self,
         common_hal_mcu_disable_interrupts();
         common_hal_rgbmatrix_timer_enable(self->timer);
         stat = _PM_begin(&self->core);
-        _PM_convert_565(&self->core, self->bufinfo.buf, self->width);
+
+        if (stat == PROTOMATTER_OK) {
+            _PM_convert_565(&self->core, self->bufinfo.buf, self->width);
+        }
         common_hal_mcu_enable_interrupts();
-        _PM_swapbuffer_maybe(&self->core);
+        if (stat == PROTOMATTER_OK) {
+            _PM_swapbuffer_maybe(&self->core);
+        }
     }
 
     if (stat != PROTOMATTER_OK) {
-        // XXX this deinit() actually makes crashy-crashy
-        // can trigger it by sending inappropriate pins
         common_hal_rgbmatrix_rgbmatrix_deinit(self);
         switch (stat) {
         case PROTOMATTER_ERR_PINS:
@@ -117,7 +121,9 @@ void common_hal_rgbmatrix_rgbmatrix_reconstruct(rgbmatrix_rgbmatrix_obj_t* self,
         case PROTOMATTER_ERR_ARG:
             mp_raise_ValueError(translate("Invalid argument"));
             break;
-        case PROTOMATTER_ERR_MALLOC: /// should have already been signaled as NLR
+        case PROTOMATTER_ERR_MALLOC:
+            mp_raise_msg(&mp_type_MemoryError, NULL);
+            break;
         default:
             mp_raise_msg_varg(&mp_type_RuntimeError,
                 translate("Internal error #%d"), (int)stat);
@@ -126,7 +132,6 @@ void common_hal_rgbmatrix_rgbmatrix_reconstruct(rgbmatrix_rgbmatrix_obj_t* self,
     }
 
     self->paused = 0;
-
 }
 
 STATIC void free_pin(uint8_t *pin) {
diff --git a/shared-module/rgbmatrix/allocator.h b/shared-module/rgbmatrix/allocator.h
index 5e6f0b41d..172d82684 100644
--- a/shared-module/rgbmatrix/allocator.h
+++ b/shared-module/rgbmatrix/allocator.h
@@ -11,7 +11,7 @@
 
 static inline void *_PM_allocator_impl(size_t sz) {
     if (gc_alloc_possible()) {
-        return m_malloc(sz + sizeof(void*), true);
+        return m_malloc_maybe(sz + sizeof(void*), true);
     } else {
         supervisor_allocation *allocation = allocate_memory(align32_size(sz), false);
         return allocation ? allocation->ptr : NULL;