wip

14 files changed, 105 insertions, 55 deletions

diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk
index 915310f83..34c097e36 100644
--- a/py/circuitpy_defns.mk
+++ b/py/circuitpy_defns.mk
@@ -229,12 +229,10 @@ SRC_COMMON_HAL_ALL = \
 	_bleio/__init__.c \
 	_bleio/Adapter.c \
 	_bleio/Attribute.c \
-	_bleio/Central.c \
 	_bleio/Characteristic.c \
 	_bleio/CharacteristicBuffer.c \
+	_bleio/Connection.c \
 	_bleio/Descriptor.c \
-	_bleio/Peripheral.c \
-	_bleio/Scanner.c \
 	_bleio/Service.c \
 	_bleio/UUID.c \
 	analogio/AnalogIn.c \
@@ -306,6 +304,7 @@ SRC_SHARED_MODULE_ALL = \
 	_bleio/Address.c \
 	_bleio/Attribute.c \
 	_bleio/ScanEntry.c \
+	_bleio/ScanResults.c \
 	_pixelbuf/PixelBuf.c \
 	_pixelbuf/__init__.c \
 	_stage/Layer.c \
diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h
index 227376ead..cf21d26dd 100644
--- a/py/circuitpy_mpconfig.h
+++ b/py/circuitpy_mpconfig.h
@@ -28,11 +28,12 @@
 // sure that the same feature set and settings are used, such as in atmel-samd
 // and nrf.
 
-#include <stdint.h>
-
 #ifndef __INCLUDED_MPCONFIG_CIRCUITPY_H
 #define __INCLUDED_MPCONFIG_CIRCUITPY_H
 
+#include <stdint.h>
+#include <stdatomic.h>
+
 // This is CircuitPython.
 #define CIRCUITPY 1
 
@@ -56,8 +57,8 @@
 #define MICROPY_COMP_MODULE_CONST        (1)
 #define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (0)
 #define MICROPY_DEBUG_PRINTERS           (0)
-#define MICROPY_EMIT_INLINE_THUMB        (0)
-#define MICROPY_EMIT_THUMB               (0)
+#define MICROPY_EMIT_INLINE_THUMB        (CIRCUITPY_ENABLE_MPY_NATIVE)
+#define MICROPY_EMIT_THUMB               (CIRCUITPY_ENABLE_MPY_NATIVE)
 #define MICROPY_EMIT_X64                 (0)
 #define MICROPY_ENABLE_DOC_STRING        (0)
 #define MICROPY_ENABLE_FINALISER         (1)
@@ -139,7 +140,6 @@
 #define MICROPY_VFS_FAT             (MICROPY_VFS)
 #define MICROPY_READER_VFS          (MICROPY_VFS)
 
-
 // type definitions for the specific machine
 
 #define BYTES_PER_WORD (4)
@@ -213,6 +213,10 @@ typedef long mp_off_t;
 #define MP_SSIZE_MAX (0x7fffffff)
 #endif
 
+#if INTERNAL_FLASH_FILESYSTEM == 0 && QSPI_FLASH_FILESYSTEM == 0 && SPI_FLASH_FILESYSTEM == 0 && !CIRCUITPY_MINIMAL_BUILD
+#error No *_FLASH_FILESYSTEM set!
+#endif
+
 // These CIRCUITPY_xxx values should all be defined in the *.mk files as being on or off.
 // So if any are not defined in *.mk, they'll throw an error here.
 
@@ -652,14 +656,14 @@ extern const struct _mp_obj_module_t ustack_module;
     FLASH_ROOT_POINTERS \
     NETWORK_ROOT_POINTERS \
 
-void run_background_tasks(void);
-#define RUN_BACKGROUND_TASKS (run_background_tasks())
+void supervisor_run_background_tasks_if_tick(void);
+#define RUN_BACKGROUND_TASKS (supervisor_run_background_tasks_if_tick())
 
 // TODO: Used in wiznet5k driver, but may not be needed in the long run.
 #define MICROPY_THREAD_YIELD()
 
-#define MICROPY_VM_HOOK_LOOP run_background_tasks();
-#define MICROPY_VM_HOOK_RETURN run_background_tasks();
+#define MICROPY_VM_HOOK_LOOP RUN_BACKGROUND_TASKS;
+#define MICROPY_VM_HOOK_RETURN RUN_BACKGROUND_TASKS;
 
 // CIRCUITPY_AUTORELOAD_DELAY_MS = 0 will completely disable autoreload.
 #ifndef CIRCUITPY_AUTORELOAD_DELAY_MS
diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk
index c0548abfa..8ab6b1288 100644
--- a/py/circuitpy_mpconfig.mk
+++ b/py/circuitpy_mpconfig.mk
@@ -33,15 +33,19 @@
 ifndef CIRCUITPY_FULL_BUILD
   ifeq ($(CIRCUITPY_SMALL_BUILD),1)
     CIRCUITPY_FULL_BUILD = 0
-    CFLAGS += -DCIRCUITPY_FULL_BUILD=0
   else
     CIRCUITPY_FULL_BUILD = 1
-    CFLAGS += -DCIRCUITPY_FULL_BUILD=1
   endif
 endif
+CFLAGS += -DCIRCUITPY_FULL_BUILD=$(CIRCUITPY_FULL_BUILD)
 
 # Setting CIRCUITPY_MINIMAL_BUILD = 1 will disable all features
 # Use for for early stage or highly restricted ports
+ifndef CIRCUITPY_MINIMAL_BUILD
+CIRCUITPY_MINIMAL_BUILD = 0
+endif
+CFLAGS += -DCIRCUITPY_MINIMAL_BUILD=$(CIRCUITPY_MINIMAL_BUILD)
+
 ifndef CIRCUITPY_DEFAULT_BUILD
   ifeq ($(CIRCUITPY_MINIMAL_BUILD),1)
         CIRCUITPY_FULL_BUILD = 0
@@ -50,12 +54,14 @@ ifndef CIRCUITPY_DEFAULT_BUILD
         CIRCUITPY_DEFAULT_BUILD = 1
   endif
 endif
+CFLAGS += -DCIRCUITPY_DEFAULT_BUILD=$(CIRCUITPY_DEFAULT_BUILD)
 
 # Some features have no unique HAL component, and thus there's never
-# a reason to not include them. 
+# a reason to not include them.
 ifndef CIRCUITPY_ALWAYS_BUILD
   CIRCUITPY_ALWAYS_BUILD = 1
 endif
+CFLAGS += -DCIRCUITPY_ALWAYS_BUILD=$(CIRCUITPY_ALWAYS_BUILD)
 
 
 # All builtin modules are listed below, with default values (0 for off, 1 for on)
@@ -169,7 +175,7 @@ CIRCUITPY_NEOPIXEL_WRITE = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_NEOPIXEL_WRITE=$(CIRCUITPY_NEOPIXEL_WRITE)
 
-# Only certain boards support NETWORK (Ethernet)
+# Enabled on SAMD51. Won't fit on SAMD21 builds. Not tested on nRF or STM32F4 builds.
 ifndef CIRCUITPY_NETWORK
 CIRCUITPY_NETWORK = 0
 endif
@@ -306,3 +312,9 @@ ifndef CIRCUITPY_SERIAL_UART
 CIRCUITPY_SERIAL_UART = 0
 endif
 CFLAGS += -DCIRCUITPY_SERIAL_UART=$(CIRCUITPY_SERIAL_UART)
+
+# Enabled micropython.native decorator (experimental)
+ifndef CIRCUITPY_ENABLE_MPY_NATIVE
+CIRCUITPY_ENABLE_MPY_NATIVE = 0
+endif
+CFLAGS += -DCIRCUITPY_ENABLE_MPY_NATIVE=$(CIRCUITPY_ENABLE_MPY_NATIVE)
diff --git a/py/compile.c b/py/compile.c
index 1218aa07e..77715d3fe 100644
--- a/py/compile.c
+++ b/py/compile.c
@@ -775,13 +775,22 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, int name_len, mp_parse_
     qstr attr = MP_PARSE_NODE_LEAF_ARG(name_nodes[1]);
     if (attr == MP_QSTR_bytecode) {
         *emit_options = MP_EMIT_OPT_BYTECODE;
-#if MICROPY_EMIT_NATIVE
+    // @micropython.native decorator.
     } else if (attr == MP_QSTR_native) {
+        // Different from MicroPython: native doesn't raise SyntaxError if native support isn't
+        // compiled, it just passes through the function unmodified.
+        #if MICROPY_EMIT_NATIVE
         *emit_options = MP_EMIT_OPT_NATIVE_PYTHON;
+        #else
+        return true;
+        #endif
+    #if MICROPY_EMIT_NATIVE
+    // @micropython.viper decorator.
     } else if (attr == MP_QSTR_viper) {
         *emit_options = MP_EMIT_OPT_VIPER;
-#endif
+    #endif
     #if MICROPY_EMIT_INLINE_ASM
+    // @micropython.asm_thumb decorator.
     } else if (attr == ASM_DECORATOR_QSTR) {
         *emit_options = MP_EMIT_OPT_ASM;
     #endif
@@ -3207,7 +3216,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind
             }
             if (pass > MP_PASS_SCOPE) {
                 mp_int_t bytesize = MP_PARSE_NODE_LEAF_SMALL_INT(pn_arg[0]);
-                for (uint j = 1; j < n_args; j++) {
+                for (int j = 1; j < n_args; j++) {
                     if (!MP_PARSE_NODE_IS_SMALL_INT(pn_arg[j])) {
                         compile_syntax_error(comp, nodes[i], translate("'data' requires integer arguments"));
                         return;
diff --git a/py/emitnative.c b/py/emitnative.c
index 67b0025c6..6d23bf097 100644
--- a/py/emitnative.c
+++ b/py/emitnative.c
@@ -58,6 +58,22 @@
 #define DEBUG_printf(...) (void)0
 #endif
 
+#ifndef N_X64
+    #define N_X64 (0)
+#endif
+#ifndef N_X86
+    #define N_X86 (0)
+#endif
+#ifndef N_THUMB
+    #define N_THUMB (0)
+#endif
+#ifndef N_ARM
+    #define N_ARM (0)
+#endif
+#ifndef N_XTENSA
+    #define N_XTENSA (0)
+#endif
+
 // wrapper around everything in this file
 #if N_X64 || N_X86 || N_THUMB || N_ARM || N_XTENSA
 
@@ -443,10 +459,12 @@ STATIC void emit_native_end_pass(emit_t *emit) {
             type_sig |= (emit->local_vtype[i] & 0xf) << (i * 4 + 4);
         }
 
+        #pragma GCC diagnostic ignored "-Wcast-align"
         mp_emit_glue_assign_native(emit->scope->raw_code,
             emit->do_viper_types ? MP_CODE_NATIVE_VIPER : MP_CODE_NATIVE_PY,
             f, f_len, (mp_uint_t*)((byte*)f + emit->const_table_offset),
             emit->scope->num_pos_args, emit->scope->scope_flags, type_sig);
+        #pragma GCC diagnostic pop
     }
 }
 
diff --git a/py/gc.c b/py/gc.c
index ac584d5d2..a8a5fde12 100755
--- a/py/gc.c
+++ b/py/gc.c
@@ -53,9 +53,6 @@
 // detect untraced object still in use
 #define CLEAR_ON_SWEEP (0)
 
-#define WORDS_PER_BLOCK ((MICROPY_BYTES_PER_GC_BLOCK) / BYTES_PER_WORD)
-#define BYTES_PER_BLOCK (MICROPY_BYTES_PER_GC_BLOCK)
-
 // ATB = allocation table byte
 // 0b00 = FREE -- free block
 // 0b01 = HEAD -- head of a chain of blocks
@@ -209,13 +206,6 @@ bool gc_is_locked(void) {
     return MP_STATE_MEM(gc_lock_depth) != 0;
 }
 
-// ptr should be of type void*
-#define VERIFY_PTR(ptr) ( \
-        ((uintptr_t)(ptr) & (BYTES_PER_BLOCK - 1)) == 0      /* must be aligned on a block */ \
-        && ptr >= (void*)MP_STATE_MEM(gc_pool_start)     /* must be above start of pool */ \
-        && ptr < (void*)MP_STATE_MEM(gc_pool_end)        /* must be below end of pool */ \
-    )
-
 #ifndef TRACE_MARK
 #if DEBUG_PRINT
 #define TRACE_MARK(block, ptr) DEBUG_printf("gc_mark(%p)\n", ptr)
diff --git a/py/gc.h b/py/gc.h
index 02bf45158..cd63ba94c 100644
--- a/py/gc.h
+++ b/py/gc.h
@@ -29,8 +29,19 @@
 #include <stdint.h>
 
 #include "py/mpconfig.h"
+#include "py/mpstate.h"
 #include "py/misc.h"
 
+#define WORDS_PER_BLOCK ((MICROPY_BYTES_PER_GC_BLOCK) / BYTES_PER_WORD)
+#define BYTES_PER_BLOCK (MICROPY_BYTES_PER_GC_BLOCK)
+
+// ptr should be of type void*
+#define VERIFY_PTR(ptr) ( \
+        ((uintptr_t)(ptr) & (BYTES_PER_BLOCK - 1)) == 0      /* must be aligned on a block */ \
+        && ptr >= (void*)MP_STATE_MEM(gc_pool_start)     /* must be above start of pool */ \
+        && ptr < (void*)MP_STATE_MEM(gc_pool_end)        /* must be below end of pool */ \
+    )
+
 void gc_init(void *start, void *end);
 void gc_deinit(void);
 
diff --git a/py/gc_long_lived.c b/py/gc_long_lived.c
index 49bf1fcd7..01c22a7af 100755
--- a/py/gc_long_lived.c
+++ b/py/gc_long_lived.c
@@ -126,6 +126,11 @@ mp_obj_t make_obj_long_lived(mp_obj_t obj, uint8_t max_depth){
     if (obj == NULL) {
         return obj;
     }
+    // If not in the GC pool, do nothing. This can happen (at least) when
+    // there are frozen mp_type_bytes objects in ROM.
+    if (!VERIFY_PTR((void *)obj)) {
+        return obj;
+    }
     if (MP_OBJ_IS_TYPE(obj, &mp_type_fun_bc)) {
         mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(obj);
         return MP_OBJ_FROM_PTR(make_fun_bc_long_lived(fun_bc, max_depth));
diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py
index 8e3835d86..0d667959d 100644
--- a/py/makeqstrdata.py
+++ b/py/makeqstrdata.py
@@ -103,14 +103,10 @@ def compute_huffman_coding(translations, qstrs, compression_filename):
     # go through each qstr and print it out
     for _, _, qstr in qstrs.values():
         all_strings.append(qstr)
-    all_strings_concat = "".join(all_strings).encode("utf-8")
+    all_strings_concat = "".join(all_strings)
     counts = collections.Counter(all_strings_concat)
-    # add other values
-    for i in range(256):
-        if i not in counts:
-            counts[i] = 0
     cb = huffman.codebook(counts.items())
-    values = bytearray()
+    values = []
     length_count = {}
     renumbered = 0
     last_l = None
@@ -124,26 +120,27 @@ def compute_huffman_coding(translations, qstrs, compression_filename):
         if last_l:
             renumbered <<= (l - last_l)
         canonical[ch] = '{0:0{width}b}'.format(renumbered, width=l)
-        if chr(ch) in C_ESCAPES:
-            s = C_ESCAPES[chr(ch)]
-        else:
-            s = chr(ch)
-        print("//", ch, s, counts[ch], canonical[ch], renumbered)
+        s = C_ESCAPES.get(ch, ch)
+        print("//", ord(ch), s, counts[ch], canonical[ch], renumbered)
         renumbered += 1
         last_l = l
     lengths = bytearray()
-    for i in range(1, max(length_count) + 1):
+    print("// length count", length_count)
+    for i in range(1, max(length_count) + 2):
         lengths.append(length_count.get(i, 0))
+    print("// values", values, "lengths", len(lengths), lengths)
+    print("// estimated total memory size", len(lengths) + 2*len(values) + sum(len(cb[u]) for u in all_strings_concat))
     print("//", values, lengths)
+    values_type = "uint16_t" if max(ord(u) for u in values) > 255 else "uint8_t"
     with open(compression_filename, "w") as f:
         f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths))))
-        f.write("const uint8_t values[256] = {{ {} }};\n".format(", ".join(map(str, values))))
+        f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values)))
     return values, lengths
 
 def decompress(encoding_table, length, encoded):
     values, lengths = encoding_table
     #print(l, encoded)
-    dec = bytearray(length)
+    dec = []
     this_byte = 0
     this_bit = 7
     b = encoded[this_byte]
@@ -173,14 +170,14 @@ def decompress(encoding_table, length, encoded):
             searched_length += lengths[bit_length]
 
         v = values[searched_length + bits - max_code]
-        dec[i] = v
-    return dec
+        dec.append(v)
+    return ''.join(dec)
 
 def compress(encoding_table, decompressed):
-    if not isinstance(decompressed, bytes):
+    if not isinstance(decompressed, str):
         raise TypeError()
     values, lengths = encoding_table
-    enc = bytearray(len(decompressed) * 2)
+    enc = bytearray(len(decompressed) * 3)
     #print(decompressed)
     #print(lengths)
     current_bit = 7
@@ -228,7 +225,7 @@ def compress(encoding_table, decompressed):
     if current_bit != 7:
         current_byte += 1
     if current_byte > len(decompressed):
-        print("Note: compression increased length", repr(decompressed.decode('utf-8')), len(decompressed), current_byte, file=sys.stderr)
+        print("Note: compression increased length", repr(decompressed), len(decompressed), current_byte, file=sys.stderr)
     return enc[:current_byte]
 
 def qstr_escape(qst):
@@ -347,9 +344,9 @@ def print_qstr_data(encoding_table, qcfgs, qstrs, i18ns):
     total_text_compressed_size = 0
     for original, translation in i18ns:
         translation_encoded = translation.encode("utf-8")
-        compressed = compress(encoding_table, translation_encoded)
+        compressed = compress(encoding_table, translation)
         total_text_compressed_size += len(compressed)
-        decompressed = decompress(encoding_table, len(translation_encoded), compressed).decode("utf-8")
+        decompressed = decompress(encoding_table, len(translation_encoded), compressed)
         for c in C_ESCAPES:
             decompressed = decompressed.replace(c, C_ESCAPES[c])
         print("TRANSLATION(\"{}\", {}, {{ {} }}) // {}".format(original, len(translation_encoded)+1, ", ".join(["0x{:02x}".format(x) for x in compressed]), decompressed))
diff --git a/py/objfun.c b/py/objfun.c
index b8364815b..7e5899456 100644
--- a/py/objfun.c
+++ b/py/objfun.c
@@ -546,7 +546,7 @@ STATIC mp_uint_t convert_obj_for_inline_asm(mp_obj_t obj) {
 STATIC mp_obj_t fun_asm_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_obj_fun_asm_t *self = self_in;
 
-    mp_arg_check_num(n_args, n_kw, self->n_args, self->n_args, false);
+    mp_arg_check_num_kw_array(n_args, n_kw, self->n_args, self->n_args, false);
 
     void *fun = MICROPY_MAKE_POINTER_CALLABLE(self->fun_data);
 
diff --git a/py/py.mk b/py/py.mk
index 17cb79264..e1367ef1d 100644
--- a/py/py.mk
+++ b/py/py.mk
@@ -307,7 +307,7 @@ $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD)
 
 # build a list of registered modules for py/objmodule.c.
 $(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
-	@$(ECHO) "GEN $@"
+	@$(STEPECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@
 
 SRC_QSTR += $(HEADER_BUILD)/moduledefs.h
diff --git a/py/ringbuf.h b/py/ringbuf.h
index 5f82cc096..7fc35d266 100644
--- a/py/ringbuf.h
+++ b/py/ringbuf.h
@@ -99,4 +99,11 @@ static inline void ringbuf_put_n(ringbuf_t* r, uint8_t* buf, uint8_t bufsize)
         }
     }
 }
+
+static inline void ringbuf_get_n(ringbuf_t* r, uint8_t* buf, uint8_t bufsize)
+{
+    for(uint8_t i=0; i < bufsize; i++) {
+        buf[i] = ringbuf_get(r);
+    }
+}
 #endif // MICROPY_INCLUDED_PY_RINGBUF_H
diff --git a/py/runtime.c b/py/runtime.c
index 9968f26e0..a786619bf 100644
--- a/py/runtime.c
+++ b/py/runtime.c
@@ -31,8 +31,6 @@
 
 #include "extmod/vfs.h"
 
-// #include "py/mpstate.h"
-// #include "py/nlr.h"
 #include "py/parsenum.h"
 #include "py/compile.h"
 #include "py/objstr.h"
diff --git a/py/vm.c b/py/vm.c
index b5f53ee9a..353fc8810 100644
--- a/py/vm.c
+++ b/py/vm.c
@@ -758,7 +758,7 @@ unwind_jump:;
                     } else {
                         PUSH(value); // push the next iteration value
                     }
-                    DISPATCH();
+                    DISPATCH_WITH_PEND_EXC_CHECK();
                 }
 
                 // matched against: SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH