Merge branch 'master' into capablerobot-usbhub

19 files changed, 267 insertions, 55 deletions

diff --git a/py/binary.c b/py/binary.c
index 0e4a1d5f6..6b46425cf 100644
--- a/py/binary.c
+++ b/py/binary.c
@@ -308,6 +308,7 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **
             bool signed_type = is_signed(val_type);
             #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
             if (MP_OBJ_IS_TYPE(val_in, &mp_type_int)) {
+                // It's a longint.
                 mp_obj_int_buffer_overflow_check(val_in, size, signed_type);
                 mp_obj_int_to_bytes_impl(val_in, struct_type == '>', size, p);
                 return;
@@ -315,7 +316,8 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **
             #endif
             {
                 val = mp_obj_get_int(val_in);
-                mp_obj_int_buffer_overflow_check(val_in, size, signed_type);
+                // Small int checking is separate, to be fast.
+                mp_small_int_buffer_overflow_check(val, size, signed_type);
                 // zero/sign extend if needed
                 if (BYTES_PER_WORD < 8 && size > sizeof(val)) {
                     int c = (is_signed(val_type) && (mp_int_t)val < 0) ? 0xff : 0x00;
@@ -353,6 +355,7 @@ void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t v
 
             #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
             if (MP_OBJ_IS_TYPE(val_in, &mp_type_int)) {
+                // It's a long int.
                 mp_obj_int_buffer_overflow_check(val_in, size, signed_type);
                 mp_obj_int_to_bytes_impl(val_in, MP_ENDIANNESS_BIG,
                     size, (uint8_t*)p + index * size);
@@ -360,7 +363,8 @@ void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t v
             }
             #endif
             mp_int_t val = mp_obj_get_int(val_in);
-            mp_obj_int_buffer_overflow_check(val_in, size, signed_type);
+            // Small int checking is separate, to be fast.
+            mp_small_int_buffer_overflow_check(val, size, signed_type);
             mp_binary_set_val_array_from_int(typecode, p, index, val);
         }
     }
diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk
index 1aca33901..a6dde61da 100644
--- a/py/circuitpy_defns.mk
+++ b/py/circuitpy_defns.mk
@@ -108,6 +108,12 @@ endif
 ifeq ($(CIRCUITPY_AUDIOIO),1)
 SRC_PATTERNS += audioio/%
 endif
+ifeq ($(CIRCUITPY_AUDIOPWMIO),1)
+SRC_PATTERNS += audiopwmio/%
+endif
+ifeq ($(CIRCUITPY_AUDIOCORE),1)
+SRC_PATTERNS += audiocore/%
+endif
 ifeq ($(CIRCUITPY_BITBANGIO),1)
 SRC_PATTERNS += bitbangio/%
 endif
@@ -162,6 +168,9 @@ endif
 ifeq ($(CIRCUITPY_PULSEIO),1)
 SRC_PATTERNS += pulseio/%
 endif
+ifeq ($(CIRCUITPY_PS2IO),1)
+SRC_PATTERNS += ps2io/%
+endif
 ifeq ($(CIRCUITPY_RANDOM),1)
 SRC_PATTERNS += random/%
 endif
@@ -217,11 +226,13 @@ $(filter $(SRC_PATTERNS), \
 	audiobusio/__init__.c \
 	audiobusio/I2SOut.c \
 	audiobusio/PDMIn.c \
+	audiopwmio/__init__.c \
+	audiopwmio/PWMAudioOut.c \
 	audioio/__init__.c \
 	audioio/AudioOut.c \
 	bleio/__init__.c \
 	bleio/Adapter.c \
-	bleio/Broadcaster.c \
+	bleio/Central.c \
 	bleio/Characteristic.c \
 	bleio/CharacteristicBuffer.c \
 	bleio/Descriptor.c \
@@ -252,6 +263,8 @@ $(filter $(SRC_PATTERNS), \
 	pulseio/PulseIn.c \
 	pulseio/PulseOut.c \
 	pulseio/__init__.c \
+	ps2io/Ps2.c \
+	ps2io/__init__.c \
 	rotaryio/IncrementalEncoder.c \
 	rotaryio/__init__.c \
 	rtc/RTC.c \
@@ -284,8 +297,6 @@ SRC_BINDINGS_ENUMS += \
 SRC_BINDINGS_ENUMS += \
 $(filter $(SRC_PATTERNS), \
 	bleio/Address.c \
-	bleio/AddressType.c \
-	bleio/AdvertisementData.c \
 	bleio/ScanEntry.c \
 )
 
@@ -297,21 +308,26 @@ $(filter $(SRC_PATTERNS), \
 	_stage/Layer.c \
 	_stage/Text.c \
 	_stage/__init__.c \
+	audiopwmio/__init__.c \
 	audioio/__init__.c \
-	audioio/Mixer.c \
-	audioio/RawSample.c \
-	audioio/WaveFile.c \
+	audiocore/__init__.c \
+	audiocore/Mixer.c \
+	audiocore/RawSample.c \
+	audiocore/WaveFile.c \
 	bitbangio/I2C.c \
 	bitbangio/OneWire.c \
 	bitbangio/SPI.c \
 	bitbangio/__init__.c \
 	board/__init__.c \
+	bleio/Address.c \
+	bleio/ScanEntry.c \
 	busio/OneWire.c \
 	displayio/Bitmap.c \
 	displayio/ColorConverter.c \
 	displayio/Display.c \
 	displayio/FourWire.c \
 	displayio/Group.c \
+	displayio/I2CDisplay.c \
 	displayio/OnDiskBitmap.c \
 	displayio/Palette.c \
 	displayio/Shape.c \
@@ -361,3 +377,8 @@ $(addprefix lib/,\
 	libm/atan2f.c \
 	)
 endif
+
+.PHONY: check-release-needs-clean-build
+
+check-release-needs-clean-build:
+	@echo "RELEASE_NEEDS_CLEAN_BUILD = $(RELEASE_NEEDS_CLEAN_BUILD)"
diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h
index f957886f4..d88d6538e 100644
--- a/py/circuitpy_mpconfig.h
+++ b/py/circuitpy_mpconfig.h
@@ -72,6 +72,7 @@
 #define MICROPY_KBD_EXCEPTION            (1)
 #define MICROPY_MEM_STATS                (0)
 #define MICROPY_NONSTANDARD_TYPECODES    (0)
+#define MICROPY_OPT_COMPUTED_GOTO        (1)
 #define MICROPY_PERSISTENT_CODE_LOAD     (1)
 
 #define MICROPY_PY_ARRAY                 (1)
@@ -89,6 +90,7 @@
 #define MICROPY_PY_BUILTINS_MIN_MAX      (1)
 #define MICROPY_PY_BUILTINS_PROPERTY     (1)
 #define MICROPY_PY_BUILTINS_REVERSED     (1)
+#define MICROPY_PY_BUILTINS_ROUND_INT    (1)
 #define MICROPY_PY_BUILTINS_SET          (1)
 #define MICROPY_PY_BUILTINS_SLICE        (1)
 #define MICROPY_PY_BUILTINS_SLICE_ATTRS  (1)
@@ -228,6 +230,13 @@ extern const struct _mp_obj_module_t audiobusio_module;
 #define AUDIOBUSIO_MODULE
 #endif
 
+#if CIRCUITPY_AUDIOCORE
+#define AUDIOCORE_MODULE         { MP_OBJ_NEW_QSTR(MP_QSTR_audiocore), (mp_obj_t)&audiocore_module },
+extern const struct _mp_obj_module_t audiocore_module;
+#else
+#define AUDIOCORE_MODULE
+#endif
+
 #if CIRCUITPY_AUDIOIO
 #define AUDIOIO_MODULE         { MP_OBJ_NEW_QSTR(MP_QSTR_audioio), (mp_obj_t)&audioio_module },
 extern const struct _mp_obj_module_t audioio_module;
@@ -235,6 +244,13 @@ extern const struct _mp_obj_module_t audioio_module;
 #define AUDIOIO_MODULE
 #endif
 
+#if CIRCUITPY_AUDIOPWMIO
+#define AUDIOPWMIO_MODULE         { MP_OBJ_NEW_QSTR(MP_QSTR_audiopwmio), (mp_obj_t)&audiopwmio_module },
+extern const struct _mp_obj_module_t audiopwmio_module;
+#else
+#define AUDIOPWMIO_MODULE
+#endif
+
 #if CIRCUITPY_BITBANGIO
 #define BITBANGIO_MODULE       { MP_OBJ_NEW_QSTR(MP_QSTR_bitbangio), (mp_obj_t)&bitbangio_module },
 extern const struct _mp_obj_module_t bitbangio_module;
@@ -415,6 +431,13 @@ extern const struct _mp_obj_module_t pulseio_module;
 #define PULSEIO_MODULE
 #endif
 
+#if CIRCUITPY_PS2IO
+extern const struct _mp_obj_module_t ps2io_module;
+#define PS2IO_MODULE         { MP_OBJ_NEW_QSTR(MP_QSTR_ps2io), (mp_obj_t)&ps2io_module },
+#else
+#define PS2IO_MODULE
+#endif
+
 #if CIRCUITPY_RANDOM
 extern const struct _mp_obj_module_t random_module;
 #define RANDOM_MODULE          { MP_OBJ_NEW_QSTR(MP_QSTR_random), (mp_obj_t)&random_module },
@@ -555,7 +578,9 @@ extern const struct _mp_obj_module_t ustack_module;
 #define MICROPY_PORT_BUILTIN_MODULES_STRONG_LINKS \
     ANALOGIO_MODULE \
     AUDIOBUSIO_MODULE \
+    AUDIOCORE_MODULE \
     AUDIOIO_MODULE \
+    AUDIOPWMIO_MODULE \
     BITBANGIO_MODULE \
     BLEIO_MODULE \
     BOARD_MODULE \
@@ -579,6 +604,7 @@ extern const struct _mp_obj_module_t ustack_module;
     PEW_MODULE \
     PIXELBUF_MODULE \
     PULSEIO_MODULE \
+    PS2IO_MODULE \
     RANDOM_MODULE \
     RE_MODULE \
     ROTARYIO_MODULE \
diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk
index b436929e0..a408cd7ac 100644
--- a/py/circuitpy_mpconfig.mk
+++ b/py/circuitpy_mpconfig.mk
@@ -32,22 +32,35 @@
 # CIRCUITPY_FULL_BUILD = 0
 
 ifndef CIRCUITPY_FULL_BUILD
-ifeq ($(CIRCUITPY_SMALL_BUILD),1)
-CIRCUITPY_FULL_BUILD = 0
-CFLAGS += -DCIRCUITPY_FULL_BUILD=0
-else
-CIRCUITPY_FULL_BUILD = 1
-CFLAGS += -DCIRCUITPY_FULL_BUILD=1
+  ifeq ($(CIRCUITPY_SMALL_BUILD),1)
+    CIRCUITPY_FULL_BUILD = 0
+    CFLAGS += -DCIRCUITPY_FULL_BUILD=0
+  else
+    CIRCUITPY_FULL_BUILD = 1
+    CFLAGS += -DCIRCUITPY_FULL_BUILD=1
+  endif
 endif
+
+# Setting CIRCUITPY_MINIMAL_BUILD = 1 will disable all features
+# Use for for early stage or highly restricted ports
+ifndef CIRCUITPY_DEFAULT_BUILD
+  ifeq ($(CIRCUITPY_MINIMAL_BUILD),1)
+  	CIRCUITPY_FULL_BUILD = 0
+  	CIRCUITPY_DEFAULT_BUILD = 0
+  else
+  	CIRCUITPY_DEFAULT_BUILD = 1
+  endif
 endif
 
+
+
 # All builtin modules are listed below, with default values (0 for off, 1 for on)
 # Some are always on, some are always off, and some depend on CIRCUITPY_FULL_BUILD.
 #
 # *** You can override any of the defaults by defining them in your mpconfigboard.mk.
 
 ifndef CIRCUITPY_ANALOGIO
-CIRCUITPY_ANALOGIO = 1
+CIRCUITPY_ANALOGIO = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_ANALOGIO=$(CIRCUITPY_ANALOGIO)
 
@@ -61,6 +74,20 @@ CIRCUITPY_AUDIOIO = $(CIRCUITPY_FULL_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_AUDIOIO=$(CIRCUITPY_AUDIOIO)
 
+ifndef CIRCUITPY_AUDIOPWMIO
+CIRCUITPY_AUDIOPWMIO = 0
+endif
+CFLAGS += -DCIRCUITPY_AUDIOPWMIO=$(CIRCUITPY_AUDIOPWMIO)
+
+ifndef CIRCUITPY_AUDIOCORE
+ifeq ($(CIRCUITPY_AUDIOPWMIO),1)
+CIRCUITPY_AUDIOCORE = $(CIRCUITPY_AUDIOPWMIO)
+else
+CIRCUITPY_AUDIOCORE = $(CIRCUITPY_AUDIOIO)
+endif
+endif
+CFLAGS += -DCIRCUITPY_AUDIOCORE=$(CIRCUITPY_AUDIOCORE)
+
 ifndef CIRCUITPY_BITBANGIO
 CIRCUITPY_BITBANGIO = $(CIRCUITPY_FULL_BUILD)
 endif
@@ -73,17 +100,17 @@ endif
 CFLAGS += -DCIRCUITPY_BLEIO=$(CIRCUITPY_BLEIO)
 
 ifndef CIRCUITPY_BOARD
-CIRCUITPY_BOARD = 1
+CIRCUITPY_BOARD = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_BOARD=$(CIRCUITPY_BOARD)
 
 ifndef CIRCUITPY_BUSIO
-CIRCUITPY_BUSIO = 1
+CIRCUITPY_BUSIO = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_BUSIO=$(CIRCUITPY_BUSIO)
 
 ifndef CIRCUITPY_DIGITALIO
-CIRCUITPY_DIGITALIO = 1
+CIRCUITPY_DIGITALIO = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_DIGITALIO=$(CIRCUITPY_DIGITALIO)
 
@@ -113,17 +140,17 @@ endif
 CFLAGS += -DCIRCUITPY_I2CSLAVE=$(CIRCUITPY_I2CSLAVE)
 
 ifndef CIRCUITPY_MATH
-CIRCUITPY_MATH = 1
+CIRCUITPY_MATH = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_MATH=$(CIRCUITPY_MATH)
 
 ifndef CIRCUITPY_MICROCONTROLLER
-CIRCUITPY_MICROCONTROLLER = 1
+CIRCUITPY_MICROCONTROLLER = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_MICROCONTROLLER=$(CIRCUITPY_MICROCONTROLLER)
 
 ifndef CIRCUITPY_NEOPIXEL_WRITE
-CIRCUITPY_NEOPIXEL_WRITE = 1
+CIRCUITPY_NEOPIXEL_WRITE = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_NEOPIXEL_WRITE=$(CIRCUITPY_NEOPIXEL_WRITE)
 
@@ -134,12 +161,12 @@ endif
 CFLAGS += -DCIRCUITPY_NETWORK=$(CIRCUITPY_NETWORK)
 
 ifndef CIRCUITPY_NVM
-CIRCUITPY_NVM = 1
+CIRCUITPY_NVM = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_NVM=$(CIRCUITPY_NVM)
 
 ifndef CIRCUITPY_OS
-CIRCUITPY_OS = 1
+CIRCUITPY_OS = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_OS=$(CIRCUITPY_OS)
 
@@ -149,22 +176,28 @@ endif
 CFLAGS += -DCIRCUITPY_PIXELBUF=$(CIRCUITPY_PIXELBUF)
 
 ifndef CIRCUITPY_PULSEIO
-CIRCUITPY_PULSEIO = 1
+CIRCUITPY_PULSEIO = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_PULSEIO=$(CIRCUITPY_PULSEIO)
 
+# Only for SAMD boards for the moment
+ifndef CIRCUITPY_PS2IO
+CIRCUITPY_PS2IO = 0
+endif
+CFLAGS += -DCIRCUITPY_PS2IO=$(CIRCUITPY_PS2IO)
+
 ifndef CIRCUITPY_RANDOM
-CIRCUITPY_RANDOM = 1
+CIRCUITPY_RANDOM = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_RANDOM=$(CIRCUITPY_RANDOM)
 
 ifndef CIRCUITPY_ROTARYIO
-CIRCUITPY_ROTARYIO = 1
+CIRCUITPY_ROTARYIO = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_ROTARYIO=$(CIRCUITPY_ROTARYIO)
 
 ifndef CIRCUITPY_RTC
-CIRCUITPY_RTC = 1
+CIRCUITPY_RTC = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_RTC=$(CIRCUITPY_RTC)
 
@@ -183,27 +216,27 @@ endif
 CFLAGS += -DCIRCUITPY_STAGE=$(CIRCUITPY_STAGE)
 
 ifndef CIRCUITPY_STORAGE
-CIRCUITPY_STORAGE = 1
+CIRCUITPY_STORAGE = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE)
 
 ifndef CIRCUITPY_STRUCT
-CIRCUITPY_STRUCT = 1
+CIRCUITPY_STRUCT = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_STRUCT=$(CIRCUITPY_STRUCT)
 
 ifndef CIRCUITPY_SUPERVISOR
-CIRCUITPY_SUPERVISOR = 1
+CIRCUITPY_SUPERVISOR = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_SUPERVISOR=$(CIRCUITPY_SUPERVISOR)
 
 ifndef CIRCUITPY_TIME
-CIRCUITPY_TIME = 1
+CIRCUITPY_TIME = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_TIME=$(CIRCUITPY_TIME)
 
 ifndef CIRCUITPY_TOUCHIO
-CIRCUITPY_TOUCHIO = 1
+CIRCUITPY_TOUCHIO = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_TOUCHIO=$(CIRCUITPY_TOUCHIO)
 
@@ -214,12 +247,12 @@ endif
 CFLAGS += -DCIRCUITPY_UHEAP=$(CIRCUITPY_UHEAP)
 
 ifndef CIRCUITPY_USB_HID
-CIRCUITPY_USB_HID = 1
+CIRCUITPY_USB_HID = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_USB_HID=$(CIRCUITPY_USB_HID)
 
 ifndef CIRCUITPY_USB_MIDI
-CIRCUITPY_USB_MIDI = 1
+CIRCUITPY_USB_MIDI = $(CIRCUITPY_DEFAULT_BUILD)
 endif
 CFLAGS += -DCIRCUITPY_USB_MIDI=$(CIRCUITPY_USB_MIDI)
 
diff --git a/py/emitglue.c b/py/emitglue.c
index 74bf8ddca..7708689dd 100644
--- a/py/emitglue.c
+++ b/py/emitglue.c
@@ -142,11 +142,12 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar
             fun = mp_obj_new_fun_asm(rc->n_pos_args, rc->data.u_native.fun_data, rc->data.u_native.type_sig);
             break;
         #endif
-        default:
-            // rc->kind should always be set and BYTECODE is the only remaining case
-            assert(rc->kind == MP_CODE_BYTECODE);
+        case MP_CODE_BYTECODE:
             fun = mp_obj_new_fun_bc(def_args, def_kw_args, rc->data.u_byte.bytecode, rc->data.u_byte.const_table);
             break;
+        default:
+            // All other kinds are invalid.
+            mp_raise_RuntimeError(translate("Corrupt raw code"));
     }
 
     // check for generator functions and if so wrap in generator object
diff --git a/py/gc.c b/py/gc.c
index e9b85fa7c..10b36950c 100755
--- a/py/gc.c
+++ b/py/gc.c
@@ -383,6 +383,10 @@ void gc_collect_start(void) {
     #endif
 }
 
+void gc_collect_ptr(void *ptr) {
+    gc_mark(ptr);
+}
+
 void gc_collect_root(void **ptrs, size_t len) {
     for (size_t i = 0; i < len; i++) {
         void *ptr = ptrs[i];
diff --git a/py/gc.h b/py/gc.h
index 757a2a6e0..02bf45158 100644
--- a/py/gc.h
+++ b/py/gc.h
@@ -43,6 +43,7 @@ bool gc_is_locked(void);
 // A given port must implement gc_collect by using the other collect functions.
 void gc_collect(void);
 void gc_collect_start(void);
+void gc_collect_ptr(void *ptr);
 void gc_collect_root(void **ptrs, size_t len);
 void gc_collect_end(void);
 
diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py
index 5b5ec1c37..8e3835d86 100644
--- a/py/makeqstrdata.py
+++ b/py/makeqstrdata.py
@@ -180,7 +180,7 @@ def compress(encoding_table, decompressed):
     if not isinstance(decompressed, bytes):
         raise TypeError()
     values, lengths = encoding_table
-    enc = bytearray(len(decompressed))
+    enc = bytearray(len(decompressed) * 2)
     #print(decompressed)
     #print(lengths)
     current_bit = 7
@@ -227,6 +227,8 @@ def compress(encoding_table, decompressed):
                 current_bit -= 1
     if current_bit != 7:
         current_byte += 1
+    if current_byte > len(decompressed):
+        print("Note: compression increased length", repr(decompressed.decode('utf-8')), len(decompressed), current_byte, file=sys.stderr)
     return enc[:current_byte]
 
 def qstr_escape(qst):
diff --git a/py/modbuiltins.c b/py/modbuiltins.c
index 124dcd05a..e764f1987 100644
--- a/py/modbuiltins.c
+++ b/py/modbuiltins.c
@@ -455,13 +455,13 @@ STATIC mp_obj_t mp_builtin_round(size_t n_args, const mp_obj_t *args) {
             return o_in;
         }
 
-        #if !MICROPY_PY_BUILTINS_ROUND_INT
-        mp_raise_NotImplementedError(NULL);
-        #else
         mp_int_t num_dig = mp_obj_get_int(args[1]);
         if (num_dig >= 0) {
             return o_in;
         }
+        #if !MICROPY_PY_BUILTINS_ROUND_INT
+        mp_raise_NotImplementedError(NULL);
+        #else
 
         mp_obj_t mult = mp_binary_op(MP_BINARY_OP_POWER, MP_OBJ_NEW_SMALL_INT(10), MP_OBJ_NEW_SMALL_INT(-num_dig));
         mp_obj_t half_mult =  mp_binary_op(MP_BINARY_OP_FLOOR_DIVIDE, mult, MP_OBJ_NEW_SMALL_INT(2));
diff --git a/py/obj.c b/py/obj.c
index fb59eec82..322a302f9 100644
--- a/py/obj.c
+++ b/py/obj.c
@@ -531,6 +531,36 @@ mp_obj_t mp_identity_getiter(mp_obj_t self, mp_obj_iter_buf_t *iter_buf) {
     return self;
 }
 
+typedef struct {
+    mp_obj_base_t base;
+    mp_fun_1_t iternext;
+    mp_obj_t obj;
+    mp_int_t cur;
+} mp_obj_generic_it_t;
+
+STATIC mp_obj_t generic_it_iternext(mp_obj_t self_in) {
+    mp_obj_generic_it_t *self = MP_OBJ_TO_PTR(self_in);
+    mp_obj_type_t *type = mp_obj_get_type(self->obj);
+    mp_obj_t current_length = type->unary_op(MP_UNARY_OP_LEN, self->obj);
+    if (self->cur < MP_OBJ_SMALL_INT_VALUE(current_length)) {
+        mp_obj_t o_out = type->subscr(self->obj, MP_OBJ_NEW_SMALL_INT(self->cur), MP_OBJ_SENTINEL);
+        self->cur += 1;
+        return o_out;
+    } else {
+        return MP_OBJ_STOP_ITERATION;
+    }
+}
+
+mp_obj_t mp_obj_new_generic_iterator(mp_obj_t obj, mp_obj_iter_buf_t *iter_buf) {
+    assert(sizeof(mp_obj_generic_it_t) <= sizeof(mp_obj_iter_buf_t));
+    mp_obj_generic_it_t *o = (mp_obj_generic_it_t*)iter_buf;
+    o->base.type = &mp_type_polymorph_iter;
+    o->iternext = generic_it_iternext;
+    o->obj = obj;
+    o->cur = 0;
+    return MP_OBJ_FROM_PTR(o);
+}
+
 bool mp_get_buffer(mp_obj_t obj, mp_buffer_info_t *bufinfo, mp_uint_t flags) {
     mp_obj_type_t *type = mp_obj_get_type(obj);
     if (type->buffer_p.get_buffer == NULL) {
diff --git a/py/obj.h b/py/obj.h
index ae2bff190..6eead377d 100644
--- a/py/obj.h
+++ b/py/obj.h
@@ -763,6 +763,7 @@ void mp_obj_tuple_del(mp_obj_t self_in);
 mp_int_t mp_obj_tuple_hash(mp_obj_t self_in);
 
 // list
+mp_obj_t mp_obj_list_clear(mp_obj_t self_in);
 mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg);
 mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value);
 void mp_obj_list_get(mp_obj_t self_in, size_t *len, mp_obj_t **items);
@@ -819,6 +820,10 @@ mp_obj_t mp_identity(mp_obj_t self);
 MP_DECLARE_CONST_FUN_OBJ_1(mp_identity_obj);
 mp_obj_t mp_identity_getiter(mp_obj_t self, mp_obj_iter_buf_t *iter_buf);
 
+// Generic iterator that uses unary op and subscr to iterate over a native type. It will be slower
+// than a custom iterator but applies broadly.
+mp_obj_t mp_obj_new_generic_iterator(mp_obj_t self, mp_obj_iter_buf_t *iter_buf);
+
 // module
 typedef struct _mp_obj_module_t {
     mp_obj_base_t base;
diff --git a/py/objint.c b/py/objint.c
index 80f6dfe55..9e11871f1 100644
--- a/py/objint.c
+++ b/py/objint.c
@@ -300,6 +300,8 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co
     return b;
 }
 
+#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
+
 void mp_obj_int_buffer_overflow_check(mp_obj_t self_in, size_t nbytes, bool is_signed)
 {
     if (is_signed) {
@@ -329,7 +331,43 @@ void mp_obj_int_buffer_overflow_check(mp_obj_t self_in, size_t nbytes, bool is_s
         }
     }
 
-    mp_raise_OverflowError_varg(translate("value would overflow a %d byte buffer"), nbytes);
+    mp_raise_OverflowError_varg(translate("value must fit in %d byte(s)"), nbytes);
+}
+
+#endif // MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
+
+void mp_small_int_buffer_overflow_check(mp_int_t val, size_t nbytes, bool is_signed) {
+    // Fast path for zero.
+    if (val == 0) {
+        return;
+    }
+
+    // Trying to store negative values in unsigned bytes falls through to failure.
+    if (is_signed || val >= 0) {
+
+        if (nbytes >= sizeof(val)) {
+            // All non-negative N bit signed integers fit in an unsigned N bit integer.
+            // This case prevents shifting too far below.
+            return;
+        }
+
+        if (is_signed) {
+            mp_int_t edge = ((mp_int_t)1 << (nbytes * 8 - 1));
+            if (-edge <= val && val < edge) {
+                return;
+            }
+            // Out of range, fall through to failure.
+        } else {
+            // Unsigned. We already know val >= 0.
+            mp_int_t edge = ((mp_int_t)1 << (nbytes * 8));
+            if (val < edge) {
+                return;
+            }
+        }
+        // Fall through to failure.
+    }
+
+    mp_raise_OverflowError_varg(translate("value must fit in %d byte(s)"), nbytes);
 }
 
 #if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_NONE
@@ -467,15 +505,16 @@ STATIC mp_obj_t int_to_bytes(size_t n_args, const mp_obj_t *args) {
     byte *data = (byte*)vstr.buf;
     memset(data, 0, len);
 
-    mp_obj_int_buffer_overflow_check(args[0], len, false);
-
     #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
     if (!MP_OBJ_IS_SMALL_INT(args[0])) {
+        mp_obj_int_buffer_overflow_check(args[0], len, false);
         mp_obj_int_to_bytes_impl(args[0], big_endian, len, data);
     } else
     #endif
     {
         mp_int_t val = MP_OBJ_SMALL_INT_VALUE(args[0]);
+        // Small int checking is separate, to be fast.
+        mp_small_int_buffer_overflow_check(val, len, false);
         size_t l = MIN((size_t)len, sizeof(val));
         mp_binary_set_int(l, big_endian, data + (big_endian ? (len - l) : 0), val);
     }
diff --git a/py/objint.h b/py/objint.h
index 654404ac9..e8c9bc3e0 100644
--- a/py/objint.h
+++ b/py/objint.h
@@ -53,7 +53,12 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co
                            int base, const char *prefix, char base_char, char comma);
 char *mp_obj_int_formatted_impl(char **buf, size_t *buf_size, size_t *fmt_size, mp_const_obj_t self_in,
                                 int base, const char *prefix, char base_char, char comma);
+#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
 void mp_obj_int_buffer_overflow_check(mp_obj_t self_in, size_t nbytes, bool is_signed);
+#endif
+
+void mp_small_int_buffer_overflow_check(mp_int_t val, size_t nbytes, bool is_signed);
+
 mp_int_t mp_obj_int_hash(mp_obj_t self_in);
 mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf);
 void mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf);
diff --git a/py/objlist.c b/py/objlist.c
index 16ca4b353..558c4c611 100644
--- a/py/objlist.c
+++ b/py/objlist.c
@@ -340,7 +340,7 @@ mp_obj_t mp_obj_list_sort(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_
     return mp_const_none;
 }
 
-STATIC mp_obj_t list_clear(mp_obj_t self_in) {
+mp_obj_t mp_obj_list_clear(mp_obj_t self_in) {
     mp_check_self(MP_OBJ_IS_TYPE(self_in, &mp_type_list));
     mp_obj_list_t *self = MP_OBJ_TO_PTR(self_in);
     self->len = 0;
@@ -418,7 +418,7 @@ STATIC mp_obj_t list_reverse(mp_obj_t self_in) {
 
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(list_append_obj, mp_obj_list_append);
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(list_extend_obj, list_extend);
-STATIC MP_DEFINE_CONST_FUN_OBJ_1(list_clear_obj, list_clear);
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(list_clear_obj, mp_obj_list_clear);
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(list_copy_obj, list_copy);
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(list_count_obj, list_count);
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(list_index_obj, 2, 4, list_index);
diff --git a/py/objtype.c b/py/objtype.c
index 88e918827..5133d849f 100644
--- a/py/objtype.c
+++ b/py/objtype.c
@@ -117,6 +117,16 @@ mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *class, const mp_obj_
     return o;
 }
 
+// When instances are first created they have the base_init wrapper as their native parent's
+// instance because make_new combines __new__ and __init__. This object is invalid for the native
+// code so it must call this method to ensure that the given object has been __init__'d and is
+// valid.
+void mp_obj_assert_native_inited(mp_obj_t native_object) {
+    if (native_object == MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj)) {
+        mp_raise_NotImplementedError(translate("Call super().__init__() before accessing native object."));
+    }
+}
+
 // TODO
 // This implements depth-first left-to-right MRO, which is not compliant with Python3 MRO
 // http://python-history.blogspot.com/2010/06/method-resolution-order.html
diff --git a/py/objtype.h b/py/objtype.h
index 13613f01f..a32c87496 100644
--- a/py/objtype.h
+++ b/py/objtype.h
@@ -37,6 +37,8 @@ typedef struct _mp_obj_instance_t {
     // TODO maybe cache __getattr__ and __setattr__ for efficient lookup of them
 } mp_obj_instance_t;
 
+void mp_obj_assert_native_inited(mp_obj_t native_object);
+
 #if MICROPY_CPYTHON_COMPAT
 // this is needed for object.__new__
 mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *cls, const mp_obj_type_t **native_base);
diff --git a/py/persistentcode.c b/py/persistentcode.c
index f6de782ed..eb69bd407 100644
--- a/py/persistentcode.c
+++ b/py/persistentcode.c
@@ -102,20 +102,35 @@ STATIC void extract_prelude(const byte **ip, const byte **ip2, bytecode_prelude_
 
 #include "py/parsenum.h"
 
+STATIC void raise_corrupt_mpy(void) {
+    mp_raise_RuntimeError(translate("Corrupt .mpy file"));
+}
+
 STATIC int read_byte(mp_reader_t *reader) {
-    return reader->readbyte(reader->data);
+    mp_uint_t b = reader->readbyte(reader->data);
+    if (b == MP_READER_EOF) {
+        raise_corrupt_mpy();
+    }
+    return b;
 }
 
 STATIC void read_bytes(mp_reader_t *reader, byte *buf, size_t len) {
     while (len-- > 0) {
-        *buf++ = reader->readbyte(reader->data);
+        mp_uint_t b =reader->readbyte(reader->data);
+        if (b == MP_READER_EOF) {
+            raise_corrupt_mpy();
+        }
+        *buf++ = b;
     }
 }
 
 STATIC size_t read_uint(mp_reader_t *reader) {
     size_t unum = 0;
     for (;;) {
-        byte b = reader->readbyte(reader->data);
+        mp_uint_t b = reader->readbyte(reader->data);
+        if (b == MP_READER_EOF) {
+            raise_corrupt_mpy();
+        }
         unum = (unum << 7) | (b & 0x7f);
         if ((b & 0x80) == 0) {
             break;
@@ -145,11 +160,12 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) {
             return mp_obj_new_str_from_vstr(obj_type == 's' ? &mp_type_str : &mp_type_bytes, &vstr);
         } else if (obj_type == 'i') {
             return mp_parse_num_integer(vstr.buf, vstr.len, 10, NULL);
-        } else {
-            assert(obj_type == 'f' || obj_type == 'c');
+        } else if (obj_type == 'f' || obj_type == 'c') {
             return mp_parse_num_decimal(vstr.buf, vstr.len, obj_type == 'c', false, NULL);
         }
     }
+    raise_corrupt_mpy();
+    return MP_OBJ_FROM_PTR(&mp_const_none_obj);
 }
 
 STATIC void load_bytecode_qstrs(mp_reader_t *reader, byte *ip, byte *ip_top) {
diff --git a/py/py.mk b/py/py.mk
index a69243b01..17cb79264 100644
--- a/py/py.mk
+++ b/py/py.mk
@@ -107,9 +107,9 @@ endif
 
 # External modules written in C.
 ifneq ($(USER_C_MODULES),)
-# pre-define USERMOD variables as expanded so that variables are immediate 
+# pre-define USERMOD variables as expanded so that variables are immediate
 # expanded as they're added to them
-SRC_USERMOD := 
+SRC_USERMOD :=
 CFLAGS_USERMOD :=
 LDFLAGS_USERMOD :=
 $(foreach module, $(wildcard $(USER_C_MODULES)/*/micropython.mk), \
@@ -343,10 +343,23 @@ $(PY_BUILD)/qstr.o: $(HEADER_BUILD)/qstrdefs.generated.h
 $(PY_BUILD)/nlr%.o: CFLAGS += -Os
 
 # optimising gc for speed; 5ms down to 4ms on pybv2
+ifndef SUPEROPT_GC
+  SUPEROPT_GC = 1
+endif
+
+ifeq ($(SUPEROPT_GC),1)
 $(PY_BUILD)/gc.o: CFLAGS += $(CSUPEROPT)
+endif
 
 # optimising vm for speed, adds only a small amount to code size but makes a huge difference to speed (20% faster)
+ifndef SUPEROPT_VM
+  SUPEROPT_VM = 1
+endif
+
+ifeq ($(SUPEROPT_VM),1)
 $(PY_BUILD)/vm.o: CFLAGS += $(CSUPEROPT)
+endif
+
 # Optimizing vm.o for modern deeply pipelined CPUs with branch predictors
 # may require disabling tail jump optimization. This will make sure that
 # each opcode has its own dispatching jump which will improve branch
diff --git a/py/vmentrytable.h b/py/vmentrytable.h
index 615f4e2ce..a0e2d4065 100644
--- a/py/vmentrytable.h
+++ b/py/vmentrytable.h
@@ -24,7 +24,7 @@
  * THE SOFTWARE.
  */
 
-#if __clang__
+#ifdef __clang__
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Winitializer-overrides"
 #endif // __clang__
@@ -113,6 +113,6 @@ static const void *const entry_table[256] = {
     [MP_BC_BINARY_OP_MULTI ... MP_BC_BINARY_OP_MULTI + MP_BINARY_OP_NUM_BYTECODE - 1] = &&entry_MP_BC_BINARY_OP_MULTI,
 };
 
-#if __clang__
+#ifdef __clang__
 #pragma clang diagnostic pop
 #endif // __clang__