Merge tag 'v1.9.1'2.0.0-alpha.0

Fixes for stmhal USB mass storage, lwIP bindings and VFS regressions

This release provides an important fix for the USB mass storage device in
the stmhal port by implementing the SCSI SYNCHRONIZE_CACHE command, which
is now require by some Operating Systems.  There are also fixes for the
lwIP bindings to improve non-blocking sockets and error codes.  The VFS has
some regressions fixed including the ability to statvfs the root.

All changes are listed below.

py core:
- modbuiltins: add core-provided version of input() function
- objstr: catch case of negative "maxsplit" arg to str.rsplit()
- persistentcode: allow to compile with complex numbers disabled
- objstr: allow to compile with obj-repr D, and unicode disabled
- modsys: allow to compile with obj-repr D and PY_ATTRTUPLE disabled
- provide mp_decode_uint_skip() to help reduce stack usage
- makeqstrdefs.py: make script run correctly with Python 2.6
- objstringio: if created from immutable object, follow copy on write policy

extmod:
- modlwip: connect: for non-blocking mode, return EINPROGRESS
- modlwip: fix error codes for duplicate calls to connect()
- modlwip: accept: fix error code for non-blocking mode
- vfs: allow to statvfs the root directory
- vfs: allow "buffering" and "encoding" args to VFS's open()
- modframebuf: fix signed/unsigned comparison pendantic warning

lib:
- libm: use isfinite instead of finitef, for C99 compatibility
- utils/interrupt_char: remove support for KBD_EXCEPTION disabled

tests:
- basics/string_rsplit: add tests for negative "maxsplit" argument
- float: convert "sys.exit()" to "raise SystemExit"
- float/builtin_float_minmax: PEP8 fixes
- basics: convert "sys.exit()" to "raise SystemExit"
- convert remaining "sys.exit()" to "raise SystemExit"

unix port:
- convert to use core-provided version of built-in import()
- Makefile: replace references to make with $(MAKE)

windows port:
- convert to use core-provided version of built-in import()

qemu-arm port:
- Makefile: adjust object-file lists to get correct dependencies
- enable micropython.mem_*() functions to allow more tests

stmhal port:
- boards: enable DAC for NUCLEO_F767ZI board
- add support for NUCLEO_F446RE board
- pass USB handler as parameter to allow more than one USB handler
- usb: use local USB handler variable in Start-of-Frame handler
- usb: make state for USB device private to top-level USB driver
- usbdev: for MSC implement SCSI SYNCHRONIZE_CACHE command
- convert from using stmhal's input() to core provided version

cc3200 port:
- convert from using stmhal's input() to core provided version

teensy port:
- convert from using stmhal's input() to core provided version

esp8266 port:
- Makefile: replace references to make with $(MAKE)
- Makefile: add clean-modules target
- convert from using stmhal's input() to core provided version

zephyr port:
- modusocket: getaddrinfo: Fix mp_obj_len() usage
- define MICROPY_PY_SYS_PLATFORM (to "zephyr")
- machine_pin: use native Zephyr types for Zephyr API calls

docs:
- machine.Pin: remove out_value() method
- machine.Pin: add on() and off() methods
- esp8266: consistently replace Pin.high/low methods with .on/off
- esp8266/quickref: polish Pin.on()/off() examples
- network: move confusingly-named cc3200 Server class to its reference
- uos: deconditionalize, remove minor port-specific details
- uos: move cc3200 port legacy VFS mounting functions to its ref doc
- machine: sort machine classes in logical order, not alphabetically
- network: first step to describe standard network class interface

examples:
- embedding: use core-provided KeyboardInterrupt object

1 files changed, 24 insertions, 19 deletions

diff --git a/py/map.c b/py/map.c
index 0916ec522..50d74f38f 100644
--- a/py/map.c
+++ b/py/map.c
@@ -56,7 +56,7 @@ STATIC const uint16_t hash_allocation_sizes[] = {
     3229, 4831, 7243, 10861, 16273, 24407, 36607, 54907, // *1.5
 };
 
-STATIC mp_uint_t get_hash_alloc_greater_or_equal_to(mp_uint_t x) {
+STATIC size_t get_hash_alloc_greater_or_equal_to(size_t x) {
     for (size_t i = 0; i < MP_ARRAY_SIZE(hash_allocation_sizes); i++) {
         if (hash_allocation_sizes[i] >= x) {
             return hash_allocation_sizes[i];
@@ -70,7 +70,7 @@ STATIC mp_uint_t get_hash_alloc_greater_or_equal_to(mp_uint_t x) {
 /******************************************************************************/
 /* map                                                                        */
 
-void mp_map_init(mp_map_t *map, mp_uint_t n) {
+void mp_map_init(mp_map_t *map, size_t n) {
     if (n == 0) {
         map->alloc = 0;
         map->table = NULL;
@@ -84,7 +84,7 @@ void mp_map_init(mp_map_t *map, mp_uint_t n) {
     map->is_ordered = 0;
 }
 
-void mp_map_init_fixed_table(mp_map_t *map, mp_uint_t n, const mp_obj_t *table) {
+void mp_map_init_fixed_table(mp_map_t *map, size_t n, const mp_obj_t *table) {
     map->alloc = n;
     map->used = n;
     map->all_keys_are_qstrs = 1;
@@ -93,7 +93,7 @@ void mp_map_init_fixed_table(mp_map_t *map, mp_uint_t n, const mp_obj_t *table)
     map->table = (mp_map_elem_t*)table;
 }
 
-mp_map_t *mp_map_new(mp_uint_t n) {
+mp_map_t *mp_map_new(size_t n) {
     mp_map_t *map = m_new(mp_map_t, 1);
     mp_map_init(map, n);
     return map;
@@ -124,8 +124,8 @@ void mp_map_clear(mp_map_t *map) {
 }
 
 STATIC void mp_map_rehash(mp_map_t *map) {
-    mp_uint_t old_alloc = map->alloc;
-    mp_uint_t new_alloc = get_hash_alloc_greater_or_equal_to(map->alloc + 1);
+    size_t old_alloc = map->alloc;
+    size_t new_alloc = get_hash_alloc_greater_or_equal_to(map->alloc + 1);
     mp_map_elem_t *old_table = map->table;
     mp_map_elem_t *new_table = m_new0(mp_map_elem_t, new_alloc);
     // If we reach this point, table resizing succeeded, now we can edit the old map.
@@ -133,7 +133,7 @@ STATIC void mp_map_rehash(mp_map_t *map) {
     map->used = 0;
     map->all_keys_are_qstrs = 1;
     map->table = new_table;
-    for (mp_uint_t i = 0; i < old_alloc; i++) {
+    for (size_t i = 0; i < old_alloc; i++) {
         if (old_table[i].key != MP_OBJ_NULL && old_table[i].key != MP_OBJ_SENTINEL) {
             mp_map_lookup(map, old_table[i].key, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = old_table[i].value;
         }
@@ -178,8 +178,14 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t
         for (mp_map_elem_t *elem = &map->table[0], *top = &map->table[map->used]; elem < top; elem++) {
             if (elem->key == index || (!compare_only_ptrs && mp_obj_equal(elem->key, index))) {
                 if (MP_UNLIKELY(lookup_kind == MP_MAP_LOOKUP_REMOVE_IF_FOUND)) {
-                    elem->key = MP_OBJ_SENTINEL;
-                    // keep elem->value so that caller can access it if needed
+                    // remove the found element by moving the rest of the array down
+                    mp_obj_t value = elem->value;
+                    --map->used;
+                    memmove(elem, elem + 1, (top - elem - 1) * sizeof(*elem));
+                    // put the found element after the end so the caller can access it if needed
+                    elem = &map->table[map->used];
+                    elem->key = MP_OBJ_NULL;
+                    elem->value = value;
                 }
                 return elem;
             }
@@ -187,7 +193,6 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t
         if (MP_LIKELY(lookup_kind != MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)) {
             return NULL;
         }
-        // TODO shrink array down over any previously-freed slots
         if (map->used == map->alloc) {
             // TODO: Alloc policy
             map->alloc += 4;
@@ -220,8 +225,8 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t
         hash = MP_OBJ_SMALL_INT_VALUE(mp_unary_op(MP_UNARY_OP_HASH, index));
     }
 
-    mp_uint_t pos = hash % map->alloc;
-    mp_uint_t start_pos = pos;
+    size_t pos = hash % map->alloc;
+    size_t start_pos = pos;
     mp_map_elem_t *avail_slot = NULL;
     for (;;) {
         mp_map_elem_t *slot = &map->table[pos];
@@ -296,19 +301,19 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t
 
 #if MICROPY_PY_BUILTINS_SET
 
-void mp_set_init(mp_set_t *set, mp_uint_t n) {
+void mp_set_init(mp_set_t *set, size_t n) {
     set->alloc = n;
     set->used = 0;
     set->table = m_new0(mp_obj_t, set->alloc);
 }
 
 STATIC void mp_set_rehash(mp_set_t *set) {
-    mp_uint_t old_alloc = set->alloc;
+    size_t old_alloc = set->alloc;
     mp_obj_t *old_table = set->table;
     set->alloc = get_hash_alloc_greater_or_equal_to(set->alloc + 1);
     set->used = 0;
     set->table = m_new0(mp_obj_t, set->alloc);
-    for (mp_uint_t i = 0; i < old_alloc; i++) {
+    for (size_t i = 0; i < old_alloc; i++) {
         if (old_table[i] != MP_OBJ_NULL && old_table[i] != MP_OBJ_SENTINEL) {
             mp_set_lookup(set, old_table[i], MP_MAP_LOOKUP_ADD_IF_NOT_FOUND);
         }
@@ -328,8 +333,8 @@ mp_obj_t mp_set_lookup(mp_set_t *set, mp_obj_t index, mp_map_lookup_kind_t looku
         }
     }
     mp_uint_t hash = MP_OBJ_SMALL_INT_VALUE(mp_unary_op(MP_UNARY_OP_HASH, index));
-    mp_uint_t pos = hash % set->alloc;
-    mp_uint_t start_pos = pos;
+    size_t pos = hash % set->alloc;
+    size_t start_pos = pos;
     mp_obj_t *avail_slot = NULL;
     for (;;) {
         mp_obj_t elem = set->table[pos];
@@ -390,7 +395,7 @@ mp_obj_t mp_set_lookup(mp_set_t *set, mp_obj_t index, mp_map_lookup_kind_t looku
 }
 
 mp_obj_t mp_set_remove_first(mp_set_t *set) {
-    for (mp_uint_t pos = 0; pos < set->alloc; pos++) {
+    for (size_t pos = 0; pos < set->alloc; pos++) {
         if (MP_SET_SLOT_IS_FILLED(set, pos)) {
             mp_obj_t elem = set->table[pos];
             // delete element
@@ -418,7 +423,7 @@ void mp_set_clear(mp_set_t *set) {
 
 #if defined(DEBUG_PRINT) && DEBUG_PRINT
 void mp_map_dump(mp_map_t *map) {
-    for (mp_uint_t i = 0; i < map->alloc; i++) {
+    for (size_t i = 0; i < map->alloc; i++) {
         if (map->table[i].key != NULL) {
             mp_obj_print(map->table[i].key, PRINT_REPR);
         } else {