Merge tag 'v1.9.1'2.0.0-alpha.0

Fixes for stmhal USB mass storage, lwIP bindings and VFS regressions

This release provides an important fix for the USB mass storage device in
the stmhal port by implementing the SCSI SYNCHRONIZE_CACHE command, which
is now require by some Operating Systems.  There are also fixes for the
lwIP bindings to improve non-blocking sockets and error codes.  The VFS has
some regressions fixed including the ability to statvfs the root.

All changes are listed below.

py core:
- modbuiltins: add core-provided version of input() function
- objstr: catch case of negative "maxsplit" arg to str.rsplit()
- persistentcode: allow to compile with complex numbers disabled
- objstr: allow to compile with obj-repr D, and unicode disabled
- modsys: allow to compile with obj-repr D and PY_ATTRTUPLE disabled
- provide mp_decode_uint_skip() to help reduce stack usage
- makeqstrdefs.py: make script run correctly with Python 2.6
- objstringio: if created from immutable object, follow copy on write policy

extmod:
- modlwip: connect: for non-blocking mode, return EINPROGRESS
- modlwip: fix error codes for duplicate calls to connect()
- modlwip: accept: fix error code for non-blocking mode
- vfs: allow to statvfs the root directory
- vfs: allow "buffering" and "encoding" args to VFS's open()
- modframebuf: fix signed/unsigned comparison pendantic warning

lib:
- libm: use isfinite instead of finitef, for C99 compatibility
- utils/interrupt_char: remove support for KBD_EXCEPTION disabled

tests:
- basics/string_rsplit: add tests for negative "maxsplit" argument
- float: convert "sys.exit()" to "raise SystemExit"
- float/builtin_float_minmax: PEP8 fixes
- basics: convert "sys.exit()" to "raise SystemExit"
- convert remaining "sys.exit()" to "raise SystemExit"

unix port:
- convert to use core-provided version of built-in import()
- Makefile: replace references to make with $(MAKE)

windows port:
- convert to use core-provided version of built-in import()

qemu-arm port:
- Makefile: adjust object-file lists to get correct dependencies
- enable micropython.mem_*() functions to allow more tests

stmhal port:
- boards: enable DAC for NUCLEO_F767ZI board
- add support for NUCLEO_F446RE board
- pass USB handler as parameter to allow more than one USB handler
- usb: use local USB handler variable in Start-of-Frame handler
- usb: make state for USB device private to top-level USB driver
- usbdev: for MSC implement SCSI SYNCHRONIZE_CACHE command
- convert from using stmhal's input() to core provided version

cc3200 port:
- convert from using stmhal's input() to core provided version

teensy port:
- convert from using stmhal's input() to core provided version

esp8266 port:
- Makefile: replace references to make with $(MAKE)
- Makefile: add clean-modules target
- convert from using stmhal's input() to core provided version

zephyr port:
- modusocket: getaddrinfo: Fix mp_obj_len() usage
- define MICROPY_PY_SYS_PLATFORM (to "zephyr")
- machine_pin: use native Zephyr types for Zephyr API calls

docs:
- machine.Pin: remove out_value() method
- machine.Pin: add on() and off() methods
- esp8266: consistently replace Pin.high/low methods with .on/off
- esp8266/quickref: polish Pin.on()/off() examples
- network: move confusingly-named cc3200 Server class to its reference
- uos: deconditionalize, remove minor port-specific details
- uos: move cc3200 port legacy VFS mounting functions to its ref doc
- machine: sort machine classes in logical order, not alphabetically
- network: first step to describe standard network class interface

examples:
- embedding: use core-provided KeyboardInterrupt object

1 files changed, 173 insertions, 116 deletions

diff --git a/stmhal/main.c b/stmhal/main.c
index 97150ecac..566c2db07 100644
--- a/stmhal/main.c
+++ b/stmhal/main.c
@@ -27,23 +27,20 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "py/nlr.h"
-#include "py/lexer.h"
-#include "py/parse.h"
-#include "py/obj.h"
 #include "py/runtime.h"
 #include "py/stackctrl.h"
 #include "py/gc.h"
 #include "py/mphal.h"
-
+#include "lib/mp-readline/readline.h"
 #include "lib/utils/pyexec.h"
-#include "lib/fatfs/ff.h"
-#include "extmod/fsusermount.h"
+#include "lib/oofatfs/ff.h"
+#include "extmod/vfs.h"
+#include "extmod/vfs_fat.h"
 
 #include "systick.h"
 #include "pendsv.h"
+#include "pybthread.h"
 #include "gccollect.h"
-#include "readline.h"
 #include "modmachine.h"
 #include "i2c.h"
 #include "spi.h"
@@ -66,18 +63,19 @@
 
 void SystemClock_Config(void);
 
+pyb_thread_t pyb_thread_main;
 fs_user_mount_t fs_user_mount_flash;
 
 void flash_error(int n) {
     for (int i = 0; i < n; i++) {
-        led_state(PYB_LED_R1, 1);
-        led_state(PYB_LED_R2, 0);
-        HAL_Delay(250);
-        led_state(PYB_LED_R1, 0);
-        led_state(PYB_LED_R2, 1);
-        HAL_Delay(250);
+        led_state(PYB_LED_RED, 1);
+        led_state(PYB_LED_GREEN, 0);
+        mp_hal_delay_ms(250);
+        led_state(PYB_LED_RED, 0);
+        led_state(PYB_LED_GREEN, 1);
+        mp_hal_delay_ms(250);
     }
-    led_state(PYB_LED_R2, 0);
+    led_state(PYB_LED_GREEN, 0);
 }
 
 void NORETURN __fatal_error(const char *msg) {
@@ -166,96 +164,91 @@ static const char fresh_readme_txt[] =
 ;
 
 // avoid inlining to avoid stack usage within main()
-MP_NOINLINE STATIC void init_flash_fs(uint reset_mode) {
+MP_NOINLINE STATIC bool init_flash_fs(uint reset_mode) {
     // init the vfs object
-    fs_user_mount_t *vfs = &fs_user_mount_flash;
-    vfs->str = "/flash";
-    vfs->len = 6;
-    vfs->flags = 0;
-    pyb_flash_init_vfs(vfs);
-
-    // put the flash device in slot 0 (it will be unused at this point)
-    MP_STATE_PORT(fs_user_mount)[0] = vfs;
+    fs_user_mount_t *vfs_fat = &fs_user_mount_flash;
+    vfs_fat->flags = 0;
+    pyb_flash_init_vfs(vfs_fat);
 
     // try to mount the flash
-    FRESULT res = f_mount(&vfs->fatfs, vfs->str, 1);
+    FRESULT res = f_mount(&vfs_fat->fatfs);
 
     if (reset_mode == 3 || res == FR_NO_FILESYSTEM) {
         // no filesystem, or asked to reset it, so create a fresh one
 
         // LED on to indicate creation of LFS
-        led_state(PYB_LED_R2, 1);
+        led_state(PYB_LED_GREEN, 1);
         uint32_t start_tick = HAL_GetTick();
 
-        res = f_mkfs("/flash", 0, 0);
+        uint8_t working_buf[_MAX_SS];
+        res = f_mkfs(&vfs_fat->fatfs, FM_FAT, 0, working_buf, sizeof(working_buf));
         if (res == FR_OK) {
             // success creating fresh LFS
         } else {
             printf("PYB: can't create flash filesystem\n");
-            MP_STATE_PORT(fs_user_mount)[0] = NULL;
-            return;
+            return false;
         }
 
         // set label
-        f_setlabel("/flash/pybflash");
+        f_setlabel(&vfs_fat->fatfs, "pybflash");
 
         // create empty main.py
         FIL fp;
-        f_open(&fp, "/flash/main.py", FA_WRITE | FA_CREATE_ALWAYS);
+        f_open(&vfs_fat->fatfs, &fp, "/main.py", FA_WRITE | FA_CREATE_ALWAYS);
         UINT n;
         f_write(&fp, fresh_main_py, sizeof(fresh_main_py) - 1 /* don't count null terminator */, &n);
         // TODO check we could write n bytes
         f_close(&fp);
 
         // create .inf driver file
-        f_open(&fp, "/flash/pybcdc.inf", FA_WRITE | FA_CREATE_ALWAYS);
+        f_open(&vfs_fat->fatfs, &fp, "/pybcdc.inf", FA_WRITE | FA_CREATE_ALWAYS);
         f_write(&fp, fresh_pybcdc_inf, sizeof(fresh_pybcdc_inf) - 1 /* don't count null terminator */, &n);
         f_close(&fp);
 
         // create readme file
-        f_open(&fp, "/flash/README.txt", FA_WRITE | FA_CREATE_ALWAYS);
+        f_open(&vfs_fat->fatfs, &fp, "/README.txt", FA_WRITE | FA_CREATE_ALWAYS);
         f_write(&fp, fresh_readme_txt, sizeof(fresh_readme_txt) - 1 /* don't count null terminator */, &n);
         f_close(&fp);
 
         // keep LED on for at least 200ms
         sys_tick_wait_at_least(start_tick, 200);
-        led_state(PYB_LED_R2, 0);
+        led_state(PYB_LED_GREEN, 0);
     } else if (res == FR_OK) {
         // mount sucessful
     } else {
+    fail:
         printf("PYB: can't mount flash\n");
-        MP_STATE_PORT(fs_user_mount)[0] = NULL;
-        return;
+        return false;
+    }
+
+    // mount the flash device (there should be no other devices mounted at this point)
+    // we allocate this structure on the heap because vfs->next is a root pointer
+    mp_vfs_mount_t *vfs = m_new_obj_maybe(mp_vfs_mount_t);
+    if (vfs == NULL) {
+        goto fail;
     }
+    vfs->str = "/flash";
+    vfs->len = 6;
+    vfs->obj = MP_OBJ_FROM_PTR(vfs_fat);
+    vfs->next = NULL;
+    MP_STATE_VM(vfs_mount_table) = vfs;
 
     // The current directory is used as the boot up directory.
     // It is set to the internal flash filesystem by default.
-    f_chdrive("/flash");
+    MP_STATE_PORT(vfs_cur) = vfs;
 
     // Make sure we have a /flash/boot.py.  Create it if needed.
     FILINFO fno;
-#if _USE_LFN
-    fno.lfname = NULL;
-    fno.lfsize = 0;
-#endif
-    res = f_stat("/flash/boot.py", &fno);
-    if (res == FR_OK) {
-        if (fno.fattrib & AM_DIR) {
-            // exists as a directory
-            // TODO handle this case
-            // see http://elm-chan.org/fsw/ff/img/app2.c for a "rm -rf" implementation
-        } else {
-            // exists as a file, good!
-        }
-    } else {
+    res = f_stat(&vfs_fat->fatfs, "/boot.py", &fno);
+    if (res != FR_OK) {
         // doesn't exist, create fresh file
 
         // LED on to indicate creation of boot.py
-        led_state(PYB_LED_R2, 1);
+        led_state(PYB_LED_GREEN, 1);
         uint32_t start_tick = HAL_GetTick();
 
         FIL fp;
-        f_open(&fp, "/flash/boot.py", FA_WRITE | FA_CREATE_ALWAYS);
+        f_open(&vfs_fat->fatfs, &fp, "/boot.py", FA_WRITE | FA_CREATE_ALWAYS);
         UINT n;
         f_write(&fp, fresh_boot_py, sizeof(fresh_boot_py) - 1 /* don't count null terminator */, &n);
         // TODO check we could write n bytes
@@ -263,9 +256,87 @@ MP_NOINLINE STATIC void init_flash_fs(uint reset_mode) {
 
         // keep LED on for at least 200ms
         sys_tick_wait_at_least(start_tick, 200);
-        led_state(PYB_LED_R2, 0);
+        led_state(PYB_LED_GREEN, 0);
+    }
+
+    return true;
+}
+
+#if MICROPY_HW_HAS_SDCARD
+STATIC bool init_sdcard_fs(bool first_soft_reset) {
+    bool first_part = true;
+    for (int part_num = 1; part_num <= 4; ++part_num) {
+        // create vfs object
+        fs_user_mount_t *vfs_fat = m_new_obj_maybe(fs_user_mount_t);
+        mp_vfs_mount_t *vfs = m_new_obj_maybe(mp_vfs_mount_t);
+        if (vfs == NULL || vfs_fat == NULL) {
+            break;
+        }
+        vfs_fat->flags = FSUSER_FREE_OBJ;
+        sdcard_init_vfs(vfs_fat, part_num);
+
+        // try to mount the partition
+        FRESULT res = f_mount(&vfs_fat->fatfs);
+
+        if (res != FR_OK) {
+            // couldn't mount
+            m_del_obj(fs_user_mount_t, vfs_fat);
+            m_del_obj(mp_vfs_mount_t, vfs);
+        } else {
+            // mounted via FatFs, now mount the SD partition in the VFS
+            if (first_part) {
+                // the first available partition is traditionally called "sd" for simplicity
+                vfs->str = "/sd";
+                vfs->len = 3;
+            } else {
+                // subsequent partitions are numbered by their index in the partition table
+                if (part_num == 2) {
+                    vfs->str = "/sd2";
+                } else if (part_num == 2) {
+                    vfs->str = "/sd3";
+                } else {
+                    vfs->str = "/sd4";
+                }
+                vfs->len = 4;
+            }
+            vfs->obj = MP_OBJ_FROM_PTR(vfs_fat);
+            vfs->next = NULL;
+            for (mp_vfs_mount_t **m = &MP_STATE_VM(vfs_mount_table);; m = &(*m)->next) {
+                if (*m == NULL) {
+                    *m = vfs;
+                    break;
+                }
+            }
+
+            if (first_soft_reset) {
+                // use SD card as medium for the USB MSD
+                #if defined(USE_DEVICE_MODE)
+                pyb_usb_storage_medium = PYB_USB_STORAGE_MEDIUM_SDCARD;
+                #endif
+            }
+
+            #if defined(USE_DEVICE_MODE)
+            // only use SD card as current directory if that's what the USB medium is
+            if (pyb_usb_storage_medium == PYB_USB_STORAGE_MEDIUM_SDCARD)
+            #endif
+            {
+                if (first_part) {
+                    // use SD card as current directory
+                    MP_STATE_PORT(vfs_cur) = vfs;
+                }
+            }
+            first_part = false;
+        }
+    }
+
+    if (first_part) {
+        printf("PYB: can't mount SD card\n");
+        return false;
+    } else {
+        return true;
     }
 }
+#endif
 
 STATIC uint update_reset_mode(uint reset_mode) {
 #if MICROPY_HW_HAS_SWITCH
@@ -278,7 +349,7 @@ STATIC uint update_reset_mode(uint reset_mode) {
             if (!switch_get()) {
                 break;
             }
-            HAL_Delay(20);
+            mp_hal_delay_ms(20);
             if (i % 30 == 29) {
                 if (++reset_mode > 3) {
                     reset_mode = 1;
@@ -293,13 +364,13 @@ STATIC uint update_reset_mode(uint reset_mode) {
             led_state(2, 0);
             led_state(3, 0);
             led_state(4, 0);
-            HAL_Delay(50);
+            mp_hal_delay_ms(50);
             led_state(2, reset_mode & 1);
             led_state(3, reset_mode & 2);
             led_state(4, reset_mode & 4);
-            HAL_Delay(50);
+            mp_hal_delay_ms(50);
         }
-        HAL_Delay(400);
+        mp_hal_delay_ms(400);
 
 #elif defined(MICROPY_HW_LED1)
 
@@ -312,11 +383,11 @@ STATIC uint update_reset_mode(uint reset_mode) {
                     break;
                 }
                 led_state(1, 1);
-                HAL_Delay(100);
+                mp_hal_delay_ms(100);
                 led_state(1, 0);
-                HAL_Delay(200);
+                mp_hal_delay_ms(200);
             }
-            HAL_Delay(400);
+            mp_hal_delay_ms(400);
             if (!switch_get()) {
                 break;
             }
@@ -328,11 +399,11 @@ STATIC uint update_reset_mode(uint reset_mode) {
         for (uint i = 0; i < 2; i++) {
             for (uint j = 0; j < reset_mode; j++) {
                 led_state(1, 1);
-                HAL_Delay(100);
+                mp_hal_delay_ms(100);
                 led_state(1, 0);
-                HAL_Delay(200);
+                mp_hal_delay_ms(200);
             }
-            HAL_Delay(400);
+            mp_hal_delay_ms(400);
         }
 #else
 #error Need a reset mode update method
@@ -345,11 +416,6 @@ STATIC uint update_reset_mode(uint reset_mode) {
 int main(void) {
     // TODO disable JTAG
 
-    // Stack limit should be less than real stack size, so we have a chance
-    // to recover from limit hit.  (Limit is measured in bytes.)
-    mp_stack_ctrl_init();
-    mp_stack_set_limit((char*)&_ram_end - (char*)&_heap_end - 1024);
-
     /* STM32F4xx HAL library initialization:
          - Configure the Flash prefetch, instruction and Data caches
          - Configure the Systick to generate an interrupt each 1 msec
@@ -383,6 +449,9 @@ int main(void) {
     #endif
 
     // basic sub-system init
+    #if MICROPY_PY_THREAD
+    pyb_thread_init(&pyb_thread_main);
+    #endif
     pendsv_init();
     led_init();
 #if MICROPY_HW_HAS_SWITCH
@@ -428,6 +497,17 @@ soft_reset:
         storage_init();
     }
 
+    // Python threading init
+    #if MICROPY_PY_THREAD
+    mp_thread_init();
+    #endif
+
+    // Stack limit should be less than real stack size, so we have a chance
+    // to recover from limit hit.  (Limit is measured in bytes.)
+    // Note: stack control relies on main thread being initialised above
+    mp_stack_set_top(&_estack);
+    mp_stack_set_limit((char*)&_estack - (char*)&_heap_end - 1024);
+
     // GC init
     gc_init(&_heap_start, &_heap_end);
 
@@ -435,8 +515,6 @@ soft_reset:
     mp_init();
     mp_obj_list_init(mp_sys_path, 0);
     mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script)
-    mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_flash));
-    mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_flash_slash_lib));
     mp_obj_list_init(mp_sys_argv, 0);
 
     // Initialise low-level sub-systems.  Here we need to very basic things like
@@ -479,53 +557,28 @@ soft_reset:
 
     // Initialise the local flash filesystem.
     // Create it if needed, mount in on /flash, and set it as current dir.
-    init_flash_fs(reset_mode);
+    bool mounted_flash = init_flash_fs(reset_mode);
 
+    bool mounted_sdcard = false;
 #if MICROPY_HW_HAS_SDCARD
     // if an SD card is present then mount it on /sd/
     if (sdcard_is_present()) {
-        // create vfs object
-        fs_user_mount_t *vfs = m_new_obj_maybe(fs_user_mount_t);
-        if (vfs == NULL) {
-            goto no_mem_for_sd;
+        // if there is a file in the flash called "SKIPSD", then we don't mount the SD card
+        if (!mounted_flash || f_stat(&fs_user_mount_flash.fatfs, "/SKIPSD", NULL) != FR_OK) {
+            mounted_sdcard = init_sdcard_fs(first_soft_reset);
         }
-        vfs->str = "/sd";
-        vfs->len = 3;
-        vfs->flags = FSUSER_FREE_OBJ;
-        sdcard_init_vfs(vfs);
-
-        // put the sd device in slot 1 (it will be unused at this point)
-        MP_STATE_PORT(fs_user_mount)[1] = vfs;
-
-        FRESULT res = f_mount(&vfs->fatfs, vfs->str, 1);
-        if (res != FR_OK) {
-            printf("PYB: can't mount SD card\n");
-            MP_STATE_PORT(fs_user_mount)[1] = NULL;
-            m_del_obj(fs_user_mount_t, vfs);
-        } else {
-            // TODO these should go before the /flash entries in the path
-            mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_sd));
-            mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_sd_slash_lib));
-
-            if (first_soft_reset) {
-                // use SD card as medium for the USB MSD
-#if defined(USE_DEVICE_MODE)
-                pyb_usb_storage_medium = PYB_USB_STORAGE_MEDIUM_SDCARD;
+    }
 #endif
-            }
 
-            #if defined(USE_DEVICE_MODE)
-            // only use SD card as current directory if that's what the USB medium is
-            if (pyb_usb_storage_medium == PYB_USB_STORAGE_MEDIUM_SDCARD)
-            #endif
-            {
-                // use SD card as current directory
-                f_chdrive("/sd");
-            }
-        }
-        no_mem_for_sd:;
+    // set sys.path based on mounted filesystems (/sd is first so it can override /flash)
+    if (mounted_sdcard) {
+        mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_sd));
+        mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_sd_slash_lib));
+    }
+    if (mounted_flash) {
+        mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_flash));
+        mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_flash_slash_lib));
     }
-#endif
 
     // reset config variables; they should be set by boot.py
     MP_STATE_PORT(pyb_config_main) = MP_OBJ_NULL;
@@ -534,9 +587,9 @@ soft_reset:
     // TODO perhaps have pyb.reboot([bootpy]) function to soft-reboot and execute custom boot.py
     if (reset_mode == 1 || reset_mode == 3) {
         const char *boot_py = "boot.py";
-        FRESULT res = f_stat(boot_py, NULL);
-        if (res == FR_OK) {
-            int ret = pyexec_file(boot_py, NULL);
+        mp_import_stat_t stat = mp_import_stat(boot_py);
+        if (stat == MP_IMPORT_STAT_FILE) {
+            int ret = pyexec_file(boot_py);
             if (ret & PYEXEC_FORCED_EXIT) {
                 goto soft_reset_exit;
             }
@@ -597,9 +650,9 @@ soft_reset:
         } else {
             main_py = mp_obj_str_get_str(MP_STATE_PORT(pyb_config_main));
         }
-        FRESULT res = f_stat(main_py, NULL);
-        if (res == FR_OK) {
-            int ret = pyexec_file(main_py, NULL);
+        mp_import_stat_t stat = mp_import_stat(main_py);
+        if (stat == MP_IMPORT_STAT_FILE) {
+            int ret = pyexec_file(main_py);
             if (ret & PYEXEC_FORCED_EXIT) {
                 goto soft_reset_exit;
             }
@@ -637,6 +690,10 @@ soft_reset_exit:
     can_deinit();
 #endif
 
+    #if MICROPY_PY_THREAD
+    pyb_thread_deinit();
+    #endif
+
     first_soft_reset = false;
     goto soft_reset;
 }