Merge tag 'v1.9.1'2.0.0-alpha.0

Fixes for stmhal USB mass storage, lwIP bindings and VFS regressions

This release provides an important fix for the USB mass storage device in
the stmhal port by implementing the SCSI SYNCHRONIZE_CACHE command, which
is now require by some Operating Systems.  There are also fixes for the
lwIP bindings to improve non-blocking sockets and error codes.  The VFS has
some regressions fixed including the ability to statvfs the root.

All changes are listed below.

py core:
- modbuiltins: add core-provided version of input() function
- objstr: catch case of negative "maxsplit" arg to str.rsplit()
- persistentcode: allow to compile with complex numbers disabled
- objstr: allow to compile with obj-repr D, and unicode disabled
- modsys: allow to compile with obj-repr D and PY_ATTRTUPLE disabled
- provide mp_decode_uint_skip() to help reduce stack usage
- makeqstrdefs.py: make script run correctly with Python 2.6
- objstringio: if created from immutable object, follow copy on write policy

extmod:
- modlwip: connect: for non-blocking mode, return EINPROGRESS
- modlwip: fix error codes for duplicate calls to connect()
- modlwip: accept: fix error code for non-blocking mode
- vfs: allow to statvfs the root directory
- vfs: allow "buffering" and "encoding" args to VFS's open()
- modframebuf: fix signed/unsigned comparison pendantic warning

lib:
- libm: use isfinite instead of finitef, for C99 compatibility
- utils/interrupt_char: remove support for KBD_EXCEPTION disabled

tests:
- basics/string_rsplit: add tests for negative "maxsplit" argument
- float: convert "sys.exit()" to "raise SystemExit"
- float/builtin_float_minmax: PEP8 fixes
- basics: convert "sys.exit()" to "raise SystemExit"
- convert remaining "sys.exit()" to "raise SystemExit"

unix port:
- convert to use core-provided version of built-in import()
- Makefile: replace references to make with $(MAKE)

windows port:
- convert to use core-provided version of built-in import()

qemu-arm port:
- Makefile: adjust object-file lists to get correct dependencies
- enable micropython.mem_*() functions to allow more tests

stmhal port:
- boards: enable DAC for NUCLEO_F767ZI board
- add support for NUCLEO_F446RE board
- pass USB handler as parameter to allow more than one USB handler
- usb: use local USB handler variable in Start-of-Frame handler
- usb: make state for USB device private to top-level USB driver
- usbdev: for MSC implement SCSI SYNCHRONIZE_CACHE command
- convert from using stmhal's input() to core provided version

cc3200 port:
- convert from using stmhal's input() to core provided version

teensy port:
- convert from using stmhal's input() to core provided version

esp8266 port:
- Makefile: replace references to make with $(MAKE)
- Makefile: add clean-modules target
- convert from using stmhal's input() to core provided version

zephyr port:
- modusocket: getaddrinfo: Fix mp_obj_len() usage
- define MICROPY_PY_SYS_PLATFORM (to "zephyr")
- machine_pin: use native Zephyr types for Zephyr API calls

docs:
- machine.Pin: remove out_value() method
- machine.Pin: add on() and off() methods
- esp8266: consistently replace Pin.high/low methods with .on/off
- esp8266/quickref: polish Pin.on()/off() examples
- network: move confusingly-named cc3200 Server class to its reference
- uos: deconditionalize, remove minor port-specific details
- uos: move cc3200 port legacy VFS mounting functions to its ref doc
- machine: sort machine classes in logical order, not alphabetically
- network: first step to describe standard network class interface

examples:
- embedding: use core-provided KeyboardInterrupt object

31 files changed, 331 insertions, 11 deletions

diff --git a/tests/micropython/const_intbig.py b/tests/micropython/const_intbig.py
new file mode 100644
index 000000000..e74902652
--- /dev/null
+++ b/tests/micropython/const_intbig.py
@@ -0,0 +1,13 @@
+# test constant optimisation, with consts that are bignums
+
+from micropython import const
+
+# check we can make consts from bignums
+Z1 = const(0xffffffff)
+Z2 = const(0xffffffffffffffff)
+print(hex(Z1), hex(Z2))
+
+# check arithmetic with bignum
+Z3 = const(Z1 + Z2)
+Z4 = const((1 << 100) + Z1)
+print(hex(Z3), hex(Z4))
diff --git a/tests/micropython/const_intbig.py.exp b/tests/micropython/const_intbig.py.exp
new file mode 100644
index 000000000..137f86cf8
--- /dev/null
+++ b/tests/micropython/const_intbig.py.exp
@@ -0,0 +1,2 @@
+0xffffffff 0xffffffffffffffff
+0x100000000fffffffe 0x100000000000000000ffffffff
diff --git a/tests/micropython/heapalloc_bytesio.py b/tests/micropython/heapalloc_bytesio.py
new file mode 100644
index 000000000..4aae2abf0
--- /dev/null
+++ b/tests/micropython/heapalloc_bytesio.py
@@ -0,0 +1,18 @@
+try:
+    import uio
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+import micropython
+
+data = b"1234" * 16
+buf = uio.BytesIO(64)
+
+micropython.heap_lock()
+
+buf.write(data)
+
+micropython.heap_unlock()
+
+print(buf.getvalue())
diff --git a/tests/micropython/heapalloc_bytesio.py.exp b/tests/micropython/heapalloc_bytesio.py.exp
new file mode 100644
index 000000000..675761c2b
--- /dev/null
+++ b/tests/micropython/heapalloc_bytesio.py.exp
@@ -0,0 +1 @@
+b'1234123412341234123412341234123412341234123412341234123412341234'
diff --git a/tests/micropython/heapalloc_bytesio2.py b/tests/micropython/heapalloc_bytesio2.py
new file mode 100644
index 000000000..cd76f5807
--- /dev/null
+++ b/tests/micropython/heapalloc_bytesio2.py
@@ -0,0 +1,20 @@
+# Creating BytesIO from immutable object should not immediately
+# copy its content.
+try:
+    import uio
+    import micropython
+    micropython.mem_total
+except (ImportError, AttributeError):
+    print("SKIP")
+    raise SystemExit
+
+
+data = b"1234" * 256
+
+before = micropython.mem_total()
+
+buf = uio.BytesIO(data)
+
+after = micropython.mem_total()
+
+print(after - before < len(data))
diff --git a/tests/micropython/heapalloc_bytesio2.py.exp b/tests/micropython/heapalloc_bytesio2.py.exp
new file mode 100644
index 000000000..0ca95142b
--- /dev/null
+++ b/tests/micropython/heapalloc_bytesio2.py.exp
@@ -0,0 +1 @@
+True
diff --git a/tests/micropython/heapalloc_exc_raise.py b/tests/micropython/heapalloc_exc_raise.py
new file mode 100644
index 000000000..d60e14ce5
--- /dev/null
+++ b/tests/micropython/heapalloc_exc_raise.py
@@ -0,0 +1,23 @@
+# Test that we can raise and catch (preallocated) exception
+# without memory allocation.
+import micropython
+
+e = ValueError("error")
+
+def func():
+    try:
+        # This works as is because traceback is not allocated
+        # if not possible (heap is locked, no memory). If heap
+        # is not locked, this would allocate a traceback entry.
+        # To avoid that, traceback should be warmed up (by raising
+        # it once after creation) and then cleared before each
+        # raise with:
+        # e.__traceback__ = None
+        raise e
+    except Exception as e2:
+        print(e2)
+
+micropython.heap_lock()
+func()
+print("ok")
+micropython.heap_unlock()
diff --git a/tests/micropython/heapalloc_exc_raise.py.exp b/tests/micropython/heapalloc_exc_raise.py.exp
new file mode 100644
index 000000000..1e9edc75d
--- /dev/null
+++ b/tests/micropython/heapalloc_exc_raise.py.exp
@@ -0,0 +1,2 @@
+error
+ok
diff --git a/tests/micropython/heapalloc_int_from_bytes.py b/tests/micropython/heapalloc_int_from_bytes.py
new file mode 100644
index 000000000..5fe50443a
--- /dev/null
+++ b/tests/micropython/heapalloc_int_from_bytes.py
@@ -0,0 +1,9 @@
+# Test that int.from_bytes() for small number of bytes generates
+# small int.
+import micropython
+
+micropython.heap_lock()
+print(int.from_bytes(b"1", "little"))
+print(int.from_bytes(b"12", "little"))
+print(int.from_bytes(b"2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", "little"))
+micropython.heap_unlock()
diff --git a/tests/micropython/heapalloc_int_from_bytes.py.exp b/tests/micropython/heapalloc_int_from_bytes.py.exp
new file mode 100644
index 000000000..5ac6a573f
--- /dev/null
+++ b/tests/micropython/heapalloc_int_from_bytes.py.exp
@@ -0,0 +1,3 @@
+49
+12849
+50
diff --git a/tests/micropython/heapalloc_iter.py b/tests/micropython/heapalloc_iter.py
new file mode 100644
index 000000000..79461f999
--- /dev/null
+++ b/tests/micropython/heapalloc_iter.py
@@ -0,0 +1,69 @@
+# test that iterating doesn't use the heap
+try:
+    import array
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+try:
+    from micropython import heap_lock, heap_unlock
+except (ImportError, AttributeError):
+    heap_lock = heap_unlock = lambda:0
+
+def do_iter(l):
+    for i in l:
+        print(i)
+
+def gen_func():
+    yield 1
+    yield 2
+
+# pre-create collections to iterate over
+ba = bytearray(b'123')
+ar = array.array('H', (123, 456))
+t = (1, 2, 3)
+l = [1, 2]
+d = {1:2}
+s = set((1,))
+fs = frozenset((1,))
+g1 = (100 + x for x in range(2))
+g2 = gen_func()
+
+# test containment (both success and failure) with the heap locked
+heap_lock()
+print(49 in b'123', 255 in b'123')
+print(1 in t, -1 in t)
+print(1 in l, -1 in l)
+print(1 in d, -1 in d)
+print(1 in s, -1 in s)
+heap_unlock()
+
+# test unpacking with the heap locked
+unp0 = unp1 = unp2 = None # preallocate slots for globals
+heap_lock()
+unp0, unp1, unp2 = t
+print(unp0, unp1, unp2)
+heap_unlock()
+
+# test certain builtins with the heap locked
+heap_lock()
+print(all(t))
+print(any(t))
+print(min(t))
+print(max(t))
+print(sum(t))
+heap_unlock()
+
+# test iterating over collections with the heap locked
+heap_lock()
+do_iter(b'123')
+do_iter(ba)
+do_iter(ar)
+do_iter(t)
+do_iter(l)
+do_iter(d)
+do_iter(s)
+do_iter(fs)
+do_iter(g1)
+do_iter(g2)
+heap_unlock()
diff --git a/tests/micropython/heapalloc_str.py b/tests/micropython/heapalloc_str.py
new file mode 100644
index 000000000..39aa56ccd
--- /dev/null
+++ b/tests/micropython/heapalloc_str.py
@@ -0,0 +1,18 @@
+# String operations which don't require allocation
+import micropython
+
+micropython.heap_lock()
+
+# Concatenating empty string returns original string
+b"" + b""
+b"" + b"1"
+b"2" + b""
+
+"" + ""
+"" + "1"
+"2" + ""
+
+# If no replacements done, returns original string
+"foo".replace(",", "_")
+
+micropython.heap_unlock()
diff --git a/tests/micropython/heapalloc_str.py.exp b/tests/micropython/heapalloc_str.py.exp
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/micropython/heapalloc_str.py.exp
diff --git a/tests/micropython/heapalloc_super.py b/tests/micropython/heapalloc_super.py
new file mode 100644
index 000000000..1cf5293d2
--- /dev/null
+++ b/tests/micropython/heapalloc_super.py
@@ -0,0 +1,17 @@
+# test super() operations which don't require allocation
+import micropython
+
+class A:
+    def foo(self):
+        print('A foo')
+        return 42
+class B(A):
+    def foo(self):
+        print('B foo')
+        print(super().foo())
+
+b = B()
+
+micropython.heap_lock()
+b.foo()
+micropython.heap_unlock()
diff --git a/tests/micropython/heapalloc_super.py.exp b/tests/micropython/heapalloc_super.py.exp
new file mode 100644
index 000000000..5dabd0c7c
--- /dev/null
+++ b/tests/micropython/heapalloc_super.py.exp
@@ -0,0 +1,3 @@
+B foo
+A foo
+42
diff --git a/tests/micropython/heapalloc_traceback.py b/tests/micropython/heapalloc_traceback.py
index 808df0225..f4212b6ce 100644
--- a/tests/micropython/heapalloc_traceback.py
+++ b/tests/micropython/heapalloc_traceback.py
@@ -2,7 +2,11 @@
 
 import micropython
 import sys
-import uio
+try:
+    import uio
+except ImportError:
+    print("SKIP")
+    raise SystemExit
 
 # preallocate exception instance with some room for a traceback
 global_exc = StopIteration()
diff --git a/tests/micropython/heapalloc_traceback.py.exp b/tests/micropython/heapalloc_traceback.py.exp
index 9ba10948d..291bbd697 100644
--- a/tests/micropython/heapalloc_traceback.py.exp
+++ b/tests/micropython/heapalloc_traceback.py.exp
@@ -1,5 +1,5 @@
 StopIteration
 Traceback (most recent call last):
-  File  , line 18, in test
+  File  , line 22, in test
 StopIteration: 
 
diff --git a/tests/micropython/kbd_intr.py b/tests/micropython/kbd_intr.py
new file mode 100644
index 000000000..879c9a229
--- /dev/null
+++ b/tests/micropython/kbd_intr.py
@@ -0,0 +1,12 @@
+# test the micropython.kbd_intr() function
+
+import micropython
+
+try:
+    micropython.kbd_intr
+except AttributeError:
+    print('SKIP')
+    raise SystemExit
+
+# just check we can actually call it
+micropython.kbd_intr(3)
diff --git a/tests/micropython/kbd_intr.py.exp b/tests/micropython/kbd_intr.py.exp
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/micropython/kbd_intr.py.exp
diff --git a/tests/micropython/native_const.py b/tests/micropython/native_const_intbig.py
index 611b39d8f..611b39d8f 100644
--- a/tests/micropython/native_const.py
+++ b/tests/micropython/native_const_intbig.py
diff --git a/tests/micropython/native_const.py.exp b/tests/micropython/native_const_intbig.py.exp
index 1d52d220f..1d52d220f 100644
--- a/tests/micropython/native_const.py.exp
+++ b/tests/micropython/native_const_intbig.py.exp
diff --git a/tests/micropython/opt_level.py b/tests/micropython/opt_level.py
index 4e2f2f4ea..5a10047f0 100644
--- a/tests/micropython/opt_level.py
+++ b/tests/micropython/opt_level.py
@@ -12,3 +12,8 @@ exec('print(__debug__)')
 micropython.opt_level(1)
 exec('print(__debug__)')
 exec('assert 0')
+
+# check that level 3 doesn't store line numbers
+# the expected output is that any line is printed as "line 1"
+micropython.opt_level(3)
+exec('try:\n xyz\nexcept NameError as er:\n import sys\n sys.print_exception(er)')
diff --git a/tests/micropython/opt_level.py.exp b/tests/micropython/opt_level.py.exp
index 74b3dd74e..9b1bb4d24 100644
--- a/tests/micropython/opt_level.py.exp
+++ b/tests/micropython/opt_level.py.exp
@@ -2,3 +2,6 @@
 1
 True
 False
+Traceback (most recent call last):
+  File "<string>", line 1, in <module>
+NameError: name 'xyz' is not defined
diff --git a/tests/micropython/schedule.py b/tests/micropython/schedule.py
new file mode 100644
index 000000000..74f90cb2d
--- /dev/null
+++ b/tests/micropython/schedule.py
@@ -0,0 +1,60 @@
+# test micropython.schedule() function
+
+import micropython
+
+try:
+    micropython.schedule
+except AttributeError:
+    print('SKIP')
+    raise SystemExit
+
+# Basic test of scheduling a function.
+
+def callback(arg):
+    global done
+    print(arg)
+    done = True
+
+done = False
+micropython.schedule(callback, 1)
+while not done:
+    pass
+
+# Test that callbacks can be scheduled from within a callback, but
+# that they don't execute until the outer callback is finished.
+
+def callback_inner(arg):
+    global done
+    print('inner')
+    done += 1
+
+def callback_outer(arg):
+    global done
+    micropython.schedule(callback_inner, 0)
+    # need a loop so that the VM can check for pending events
+    for i in range(2):
+        pass
+    print('outer')
+    done += 1
+
+done = 0
+micropython.schedule(callback_outer, 0)
+while done != 2:
+    pass
+
+# Test that scheduling too many callbacks leads to an exception.  To do this we
+# must schedule from within a callback to guarantee that the scheduler is locked.
+
+def callback(arg):
+    global done
+    try:
+        for i in range(100):
+            micropython.schedule(lambda x:x, None)
+    except RuntimeError:
+        print('RuntimeError')
+    done = True
+
+done = False
+micropython.schedule(callback, None)
+while not done:
+    pass
diff --git a/tests/micropython/schedule.py.exp b/tests/micropython/schedule.py.exp
new file mode 100644
index 000000000..c4a3e1227
--- /dev/null
+++ b/tests/micropython/schedule.py.exp
@@ -0,0 +1,4 @@
+1
+outer
+inner
+RuntimeError
diff --git a/tests/micropython/viper_error.py b/tests/micropython/viper_error.py
index 116bd4ea0..847257285 100644
--- a/tests/micropython/viper_error.py
+++ b/tests/micropython/viper_error.py
@@ -3,13 +3,19 @@
 def test(code):
     try:
         exec(code)
-    except (SyntaxError, ViperTypeError) as e:
+    except (SyntaxError, ViperTypeError, NotImplementedError) as e:
         print(repr(e))
 
 # viper: annotations must be identifiers
 test("@micropython.viper\ndef f(a:1): pass")
 test("@micropython.viper\ndef f() -> 1: pass")
 
+# unknown type
+test("@micropython.viper\ndef f(x:unknown_type): pass")
+
+# too many arguments
+test("@micropython.viper\ndef f(a, b, c, d, e): pass")
+
 # local used before type known
 test("""
 @micropython.viper
@@ -49,6 +55,9 @@ test("@micropython.viper\ndef f(): 1[x]")
 # can't store
 test("@micropython.viper\ndef f(): 1[0] = 1")
 test("@micropython.viper\ndef f(): 1[x] = 1")
+test("@micropython.viper\ndef f(x:int): x[0] = x")
+test("@micropython.viper\ndef f(x:ptr32): x[0] = None")
+test("@micropython.viper\ndef f(x:ptr32): x[x] = None")
 
 # must raise an object
 test("@micropython.viper\ndef f(): raise 1")
@@ -57,3 +66,16 @@ test("@micropython.viper\ndef f(): raise 1")
 test("@micropython.viper\ndef f(x:int): +x")
 test("@micropython.viper\ndef f(x:int): -x")
 test("@micropython.viper\ndef f(x:int): ~x")
+
+# binary op not implemented
+test("@micropython.viper\ndef f(x:int): res = x in x")
+
+# yield (from) not implemented
+test("@micropython.viper\ndef f(): yield")
+test("@micropython.viper\ndef f(): yield from f")
+
+# passing a ptr to a Python function not implemented
+test("@micropython.viper\ndef f(): print(ptr(1))")
+
+# cast of a casting identifier not implemented
+test("@micropython.viper\ndef f(): int(int)")
diff --git a/tests/micropython/viper_error.py.exp b/tests/micropython/viper_error.py.exp
index 1afcd4bdb..96be5a590 100644
--- a/tests/micropython/viper_error.py.exp
+++ b/tests/micropython/viper_error.py.exp
@@ -1,5 +1,7 @@
 SyntaxError('parameter annotation must be an identifier',)
 SyntaxError('return annotation must be an identifier',)
+ViperTypeError("unknown type 'unknown_type'",)
+ViperTypeError("Viper functions don't currently support more than 4 arguments",)
 ViperTypeError("local 'x' used before type known",)
 ViperTypeError("local 'x' has type 'int' but source is 'object'",)
 ViperTypeError("can't implicitly convert 'ptr' to 'bool'",)
@@ -9,7 +11,15 @@ ViperTypeError("can't load from 'int'",)
 ViperTypeError("can't load from 'int'",)
 ViperTypeError("can't store to 'int'",)
 ViperTypeError("can't store to 'int'",)
+ViperTypeError("can't store to 'int'",)
+ViperTypeError("can't store 'None'",)
+ViperTypeError("can't store 'None'",)
 ViperTypeError('must raise an object',)
 ViperTypeError('unary op __pos__ not implemented',)
 ViperTypeError('unary op __neg__ not implemented',)
 ViperTypeError('unary op __invert__ not implemented',)
+ViperTypeError('binary op __contains__ not implemented',)
+NotImplementedError('native yield',)
+NotImplementedError('native yield from',)
+NotImplementedError('conversion to object',)
+NotImplementedError('casting',)
diff --git a/tests/micropython/viper_misc.py b/tests/micropython/viper_misc.py
index 8bf6cc638..021e03f2c 100644
--- a/tests/micropython/viper_misc.py
+++ b/tests/micropython/viper_misc.py
@@ -50,13 +50,6 @@ def viper_no_annotation(x, y):
     return x * y
 print(viper_no_annotation(4, 5))
 
-# unsigned ints
-@micropython.viper
-def viper_uint() -> uint:
-    return uint(-1)
-import sys
-print(viper_uint() == (sys.maxsize << 1 | 1))
-
 # a for loop
 @micropython.viper
 def viper_for(a:int, b:int) -> int:
diff --git a/tests/micropython/viper_misc.py.exp b/tests/micropython/viper_misc.py.exp
index 4800050d1..e4462771b 100644
--- a/tests/micropython/viper_misc.py.exp
+++ b/tests/micropython/viper_misc.py.exp
@@ -5,7 +5,6 @@ Ellipsis
 6
 7
 20
-True
 49994955
 1 1
 1 3
diff --git a/tests/micropython/viper_misc_intbig.py b/tests/micropython/viper_misc_intbig.py
new file mode 100644
index 000000000..e036435c7
--- /dev/null
+++ b/tests/micropython/viper_misc_intbig.py
@@ -0,0 +1,8 @@
+import micropython
+
+# unsigned ints
+@micropython.viper
+def viper_uint() -> uint:
+    return uint(-1)
+import sys
+print(viper_uint() == (sys.maxsize << 1 | 1))
diff --git a/tests/micropython/viper_misc_intbig.py.exp b/tests/micropython/viper_misc_intbig.py.exp
new file mode 100644
index 000000000..0ca95142b
--- /dev/null
+++ b/tests/micropython/viper_misc_intbig.py.exp
@@ -0,0 +1 @@
+True