From 54d97251fe2dd4939652a186bf703885e654b4d1 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 12 Sep 2020 13:57:31 -0500 Subject: modstruct: Improve compliance with python3 While checking whether we can enable -Wimplicit-fallthrough, I encountered a diagnostic in mp_binary_set_val_array_from_int which led to discovering the following bug: ``` >>> struct.pack("xb", 3) b'\x03\x03' ``` That is, the next value (3) was used as the value of a padding byte, while standard Python always fills "x" bytes with zeros. I initially thought this had to do with the unintentional fallthrough, but it doesn't. Instead, this code would relate to an array.array with a typecode of padding ('x'), which is ALSO not desktop Python compliant: ``` >>> array.array('x', (1, 2, 3)) array('x', [1, 0, 0]) ``` Possibly this is dead code that used to be shared between struct-setting and array-setting, but it no longer is. I also discovered that the argument list length for struct.pack and struct.pack_into were not checked, and that the length of binary data passed to array.array was not checked to be a multiple of the element size. I have corrected all of these to conform more closely to standard Python and revised some tests where necessary. Some tests for micropython-specific behavior that does not conform to standard Python and is not present in CircuitPython was deleted outright. --- tests/basics/struct1.py | 25 +++++++++++++++++++++ tests/basics/struct_micropython.py | 41 ---------------------------------- tests/basics/struct_micropython.py.exp | 2 -- tests/misc/non_compliant.py | 6 ----- tests/misc/non_compliant.py.exp | 2 -- 5 files changed, 25 insertions(+), 51 deletions(-) delete mode 100644 tests/basics/struct_micropython.py delete mode 100644 tests/basics/struct_micropython.py.exp (limited to 'tests') diff --git a/tests/basics/struct1.py b/tests/basics/struct1.py index db34342a1..107006cc3 100644 --- a/tests/basics/struct1.py +++ b/tests/basics/struct1.py @@ -39,6 +39,28 @@ print(v == (10, 100, 200, 300)) # network byte order print(struct.pack('!i', 123)) +# too short / too long arguments +buf = bytearray(b'>>>123<<<') +try: + struct.pack_into('bb', buf, 0, 3) +except: + print('struct.error') + +try: + struct.pack_into('bb', buf, 0, 3, 1, 4) +except: + print('struct.error') + +try: + struct.pack('bb', 3) +except: + print('struct.error') + +try: + struct.pack('bb', 3, 1, 4) +except: + print('struct.error') + # check that we get an error if the buffer is too small try: struct.unpack('I', b'\x00\x00\x00') @@ -96,3 +118,6 @@ try: print(struct.unpack_from(' Date: Sat, 10 Oct 2020 23:39:32 -0700 Subject: update async tests with less upython workaround and more cpython compatibility --- py/compile.c | 12 ++++++++---- py/objgenerator.c | 8 ++------ tests/basics/async_await2.py | 28 ++++++++++++++-------------- tests/basics/async_for2.py | 30 +++++++++++++++--------------- tests/basics/async_with2.py | 32 ++++++++++++++++---------------- 5 files changed, 55 insertions(+), 55 deletions(-) (limited to 'tests') diff --git a/py/compile.c b/py/compile.c index da31846e2..45a586594 100644 --- a/py/compile.c +++ b/py/compile.c @@ -40,6 +40,8 @@ #if MICROPY_ENABLE_COMPILER +#define DEBUG_PRINT(...) mp_printf(&mp_plat_print __VA_OPT__(,) __VA_ARGS__) + // TODO need to mangle __attr names #define INVALID_LABEL (0xffff) @@ -853,7 +855,7 @@ STATIC void compile_decorated(compiler_t *comp, mp_parse_node_struct_t *pns) { mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns_body->nodes[0]; body_name = compile_funcdef_helper(comp, pns0, emit_options); scope_t *fscope = (scope_t*)pns0->nodes[4]; - fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR; + fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR | MP_SCOPE_FLAG_ASYNC; #endif } else { assert(MP_PARSE_NODE_STRUCT_KIND(pns_body) == PN_classdef); // should be @@ -2655,11 +2657,13 @@ STATIC void compile_atom_expr_await(compiler_t *comp, mp_parse_node_struct_t *pn compile_require_async_context(comp, pns); compile_atom_expr_normal(comp, pns); - + // If it's an awaitable thing, need to reach for the __await__ method for the coroutine. + // async def functions' __await__ return themselves, which are able to receive a send(), + // while other types with custom __await__ implementations return async generators. EMIT_ARG(load_method, MP_QSTR___await__, false); EMIT_ARG(call_method, 0, 0, 0); - // EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); // don't yield anything from an awaitable; only return the final result. - // EMIT_ARG(yield, MP_EMIT_YIELD_FROM); + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT_ARG(yield, MP_EMIT_YIELD_FROM); } #endif diff --git a/py/objgenerator.c b/py/objgenerator.c index baa6a9849..5e651ac26 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -234,12 +234,8 @@ STATIC mp_obj_t gen_instance_await(mp_obj_t self_in) { nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_AttributeError, translate("type object 'generator' has no attribute '__await__'"))); } - mp_obj_t ret = gen_resume_and_raise(self_in, mp_const_none, MP_OBJ_NULL); - if (ret == MP_OBJ_STOP_ITERATION) { - nlr_raise(mp_obj_new_exception(&mp_type_StopIteration)); - } else { - return ret; - } + // You can directly call send on a coroutine generator or you can __await__ then send on the return of that. + return self; } STATIC MP_DEFINE_CONST_FUN_OBJ_1(gen_instance_await_obj, gen_instance_await); #endif diff --git a/tests/basics/async_await2.py b/tests/basics/async_await2.py index 7097ce85a..2c5015fb4 100644 --- a/tests/basics/async_await2.py +++ b/tests/basics/async_await2.py @@ -1,22 +1,22 @@ # test await expression -import sys -if sys.implementation.name in ('micropython', 'circuitpython'): - # uPy allows normal generators to be awaitables - coroutine = lambda f: f -else: - import types - coroutine = types.coroutine +# uPy allows normal generators to be awaitables. +# CircuitPython does not. +# In CircuitPython you need to have an __await__ method on an awaitable like in CPython; +# and like in CPython, generators do not have __await__. -@coroutine -def wait(value): - print('wait value:', value) - msg = yield 'message from wait(%u)' % value - print('wait got back:', msg) - return 10 +class Awaitable: + def __init__(self, value): + self.value = value + + def __await__(self): + print('wait value:', self.value) + msg = yield 'message from wait(%u)' % self.value + print('wait got back:', msg) + return 10 async def f(): - x = await wait(1)**2 + x = await Awaitable(1)**2 print('x =', x) coro = f() diff --git a/tests/basics/async_for2.py b/tests/basics/async_for2.py index add74dd38..d7b97cb38 100644 --- a/tests/basics/async_for2.py +++ b/tests/basics/async_for2.py @@ -1,19 +1,19 @@ # test waiting within "async for" __anext__ function -import sys -if sys.implementation.name in ('micropython', 'circuitpython'): - # uPy allows normal generators to be awaitables - coroutine = lambda f: f -else: - import types - coroutine = types.coroutine - -@coroutine -def f(x): - print('f start:', x) - yield x + 1 - yield x + 2 - return x + 3 +# uPy allows normal generators to be awaitables. +# CircuitPython does not. +# In CircuitPython you need to have an __await__ method on an awaitable like in CPython; +# and like in CPython, generators do not have __await__. + +class Awaitable: + def __init__(self, x): + self.x = x + + def __await__(self): + print('f start:', self.x) + yield self.x + 1 + yield self.x + 2 + return self.x + 3 class ARange: def __init__(self, high): @@ -27,7 +27,7 @@ class ARange: async def __anext__(self): print('anext') - print('f returned:', await f(20)) + print('f returned:', await Awaitable(20)) if self.cur < self.high: val = self.cur self.cur += 1 diff --git a/tests/basics/async_with2.py b/tests/basics/async_with2.py index 5fb06ba60..2e9e9e5a7 100644 --- a/tests/basics/async_with2.py +++ b/tests/basics/async_with2.py @@ -1,32 +1,32 @@ # test waiting within async with enter/exit functions -import sys -if sys.implementation.name in ('micropython', 'circuitpython'): - # uPy allows normal generators to be awaitables - coroutine = lambda f: f -else: - import types - coroutine = types.coroutine +# uPy allows normal generators to be awaitables. +# CircuitPython does not. +# In CircuitPython you need to have an __await__ method on an awaitable like in CPython; +# and like in CPython, generators do not have __await__. -@coroutine -def f(x): - print('f start:', x) - yield x + 1 - yield x + 2 - return x + 3 +class Awaitable: + def __init__(self, x): + self.x = x + + def __await__(self): + print('f start:', self.x) + yield self.x + 1 + yield self.x + 2 + return self.x + 3 class AContext: async def __aenter__(self): print('enter') - print('f returned:', await f(10)) + print('f returned:', await Awaitable(10)) async def __aexit__(self, exc_type, exc, tb): print('exit', exc_type, exc) - print('f returned:', await f(20)) + print('f returned:', await Awaitable(20)) async def coro(): async with AContext(): print('body start') - print('body f returned:', await f(30)) + print('body f returned:', await Awaitable(30)) print('body end') o = coro() -- cgit v1.2.3