From c5dbdbe88c099494cd14790b8c645020538eed27 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 21 Feb 2019 00:33:36 -0500 Subject: rename ure, ujson, and uerrno to re, json, and errno --- extmod/modujson.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'extmod/modujson.c') diff --git a/extmod/modujson.c b/extmod/modujson.c index ea6c8b40f..80fccc8a1 100644 --- a/extmod/modujson.c +++ b/extmod/modujson.c @@ -292,7 +292,7 @@ STATIC mp_obj_t mod_ujson_loads(mp_obj_t obj) { STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_loads_obj, mod_ujson_loads); STATIC const mp_rom_map_elem_t mp_module_ujson_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ujson) }, + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_json) }, { MP_ROM_QSTR(MP_QSTR_dump), MP_ROM_PTR(&mod_ujson_dump_obj) }, { MP_ROM_QSTR(MP_QSTR_dumps), MP_ROM_PTR(&mod_ujson_dumps_obj) }, { MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&mod_ujson_load_obj) }, -- cgit v1.2.3 From 10d3a20f8a639ad8ef5555e2142052190e8a7e13 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 21 Feb 2019 11:09:44 -0500 Subject: Add CIRCUITPY macro; rename u* only when CIRCUITPY=1 --- extmod/modujson.c | 4 ++++ extmod/modure.c | 4 ++++ py/circuitpy_mpconfig.h | 3 +++ py/moduerrno.c | 4 ++++ py/mpconfig.h | 5 +++++ py/objmodule.c | 19 ++++++++++++++++--- 6 files changed, 36 insertions(+), 3 deletions(-) (limited to 'extmod/modujson.c') diff --git a/extmod/modujson.c b/extmod/modujson.c index 80fccc8a1..6b24bf578 100644 --- a/extmod/modujson.c +++ b/extmod/modujson.c @@ -292,7 +292,11 @@ STATIC mp_obj_t mod_ujson_loads(mp_obj_t obj) { STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_loads_obj, mod_ujson_loads); STATIC const mp_rom_map_elem_t mp_module_ujson_globals_table[] = { +#if CIRCUITPY { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_json) }, +#else + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ujson) }, +#endif { MP_ROM_QSTR(MP_QSTR_dump), MP_ROM_PTR(&mod_ujson_dump_obj) }, { MP_ROM_QSTR(MP_QSTR_dumps), MP_ROM_PTR(&mod_ujson_dumps_obj) }, { MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&mod_ujson_load_obj) }, diff --git a/extmod/modure.c b/extmod/modure.c index 6561362a8..e2556c41b 100644 --- a/extmod/modure.c +++ b/extmod/modure.c @@ -462,7 +462,11 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_re_sub_obj, 3, 5, mod_re_sub); #endif STATIC const mp_rom_map_elem_t mp_module_re_globals_table[] = { +#if CIRCUITPY { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_re) }, +#else + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ure) }, +#endif { MP_ROM_QSTR(MP_QSTR_compile), MP_ROM_PTR(&mod_re_compile_obj) }, { MP_ROM_QSTR(MP_QSTR_match), MP_ROM_PTR(&mod_re_match_obj) }, { MP_ROM_QSTR(MP_QSTR_search), MP_ROM_PTR(&mod_re_search_obj) }, diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index c70d52a7b..bb43201d8 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -33,6 +33,9 @@ #ifndef __INCLUDED_MPCONFIG_CIRCUITPY_H #define __INCLUDED_MPCONFIG_CIRCUITPY_H +// This is CircuitPython. +#define CIRCUITPY 1 + // REPR_C encodes qstrs, 31-bit ints, and 30-bit floats in a single 32-bit word. #define MICROPY_OBJ_REPR (MICROPY_OBJ_REPR_C) diff --git a/py/moduerrno.c b/py/moduerrno.c index c5daae468..7915603e4 100644 --- a/py/moduerrno.c +++ b/py/moduerrno.c @@ -84,7 +84,11 @@ STATIC const mp_obj_dict_t errorcode_dict = { #endif STATIC const mp_rom_map_elem_t mp_module_uerrno_globals_table[] = { +#if CIRCUITPY { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_errno) }, +#else + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uerrno) }, +#endif #if MICROPY_PY_UERRNO_ERRORCODE { MP_ROM_QSTR(MP_QSTR_errorcode), MP_ROM_PTR(&errorcode_dict) }, #endif diff --git a/py/mpconfig.h b/py/mpconfig.h index bb7952a8b..3ec383817 100755 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -45,6 +45,11 @@ #include #endif +// Is this a CircuitPython build? +#ifndef CIRCUITPY +#define CIRCUITPY 0 +#endif + // Any options not explicitly set in mpconfigport.h will get default // values below. diff --git a/py/objmodule.c b/py/objmodule.c index 9a56f4851..469a95976 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -148,7 +148,11 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = { { MP_ROM_QSTR(MP_QSTR_array), MP_ROM_PTR(&mp_module_array) }, #endif #if MICROPY_PY_IO +#if CIRCUITPY { MP_ROM_QSTR(MP_QSTR_io), MP_ROM_PTR(&mp_module_io) }, +#else + { MP_ROM_QSTR(MP_QSTR_uio), MP_ROM_PTR(&mp_module_io) }, +#endif #endif #if MICROPY_PY_COLLECTIONS { MP_ROM_QSTR(MP_QSTR_collections), MP_ROM_PTR(&mp_module_collections) }, @@ -179,9 +183,12 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = { // extmod modules #if MICROPY_PY_UERRNO +#if CIRCUITPY // CircuitPython: Defined in MICROPY_PORT_BUILTIN_MODULES, so not defined here. // TODO: move to shared-bindings/ -// { MP_ROM_QSTR(MP_QSTR_uerrno), MP_ROM_PTR(&mp_module_uerrno) }, +#else + { MP_ROM_QSTR(MP_QSTR_uerrno), MP_ROM_PTR(&mp_module_uerrno) }, +#endif #endif #if MICROPY_PY_UCTYPES { MP_ROM_QSTR(MP_QSTR_uctypes), MP_ROM_PTR(&mp_module_uctypes) }, @@ -190,14 +197,20 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = { { MP_ROM_QSTR(MP_QSTR_uzlib), MP_ROM_PTR(&mp_module_uzlib) }, #endif #if MICROPY_PY_UJSON +#if CIRCUITPY // CircuitPython: Defined in MICROPY_PORT_BUILTIN_MODULES, so not defined here. // TODO: move to shared-bindings/ -// { MP_ROM_QSTR(MP_QSTR_ujson), MP_ROM_PTR(&mp_module_ujson) }, +#else + { MP_ROM_QSTR(MP_QSTR_ujson), MP_ROM_PTR(&mp_module_ujson) }, +#endif #endif #if MICROPY_PY_URE +#if CIRCUITPY // CircuitPython: Defined in MICROPY_PORT_BUILTIN_MODULES, so not defined here. // TODO: move to shared-bindings/ -// { MP_ROM_QSTR(MP_QSTR_ure), MP_ROM_PTR(&mp_module_ure) }, +#else + { MP_ROM_QSTR(MP_QSTR_ure), MP_ROM_PTR(&mp_module_ure) }, +#endif #endif #if MICROPY_PY_UHEAPQ { MP_ROM_QSTR(MP_QSTR_uheapq), MP_ROM_PTR(&mp_module_uheapq) }, -- cgit v1.2.3 From eb3d5fa453bd04cc116e704c36a97250c93b7b4b Mon Sep 17 00:00:00 2001 From: warriorofwire <3454741+WarriorOfWire@users.noreply.github.com> Date: Sun, 10 May 2020 20:45:42 -0700 Subject: ujson: do not eat trailing whitespace Ujson should only worry about whitespace before JSON. This becomes apparent when you are using MP stream protocol to read directly from input buffers. When you attempt to read(1) on a UART (and possibly other protocols) you have to wait for either the byte or the timeout. Fixes: - Waiting for a timeout after you have completed reading a correct and complete JSON off the input. - Raising an OSError after reading a correct and complete JSON off the input. - Eating more data than semantically owned off the input buffer. - Blocking to start parsing JSON until the entire JSON body has been loaded into a potentially large, contiguous Python object. Code you would write before: ``` line = board_busio_uart_port.read_line() json_dict = json.loads(line) ``` or reaching for fixed buffers and swapping them around in Python. Code that did not work before that does now: ``` json_dict = json.load(board_busio_uart_port) ``` - This removes the need for intermediate copies of data when reading JSON from micropython stream protocol inputs. - It also increases total application speed by parsing JSON concurrently with receiving on boards that read from UART via DMA. - It simplifies code that users write while improving their apps. --- extmod/modujson.c | 26 ++++++++++++++++++-------- ports/atmel-samd/common-hal/busio/UART.c | 3 +++ shared-bindings/busio/UART.c | 4 ++++ 3 files changed, 25 insertions(+), 8 deletions(-) (limited to 'extmod/modujson.c') diff --git a/extmod/modujson.c b/extmod/modujson.c index 6b24bf578..daefe18ba 100644 --- a/extmod/modujson.c +++ b/extmod/modujson.c @@ -53,6 +53,10 @@ STATIC mp_obj_t mod_ujson_dumps(mp_obj_t obj) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_dumps_obj, mod_ujson_dumps); +#define JSON_DEBUG(...) (void)0 +// #define JSON_DEBUG(...) mp_printf(&mp_plat_print __VA_OPT__(,) __VA_ARGS__) + + // The function below implements a simple non-recursive JSON parser. // // The JSON specification is at http://www.ietf.org/rfc/rfc4627.txt @@ -80,6 +84,7 @@ typedef struct _ujson_stream_t { STATIC byte ujson_stream_next(ujson_stream_t *s) { mp_uint_t ret = s->read(s->stream_obj, &s->cur, 1, &s->errcode); + JSON_DEBUG(" usjon_stream_next err:%2d cur: %c \n", s->errcode, s->cur); if (s->errcode != 0) { mp_raise_OSError(s->errcode); } @@ -92,6 +97,7 @@ STATIC byte ujson_stream_next(ujson_stream_t *s) { STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { const mp_stream_p_t *stream_p = mp_get_stream_raise(stream_obj, MP_STREAM_OP_READ); ujson_stream_t s = {stream_obj, stream_p->read, 0, 0}; + JSON_DEBUG("got JSON stream\n"); vstr_t vstr; vstr_init(&vstr, 8); mp_obj_list_t stack; // we use a list as a simple stack for nested JSON @@ -101,6 +107,15 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { mp_obj_type_t *stack_top_type = NULL; mp_obj_t stack_key = MP_OBJ_NULL; S_NEXT(s); + // Eat _leading_ whitespace. + // If we eat trailing whitespace we will block for timeout on streams like UART that + // wait for requested data. Furthermore, it is an OSError to read(1) and incur + // a timeout on those APIs. + // For these reasons, we must only eat _leading_ whitespace. + while (unichar_isspace(S_CUR(s))) { + JSON_DEBUG("Eating leading whitespace"); + S_NEXT(s); + } for (;;) { cont: if (S_END(s)) { @@ -262,14 +277,9 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { } } success: - // eat trailing whitespace - while (unichar_isspace(S_CUR(s))) { - S_NEXT(s); - } - if (!S_END(s)) { - // unexpected chars - goto fail; - } + // It is legal for a stream to have contents before and after JSON. + // If this parser has consumed a full successful JSON and its parse + // stack is empty, the parse has succeeded. if (stack_top == MP_OBJ_NULL || stack.len != 0) { // not exactly 1 object goto fail; diff --git a/ports/atmel-samd/common-hal/busio/UART.c b/ports/atmel-samd/common-hal/busio/UART.c index 1f6b75f97..d6c13eb7e 100644 --- a/ports/atmel-samd/common-hal/busio/UART.c +++ b/ports/atmel-samd/common-hal/busio/UART.c @@ -45,6 +45,9 @@ #include "samd/sercom.h" +#define UART_DEBUG(...) (void)0 +// #define UART_DEBUG(...) mp_printf(&mp_plat_print __VA_OPT__(,) __VA_ARGS__) + // Do-nothing callback needed so that usart_async code will enable rx interrupts. // See comment below re usart_async_register_callback() static void usart_async_rxc_callback(const struct usart_async_descriptor *const descr) { diff --git a/shared-bindings/busio/UART.c b/shared-bindings/busio/UART.c index f231924d5..883b4630f 100644 --- a/shared-bindings/busio/UART.c +++ b/shared-bindings/busio/UART.c @@ -39,6 +39,9 @@ #include "py/stream.h" #include "supervisor/shared/translate.h" +#define STREAM_DEBUG(...) (void)0 +// #define STREAM_DEBUG(...) mp_printf(&mp_plat_print __VA_OPT__(,) __VA_ARGS__) + //| .. currentmodule:: busio //| @@ -219,6 +222,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(busio_uart___exit___obj, 4, 4, busio_ // These three methods are used by the shared stream methods. STATIC mp_uint_t busio_uart_read(mp_obj_t self_in, void *buf_in, mp_uint_t size, int *errcode) { + STREAM_DEBUG("busio_uart_read stream %d\n", size); busio_uart_obj_t *self = MP_OBJ_TO_PTR(self_in); check_for_deinit(self); byte *buf = buf_in; -- cgit v1.2.3 From d8491f31768b04322728bc436b2f8e6c5f2bf0d6 Mon Sep 17 00:00:00 2001 From: warriorofwire <3454741+WarriorOfWire@users.noreply.github.com> Date: Sun, 10 May 2020 21:56:01 -0700 Subject: ujson: back out overeager loads() change; only change load() --- extmod/modujson.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) (limited to 'extmod/modujson.c') diff --git a/extmod/modujson.c b/extmod/modujson.c index daefe18ba..0f93ccb11 100644 --- a/extmod/modujson.c +++ b/extmod/modujson.c @@ -94,7 +94,7 @@ STATIC byte ujson_stream_next(ujson_stream_t *s) { return s->cur; } -STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { +STATIC mp_obj_t _mod_ujson_load(mp_obj_t stream_obj, bool return_first_json) { const mp_stream_p_t *stream_p = mp_get_stream_raise(stream_obj, MP_STREAM_OP_READ); ujson_stream_t s = {stream_obj, stream_p->read, 0, 0}; JSON_DEBUG("got JSON stream\n"); @@ -107,15 +107,6 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { mp_obj_type_t *stack_top_type = NULL; mp_obj_t stack_key = MP_OBJ_NULL; S_NEXT(s); - // Eat _leading_ whitespace. - // If we eat trailing whitespace we will block for timeout on streams like UART that - // wait for requested data. Furthermore, it is an OSError to read(1) and incur - // a timeout on those APIs. - // For these reasons, we must only eat _leading_ whitespace. - while (unichar_isspace(S_CUR(s))) { - JSON_DEBUG("Eating leading whitespace"); - S_NEXT(s); - } for (;;) { cont: if (S_END(s)) { @@ -277,9 +268,19 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { } } success: - // It is legal for a stream to have contents before and after JSON. - // If this parser has consumed a full successful JSON and its parse - // stack is empty, the parse has succeeded. + // It is legal for a stream to have contents after JSON. + // E.g., A UART is not closed after receiving an object; in load() we will + // return the first complete JSON object, while in loads() we will retain + // strict adherence to the buffer's complete semantic. + if (!return_first_json) { + while (unichar_isspace(S_CUR(s))) { + S_NEXT(s); + } + if (!S_END(s)) { + // unexpected chars + goto fail; + } + } if (stack_top == MP_OBJ_NULL || stack.len != 0) { // not exactly 1 object goto fail; @@ -290,6 +291,10 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { fail: mp_raise_ValueError(translate("syntax error in JSON")); } + +STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { + return _mod_ujson_load(stream_obj, true); +} STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_load_obj, mod_ujson_load); STATIC mp_obj_t mod_ujson_loads(mp_obj_t obj) { @@ -297,7 +302,7 @@ STATIC mp_obj_t mod_ujson_loads(mp_obj_t obj) { const char *buf = mp_obj_str_get_data(obj, &len); vstr_t vstr = {len, len, (char*)buf, true}; mp_obj_stringio_t sio = {{&mp_type_stringio}, &vstr, 0, MP_OBJ_NULL}; - return mod_ujson_load(MP_OBJ_FROM_PTR(&sio)); + return _mod_ujson_load(MP_OBJ_FROM_PTR(&sio), false); } STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_loads_obj, mod_ujson_loads); -- cgit v1.2.3