From 487dbdb26748b86cf247600af187437310311145 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 1 Nov 2017 13:16:16 +1100 Subject: py/compile: Use alloca instead of qstr_build when compiling import name. The technique of using alloca is how dotted import names are composed in mp_import_from and mp_builtin___import__, so use the same technique in the compiler. This puts less pressure on the heap (only the stack is used if the qstr already exists, and if it doesn't exist then the standard qstr block memory is used for the new qstr rather than a separate chunk of the heap) and reduces overall code size. --- py/compile.c | 6 +++--- py/qstr.c | 23 ----------------------- py/qstr.h | 3 --- 3 files changed, 3 insertions(+), 29 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 4e704abfb..ee017498a 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1050,8 +1050,8 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { for (int i = 0; i < n; i++) { len += qstr_len(MP_PARSE_NODE_LEAF_ARG(pns->nodes[i])); } - byte *q_ptr; - byte *str_dest = qstr_build_start(len, &q_ptr); + char *q_ptr = alloca(len); + char *str_dest = q_ptr; for (int i = 0; i < n; i++) { if (i > 0) { *str_dest++ = '.'; @@ -1061,7 +1061,7 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { memcpy(str_dest, str_src, str_src_len); str_dest += str_src_len; } - qstr q_full = qstr_build_end(q_ptr); + qstr q_full = qstr_from_strn(q_ptr, len); EMIT_ARG(import_name, q_full); if (is_as) { for (int i = 1; i < n; i++) { diff --git a/py/qstr.c b/py/qstr.c index 95c9b6835..a3c9612c6 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -243,29 +243,6 @@ qstr qstr_from_strn(const char *str, size_t len) { return q; } -byte *qstr_build_start(size_t len, byte **q_ptr) { - assert(len < (1 << (8 * MICROPY_QSTR_BYTES_IN_LEN))); - *q_ptr = m_new(byte, MICROPY_QSTR_BYTES_IN_HASH + MICROPY_QSTR_BYTES_IN_LEN + len + 1); - Q_SET_LENGTH(*q_ptr, len); - return Q_GET_DATA(*q_ptr); -} - -qstr qstr_build_end(byte *q_ptr) { - QSTR_ENTER(); - qstr q = qstr_find_strn((const char*)Q_GET_DATA(q_ptr), Q_GET_LENGTH(q_ptr)); - if (q == 0) { - size_t len = Q_GET_LENGTH(q_ptr); - mp_uint_t hash = qstr_compute_hash(Q_GET_DATA(q_ptr), len); - Q_SET_HASH(q_ptr, hash); - q_ptr[MICROPY_QSTR_BYTES_IN_HASH + MICROPY_QSTR_BYTES_IN_LEN + len] = '\0'; - q = qstr_add(q_ptr); - } else { - m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr)); - } - QSTR_EXIT(); - return q; -} - mp_uint_t qstr_hash(qstr q) { return Q_GET_HASH(find_qstr(q)); } diff --git a/py/qstr.h b/py/qstr.h index e2bdcc351..63fd0c369 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -65,9 +65,6 @@ qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTR_NULL if qstr qstr_from_str(const char *str); qstr qstr_from_strn(const char *str, size_t len); -byte *qstr_build_start(size_t len, byte **q_ptr); -qstr qstr_build_end(byte *q_ptr); - mp_uint_t qstr_hash(qstr q); const char *qstr_str(qstr q); size_t qstr_len(qstr q); -- cgit v1.2.3 From 1b146e9de97708b70c3ad96022886e2ce44d246d Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 8 Nov 2017 19:45:18 +0200 Subject: py/mpconfig: Introduce reusable MP_HTOBE32(), etc. macros. Macros to convert big-endian values to host byte order and vice-versa. These were defined in adhoc way for some ports (e.g. esp8266), allow reuse, provide default implementations, while allow ports to override. --- ports/esp8266/posix_helpers.c | 5 ++--- py/mpconfig.h | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/ports/esp8266/posix_helpers.c b/ports/esp8266/posix_helpers.c index 1fc677c5c..b72c4ff9d 100644 --- a/ports/esp8266/posix_helpers.c +++ b/ports/esp8266/posix_helpers.c @@ -55,14 +55,13 @@ void *realloc(void *ptr, size_t size) { return p; } -#define PLATFORM_HTONL(_n) ((uint32_t)( (((_n) & 0xff) << 24) | (((_n) & 0xff00) << 8) | (((_n) >> 8) & 0xff00) | (((_n) >> 24) & 0xff) )) #undef htonl #undef ntohl uint32_t ntohl(uint32_t netlong) { - return PLATFORM_HTONL(netlong); + return MP_BE32TOH(netlong); } uint32_t htonl(uint32_t netlong) { - return PLATFORM_HTONL(netlong); + return MP_HTOBE32(netlong); } time_t time(time_t *t) { diff --git a/py/mpconfig.h b/py/mpconfig.h index 6a32ea2a6..fa1094bfc 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1293,4 +1293,24 @@ typedef double mp_float_t; #define MP_UNLIKELY(x) __builtin_expect((x), 0) #endif +#ifndef MP_HTOBE16 +#if MP_ENDIANNESS_LITTLE +# define MP_HTOBE16(x) ((uint16_t)( (((x) & 0xff) << 8) | (((x) >> 8) & 0xff) )) +# define MP_BE16TOH(x) MP_HTOBE16(x) +#else +# define MP_HTOBE16(x) (x) +# define MP_BE16TOH(x) (x) +#endif +#endif + +#ifndef MP_HTOBE32 +#if MP_ENDIANNESS_LITTLE +# define MP_HTOBE32(x) ((uint32_t)( (((x) & 0xff) << 24) | (((x) & 0xff00) << 8) | (((x) >> 8) & 0xff00) | (((x) >> 24) & 0xff) )) +# define MP_BE32TOH(x) MP_HTOBE32(x) +#else +# define MP_HTOBE32(x) (x) +# define MP_BE32TOH(x) (x) +#endif +#endif + #endif // MICROPY_INCLUDED_PY_MPCONFIG_H -- cgit v1.2.3 From cada971113e6db0cf9e0751e95dbe9217dd707b5 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 11 Nov 2017 00:11:24 +0200 Subject: py/objtype: mp_obj_new_type: Name base types related vars more clearly. As vars contains array of base types and its length, name them as such, avoid generic "items" and "len" names. --- py/objtype.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'py') diff --git a/py/objtype.c b/py/objtype.c index 6e2ab6c9a..a8376a306 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -983,12 +983,12 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) // TODO might need to make a copy of locals_dict; at least that's how CPython does it // Basic validation of base classes - size_t len; - mp_obj_t *items; - mp_obj_tuple_get(bases_tuple, &len, &items); - for (size_t i = 0; i < len; i++) { - assert(MP_OBJ_IS_TYPE(items[i], &mp_type_type)); - mp_obj_type_t *t = MP_OBJ_TO_PTR(items[i]); + size_t bases_len; + mp_obj_t *bases_items; + mp_obj_tuple_get(bases_tuple, &bases_len, &bases_items); + for (size_t i = 0; i < bases_len; i++) { + assert(MP_OBJ_IS_TYPE(bases_items[i], &mp_type_type)); + mp_obj_type_t *t = MP_OBJ_TO_PTR(bases_items[i]); // TODO: Verify with CPy, tested on function type if (t->make_new == NULL) { if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { @@ -1014,17 +1014,17 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) //o->iternext = ; not implemented o->buffer_p.get_buffer = instance_get_buffer; - if (len > 0) { + if (bases_len > 0) { // Inherit protocol from a base class. This allows to define an // abstract base class which would translate C-level protocol to // Python method calls, and any subclass inheriting from it will // support this feature. - o->protocol = ((mp_obj_type_t*)MP_OBJ_TO_PTR(items[0]))->protocol; + o->protocol = ((mp_obj_type_t*)MP_OBJ_TO_PTR(bases_items[0]))->protocol; - if (len >= 2) { + if (bases_len >= 2) { o->parent = MP_OBJ_TO_PTR(bases_tuple); } else { - o->parent = MP_OBJ_TO_PTR(items[0]); + o->parent = MP_OBJ_TO_PTR(bases_items[0]); } } -- cgit v1.2.3 From 79ed58f87b008ed1ad75c611686ca8bebfe2a817 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 6 Nov 2017 12:21:53 +0100 Subject: py/objnamedtuple: Add _asdict function if OrderedDict is supported --- ports/unix/mpconfigport_coverage.h | 1 + py/mpconfig.h | 5 +++++ py/objnamedtuple.c | 24 ++++++++++++++++++++++++ tests/basics/namedtuple_asdict.py | 20 ++++++++++++++++++++ 4 files changed, 50 insertions(+) create mode 100644 tests/basics/namedtuple_asdict.py (limited to 'py') diff --git a/ports/unix/mpconfigport_coverage.h b/ports/unix/mpconfigport_coverage.h index 367b4853a..0dcfdd5fd 100644 --- a/ports/unix/mpconfigport_coverage.h +++ b/ports/unix/mpconfigport_coverage.h @@ -44,3 +44,4 @@ #undef MICROPY_VFS_FAT #define MICROPY_VFS_FAT (1) #define MICROPY_PY_FRAMEBUF (1) +#define MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT (1) diff --git a/py/mpconfig.h b/py/mpconfig.h index fa1094bfc..4209b32c6 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -894,6 +894,11 @@ typedef double mp_float_t; #define MICROPY_PY_COLLECTIONS_ORDEREDDICT (0) #endif +// Whether to provide the _asdict function for namedtuple +#ifndef MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT +#define MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT (0) +#endif + // Whether to provide "math" module #ifndef MICROPY_PY_MATH #define MICROPY_PY_MATH (1) diff --git a/py/objnamedtuple.c b/py/objnamedtuple.c index 38daccdf2..94f02dd69 100644 --- a/py/objnamedtuple.c +++ b/py/objnamedtuple.c @@ -52,6 +52,23 @@ STATIC size_t namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr n return (size_t)-1; } +#if MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT +STATIC mp_obj_t namedtuple_asdict(mp_obj_t self_in) { + mp_obj_namedtuple_t *self = MP_OBJ_TO_PTR(self_in); + const qstr *fields = ((mp_obj_namedtuple_type_t*)self->tuple.base.type)->fields; + mp_obj_t dict = mp_obj_new_dict(self->tuple.len); + //make it an OrderedDict + mp_obj_dict_t *dictObj = MP_OBJ_TO_PTR(dict); + dictObj->base.type = &mp_type_ordereddict; + dictObj->map.is_ordered = 1; + for (size_t i = 0; i < self->tuple.len; ++i) { + mp_obj_dict_store(dict, MP_OBJ_NEW_QSTR(fields[i]), self->tuple.items[i]); + } + return dict; +} +MP_DEFINE_CONST_FUN_OBJ_1(namedtuple_asdict_obj, namedtuple_asdict); +#endif + STATIC void namedtuple_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; mp_obj_namedtuple_t *o = MP_OBJ_TO_PTR(o_in); @@ -64,6 +81,13 @@ STATIC void namedtuple_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (dest[0] == MP_OBJ_NULL) { // load attribute mp_obj_namedtuple_t *self = MP_OBJ_TO_PTR(self_in); + #if MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT + if (attr == MP_QSTR__asdict) { + dest[0] = MP_OBJ_FROM_PTR(&namedtuple_asdict_obj); + dest[1] = self_in; + return; + } + #endif size_t id = namedtuple_find_field((mp_obj_namedtuple_type_t*)self->tuple.base.type, attr); if (id == (size_t)-1) { return; diff --git a/tests/basics/namedtuple_asdict.py b/tests/basics/namedtuple_asdict.py new file mode 100644 index 000000000..c5681376f --- /dev/null +++ b/tests/basics/namedtuple_asdict.py @@ -0,0 +1,20 @@ +try: + try: + from collections import namedtuple + except ImportError: + from ucollections import namedtuple +except ImportError: + print("SKIP") + raise SystemExit + +t = namedtuple("Tup", ["baz", "foo", "bar"])(3, 2, 5) + +try: + t._asdict +except AttributeError: + print("SKIP") + raise SystemExit + +d = t._asdict() +print(list(d.keys())) +print(list(d.values())) -- cgit v1.2.3 From 564a95cb040fc425b08a9d3696700364185c3e23 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Nov 2017 11:46:49 +1100 Subject: py/emitnative: Clean up asm macro names so they have dest as first arg. All the asm macro names that convert a particular architecture to a generic interface now follow the convention whereby the "destination" (usually a register) is specified first. --- py/asmarm.h | 15 +++----- py/asmthumb.h | 15 +++----- py/asmx64.h | 15 +++----- py/asmx86.h | 15 +++----- py/asmxtensa.h | 15 +++----- py/emitnative.c | 105 ++++++++++++++++++++++++++++++-------------------------- 6 files changed, 81 insertions(+), 99 deletions(-) (limited to 'py') diff --git a/py/asmarm.h b/py/asmarm.h index a302b1590..871e35820 100644 --- a/py/asmarm.h +++ b/py/asmarm.h @@ -167,17 +167,12 @@ void asm_arm_bl_ind(asm_arm_t *as, void *fun_ptr, uint fun_id, uint reg_temp); } while (0) #define ASM_CALL_IND(as, ptr, idx) asm_arm_bl_ind(as, ptr, idx, ASM_ARM_REG_R3) -#define ASM_MOV_REG_TO_LOCAL(as, reg, local_num) asm_arm_mov_local_reg(as, (local_num), (reg)) -#define ASM_MOV_IMM_TO_REG(as, imm, reg) asm_arm_mov_reg_i32(as, (reg), (imm)) -#define ASM_MOV_ALIGNED_IMM_TO_REG(as, imm, reg) asm_arm_mov_reg_i32(as, (reg), (imm)) -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_arm_mov_reg_i32(as, (reg_temp), (imm)); \ - asm_arm_mov_local_reg(as, (local_num), (reg_temp)); \ - } while (false) -#define ASM_MOV_LOCAL_TO_REG(as, local_num, reg) asm_arm_mov_reg_local(as, (reg), (local_num)) +#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_arm_mov_local_reg((as), (local_num), (reg_src)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_arm_mov_reg_local((as), (reg_dest), (local_num)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_arm_mov_reg_reg((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG(as, local_num, reg) asm_arm_mov_reg_local_addr(as, (reg), (local_num)) +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_arm_mov_reg_local_addr((as), (reg_dest), (local_num)) #define ASM_LSL_REG_REG(as, reg_dest, reg_shift) asm_arm_lsl_reg_reg((as), (reg_dest), (reg_shift)) #define ASM_ASR_REG_REG(as, reg_dest, reg_shift) asm_arm_asr_reg_reg((as), (reg_dest), (reg_shift)) diff --git a/py/asmthumb.h b/py/asmthumb.h index 7070e03ac..552ad75fc 100644 --- a/py/asmthumb.h +++ b/py/asmthumb.h @@ -283,17 +283,12 @@ void asm_thumb_bl_ind(asm_thumb_t *as, void *fun_ptr, uint fun_id, uint reg_temp } while (0) #define ASM_CALL_IND(as, ptr, idx) asm_thumb_bl_ind(as, ptr, idx, ASM_THUMB_REG_R3) -#define ASM_MOV_REG_TO_LOCAL(as, reg, local_num) asm_thumb_mov_local_reg(as, (local_num), (reg)) -#define ASM_MOV_IMM_TO_REG(as, imm, reg) asm_thumb_mov_reg_i32_optimised(as, (reg), (imm)) -#define ASM_MOV_ALIGNED_IMM_TO_REG(as, imm, reg) asm_thumb_mov_reg_i32_aligned(as, (reg), (imm)) -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_thumb_mov_reg_i32_optimised(as, (reg_temp), (imm)); \ - asm_thumb_mov_local_reg(as, (local_num), (reg_temp)); \ - } while (false) -#define ASM_MOV_LOCAL_TO_REG(as, local_num, reg) asm_thumb_mov_reg_local(as, (reg), (local_num)) +#define ASM_MOV_LOCAL_REG(as, local_num, reg) asm_thumb_mov_local_reg((as), (local_num), (reg)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_thumb_mov_reg_i32_optimised((as), (reg_dest), (imm)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_thumb_mov_reg_i32_aligned((as), (reg_dest), (imm)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_thumb_mov_reg_local((as), (reg_dest), (local_num)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_thumb_mov_reg_reg((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG(as, local_num, reg) asm_thumb_mov_reg_local_addr(as, (reg), (local_num)) +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_thumb_mov_reg_local_addr((as), (reg_dest), (local_num)) #define ASM_LSL_REG_REG(as, reg_dest, reg_shift) asm_thumb_format_4((as), ASM_THUMB_FORMAT_4_LSL, (reg_dest), (reg_shift)) #define ASM_ASR_REG_REG(as, reg_dest, reg_shift) asm_thumb_format_4((as), ASM_THUMB_FORMAT_4_ASR, (reg_dest), (reg_shift)) diff --git a/py/asmx64.h b/py/asmx64.h index 425bdf2d3..2fbbfa9ff 100644 --- a/py/asmx64.h +++ b/py/asmx64.h @@ -162,17 +162,12 @@ void asm_x64_call_ind(asm_x64_t* as, void* ptr, int temp_r32); } while (0) #define ASM_CALL_IND(as, ptr, idx) asm_x64_call_ind(as, ptr, ASM_X64_REG_RAX) -#define ASM_MOV_REG_TO_LOCAL asm_x64_mov_r64_to_local -#define ASM_MOV_IMM_TO_REG asm_x64_mov_i64_to_r64_optimised -#define ASM_MOV_ALIGNED_IMM_TO_REG asm_x64_mov_i64_to_r64_aligned -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_x64_mov_i64_to_r64_optimised(as, (imm), (reg_temp)); \ - asm_x64_mov_r64_to_local(as, (reg_temp), (local_num)); \ - } while (false) -#define ASM_MOV_LOCAL_TO_REG asm_x64_mov_local_to_r64 +#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x64_mov_r64_to_local((as), (reg_src), (local_num)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x64_mov_i64_to_r64_optimised((as), (imm), (reg_dest)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_x64_mov_i64_to_r64_aligned((as), (imm), (reg_dest)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x64_mov_local_to_r64((as), (local_num), (reg_dest)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x64_mov_r64_r64((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG asm_x64_mov_local_addr_to_r64 +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x64_mov_local_addr_to_r64((as), (local_num), (reg_dest)) #define ASM_LSL_REG(as, reg) asm_x64_shl_r64_cl((as), (reg)) #define ASM_ASR_REG(as, reg) asm_x64_sar_r64_cl((as), (reg)) diff --git a/py/asmx86.h b/py/asmx86.h index 0a00e2e7c..bd5895453 100644 --- a/py/asmx86.h +++ b/py/asmx86.h @@ -160,17 +160,12 @@ void asm_x86_call_ind(asm_x86_t* as, void* ptr, mp_uint_t n_args, int temp_r32); } while (0) #define ASM_CALL_IND(as, ptr, idx) asm_x86_call_ind(as, ptr, mp_f_n_args[idx], ASM_X86_REG_EAX) -#define ASM_MOV_REG_TO_LOCAL asm_x86_mov_r32_to_local -#define ASM_MOV_IMM_TO_REG asm_x86_mov_i32_to_r32 -#define ASM_MOV_ALIGNED_IMM_TO_REG asm_x86_mov_i32_to_r32_aligned -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_x86_mov_i32_to_r32(as, (imm), (reg_temp)); \ - asm_x86_mov_r32_to_local(as, (reg_temp), (local_num)); \ - } while (false) -#define ASM_MOV_LOCAL_TO_REG asm_x86_mov_local_to_r32 +#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x86_mov_r32_to_local((as), (reg_src), (local_num)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_x86_mov_i32_to_r32_aligned((as), (imm), (reg_dest)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x86_mov_local_to_r32((as), (local_num), (reg_dest)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x86_mov_r32_r32((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG asm_x86_mov_local_addr_to_r32 +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x86_mov_local_addr_to_r32((as), (local_num), (reg_dest)) #define ASM_LSL_REG(as, reg) asm_x86_shl_r32_cl((as), (reg)) #define ASM_ASR_REG(as, reg) asm_x86_sar_r32_cl((as), (reg)) diff --git a/py/asmxtensa.h b/py/asmxtensa.h index 7db6c0d3d..e6d4158cb 100644 --- a/py/asmxtensa.h +++ b/py/asmxtensa.h @@ -280,17 +280,12 @@ void asm_xtensa_mov_reg_local_addr(asm_xtensa_t *as, uint reg_dest, int local_nu asm_xtensa_op_callx0(as, ASM_XTENSA_REG_A0); \ } while (0) -#define ASM_MOV_REG_TO_LOCAL(as, reg, local_num) asm_xtensa_mov_local_reg(as, (local_num), (reg)) -#define ASM_MOV_IMM_TO_REG(as, imm, reg) asm_xtensa_mov_reg_i32(as, (reg), (imm)) -#define ASM_MOV_ALIGNED_IMM_TO_REG(as, imm, reg) asm_xtensa_mov_reg_i32(as, (reg), (imm)) -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_xtensa_mov_reg_i32(as, (reg_temp), (imm)); \ - asm_xtensa_mov_local_reg(as, (local_num), (reg_temp)); \ - } while (0) -#define ASM_MOV_LOCAL_TO_REG(as, local_num, reg) asm_xtensa_mov_reg_local(as, (reg), (local_num)) +#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_xtensa_mov_local_reg((as), (local_num), (reg_src)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_xtensa_mov_reg_local((as), (reg_dest), (local_num)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_mov_n((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG(as, local_num, reg) asm_xtensa_mov_reg_local_addr(as, (reg), (local_num)) +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_xtensa_mov_reg_local_addr((as), (reg_dest), (local_num)) #define ASM_LSL_REG_REG(as, reg_dest, reg_shift) \ do { \ diff --git a/py/emitnative.c b/py/emitnative.c index 8e97dda11..964db9552 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -66,6 +66,13 @@ // this is defined so that the assembler exports generic assembler API macros #define GENERIC_ASM_API (1) +// define additional generic helper macros +#define ASM_MOV_LOCAL_IMM_VIA(as, local_num, imm, reg_temp) \ + do { \ + ASM_MOV_REG_IMM((as), (reg_temp), (imm)); \ + ASM_MOV_LOCAL_REG((as), (local_num), (reg_temp)); \ + } while (false) + #if N_X64 // x64 specific stuff @@ -389,7 +396,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop ASM_MOV_REG_REG(emit->as, REG_LOCAL_3, REG_ARG_3); } else { assert(i == 3); // should be true; max 4 args is checked above - ASM_MOV_REG_TO_LOCAL(emit->as, REG_ARG_4, i - REG_LOCAL_NUM); + ASM_MOV_LOCAL_REG(emit->as, i - REG_LOCAL_NUM, REG_ARG_4); } } #endif @@ -418,14 +425,14 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // set code_state.fun_bc - ASM_MOV_REG_TO_LOCAL(emit->as, REG_ARG_1, offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t)); + ASM_MOV_LOCAL_REG(emit->as, offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t), REG_ARG_1); // set code_state.ip (offset from start of this function to prelude info) // XXX this encoding may change size - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, emit->prelude_offset, offsetof(mp_code_state_t, ip) / sizeof(uintptr_t), REG_ARG_1); + ASM_MOV_LOCAL_IMM_VIA(emit->as, offsetof(mp_code_state_t, ip) / sizeof(uintptr_t), emit->prelude_offset, REG_ARG_1); // put address of code_state into first arg - ASM_MOV_LOCAL_ADDR_TO_REG(emit->as, 0, REG_ARG_1); + ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 0); // call mp_setup_code_state to prepare code_state structure #if N_THUMB @@ -438,11 +445,11 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // cache some locals in registers if (scope->num_locals > 0) { - ASM_MOV_LOCAL_TO_REG(emit->as, STATE_START + emit->n_state - 1 - 0, REG_LOCAL_1); + ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_1, STATE_START + emit->n_state - 1 - 0); if (scope->num_locals > 1) { - ASM_MOV_LOCAL_TO_REG(emit->as, STATE_START + emit->n_state - 1 - 1, REG_LOCAL_2); + ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_2, STATE_START + emit->n_state - 1 - 1); if (scope->num_locals > 2) { - ASM_MOV_LOCAL_TO_REG(emit->as, STATE_START + emit->n_state - 1 - 2, REG_LOCAL_3); + ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_3, STATE_START + emit->n_state - 1 - 2); } } } @@ -606,7 +613,7 @@ STATIC void need_reg_single(emit_t *emit, int reg_needed, int skip_stack_pos) { stack_info_t *si = &emit->stack_info[i]; if (si->kind == STACK_REG && si->data.u_reg == reg_needed) { si->kind = STACK_VALUE; - ASM_MOV_REG_TO_LOCAL(emit->as, si->data.u_reg, emit->stack_start + i); + ASM_MOV_LOCAL_REG(emit->as, emit->stack_start + i, si->data.u_reg); } } } @@ -617,7 +624,7 @@ STATIC void need_reg_all(emit_t *emit) { stack_info_t *si = &emit->stack_info[i]; if (si->kind == STACK_REG) { si->kind = STACK_VALUE; - ASM_MOV_REG_TO_LOCAL(emit->as, si->data.u_reg, emit->stack_start + i); + ASM_MOV_LOCAL_REG(emit->as, emit->stack_start + i, si->data.u_reg); } } } @@ -629,7 +636,7 @@ STATIC void need_stack_settled(emit_t *emit) { if (si->kind == STACK_REG) { DEBUG_printf(" reg(%u) to local(%u)\n", si->data.u_reg, emit->stack_start + i); si->kind = STACK_VALUE; - ASM_MOV_REG_TO_LOCAL(emit->as, si->data.u_reg, emit->stack_start + i); + ASM_MOV_LOCAL_REG(emit->as, emit->stack_start + i, si->data.u_reg); } } for (int i = 0; i < emit->stack_size; i++) { @@ -637,7 +644,7 @@ STATIC void need_stack_settled(emit_t *emit) { if (si->kind == STACK_IMM) { DEBUG_printf(" imm(" INT_FMT ") to local(%u)\n", si->data.u_imm, emit->stack_start + i); si->kind = STACK_VALUE; - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, si->data.u_imm, emit->stack_start + i, REG_TEMP0); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + i, si->data.u_imm, REG_TEMP0); } } } @@ -649,7 +656,7 @@ STATIC void emit_access_stack(emit_t *emit, int pos, vtype_kind_t *vtype, int re *vtype = si->vtype; switch (si->kind) { case STACK_VALUE: - ASM_MOV_LOCAL_TO_REG(emit->as, emit->stack_start + emit->stack_size - pos, reg_dest); + ASM_MOV_REG_LOCAL(emit->as, reg_dest, emit->stack_start + emit->stack_size - pos); break; case STACK_REG: @@ -659,7 +666,7 @@ STATIC void emit_access_stack(emit_t *emit, int pos, vtype_kind_t *vtype, int re break; case STACK_IMM: - ASM_MOV_IMM_TO_REG(emit->as, si->data.u_imm, reg_dest); + ASM_MOV_REG_IMM(emit->as, reg_dest, si->data.u_imm); break; } } @@ -671,7 +678,7 @@ STATIC void emit_fold_stack_top(emit_t *emit, int reg_dest) { si[0] = si[1]; if (si->kind == STACK_VALUE) { // if folded element was on the stack we need to put it in a register - ASM_MOV_LOCAL_TO_REG(emit->as, emit->stack_start + emit->stack_size - 1, reg_dest); + ASM_MOV_REG_LOCAL(emit->as, reg_dest, emit->stack_start + emit->stack_size - 1); si->kind = STACK_REG; si->data.u_reg = reg_dest; } @@ -765,30 +772,30 @@ STATIC void emit_call(emit_t *emit, mp_fun_kind_t fun_kind) { STATIC void emit_call_with_imm_arg(emit_t *emit, mp_fun_kind_t fun_kind, mp_int_t arg_val, int arg_reg) { need_reg_all(emit); - ASM_MOV_IMM_TO_REG(emit->as, arg_val, arg_reg); + ASM_MOV_REG_IMM(emit->as, arg_reg, arg_val); ASM_CALL_IND(emit->as, mp_fun_table[fun_kind], fun_kind); } // the first arg is stored in the code aligned on a mp_uint_t boundary STATIC void emit_call_with_imm_arg_aligned(emit_t *emit, mp_fun_kind_t fun_kind, mp_int_t arg_val, int arg_reg) { need_reg_all(emit); - ASM_MOV_ALIGNED_IMM_TO_REG(emit->as, arg_val, arg_reg); + ASM_MOV_REG_ALIGNED_IMM(emit->as, arg_reg, arg_val); ASM_CALL_IND(emit->as, mp_fun_table[fun_kind], fun_kind); } STATIC void emit_call_with_2_imm_args(emit_t *emit, mp_fun_kind_t fun_kind, mp_int_t arg_val1, int arg_reg1, mp_int_t arg_val2, int arg_reg2) { need_reg_all(emit); - ASM_MOV_IMM_TO_REG(emit->as, arg_val1, arg_reg1); - ASM_MOV_IMM_TO_REG(emit->as, arg_val2, arg_reg2); + ASM_MOV_REG_IMM(emit->as, arg_reg1, arg_val1); + ASM_MOV_REG_IMM(emit->as, arg_reg2, arg_val2); ASM_CALL_IND(emit->as, mp_fun_table[fun_kind], fun_kind); } // the first arg is stored in the code aligned on a mp_uint_t boundary STATIC void emit_call_with_3_imm_args_and_first_aligned(emit_t *emit, mp_fun_kind_t fun_kind, mp_int_t arg_val1, int arg_reg1, mp_int_t arg_val2, int arg_reg2, mp_int_t arg_val3, int arg_reg3) { need_reg_all(emit); - ASM_MOV_ALIGNED_IMM_TO_REG(emit->as, arg_val1, arg_reg1); - ASM_MOV_IMM_TO_REG(emit->as, arg_val2, arg_reg2); - ASM_MOV_IMM_TO_REG(emit->as, arg_val3, arg_reg3); + ASM_MOV_REG_ALIGNED_IMM(emit->as, arg_reg1, arg_val1); + ASM_MOV_REG_IMM(emit->as, arg_reg2, arg_val2); + ASM_MOV_REG_IMM(emit->as, arg_reg3, arg_val3); ASM_CALL_IND(emit->as, mp_fun_table[fun_kind], fun_kind); } @@ -808,19 +815,19 @@ STATIC void emit_get_stack_pointer_to_reg_for_pop(emit_t *emit, mp_uint_t reg_de si->kind = STACK_VALUE; switch (si->vtype) { case VTYPE_PYOBJ: - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, si->data.u_imm, emit->stack_start + emit->stack_size - 1 - i, reg_dest); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + emit->stack_size - 1 - i, si->data.u_imm, reg_dest); break; case VTYPE_BOOL: if (si->data.u_imm == 0) { - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, (mp_uint_t)mp_const_false, emit->stack_start + emit->stack_size - 1 - i, reg_dest); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + emit->stack_size - 1 - i, (mp_uint_t)mp_const_false, reg_dest); } else { - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, (mp_uint_t)mp_const_true, emit->stack_start + emit->stack_size - 1 - i, reg_dest); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + emit->stack_size - 1 - i, (mp_uint_t)mp_const_true, reg_dest); } si->vtype = VTYPE_PYOBJ; break; case VTYPE_INT: case VTYPE_UINT: - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, (uintptr_t)MP_OBJ_NEW_SMALL_INT(si->data.u_imm), emit->stack_start + emit->stack_size - 1 - i, reg_dest); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + emit->stack_size - 1 - i, (uintptr_t)MP_OBJ_NEW_SMALL_INT(si->data.u_imm), reg_dest); si->vtype = VTYPE_PYOBJ; break; default: @@ -838,9 +845,9 @@ STATIC void emit_get_stack_pointer_to_reg_for_pop(emit_t *emit, mp_uint_t reg_de stack_info_t *si = &emit->stack_info[emit->stack_size - 1 - i]; if (si->vtype != VTYPE_PYOBJ) { mp_uint_t local_num = emit->stack_start + emit->stack_size - 1 - i; - ASM_MOV_LOCAL_TO_REG(emit->as, local_num, REG_ARG_1); + ASM_MOV_REG_LOCAL(emit->as, REG_ARG_1, local_num); emit_call_with_imm_arg(emit, MP_F_CONVERT_NATIVE_TO_OBJ, si->vtype, REG_ARG_2); // arg2 = type - ASM_MOV_REG_TO_LOCAL(emit->as, REG_RET, local_num); + ASM_MOV_LOCAL_REG(emit->as, local_num, REG_RET); si->vtype = VTYPE_PYOBJ; DEBUG_printf(" convert_native_to_obj(local_num=" UINT_FMT ")\n", local_num); } @@ -848,7 +855,7 @@ STATIC void emit_get_stack_pointer_to_reg_for_pop(emit_t *emit, mp_uint_t reg_de // Adujust the stack for a pop of n_pop items, and load the stack pointer into reg_dest. adjust_stack(emit, -n_pop); - ASM_MOV_LOCAL_ADDR_TO_REG(emit->as, emit->stack_start + emit->stack_size, reg_dest); + ASM_MOV_REG_LOCAL_ADDR(emit->as, reg_dest, emit->stack_start + emit->stack_size); } // vtype of all n_push objects is VTYPE_PYOBJ @@ -858,7 +865,7 @@ STATIC void emit_get_stack_pointer_to_reg_for_push(emit_t *emit, mp_uint_t reg_d emit->stack_info[emit->stack_size + i].kind = STACK_VALUE; emit->stack_info[emit->stack_size + i].vtype = VTYPE_PYOBJ; } - ASM_MOV_LOCAL_ADDR_TO_REG(emit->as, emit->stack_start + emit->stack_size, reg_dest); + ASM_MOV_REG_LOCAL_ADDR(emit->as, reg_dest, emit->stack_start + emit->stack_size); adjust_stack(emit, n_push); } @@ -881,7 +888,7 @@ STATIC void emit_native_import_name(emit_t *emit, qstr qst) { stack_info_t *top = peek_stack(emit, 0); if (top->vtype == VTYPE_PTR_NONE) { emit_pre_pop_discard(emit); - ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)mp_const_none, REG_ARG_2); + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, (mp_uint_t)mp_const_none); } else { vtype_kind_t vtype_fromlist; emit_pre_pop_reg(emit, &vtype_fromlist, REG_ARG_2); @@ -891,7 +898,7 @@ STATIC void emit_native_import_name(emit_t *emit, qstr qst) { // level argument should be an immediate integer top = peek_stack(emit, 0); assert(top->vtype == VTYPE_INT && top->kind == STACK_IMM); - ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)MP_OBJ_NEW_SMALL_INT(top->data.u_imm), REG_ARG_3); + ASM_MOV_REG_IMM(emit->as, REG_ARG_3, (mp_uint_t)MP_OBJ_NEW_SMALL_INT(top->data.u_imm)); emit_pre_pop_discard(emit); } else { @@ -981,7 +988,7 @@ STATIC void emit_native_load_const_str(emit_t *emit, qstr qst) { STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj) { emit_native_pre(emit); need_reg_single(emit, REG_RET, 0); - ASM_MOV_ALIGNED_IMM_TO_REG(emit->as, (mp_uint_t)obj, REG_RET); + ASM_MOV_REG_ALIGNED_IMM(emit->as, REG_RET, (mp_uint_t)obj); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -1006,9 +1013,9 @@ STATIC void emit_native_load_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { } else { need_reg_single(emit, REG_TEMP0, 0); if (emit->do_viper_types) { - ASM_MOV_LOCAL_TO_REG(emit->as, local_num - REG_LOCAL_NUM, REG_TEMP0); + ASM_MOV_REG_LOCAL(emit->as, REG_TEMP0, local_num - REG_LOCAL_NUM); } else { - ASM_MOV_LOCAL_TO_REG(emit->as, STATE_START + emit->n_state - 1 - local_num, REG_TEMP0); + ASM_MOV_REG_LOCAL(emit->as, REG_TEMP0, STATE_START + emit->n_state - 1 - local_num); } emit_post_push_reg(emit, vtype, REG_TEMP0); } @@ -1134,7 +1141,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add index to base reg_base = reg_index; } @@ -1151,7 +1158,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value << 1, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 2*index to base reg_base = reg_index; } @@ -1168,7 +1175,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value << 2, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 2); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 4*index to base reg_base = reg_index; } @@ -1233,9 +1240,9 @@ STATIC void emit_native_store_fast(emit_t *emit, qstr qst, mp_uint_t local_num) } else { emit_pre_pop_reg(emit, &vtype, REG_TEMP0); if (emit->do_viper_types) { - ASM_MOV_REG_TO_LOCAL(emit->as, REG_TEMP0, local_num - REG_LOCAL_NUM); + ASM_MOV_LOCAL_REG(emit->as, local_num - REG_LOCAL_NUM, REG_TEMP0); } else { - ASM_MOV_REG_TO_LOCAL(emit->as, REG_TEMP0, STATE_START + emit->n_state - 1 - local_num); + ASM_MOV_LOCAL_REG(emit->as, STATE_START + emit->n_state - 1 - local_num, REG_TEMP0); } } emit_post(emit); @@ -1354,7 +1361,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value); #if N_ARM asm_arm_strb_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); return; @@ -1375,7 +1382,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value << 1, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); #if N_ARM asm_arm_strh_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); return; @@ -1396,7 +1403,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value << 2, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 2); #if N_ARM asm_arm_str_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); return; @@ -1773,7 +1780,7 @@ STATIC void emit_native_get_iter(emit_t *emit, bool use_stack) { emit_call(emit, MP_F_NATIVE_GETITER); } else { // mp_getiter will allocate the iter_buf on the heap - ASM_MOV_IMM_TO_REG(emit->as, 0, REG_ARG_2); + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, 0); emit_call(emit, MP_F_NATIVE_GETITER); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -1784,7 +1791,7 @@ STATIC void emit_native_for_iter(emit_t *emit, mp_uint_t label) { emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_1, MP_OBJ_ITER_BUF_NSLOTS); adjust_stack(emit, MP_OBJ_ITER_BUF_NSLOTS); emit_call(emit, MP_F_NATIVE_ITERNEXT); - ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)MP_OBJ_STOP_ITERATION, REG_TEMP1); + ASM_MOV_REG_IMM(emit->as, REG_TEMP1, (mp_uint_t)MP_OBJ_STOP_ITERATION); ASM_JUMP_IF_REG_EQ(emit->as, REG_RET, REG_TEMP1, label); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -2128,12 +2135,12 @@ STATIC void emit_native_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_c emit_native_pre(emit); if (n_pos_defaults == 0 && n_kw_defaults == 0) { emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over); - ASM_MOV_IMM_TO_REG(emit->as, n_closed_over, REG_ARG_2); + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, n_closed_over); } else { emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over + 2); - ASM_MOV_IMM_TO_REG(emit->as, 0x100 | n_closed_over, REG_ARG_2); + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, 0x100 | n_closed_over); } - ASM_MOV_ALIGNED_IMM_TO_REG(emit->as, (mp_uint_t)scope->raw_code, REG_ARG_1); + ASM_MOV_REG_ALIGNED_IMM(emit->as, REG_ARG_1, (mp_uint_t)scope->raw_code); ASM_CALL_IND(emit->as, mp_fun_table[MP_F_MAKE_CLOSURE_FROM_RAW_CODE], MP_F_MAKE_CLOSURE_FROM_RAW_CODE); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -2212,9 +2219,9 @@ STATIC void emit_native_return_value(emit_t *emit) { if (peek_vtype(emit, 0) == VTYPE_PTR_NONE) { emit_pre_pop_discard(emit); if (emit->return_vtype == VTYPE_PYOBJ) { - ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)mp_const_none, REG_RET); + ASM_MOV_REG_IMM(emit->as, REG_RET, (mp_uint_t)mp_const_none); } else { - ASM_MOV_IMM_TO_REG(emit->as, 0, REG_RET); + ASM_MOV_REG_IMM(emit->as, REG_RET, 0); } } else { vtype_kind_t vtype; -- cgit v1.2.3 From 1871a924c97cec16d0670d136e9f7056f99865df Mon Sep 17 00:00:00 2001 From: Christopher Arndt Date: Tue, 31 Oct 2017 17:19:12 +0100 Subject: py/mkenv.mk: Use $(PYTHON) consistently when calling Python tools. Rationale: * Calling Python build tool scripts from makefiles should be done consistently using `python `, instead of relying on the correct she-bang line in the script [1] and the executable bit on the script being set. This is more platform-independent. * The name/path of the Python executable should always be used via the makefile variable `PYTHON` set in `py/mkenv.mk`. This way it can be easily overwritten by the user with `make PYTHON=/path/to/my/python`. * The Python executable name should be part of the value of the makefile variable, which stands for the build tool command (e.g. `MAKE_FROZEN` and `MPY_TOOL`), not part of the command line where it is used. If a Python tool is substituted by another (non-python) program, no change to the Makefiles is necessary, except in `py/mkenv.mk`. * This also solves #3369 and #1616. [1] There are systems, where even the assumption that `/usr/bin/env` always exists, doesn't hold true, for example on Android (where otherwise the unix port compiles perfectly well). --- py/mkenv.mk | 4 ++-- py/mkrules.mk | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/mkenv.mk b/py/mkenv.mk index b167b2533..2c9c86a7a 100644 --- a/py/mkenv.mk +++ b/py/mkenv.mk @@ -58,9 +58,9 @@ CXX += -m32 LD += -m32 endif -MAKE_FROZEN = $(TOP)/tools/make-frozen.py +MAKE_FROZEN = $(PYTHON) $(TOP)/tools/make-frozen.py MPY_CROSS = $(TOP)/mpy-cross/mpy-cross -MPY_TOOL = $(TOP)/tools/mpy-tool.py +MPY_TOOL = $(PYTHON) $(TOP)/tools/mpy-tool.py all: .PHONY: all diff --git a/py/mkrules.mk b/py/mkrules.mk index 13545eb6f..72a5c3de0 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -119,7 +119,7 @@ $(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py $(TOP)/mpy-cross/mpy-cross # to build frozen_mpy.c from all .mpy files $(BUILD)/frozen_mpy.c: $(FROZEN_MPY_MPY_FILES) $(BUILD)/genhdr/qstrdefs.generated.h @$(ECHO) "Creating $@" - $(Q)$(PYTHON) $(MPY_TOOL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(FROZEN_MPY_MPY_FILES) > $@ + $(Q)$(MPY_TOOL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(FROZEN_MPY_MPY_FILES) > $@ endif ifneq ($(PROG),) -- cgit v1.2.3 From 4601759bf59e16b860a3f082e9aa4ea78356bf92 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Nov 2017 13:17:51 +1100 Subject: py/objstr: Remove "make_qstr_if_not_already" arg from mp_obj_new_str. This patch simplifies the str creation API to favour the common case of creating a str object that is not forced to be interned. To force interning of a new str the new mp_obj_new_str_via_qstr function is added, and should only be used if warranted. Apart from simplifying the mp_obj_new_str function (and making it have the same signature as mp_obj_new_bytes), this patch also reduces code size by a bit (-16 bytes for bare-arm and roughly -40 bytes on the bare-metal archs). --- extmod/modujson.c | 2 +- extmod/modwebrepl.c | 2 +- extmod/vfs_fat.c | 2 +- extmod/vfs_fat_misc.c | 2 +- extmod/vfs_reader.c | 2 +- lib/netutils/netutils.c | 2 +- ports/cc3200/mods/modwlan.c | 10 +++++----- ports/esp8266/modnetwork.c | 4 ++-- ports/esp8266/moduos.c | 2 +- ports/stm32/modnwcc3k.c | 4 ++-- ports/unix/coverage.c | 6 +++--- ports/unix/main.c | 6 +++--- ports/unix/modffi.c | 2 +- ports/unix/modjni.c | 2 +- ports/unix/modos.c | 4 ++-- ports/zephyr/modusocket.c | 2 +- py/binary.c | 2 +- py/builtinhelp.c | 2 +- py/builtinimport.c | 2 +- py/modbuiltins.c | 4 ++-- py/modio.c | 2 +- py/obj.h | 3 ++- py/objstr.c | 36 ++++++++++++++++++------------------ py/objstrunicode.c | 4 ++-- 24 files changed, 55 insertions(+), 54 deletions(-) (limited to 'py') diff --git a/extmod/modujson.c b/extmod/modujson.c index f14682d26..6eeba4ed6 100644 --- a/extmod/modujson.c +++ b/extmod/modujson.c @@ -166,7 +166,7 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) { goto fail; } S_NEXT(s); - next = mp_obj_new_str(vstr.buf, vstr.len, false); + next = mp_obj_new_str(vstr.buf, vstr.len); break; case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { diff --git a/extmod/modwebrepl.c b/extmod/modwebrepl.c index 3aba5c0f1..42b30f5ea 100644 --- a/extmod/modwebrepl.c +++ b/extmod/modwebrepl.c @@ -141,7 +141,7 @@ STATIC void handle_op(mp_obj_webrepl_t *self) { // Handle operations requiring opened file mp_obj_t open_args[2] = { - mp_obj_new_str(self->hdr.fname, strlen(self->hdr.fname), false), + mp_obj_new_str(self->hdr.fname, strlen(self->hdr.fname)), MP_OBJ_NEW_QSTR(MP_QSTR_rb) }; diff --git a/extmod/vfs_fat.c b/extmod/vfs_fat.c index 22346bdf1..9942ddeb5 100644 --- a/extmod/vfs_fat.c +++ b/extmod/vfs_fat.c @@ -197,7 +197,7 @@ STATIC mp_obj_t fat_vfs_getcwd(mp_obj_t vfs_in) { if (res != FR_OK) { mp_raise_OSError(fresult_to_errno_table[res]); } - return mp_obj_new_str(buf, strlen(buf), false); + return mp_obj_new_str(buf, strlen(buf)); } STATIC MP_DEFINE_CONST_FUN_OBJ_1(fat_vfs_getcwd_obj, fat_vfs_getcwd); diff --git a/extmod/vfs_fat_misc.c b/extmod/vfs_fat_misc.c index 9a26b4a2f..1f90ac14c 100644 --- a/extmod/vfs_fat_misc.c +++ b/extmod/vfs_fat_misc.c @@ -57,7 +57,7 @@ STATIC mp_obj_t mp_vfs_fat_ilistdir_it_iternext(mp_obj_t self_in) { // make 3-tuple with info about this entry mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(3, NULL)); if (self->is_str) { - t->items[0] = mp_obj_new_str(fn, strlen(fn), false); + t->items[0] = mp_obj_new_str(fn, strlen(fn)); } else { t->items[0] = mp_obj_new_bytes((const byte*)fn, strlen(fn)); } diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index 891098aa1..e1ee45a3c 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -71,7 +71,7 @@ STATIC void mp_reader_vfs_close(void *data) { void mp_reader_new_file(mp_reader_t *reader, const char *filename) { mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t); - mp_obj_t arg = mp_obj_new_str(filename, strlen(filename), false); + mp_obj_t arg = mp_obj_new_str(filename, strlen(filename)); rf->file = mp_vfs_open(1, &arg, (mp_map_t*)&mp_const_empty_map); int errcode; rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE); diff --git a/lib/netutils/netutils.c b/lib/netutils/netutils.c index 06c3ff9b0..073f46b19 100644 --- a/lib/netutils/netutils.c +++ b/lib/netutils/netutils.c @@ -41,7 +41,7 @@ mp_obj_t netutils_format_ipv4_addr(uint8_t *ip, netutils_endian_t endian) { } else { ip_len = snprintf(ip_str, 16, "%u.%u.%u.%u", ip[0], ip[1], ip[2], ip[3]); } - return mp_obj_new_str(ip_str, ip_len, false); + return mp_obj_new_str(ip_str, ip_len); } // Takes an array with a raw IP address, and a port, and returns a net-address diff --git a/ports/cc3200/mods/modwlan.c b/ports/cc3200/mods/modwlan.c index f9c7111b3..8acc89da3 100644 --- a/ports/cc3200/mods/modwlan.c +++ b/ports/cc3200/mods/modwlan.c @@ -885,7 +885,7 @@ STATIC mp_obj_t wlan_scan(mp_obj_t self_in) { } mp_obj_t tuple[5]; - tuple[0] = mp_obj_new_str((const char *)wlanEntry.ssid, wlanEntry.ssid_len, false); + tuple[0] = mp_obj_new_str((const char *)wlanEntry.ssid, wlanEntry.ssid_len); tuple[1] = mp_obj_new_bytes((const byte *)wlanEntry.bssid, SL_BSSID_LENGTH); // 'normalize' the security type if (wlanEntry.sec_type > 2) { @@ -1075,7 +1075,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(wlan_mode_obj, 1, 2, wlan_mode); STATIC mp_obj_t wlan_ssid(size_t n_args, const mp_obj_t *args) { wlan_obj_t *self = args[0]; if (n_args == 1) { - return mp_obj_new_str((const char *)self->ssid, strlen((const char *)self->ssid), false); + return mp_obj_new_str((const char *)self->ssid, strlen((const char *)self->ssid)); } else { size_t len; const char *ssid = mp_obj_str_get_data(args[1], &len); @@ -1095,7 +1095,7 @@ STATIC mp_obj_t wlan_auth(size_t n_args, const mp_obj_t *args) { } else { mp_obj_t security[2]; security[0] = mp_obj_new_int(self->auth); - security[1] = mp_obj_new_str((const char *)self->key, strlen((const char *)self->key), false); + security[1] = mp_obj_new_str((const char *)self->key, strlen((const char *)self->key)); return mp_obj_new_tuple(2, security); } } else { @@ -1199,7 +1199,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_KW(wlan_irq_obj, 1, wlan_irq); // mp_obj_t connections = mp_obj_new_list(0, NULL); // // if (wlan_is_connected()) { -// device[0] = mp_obj_new_str((const char *)wlan_obj.ssid_o, strlen((const char *)wlan_obj.ssid_o), false); +// device[0] = mp_obj_new_str((const char *)wlan_obj.ssid_o, strlen((const char *)wlan_obj.ssid_o)); // device[1] = mp_obj_new_bytes((const byte *)wlan_obj.bssid, SL_BSSID_LENGTH); // // add the device to the list // mp_obj_list_append(connections, mp_obj_new_tuple(MP_ARRAY_SIZE(device), device)); @@ -1232,7 +1232,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_KW(wlan_irq_obj, 1, wlan_irq); // if (sl_NetAppGet(SL_NET_APP_DEVICE_CONFIG_ID, NETAPP_SET_GET_DEV_CONF_OPT_DEVICE_URN, &len, (uint8_t *)urn) < 0) { // mp_raise_OSError(MP_EIO); // } -// return mp_obj_new_str(urn, (len - 1), false); +// return mp_obj_new_str(urn, (len - 1)); // } // // return mp_const_none; diff --git a/ports/esp8266/modnetwork.c b/ports/esp8266/modnetwork.c index b41a11f59..4066c969c 100644 --- a/ports/esp8266/modnetwork.c +++ b/ports/esp8266/modnetwork.c @@ -411,7 +411,7 @@ STATIC mp_obj_t esp_config(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs } case QS(MP_QSTR_essid): req_if = SOFTAP_IF; - val = mp_obj_new_str((char*)cfg.ap.ssid, cfg.ap.ssid_len, false); + val = mp_obj_new_str((char*)cfg.ap.ssid, cfg.ap.ssid_len); break; case QS(MP_QSTR_hidden): req_if = SOFTAP_IF; @@ -428,7 +428,7 @@ STATIC mp_obj_t esp_config(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs case QS(MP_QSTR_dhcp_hostname): { req_if = STATION_IF; char* s = wifi_station_get_hostname(); - val = mp_obj_new_str(s, strlen(s), false); + val = mp_obj_new_str(s, strlen(s)); break; } default: diff --git a/ports/esp8266/moduos.c b/ports/esp8266/moduos.c index d0554096e..93f7aa712 100644 --- a/ports/esp8266/moduos.c +++ b/ports/esp8266/moduos.c @@ -60,7 +60,7 @@ STATIC mp_obj_tuple_t os_uname_info_obj = { STATIC mp_obj_t os_uname(void) { // We must populate the "release" field each time in case it was GC'd since the last call. const char *ver = system_get_sdk_version(); - os_uname_info_obj.items[2] = mp_obj_new_str(ver, strlen(ver), false); + os_uname_info_obj.items[2] = mp_obj_new_str(ver, strlen(ver)); return (mp_obj_t)&os_uname_info_obj; } STATIC MP_DEFINE_CONST_FUN_OBJ_0(os_uname_obj, os_uname); diff --git a/ports/stm32/modnwcc3k.c b/ports/stm32/modnwcc3k.c index 8cc0a613d..4f1af7354 100644 --- a/ports/stm32/modnwcc3k.c +++ b/ports/stm32/modnwcc3k.c @@ -531,8 +531,8 @@ STATIC mp_obj_t cc3k_ifconfig(mp_obj_t self_in) { netutils_format_ipv4_addr(ipconfig.aucDefaultGateway, NETUTILS_LITTLE), netutils_format_ipv4_addr(ipconfig.aucDNSServer, NETUTILS_LITTLE), netutils_format_ipv4_addr(ipconfig.aucDHCPServer, NETUTILS_LITTLE), - mp_obj_new_str(mac_vstr.buf, mac_vstr.len, false), - mp_obj_new_str((const char*)ipconfig.uaSSID, strlen((const char*)ipconfig.uaSSID), false), + mp_obj_new_str(mac_vstr.buf, mac_vstr.len), + mp_obj_new_str((const char*)ipconfig.uaSSID, strlen((const char*)ipconfig.uaSSID)), }; return mp_obj_new_tuple(MP_ARRAY_SIZE(tuple), tuple); } diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index 651db0a94..e2896c2dd 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -227,7 +227,7 @@ STATIC mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "# str\n"); // intern string - mp_printf(&mp_plat_print, "%d\n", MP_OBJ_IS_QSTR(mp_obj_str_intern(mp_obj_new_str("intern me", 9, false)))); + mp_printf(&mp_plat_print, "%d\n", MP_OBJ_IS_QSTR(mp_obj_str_intern(mp_obj_new_str("intern me", 9)))); } // mpz @@ -260,12 +260,12 @@ STATIC mp_obj_t extra_coverage(void) { // call mp_call_function_1_protected mp_call_function_1_protected(MP_OBJ_FROM_PTR(&mp_builtin_abs_obj), MP_OBJ_NEW_SMALL_INT(1)); // call mp_call_function_1_protected with invalid args - mp_call_function_1_protected(MP_OBJ_FROM_PTR(&mp_builtin_abs_obj), mp_obj_new_str("abc", 3, false)); + mp_call_function_1_protected(MP_OBJ_FROM_PTR(&mp_builtin_abs_obj), mp_obj_new_str("abc", 3)); // call mp_call_function_2_protected mp_call_function_2_protected(MP_OBJ_FROM_PTR(&mp_builtin_divmod_obj), MP_OBJ_NEW_SMALL_INT(1), MP_OBJ_NEW_SMALL_INT(1)); // call mp_call_function_2_protected with invalid args - mp_call_function_2_protected(MP_OBJ_FROM_PTR(&mp_builtin_divmod_obj), mp_obj_new_str("abc", 3, false), mp_obj_new_str("abc", 3, false)); + mp_call_function_2_protected(MP_OBJ_FROM_PTR(&mp_builtin_divmod_obj), mp_obj_new_str("abc", 3), mp_obj_new_str("abc", 3)); } // warning diff --git a/ports/unix/main.c b/ports/unix/main.c index e1cd33fc1..25f3e04fb 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -481,7 +481,7 @@ MP_NOINLINE int main_(int argc, char **argv) { vstr_add_strn(&vstr, p + 1, p1 - p - 1); path_items[i] = mp_obj_new_str_from_vstr(&mp_type_str, &vstr); } else { - path_items[i] = MP_OBJ_NEW_QSTR(qstr_from_strn(p, p1 - p)); + path_items[i] = mp_obj_new_str_via_qstr(p, p1 - p); } p = p1 + 1; } @@ -537,7 +537,7 @@ MP_NOINLINE int main_(int argc, char **argv) { return usage(argv); } mp_obj_t import_args[4]; - import_args[0] = mp_obj_new_str(argv[a + 1], strlen(argv[a + 1]), false); + import_args[0] = mp_obj_new_str(argv[a + 1], strlen(argv[a + 1])); import_args[1] = import_args[2] = mp_const_none; // Ask __import__ to handle imported module specially - set its __name__ // to __main__, and also return this leaf module, not top-level package @@ -603,7 +603,7 @@ MP_NOINLINE int main_(int argc, char **argv) { // Set base dir of the script as first entry in sys.path char *p = strrchr(basedir, '/'); - path_items[0] = MP_OBJ_NEW_QSTR(qstr_from_strn(basedir, p - basedir)); + path_items[0] = mp_obj_new_str_via_qstr(basedir, p - basedir); free(pathbuf); set_sys_argv(argv, argc, a); diff --git a/ports/unix/modffi.c b/ports/unix/modffi.c index 78adccac1..024f83c14 100644 --- a/ports/unix/modffi.c +++ b/ports/unix/modffi.c @@ -144,7 +144,7 @@ STATIC mp_obj_t return_ffi_value(ffi_arg val, char type) if (!s) { return mp_const_none; } - return mp_obj_new_str(s, strlen(s), false); + return mp_obj_new_str(s, strlen(s)); } case 'v': return mp_const_none; diff --git a/ports/unix/modjni.c b/ports/unix/modjni.c index f29c095cf..8ec5ae54d 100644 --- a/ports/unix/modjni.c +++ b/ports/unix/modjni.c @@ -337,7 +337,7 @@ STATIC mp_obj_t new_jobject(jobject jo) { return mp_const_none; } else if (JJ(IsInstanceOf, jo, String_class)) { const char *s = JJ(GetStringUTFChars, jo, NULL); - mp_obj_t ret = mp_obj_new_str(s, strlen(s), false); + mp_obj_t ret = mp_obj_new_str(s, strlen(s)); JJ(ReleaseStringUTFChars, jo, s); return ret; } else if (JJ(IsInstanceOf, jo, Class_class)) { diff --git a/ports/unix/modos.c b/ports/unix/modos.c index 327116a0a..808d12adb 100644 --- a/ports/unix/modos.c +++ b/ports/unix/modos.c @@ -133,7 +133,7 @@ STATIC mp_obj_t mod_os_getenv(mp_obj_t var_in) { if (s == NULL) { return mp_const_none; } - return mp_obj_new_str(s, strlen(s), false); + return mp_obj_new_str(s, strlen(s)); } STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_os_getenv_obj, mod_os_getenv); @@ -171,7 +171,7 @@ STATIC mp_obj_t listdir_next(mp_obj_t self_in) { } mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(3, NULL)); - t->items[0] = mp_obj_new_str(dirent->d_name, strlen(dirent->d_name), false); + t->items[0] = mp_obj_new_str(dirent->d_name, strlen(dirent->d_name)); #ifdef _DIRENT_HAVE_D_TYPE t->items[1] = MP_OBJ_NEW_SMALL_INT(dirent->d_type); #else diff --git a/ports/zephyr/modusocket.c b/ports/zephyr/modusocket.c index e021c3a44..95414a49b 100644 --- a/ports/zephyr/modusocket.c +++ b/ports/zephyr/modusocket.c @@ -91,7 +91,7 @@ STATIC mp_obj_t format_inet_addr(struct sockaddr *addr, mp_obj_t port) { net_addr_ntop(addr->sa_family, &sockaddr_in6->sin6_addr, buf, sizeof(buf)); mp_obj_tuple_t *tuple = mp_obj_new_tuple(addr->sa_family == AF_INET ? 2 : 4, NULL); - tuple->items[0] = mp_obj_new_str(buf, strlen(buf), false); + tuple->items[0] = mp_obj_new_str(buf, strlen(buf)); // We employ the fact that port offset is the same for IPv4 & IPv6 // not filled in //tuple->items[1] = mp_obj_new_int(ntohs(((struct sockaddr_in*)addr)->sin_port)); diff --git a/py/binary.c b/py/binary.c index 870a0942b..f509ff010 100644 --- a/py/binary.c +++ b/py/binary.c @@ -206,7 +206,7 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) { return (mp_obj_t)(mp_uint_t)val; } else if (val_type == 'S') { const char *s_val = (const char*)(uintptr_t)(mp_uint_t)val; - return mp_obj_new_str(s_val, strlen(s_val), false); + return mp_obj_new_str(s_val, strlen(s_val)); #if MICROPY_PY_BUILTINS_FLOAT } else if (val_type == 'f') { union { uint32_t i; float f; } fpu = {val}; diff --git a/py/builtinhelp.c b/py/builtinhelp.c index c9992906d..7106f3ced 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -69,7 +69,7 @@ STATIC void mp_help_add_from_names(mp_obj_t list, const char *name) { while (*name) { size_t l = strlen(name); // name should end in '.py' and we strip it off - mp_obj_list_append(list, mp_obj_new_str(name, l - 3, false)); + mp_obj_list_append(list, mp_obj_new_str(name, l - 3)); name += l + 1; } } diff --git a/py/builtinimport.c b/py/builtinimport.c index 04ce66723..9235e946c 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -434,7 +434,7 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { DEBUG_printf("%.*s is dir\n", vstr_len(&path), vstr_str(&path)); // https://docs.python.org/3/reference/import.html // "Specifically, any module that contains a __path__ attribute is considered a package." - mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str(vstr_str(&path), vstr_len(&path), false)); + mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str(vstr_str(&path), vstr_len(&path))); size_t orig_path_len = path.len; vstr_add_char(&path, PATH_SEP_CHAR); vstr_add_str(&path, "__init__.py"); diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 65c03d523..ebff5f5c0 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -159,12 +159,12 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { } else { mp_raise_ValueError("chr() arg not in range(0x110000)"); } - return mp_obj_new_str(str, len, true); + return mp_obj_new_str_via_qstr(str, len); #else mp_int_t ord = mp_obj_get_int(o_in); if (0 <= ord && ord <= 0xff) { char str[1] = {ord}; - return mp_obj_new_str(str, 1, true); + return mp_obj_new_str_via_qstr(str, 1); } else { mp_raise_ValueError("chr() arg not in range(256)"); } diff --git a/py/modio.c b/py/modio.c index 353a00286..828bcec46 100644 --- a/py/modio.c +++ b/py/modio.c @@ -176,7 +176,7 @@ STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) { return MP_OBJ_FROM_PTR(o); } - mp_obj_t path_out = mp_obj_new_str(path_buf.buf, path_buf.len, false); + mp_obj_t path_out = mp_obj_new_str(path_buf.buf, path_buf.len); return mp_builtin_open(1, &path_out, (mp_map_t*)&mp_const_empty_map); } MP_DEFINE_CONST_FUN_OBJ_2(resource_stream_obj, resource_stream); diff --git a/py/obj.h b/py/obj.h index 77f0f2298..31c3ce95c 100644 --- a/py/obj.h +++ b/py/obj.h @@ -637,7 +637,8 @@ mp_obj_t mp_obj_new_int_from_uint(mp_uint_t value); mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base); mp_obj_t mp_obj_new_int_from_ll(long long val); // this must return a multi-precision integer object (or raise an overflow exception) mp_obj_t mp_obj_new_int_from_ull(unsigned long long val); // this must return a multi-precision integer object (or raise an overflow exception) -mp_obj_t mp_obj_new_str(const char* data, size_t len, bool make_qstr_if_not_already); +mp_obj_t mp_obj_new_str(const char* data, size_t len); +mp_obj_t mp_obj_new_str_via_qstr(const char* data, size_t len); mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr); mp_obj_t mp_obj_new_bytes(const byte* data, size_t len); mp_obj_t mp_obj_new_bytearray(size_t n, void *items); diff --git a/py/objstr.c b/py/objstr.c index 51da7a418..5c464ba7d 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -176,7 +176,7 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ mp_raise_msg(&mp_type_UnicodeError, NULL); } #endif - return mp_obj_new_str(bufinfo.buf, bufinfo.len, false); + return mp_obj_new_str(bufinfo.buf, bufinfo.len); } } } @@ -423,7 +423,7 @@ STATIC mp_obj_t bytes_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { if (MICROPY_PY_BUILTINS_STR_UNICODE || type == &mp_type_bytes) { return MP_OBJ_NEW_SMALL_INT(self_data[index_val]); } else { - return mp_obj_new_str((char*)&self_data[index_val], 1, true); + return mp_obj_new_str_via_qstr((char*)&self_data[index_val], 1); } } else { return MP_OBJ_NULL; // op not supported @@ -1046,7 +1046,7 @@ STATIC vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *ar } else { const char *lookup; for (lookup = field_name; lookup < field_name_top && *lookup != '.' && *lookup != '['; lookup++); - mp_obj_t field_q = mp_obj_new_str(field_name, lookup - field_name, true/*?*/); + mp_obj_t field_q = mp_obj_new_str_via_qstr(field_name, lookup - field_name); // should it be via qstr? field_name = lookup; mp_map_elem_t *key_elem = mp_map_lookup(kwargs, field_q, MP_MAP_LOOKUP); if (key_elem == NULL) { @@ -1413,7 +1413,7 @@ STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, size_t n_args, const mp_obj_ } ++str; } - mp_obj_t k_obj = mp_obj_new_str((const char*)key, str - key, true); + mp_obj_t k_obj = mp_obj_new_str_via_qstr((const char*)key, str - key); arg = mp_obj_dict_get(dict, k_obj); str++; } @@ -1992,6 +1992,11 @@ mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, siz return MP_OBJ_FROM_PTR(o); } +// Create a str using a qstr to store the data; may use existing or new qstr. +mp_obj_t mp_obj_new_str_via_qstr(const char* data, size_t len) { + return MP_OBJ_NEW_QSTR(qstr_from_strn(data, len)); +} + // Create a str/bytes object from the given vstr. The vstr buffer is resized to // the exact length required and then reused for the str/bytes object. The vstr // is cleared and can safely be passed to vstr_free if it was heap allocated. @@ -2022,25 +2027,20 @@ mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr) { return MP_OBJ_FROM_PTR(o); } -mp_obj_t mp_obj_new_str(const char* data, size_t len, bool make_qstr_if_not_already) { - if (make_qstr_if_not_already) { - // use existing, or make a new qstr - return MP_OBJ_NEW_QSTR(qstr_from_strn(data, len)); +mp_obj_t mp_obj_new_str(const char* data, size_t len) { + qstr q = qstr_find_strn(data, len); + if (q != MP_QSTR_NULL) { + // qstr with this data already exists + return MP_OBJ_NEW_QSTR(q); } else { - qstr q = qstr_find_strn(data, len); - if (q != MP_QSTR_NULL) { - // qstr with this data already exists - return MP_OBJ_NEW_QSTR(q); - } else { - // no existing qstr, don't make one - return mp_obj_new_str_of_type(&mp_type_str, (const byte*)data, len); - } + // no existing qstr, don't make one + return mp_obj_new_str_of_type(&mp_type_str, (const byte*)data, len); } } mp_obj_t mp_obj_str_intern(mp_obj_t str) { GET_STR_DATA_LEN(str, data, len); - return MP_OBJ_NEW_QSTR(qstr_from_strn((const char*)data, len)); + return mp_obj_new_str_via_qstr((const char*)data, len); } mp_obj_t mp_obj_new_bytes(const byte* data, size_t len) { @@ -2138,7 +2138,7 @@ STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { mp_obj_str8_it_t *self = MP_OBJ_TO_PTR(self_in); GET_STR_DATA_LEN(self->str, str, len); if (self->cur < len) { - mp_obj_t o_out = mp_obj_new_str((const char*)str + self->cur, 1, true); + mp_obj_t o_out = mp_obj_new_str_via_qstr((const char*)str + self->cur, 1); self->cur += 1; return o_out; } else { diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 29f7695b7..a1f54b8a2 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -216,7 +216,7 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { ++len; } } - return mp_obj_new_str((const char*)s, len, true); // This will create a one-character string + return mp_obj_new_str_via_qstr((const char*)s, len); // This will create a one-character string } else { return MP_OBJ_NULL; // op not supported } @@ -291,7 +291,7 @@ STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { if (self->cur < len) { const byte *cur = str + self->cur; const byte *end = utf8_next_char(str + self->cur); - mp_obj_t o_out = mp_obj_new_str((const char*)cur, end - cur, true); + mp_obj_t o_out = mp_obj_new_str_via_qstr((const char*)cur, end - cur); self->cur += end - cur; return o_out; } else { -- cgit v1.2.3 From 1f1d5194d775ad996f1d341c1a44b56af7ea4d4c Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Nov 2017 13:53:04 +1100 Subject: py/objstr: Make mp_obj_new_str_of_type check for existing interned qstr. The function mp_obj_new_str_of_type is a general str object constructor used in many places in the code to create either a str or bytes object. When creating a str it should first check if the string data already exists as an interned qstr, and if so then return the qstr object. This patch makes the function have such behaviour, which helps to reduce heap usage by reusing existing interned data where possible. The old behaviour of mp_obj_new_str_of_type (which didn't check for existing interned data) is made available through the function mp_obj_new_str_copy, but should only be used in very special cases. One consequence of this patch is that the following expression is now True: 'abc' is ' abc '.split()[0] --- py/objstr.c | 26 +++++++++++++++++++------- py/objstr.h | 1 + py/parse.c | 2 +- 3 files changed, 21 insertions(+), 8 deletions(-) (limited to 'py') diff --git a/py/objstr.c b/py/objstr.c index 5c464ba7d..bca2af801 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -164,7 +164,7 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ mp_raise_msg(&mp_type_UnicodeError, NULL); } #endif - mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_of_type(type, NULL, str_len)); + mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(type, NULL, str_len)); o->data = str_data; o->hash = str_hash; return MP_OBJ_FROM_PTR(o); @@ -205,7 +205,7 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size if (str_hash == 0) { str_hash = qstr_compute_hash(str_data, str_len); } - mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_of_type(&mp_type_bytes, NULL, str_len)); + mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(&mp_type_bytes, NULL, str_len)); o->data = str_data; o->hash = str_hash; return MP_OBJ_FROM_PTR(o); @@ -226,7 +226,7 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size // check if argument has the buffer protocol mp_buffer_info_t bufinfo; if (mp_get_buffer(args[0], &bufinfo, MP_BUFFER_READ)) { - return mp_obj_new_str_of_type(&mp_type_bytes, bufinfo.buf, bufinfo.len); + return mp_obj_new_bytes(bufinfo.buf, bufinfo.len); } vstr_t vstr; @@ -1977,8 +1977,9 @@ const mp_obj_type_t mp_type_bytes = { const mp_obj_str_t mp_const_empty_bytes_obj = {{&mp_type_bytes}, 0, 0, (const byte*)""}; // Create a str/bytes object using the given data. New memory is allocated and -// the data is copied across. -mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, size_t len) { +// the data is copied across. This function should only be used if the type is bytes, +// or if the type is str and the string data is known to be not interned. +mp_obj_t mp_obj_new_str_copy(const mp_obj_type_t *type, const byte* data, size_t len) { mp_obj_str_t *o = m_new_obj(mp_obj_str_t); o->base.type = type; o->len = len; @@ -1992,6 +1993,17 @@ mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, siz return MP_OBJ_FROM_PTR(o); } +// Create a str/bytes object using the given data. If the type is str and the string +// data is already interned, then a qstr object is returned. Otherwise new memory is +// allocated for the object and the data is copied across. +mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, size_t len) { + if (type == &mp_type_str) { + return mp_obj_new_str((const char*)data, len); + } else { + return mp_obj_new_bytes(data, len); + } +} + // Create a str using a qstr to store the data; may use existing or new qstr. mp_obj_t mp_obj_new_str_via_qstr(const char* data, size_t len) { return MP_OBJ_NEW_QSTR(qstr_from_strn(data, len)); @@ -2034,7 +2046,7 @@ mp_obj_t mp_obj_new_str(const char* data, size_t len) { return MP_OBJ_NEW_QSTR(q); } else { // no existing qstr, don't make one - return mp_obj_new_str_of_type(&mp_type_str, (const byte*)data, len); + return mp_obj_new_str_copy(&mp_type_str, (const byte*)data, len); } } @@ -2044,7 +2056,7 @@ mp_obj_t mp_obj_str_intern(mp_obj_t str) { } mp_obj_t mp_obj_new_bytes(const byte* data, size_t len) { - return mp_obj_new_str_of_type(&mp_type_bytes, data, len); + return mp_obj_new_str_copy(&mp_type_bytes, data, len); } bool mp_obj_str_equal(mp_obj_t s1, mp_obj_t s2) { diff --git a/py/objstr.h b/py/objstr.h index 82501a763..4e55cad09 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -65,6 +65,7 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t void mp_str_print_json(const mp_print_t *print, const byte *str_data, size_t str_len); mp_obj_t mp_obj_str_format(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs); mp_obj_t mp_obj_str_split(size_t n_args, const mp_obj_t *args); +mp_obj_t mp_obj_new_str_copy(const mp_obj_type_t *type, const byte* data, size_t len); mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, size_t len); mp_obj_t mp_obj_str_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in); diff --git a/py/parse.c b/py/parse.c index 8c51b0349..f7fe30418 100644 --- a/py/parse.c +++ b/py/parse.c @@ -417,7 +417,7 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) { pn = mp_parse_node_new_leaf(lex->tok_kind == MP_TOKEN_STRING ? MP_PARSE_NODE_STRING : MP_PARSE_NODE_BYTES, qst); } else { // not interned, make a node holding a pointer to the string/bytes object - mp_obj_t o = mp_obj_new_str_of_type( + mp_obj_t o = mp_obj_new_str_copy( lex->tok_kind == MP_TOKEN_STRING ? &mp_type_str : &mp_type_bytes, (const byte*)lex->vstr.buf, lex->vstr.len); pn = make_node_const_object(parser, lex->tok_line, o); -- cgit v1.2.3 From 8d956c26d150749375115346f4ca319455107587 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Nov 2017 14:02:28 +1100 Subject: py/objstr: When constructing str from bytes, check for existing qstr. This patch uses existing qstr data where possible when constructing a str from a bytes object. --- py/objstr.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'py') diff --git a/py/objstr.c b/py/objstr.c index bca2af801..1ff5132d2 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -164,6 +164,13 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ mp_raise_msg(&mp_type_UnicodeError, NULL); } #endif + + // Check if a qstr with this data already exists + qstr q = qstr_find_strn((const char*)str_data, str_len); + if (q != MP_QSTR_NULL) { + return MP_OBJ_NEW_QSTR(q); + } + mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(type, NULL, str_len)); o->data = str_data; o->hash = str_hash; -- cgit v1.2.3 From da154fdaf933e6546ae4d6888af1a79a76d71b4c Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 1 Apr 2017 23:52:24 +1100 Subject: py: Add config option to disable multiple inheritance. This patch introduces a new compile-time config option to disable multiple inheritance at the Python level: MICROPY_MULTIPLE_INHERITANCE. It is enabled by default. Disabling multiple inheritance eliminates a lot of recursion in the call graph (which is important for some embedded systems), and can be used to reduce code size for ports that are really constrained (by around 200 bytes for Thumb2 archs). With multiple inheritance disabled all tests in the test-suite pass except those that explicitly test for multiple inheritance. --- py/mpconfig.h | 7 +++++++ py/objtype.c | 14 +++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/mpconfig.h b/py/mpconfig.h index 4209b32c6..095b7ef7b 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -666,6 +666,13 @@ typedef double mp_float_t; /*****************************************************************************/ /* Fine control over Python builtins, classes, modules, etc */ +// Whether to support multiple inheritance of Python classes. Multiple +// inheritance makes some C functions inherently recursive, and adds a bit of +// code overhead. +#ifndef MICROPY_MULTIPLE_INHERITANCE +#define MICROPY_MULTIPLE_INHERITANCE (1) +#endif + // Whether to implement attributes on functions #ifndef MICROPY_PY_FUNCTION_ATTRS #define MICROPY_PY_FUNCTION_ATTRS (0) diff --git a/py/objtype.c b/py/objtype.c index a8376a306..01d248256 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -67,6 +67,7 @@ STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_t } else if (type->parent == NULL) { // No parents so end search here. return count; + #if MICROPY_MULTIPLE_INHERITANCE } else if (((mp_obj_base_t*)type->parent)->type == &mp_type_tuple) { // Multiple parents, search through them all recursively. const mp_obj_tuple_t *parent_tuple = type->parent; @@ -78,6 +79,7 @@ STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_t count += instance_count_native_bases(bt, last_native_base); } return count; + #endif } else { // A single parent, use iteration to continue the search. type = type->parent; @@ -172,6 +174,7 @@ STATIC void mp_obj_class_lookup(struct class_lookup_data *lookup, const mp_obj_ if (type->parent == NULL) { DEBUG_printf("mp_obj_class_lookup: No more parents\n"); return; + #if MICROPY_MULTIPLE_INHERITANCE } else if (((mp_obj_base_t*)type->parent)->type == &mp_type_tuple) { const mp_obj_tuple_t *parent_tuple = type->parent; const mp_obj_t *item = parent_tuple->items; @@ -192,6 +195,7 @@ STATIC void mp_obj_class_lookup(struct class_lookup_data *lookup, const mp_obj_ // search last base (simple tail recursion elimination) assert(MP_OBJ_IS_TYPE(*item, &mp_type_type)); type = (mp_obj_type_t*)MP_OBJ_TO_PTR(*item); + #endif } else { type = type->parent; } @@ -247,7 +251,7 @@ STATIC void instance_print(const mp_print_t *print, mp_obj_t self_in, mp_print_k mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size_t n_kw, const mp_obj_t *args) { assert(mp_obj_is_instance_type(self)); - const mp_obj_type_t *native_base; + const mp_obj_type_t *native_base = NULL; size_t num_native_bases = instance_count_native_bases(self, &native_base); assert(num_native_bases < 2); @@ -1022,7 +1026,11 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) o->protocol = ((mp_obj_type_t*)MP_OBJ_TO_PTR(bases_items[0]))->protocol; if (bases_len >= 2) { + #if MICROPY_MULTIPLE_INHERITANCE o->parent = MP_OBJ_TO_PTR(bases_tuple); + #else + mp_raise_NotImplementedError("multiple inheritance not supported"); + #endif } else { o->parent = MP_OBJ_TO_PTR(bases_items[0]); } @@ -1101,6 +1109,7 @@ STATIC void super_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (type->parent == NULL) { // no parents, do nothing + #if MICROPY_MULTIPLE_INHERITANCE } else if (((mp_obj_base_t*)type->parent)->type == &mp_type_tuple) { const mp_obj_tuple_t *parent_tuple = type->parent; size_t len = parent_tuple->len; @@ -1112,6 +1121,7 @@ STATIC void super_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { return; } } + #endif } else { mp_obj_class_lookup(&lookup, type->parent); if (dest[0] != MP_OBJ_NULL) { @@ -1158,6 +1168,7 @@ bool mp_obj_is_subclass_fast(mp_const_obj_t object, mp_const_obj_t classinfo) { if (self->parent == NULL) { // type has no parents return false; + #if MICROPY_MULTIPLE_INHERITANCE } else if (((mp_obj_base_t*)self->parent)->type == &mp_type_tuple) { // get the base objects (they should be type objects) const mp_obj_tuple_t *parent_tuple = self->parent; @@ -1173,6 +1184,7 @@ bool mp_obj_is_subclass_fast(mp_const_obj_t object, mp_const_obj_t classinfo) { // search last base (simple tail recursion elimination) object = *item; + #endif } else { // type has 1 parent object = MP_OBJ_FROM_PTR(self->parent); -- cgit v1.2.3 From 8667a5f0534238e3144adc35d487deb0ac5be5d6 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 28 Oct 2017 18:37:30 +0300 Subject: py/objnamedtuple: Allow to reuse namedtuple basic functionality. By declaring interface in objnamedtuple.h and introducing a helper allocation function. --- py/objnamedtuple.c | 32 ++++++++++++++------------------ py/objnamedtuple.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 18 deletions(-) create mode 100644 py/objnamedtuple.h (limited to 'py') diff --git a/py/objnamedtuple.c b/py/objnamedtuple.c index 94f02dd69..e7de899cf 100644 --- a/py/objnamedtuple.c +++ b/py/objnamedtuple.c @@ -30,20 +30,11 @@ #include "py/objtuple.h" #include "py/runtime.h" #include "py/objstr.h" +#include "py/objnamedtuple.h" #if MICROPY_PY_COLLECTIONS -typedef struct _mp_obj_namedtuple_type_t { - mp_obj_type_t base; - size_t n_fields; - qstr fields[]; -} mp_obj_namedtuple_type_t; - -typedef struct _mp_obj_namedtuple_t { - mp_obj_tuple_t tuple; -} mp_obj_namedtuple_t; - -STATIC size_t namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr name) { +size_t mp_obj_namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr name) { for (size_t i = 0; i < type->n_fields; i++) { if (type->fields[i] == name) { return i; @@ -88,7 +79,7 @@ STATIC void namedtuple_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { return; } #endif - size_t id = namedtuple_find_field((mp_obj_namedtuple_type_t*)self->tuple.base.type, attr); + size_t id = mp_obj_namedtuple_find_field((mp_obj_namedtuple_type_t*)self->tuple.base.type, attr); if (id == (size_t)-1) { return; } @@ -128,7 +119,7 @@ STATIC mp_obj_t namedtuple_make_new(const mp_obj_type_t *type_in, size_t n_args, memset(&tuple->items[n_args], 0, sizeof(mp_obj_t) * n_kw); for (size_t i = n_args; i < n_args + 2 * n_kw; i += 2) { qstr kw = mp_obj_str_get_qstr(args[i]); - size_t id = namedtuple_find_field(type, kw); + size_t id = mp_obj_namedtuple_find_field(type, kw); if (id == (size_t)-1) { if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { mp_arg_error_terse_mismatch(); @@ -151,9 +142,18 @@ STATIC mp_obj_t namedtuple_make_new(const mp_obj_type_t *type_in, size_t n_args, return MP_OBJ_FROM_PTR(tuple); } -STATIC mp_obj_t mp_obj_new_namedtuple_type(qstr name, size_t n_fields, mp_obj_t *fields) { +mp_obj_namedtuple_type_t *mp_obj_new_namedtuple_base(size_t n_fields, mp_obj_t *fields) { mp_obj_namedtuple_type_t *o = m_new_obj_var(mp_obj_namedtuple_type_t, qstr, n_fields); memset(&o->base, 0, sizeof(o->base)); + o->n_fields = n_fields; + for (size_t i = 0; i < n_fields; i++) { + o->fields[i] = mp_obj_str_get_qstr(fields[i]); + } + return o; +} + +STATIC mp_obj_t mp_obj_new_namedtuple_type(qstr name, size_t n_fields, mp_obj_t *fields) { + mp_obj_namedtuple_type_t *o = mp_obj_new_namedtuple_base(n_fields, fields); o->base.base.type = &mp_type_type; o->base.name = name; o->base.print = namedtuple_print; @@ -164,10 +164,6 @@ STATIC mp_obj_t mp_obj_new_namedtuple_type(qstr name, size_t n_fields, mp_obj_t o->base.subscr = mp_obj_tuple_subscr; o->base.getiter = mp_obj_tuple_getiter; o->base.parent = &mp_type_tuple; - o->n_fields = n_fields; - for (size_t i = 0; i < n_fields; i++) { - o->fields[i] = mp_obj_str_get_qstr(fields[i]); - } return MP_OBJ_FROM_PTR(o); } diff --git a/py/objnamedtuple.h b/py/objnamedtuple.h new file mode 100644 index 000000000..d32af35af --- /dev/null +++ b/py/objnamedtuple.h @@ -0,0 +1,44 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014-2017 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_PY_OBJNAMEDTUPLE_H +#define MICROPY_INCLUDED_PY_OBJNAMEDTUPLE_H + +#include "py/objtuple.h" + +typedef struct _mp_obj_namedtuple_type_t { + mp_obj_type_t base; + size_t n_fields; + qstr fields[]; +} mp_obj_namedtuple_type_t; + +typedef struct _mp_obj_namedtuple_t { + mp_obj_tuple_t tuple; +} mp_obj_namedtuple_t; + +size_t mp_obj_namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr name); +mp_obj_namedtuple_type_t *mp_obj_new_namedtuple_base(size_t n_fields, mp_obj_t *fields); + +#endif // MICROPY_INCLUDED_PY_OBJNAMEDTUPLE_H -- cgit v1.2.3 From a07fc5b6403b9a8bf7e7cb64f857272e5346d7e2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 21 Nov 2017 15:01:38 +1100 Subject: py/objfloat: Allow float() to parse anything with the buffer protocol. This generalises and simplifies the code and follows CPython behaviour. --- py/objfloat.c | 12 ++++++------ tests/float/float1.py | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'py') diff --git a/py/objfloat.c b/py/objfloat.c index 743287be6..75212a4d2 100644 --- a/py/objfloat.c +++ b/py/objfloat.c @@ -137,12 +137,11 @@ STATIC mp_obj_t float_make_new(const mp_obj_type_t *type_in, size_t n_args, size return mp_obj_new_float(0); case 1: - default: - if (MP_OBJ_IS_STR(args[0])) { - // a string, parse it - size_t l; - const char *s = mp_obj_str_get_data(args[0], &l); - return mp_parse_num_decimal(s, l, false, false, NULL); + default: { + mp_buffer_info_t bufinfo; + if (mp_get_buffer(args[0], &bufinfo, MP_BUFFER_READ)) { + // a textual representation, parse it + return mp_parse_num_decimal(bufinfo.buf, bufinfo.len, false, false, NULL); } else if (mp_obj_is_float(args[0])) { // a float, just return it return args[0]; @@ -150,6 +149,7 @@ STATIC mp_obj_t float_make_new(const mp_obj_type_t *type_in, size_t n_args, size // something else, try to cast it to a float return mp_obj_new_float(mp_obj_get_float(args[0])); } + } } } diff --git a/tests/float/float1.py b/tests/float/float1.py index c64f965a7..54807e5ac 100644 --- a/tests/float/float1.py +++ b/tests/float/float1.py @@ -36,6 +36,10 @@ try: except ValueError: print("ValueError") +# construct from something with the buffer protocol +print(float(b"1.2")) +print(float(bytearray(b"3.4"))) + # unary operators print(bool(0.0)) print(bool(1.2)) -- cgit v1.2.3 From d5cf5f70fdefa793d3e1fee9a26f03a1dd8c1d1e Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 22 Nov 2017 15:51:51 +1100 Subject: py/modbuiltins: Slightly simplify code in builtin round(). --- py/modbuiltins.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index ebff5f5c0..0c78832ac 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -457,16 +457,14 @@ STATIC mp_obj_t mp_builtin_round(size_t n_args, const mp_obj_t *args) { return o_in; } #if MICROPY_PY_BUILTINS_FLOAT - mp_int_t num_dig = 0; + mp_float_t val = mp_obj_get_float(o_in); if (n_args > 1) { - num_dig = mp_obj_get_int(args[1]); - mp_float_t val = mp_obj_get_float(o_in); + mp_int_t num_dig = mp_obj_get_int(args[1]); mp_float_t mult = MICROPY_FLOAT_C_FUN(pow)(10, num_dig); // TODO may lead to overflow mp_float_t rounded = MICROPY_FLOAT_C_FUN(nearbyint)(val * mult) / mult; return mp_obj_new_float(rounded); } - mp_float_t val = mp_obj_get_float(o_in); mp_float_t rounded = MICROPY_FLOAT_C_FUN(nearbyint)(val); return mp_obj_new_int_from_float(rounded); #else -- cgit v1.2.3 From 9783ac282ebe216d37be39fdf07113e6880c19b7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 24 Nov 2017 12:07:12 +1100 Subject: py/runtime: Simplify handling of containment binary operator. In mp_binary_op, there is no need to explicitly check for type->getiter being non-null and raising an exception because this is handled exactly by mp_getiter(). So just call the latter unconditionally. --- py/runtime.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) (limited to 'py') diff --git a/py/runtime.c b/py/runtime.c index 17e5d235c..08a35c2e6 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -536,25 +536,17 @@ mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { return res; } } - if (type->getiter != NULL) { - /* second attempt, walk the iterator */ - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(rhs, &iter_buf); - mp_obj_t next; - while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { - if (mp_obj_equal(next, lhs)) { - return mp_const_true; - } - } - return mp_const_false; - } - if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { - mp_raise_TypeError("object not iterable"); - } else { - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, - "'%s' object is not iterable", mp_obj_get_type_str(rhs))); + // final attempt, walk the iterator (will raise if rhs is not iterable) + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(rhs, &iter_buf); + mp_obj_t next; + while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { + if (mp_obj_equal(next, lhs)) { + return mp_const_true; + } } + return mp_const_false; } // generic binary_op supplied by type -- cgit v1.2.3 From 5b2f62aff312949c9c7edec6cfaaf4f97d93c442 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 24 Nov 2017 12:16:21 +1100 Subject: py/opmethods: Include the correct header for binary op enums. By directly including runtime0.h the mpconfig.h settings are not included and so the enums in runtime0.h can be incorrect. --- py/opmethods.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/opmethods.c b/py/opmethods.c index 1200ba39e..31901bb52 100644 --- a/py/opmethods.c +++ b/py/opmethods.c @@ -24,7 +24,7 @@ * THE SOFTWARE. */ -#include "py/runtime0.h" +#include "py/obj.h" #include "py/builtin.h" STATIC mp_obj_t op_getitem(mp_obj_t self_in, mp_obj_t key_in) { -- cgit v1.2.3 From 5e34a113eaaf736fb4f703a3ee0892e1705d0a63 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 24 Nov 2017 13:04:24 +1100 Subject: py/runtime: Add MP_BINARY_OP_CONTAINS as reverse of MP_BINARY_OP_IN. Before this patch MP_BINARY_OP_IN had two meanings: coming from bytecode it meant that the args needed to be swapped, but coming from within the runtime meant that the args were already in the correct order. This lead to some confusion in the code and comments stating how args were reversed. It also lead to 2 bugs: 1) containment for a subclass of a native type didn't work; 2) the expression "{True} in True" would illegally succeed and return True. In both of these cases it was because the args to MP_BINARY_OP_IN ended up being reversed twice. To fix these things this patch introduces MP_BINARY_OP_CONTAINS which corresponds exactly to the __contains__ special method, and this is the operator that built-in types should implement. MP_BINARY_OP_IN is now only emitted by the compiler and is converted to MP_BINARY_OP_CONTAINS by swapping the arguments. --- extmod/modbtree.c | 2 +- py/objarray.c | 3 +-- py/objdict.c | 4 ++-- py/objint_mpz.c | 2 +- py/objset.c | 4 ++-- py/objstr.c | 3 +-- py/objtype.c | 2 +- py/opmethods.c | 2 +- py/runtime.c | 42 ++++++++++++++++-------------------- py/runtime0.h | 4 ++++ tests/micropython/viper_error.py.exp | 2 +- 11 files changed, 34 insertions(+), 36 deletions(-) (limited to 'py') diff --git a/extmod/modbtree.c b/extmod/modbtree.c index 5c1311532..8b7688580 100644 --- a/extmod/modbtree.c +++ b/extmod/modbtree.c @@ -282,7 +282,7 @@ STATIC mp_obj_t btree_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { STATIC mp_obj_t btree_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { mp_obj_btree_t *self = MP_OBJ_TO_PTR(lhs_in); switch (op) { - case MP_BINARY_OP_IN: { + case MP_BINARY_OP_CONTAINS: { DBT key, val; key.data = (void*)mp_obj_str_get_data(rhs_in, &key.size); int res = __bt_get(self->db, &key, &val, 0); diff --git a/py/objarray.c b/py/objarray.c index 7003ec9e7..a35539484 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -269,8 +269,7 @@ STATIC mp_obj_t array_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs return lhs_in; } - case MP_BINARY_OP_IN: { - /* NOTE `a in b` is `b.__contains__(a)` */ + case MP_BINARY_OP_CONTAINS: { mp_buffer_info_t lhs_bufinfo; mp_buffer_info_t rhs_bufinfo; diff --git a/py/objdict.c b/py/objdict.c index 1553a83b4..d0f95e41a 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -115,7 +115,7 @@ STATIC mp_obj_t dict_unary_op(mp_unary_op_t op, mp_obj_t self_in) { STATIC mp_obj_t dict_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { mp_obj_dict_t *o = MP_OBJ_TO_PTR(lhs_in); switch (op) { - case MP_BINARY_OP_IN: { + case MP_BINARY_OP_CONTAINS: { mp_map_elem_t *elem = mp_map_lookup(&o->map, rhs_in, MP_MAP_LOOKUP); return mp_obj_new_bool(elem != NULL); } @@ -485,7 +485,7 @@ STATIC mp_obj_t dict_view_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t if (o->kind != MP_DICT_VIEW_KEYS) { return MP_OBJ_NULL; // op not supported } - if (op != MP_BINARY_OP_IN) { + if (op != MP_BINARY_OP_CONTAINS) { return MP_OBJ_NULL; // op not supported } return dict_binary_op(op, o->dict, rhs_in); diff --git a/py/objint_mpz.c b/py/objint_mpz.c index 7b5cb0b9d..17e3ee6d2 100644 --- a/py/objint_mpz.c +++ b/py/objint_mpz.c @@ -207,7 +207,7 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i return mp_obj_new_float(flhs / frhs); #endif - } else if (op >= MP_BINARY_OP_INPLACE_OR) { + } else if (op >= MP_BINARY_OP_INPLACE_OR && op < MP_BINARY_OP_CONTAINS) { mp_obj_int_t *res = mp_obj_int_new_mpz(); switch (op) { diff --git a/py/objset.c b/py/objset.c index 6ed15c791..3e98c30e8 100644 --- a/py/objset.c +++ b/py/objset.c @@ -461,7 +461,7 @@ STATIC mp_obj_t set_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { #else bool update = true; #endif - if (op != MP_BINARY_OP_IN && !is_set_or_frozenset(rhs)) { + if (op != MP_BINARY_OP_CONTAINS && !is_set_or_frozenset(rhs)) { // For all ops except containment the RHS must be a set/frozenset return MP_OBJ_NULL; } @@ -507,7 +507,7 @@ STATIC mp_obj_t set_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { return set_issubset(lhs, rhs); case MP_BINARY_OP_MORE_EQUAL: return set_issuperset(lhs, rhs); - case MP_BINARY_OP_IN: { + case MP_BINARY_OP_CONTAINS: { mp_obj_set_t *o = MP_OBJ_TO_PTR(lhs); mp_obj_t elem = mp_set_lookup(&o->set, rhs, MP_MAP_LOOKUP); return mp_obj_new_bool(elem != MP_OBJ_NULL); diff --git a/py/objstr.c b/py/objstr.c index 1ff5132d2..b4f15b38d 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -384,8 +384,7 @@ mp_obj_t mp_obj_str_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i return mp_obj_new_str_from_vstr(lhs_type, &vstr); } - case MP_BINARY_OP_IN: - /* NOTE `a in b` is `b.__contains__(a)` */ + case MP_BINARY_OP_CONTAINS: return mp_obj_new_bool(find_subbytes(lhs_data, lhs_len, rhs_data, rhs_len, 1) != NULL); //case MP_BINARY_OP_NOT_EQUAL: // This is never passed here diff --git a/py/objtype.c b/py/objtype.c index 01d248256..267cae815 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -424,7 +424,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = { [MP_BINARY_OP_LESS_EQUAL] = MP_QSTR___le__, [MP_BINARY_OP_MORE_EQUAL] = MP_QSTR___ge__, // MP_BINARY_OP_NOT_EQUAL, // a != b calls a == b and inverts result - [MP_BINARY_OP_IN] = MP_QSTR___contains__, + [MP_BINARY_OP_CONTAINS] = MP_QSTR___contains__, // All inplace methods are optional, and normal methods will be used // as a fallback. diff --git a/py/opmethods.c b/py/opmethods.c index 31901bb52..247fa5bbc 100644 --- a/py/opmethods.c +++ b/py/opmethods.c @@ -47,6 +47,6 @@ MP_DEFINE_CONST_FUN_OBJ_2(mp_op_delitem_obj, op_delitem); STATIC mp_obj_t op_contains(mp_obj_t lhs_in, mp_obj_t rhs_in) { mp_obj_type_t *type = mp_obj_get_type(lhs_in); - return type->binary_op(MP_BINARY_OP_IN, lhs_in, rhs_in); + return type->binary_op(MP_BINARY_OP_CONTAINS, lhs_in, rhs_in); } MP_DEFINE_CONST_FUN_OBJ_2(mp_op_contains_obj, op_contains); diff --git a/py/runtime.c b/py/runtime.c index 08a35c2e6..c7fe39367 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -523,30 +523,12 @@ mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { } } - /* deal with `in` - * - * NOTE `a in b` is `b.__contains__(a)`, hence why the generic dispatch - * needs to go below with swapped arguments - */ + // Convert MP_BINARY_OP_IN to MP_BINARY_OP_CONTAINS with swapped args. if (op == MP_BINARY_OP_IN) { - mp_obj_type_t *type = mp_obj_get_type(rhs); - if (type->binary_op != NULL) { - mp_obj_t res = type->binary_op(op, rhs, lhs); - if (res != MP_OBJ_NULL) { - return res; - } - } - - // final attempt, walk the iterator (will raise if rhs is not iterable) - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(rhs, &iter_buf); - mp_obj_t next; - while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { - if (mp_obj_equal(next, lhs)) { - return mp_const_true; - } - } - return mp_const_false; + op = MP_BINARY_OP_CONTAINS; + mp_obj_t temp = lhs; + lhs = rhs; + rhs = temp; } // generic binary_op supplied by type @@ -575,6 +557,20 @@ generic_binary_op: } #endif + if (op == MP_BINARY_OP_CONTAINS) { + // If type didn't support containment then explicitly walk the iterator. + // mp_getiter will raise the appropriate exception if lhs is not iterable. + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(lhs, &iter_buf); + mp_obj_t next; + while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { + if (mp_obj_equal(next, rhs)) { + return mp_const_true; + } + } + return mp_const_false; + } + unsupported_op: if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { mp_raise_TypeError("unsupported type for operator"); diff --git a/py/runtime0.h b/py/runtime0.h index a72b7feb7..960532d17 100644 --- a/py/runtime0.h +++ b/py/runtime0.h @@ -131,6 +131,10 @@ typedef enum { #endif , + // The runtime will convert MP_BINARY_OP_IN to this operator with swapped args. + // A type should implement this containment operator instead of MP_BINARY_OP_IN. + MP_BINARY_OP_CONTAINS, + MP_BINARY_OP_NUM_RUNTIME, // These 2 are not supported by the runtime and must be synthesised by the emitter diff --git a/tests/micropython/viper_error.py.exp b/tests/micropython/viper_error.py.exp index 96be5a590..a44fb3ff0 100644 --- a/tests/micropython/viper_error.py.exp +++ b/tests/micropython/viper_error.py.exp @@ -18,7 +18,7 @@ ViperTypeError('must raise an object',) ViperTypeError('unary op __pos__ not implemented',) ViperTypeError('unary op __neg__ not implemented',) ViperTypeError('unary op __invert__ not implemented',) -ViperTypeError('binary op __contains__ not implemented',) +ViperTypeError('binary op not implemented',) NotImplementedError('native yield',) NotImplementedError('native yield from',) NotImplementedError('conversion to object',) -- cgit v1.2.3 From 84895f1a210d0037a86887f0f647570bdf40afa2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 27 Nov 2017 12:51:52 +1100 Subject: py/parsenum: Improve parsing of floating point numbers. This patch improves parsing of floating point numbers by converting all the digits (integer and fractional) together into a number 1 or greater, and then applying the correct power of 10 at the very end. In particular the multiple "multiply by 0.1" operations to build a fraction are now combined together and applied at the same time as the exponent, at the very end. This helps to retain precision during parsing of floats, and also includes a check that the number doesn't overflow during the parsing. One benefit is that a float will have the same value no matter where the decimal point is located, eg 1.23 == 123e-2. --- py/parsenum.c | 27 +++++++++++++++++++++------ tests/float/float_parse.py | 22 ++++++++++++++++++++++ tests/float/float_parse_doubleprec.py | 16 ++++++++++++++++ tests/run-tests | 1 + 4 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 tests/float/float_parse.py create mode 100644 tests/float/float_parse_doubleprec.py (limited to 'py') diff --git a/py/parsenum.c b/py/parsenum.c index b62029f7c..98e773685 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -170,6 +170,14 @@ typedef enum { mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool force_complex, mp_lexer_t *lex) { #if MICROPY_PY_BUILTINS_FLOAT + +// DEC_VAL_MAX only needs to be rough and is used to retain precision while not overflowing +#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT +#define DEC_VAL_MAX 1e20F +#elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE +#define DEC_VAL_MAX 1e200 +#endif + const char *top = str + len; mp_float_t dec_val = 0; bool dec_neg = false; @@ -214,8 +222,8 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool // string should be a decimal number parse_dec_in_t in = PARSE_DEC_IN_INTG; bool exp_neg = false; - mp_float_t frac_mult = 0.1; mp_int_t exp_val = 0; + mp_int_t exp_extra = 0; while (str < top) { mp_uint_t dig = *str++; if ('0' <= dig && dig <= '9') { @@ -223,11 +231,18 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool if (in == PARSE_DEC_IN_EXP) { exp_val = 10 * exp_val + dig; } else { - if (in == PARSE_DEC_IN_FRAC) { - dec_val += dig * frac_mult; - frac_mult *= MICROPY_FLOAT_CONST(0.1); - } else { + if (dec_val < DEC_VAL_MAX) { + // dec_val won't overflow so keep accumulating dec_val = 10 * dec_val + dig; + if (in == PARSE_DEC_IN_FRAC) { + --exp_extra; + } + } else { + // dec_val might overflow and we anyway can't represent more digits + // of precision, so ignore the digit and just adjust the exponent + if (in == PARSE_DEC_IN_INTG) { + ++exp_extra; + } } } } else if (in == PARSE_DEC_IN_INTG && dig == '.') { @@ -261,7 +276,7 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool } // apply the exponent - dec_val *= MICROPY_FLOAT_C_FUN(pow)(10, exp_val); + dec_val *= MICROPY_FLOAT_C_FUN(pow)(10, exp_val + exp_extra); } // negate value if needed diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py new file mode 100644 index 000000000..448eff3bc --- /dev/null +++ b/tests/float/float_parse.py @@ -0,0 +1,22 @@ +# test parsing of floats + +inf = float('inf') + +# it shouldn't matter where the decimal point is if the exponent balances the value +print(float('1234') - float('0.1234e4')) +print(float('1.015625') - float('1015625e-6')) + +# very large integer part with a very negative exponent should cancel out +print(float('9' * 60 + 'e-60')) +print(float('9' * 60 + 'e-40')) +print(float('9' * 60 + 'e-20') == float('1e40')) + +# many fractional digits +print(float('.' + '9' * 70)) +print(float('.' + '9' * 70 + 'e20')) +print(float('.' + '9' * 70 + 'e-50') == float('1e-50')) + +# tiny fraction with large exponent +print(float('.' + '0' * 60 + '1e10') == float('1e-51')) +print(float('.' + '0' * 60 + '9e25')) +print(float('.' + '0' * 60 + '9e40')) diff --git a/tests/float/float_parse_doubleprec.py b/tests/float/float_parse_doubleprec.py new file mode 100644 index 000000000..356601130 --- /dev/null +++ b/tests/float/float_parse_doubleprec.py @@ -0,0 +1,16 @@ +# test parsing of floats, requiring double-precision + +# very large integer part with a very negative exponent should cancel out +print(float('9' * 400 + 'e-100')) +print(float('9' * 400 + 'e-200')) +print(float('9' * 400 + 'e-400')) + +# many fractional digits +print(float('.' + '9' * 400)) +print(float('.' + '9' * 400 + 'e100')) +print(float('.' + '9' * 400 + 'e-100')) + +# tiny fraction with large exponent +print(float('.' + '0' * 400 + '9e100')) +print(float('.' + '0' * 400 + '9e200')) +print(float('.' + '0' * 400 + '9e400')) diff --git a/tests/run-tests b/tests/run-tests index 6280a5182..3c763512c 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -271,6 +271,7 @@ def run_tests(pyb, tests, args, base_path="."): if upy_float_precision < 64: skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead skip_tests.add('float/float2int_doubleprec_intbig.py') + skip_tests.add('float/float_parse_doubleprec.py') if not has_complex: skip_tests.add('float/complex1.py') -- cgit v1.2.3 From 2161d6b603e2dfc444a597c4346f7e2301f9c05e Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 27 Nov 2017 23:40:31 +1100 Subject: py/objdict: Reuse dict-view key iterator for standard dict iterator. It has equivalent behaviour and reusing it saves some code bytes. --- py/objdict.c | 45 ++++++++++++++------------------------------- 1 file changed, 14 insertions(+), 31 deletions(-) (limited to 'py') diff --git a/py/objdict.c b/py/objdict.c index d0f95e41a..f6fb594b3 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -192,37 +192,6 @@ STATIC mp_obj_t dict_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { } } -/******************************************************************************/ -/* dict iterator */ - -typedef struct _mp_obj_dict_it_t { - mp_obj_base_t base; - mp_fun_1_t iternext; - mp_obj_t dict; - size_t cur; -} mp_obj_dict_it_t; - -STATIC mp_obj_t dict_it_iternext(mp_obj_t self_in) { - mp_obj_dict_it_t *self = MP_OBJ_TO_PTR(self_in); - mp_map_elem_t *next = dict_iter_next(MP_OBJ_TO_PTR(self->dict), &self->cur); - - if (next == NULL) { - return MP_OBJ_STOP_ITERATION; - } else { - return next->key; - } -} - -STATIC mp_obj_t dict_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { - assert(sizeof(mp_obj_dict_it_t) <= sizeof(mp_obj_iter_buf_t)); - mp_obj_dict_it_t *o = (mp_obj_dict_it_t*)iter_buf; - o->base.type = &mp_type_polymorph_iter; - o->iternext = dict_it_iternext; - o->dict = self_in; - o->cur = 0; - return MP_OBJ_FROM_PTR(o); -} - /******************************************************************************/ /* dict methods */ @@ -527,6 +496,20 @@ STATIC mp_obj_t dict_values(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(dict_values_obj, dict_values); +/******************************************************************************/ +/* dict iterator */ + +STATIC mp_obj_t dict_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_dict_view_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); + mp_obj_dict_view_it_t *o = (mp_obj_dict_view_it_t*)iter_buf; + o->base.type = &dict_view_it_type; + o->kind = MP_DICT_VIEW_KEYS; + o->dict = self_in; + o->cur = 0; + return MP_OBJ_FROM_PTR(o); +} + /******************************************************************************/ /* dict constructors & public C API */ -- cgit v1.2.3 From 3990a52c0f071a7a48cc276f30785930f31eed39 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 29 Nov 2017 15:43:40 +1100 Subject: py: Annotate func defs with NORETURN when their corresp decls have it. --- py/objstr.c | 2 +- py/stackctrl.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/objstr.c b/py/objstr.c index b4f15b38d..30153813d 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -2084,7 +2084,7 @@ bool mp_obj_str_equal(mp_obj_t s1, mp_obj_t s2) { } } -STATIC void bad_implicit_conversion(mp_obj_t self_in) { +STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in) { if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { mp_raise_TypeError("can't convert to str implicitly"); } else { diff --git a/py/stackctrl.c b/py/stackctrl.c index 7cd35fee2..11165b9a6 100644 --- a/py/stackctrl.c +++ b/py/stackctrl.c @@ -48,7 +48,7 @@ void mp_stack_set_limit(mp_uint_t limit) { MP_STATE_THREAD(stack_limit) = limit; } -void mp_exc_recursion_depth(void) { +NORETURN void mp_exc_recursion_depth(void) { nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError, MP_OBJ_NEW_QSTR(MP_QSTR_maximum_space_recursion_space_depth_space_exceeded))); } -- cgit v1.2.3 From 8e323b8fa84277531685985327fb682761abd53d Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 29 Nov 2017 16:58:27 +1100 Subject: py/qstr: Rewrite find_qstr to make manifest that it returns a valid ptr. So long as the input qstr identifier is valid (below the maximum number of qstrs) the function will always return a valid pointer. This patch eliminates the "return 0" dead-code. --- py/qstr.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'py') diff --git a/py/qstr.c b/py/qstr.c index a3c9612c6..08c3e2505 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -127,14 +127,12 @@ void qstr_init(void) { STATIC const byte *find_qstr(qstr q) { // search pool for this qstr - for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL; pool = pool->prev) { - if (q >= pool->total_prev_len) { - return pool->qstrs[q - pool->total_prev_len]; - } + // total_prev_len==0 in the final pool, so the loop will always terminate + qstr_pool_t *pool = MP_STATE_VM(last_pool); + while (q < pool->total_prev_len) { + pool = pool->prev; } - - // not found - return 0; + return pool->qstrs[q - pool->total_prev_len]; } // qstr_mutex must be taken while in this function -- cgit v1.2.3 From 74fad3536b961e2ffe08e545b1cf787c0c0b4772 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 29 Nov 2017 17:17:08 +1100 Subject: py/gc: In gc_realloc, convert pointer sanity checks to assertions. These checks are assumed to be true in all cases where gc_realloc is called with a valid pointer, so no need to waste code space and time checking them in a non-debug build. --- py/gc.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 9752b3532..172a5e8fc 100644 --- a/py/gc.c +++ b/py/gc.c @@ -628,27 +628,18 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { void *ptr = ptr_in; - // sanity check the ptr - if (!VERIFY_PTR(ptr)) { - return NULL; - } - - // get first block - size_t block = BLOCK_FROM_PTR(ptr); - GC_ENTER(); - // sanity check the ptr is pointing to the head of a block - if (ATB_GET_KIND(block) != AT_HEAD) { - GC_EXIT(); - return NULL; - } - if (MP_STATE_MEM(gc_lock_depth) > 0) { GC_EXIT(); return NULL; } + // get the GC block number corresponding to this pointer + assert(VERIFY_PTR(ptr)); + size_t block = BLOCK_FROM_PTR(ptr); + assert(ATB_GET_KIND(block) == AT_HEAD); + // compute number of new blocks that are requested size_t new_blocks = (n_bytes + BYTES_PER_BLOCK - 1) / BYTES_PER_BLOCK; -- cgit v1.2.3 From 64f11470bec3c4ce9122593d3dd968e4127eb545 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 30 Nov 2017 12:06:41 +1100 Subject: py/objgenerator: Remove unreachable code for STOP_ITERATION case. This commit essentially reverts aa9dbb1b033a8163e07fcf5763fc20146354cc48 where this if-condition was added. It seems that even when that commit was made the code was never reached by any tests, nor reachable by analysis (see below). The same is true with the code as it currently stands: no test triggers this if-condition, nor any uasyncio examples. Analysing the flow of the program also shows that it's not reachable: ==START== -> to trigger this if condition mp_execute_bytecode() must return MP_VM_RETURN_YIELD with *sp==MP_OBJ_STOP_ITERATION -> mp_execute_bytecode() can only return MP_VM_RETURN_YIELD from the MP_BC_YIELD_VALUE bytecode, which can happen in 2 ways: -> 1) from a "yield " in bytecode, but must always be a proper object, never MP_OBJ_STOP_ITERATION; ==END1== -> 2) via yield from, via mp_resume() which must return MP_VM_RETURN_YIELD with ret_value==MP_OBJ_STOP_ITERATION, which can happen in 3 ways: -> 1) it delegates to mp_obj_gen_resume(); go back to ==START== -> 2) it returns MP_VM_RETURN_YIELD directly but with a guard that ret_val!=MP_OBJ_STOP_ITERATION; ==END2== -> 3) it returns MP_VM_RETURN_YIELD with ret_val set from mp_call_method_n_kw(), but mp_call_method_n_kw() must return a proper object, never MP_OBJ_STOP_ITERATION; ==END3== The above shows there is no way to trigger the if-condition and it can be removed. --- py/objgenerator.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'py') diff --git a/py/objgenerator.c b/py/objgenerator.c index bf0bbb0e6..9a294debb 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -125,9 +125,6 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ case MP_VM_RETURN_YIELD: *ret_val = *self->code_state.sp; - if (*ret_val == MP_OBJ_STOP_ITERATION) { - self->code_state.ip = 0; - } break; case MP_VM_RETURN_EXCEPTION: { -- cgit v1.2.3 From 75d3c046dafa84dad51ed757f832c2dad2934bf9 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Mon, 4 Dec 2017 11:05:49 +0200 Subject: py/misc.h: Add m_new_obj_var_with_finaliser(). Similar to existing m_new_obj_with_finaliser(). --- py/misc.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'py') diff --git a/py/misc.h b/py/misc.h index b9f2dae90..5d557db6f 100644 --- a/py/misc.h +++ b/py/misc.h @@ -63,8 +63,10 @@ typedef unsigned int uint; #define m_new_obj_var_maybe(obj_type, var_type, var_num) ((obj_type*)m_malloc_maybe(sizeof(obj_type) + sizeof(var_type) * (var_num))) #if MICROPY_ENABLE_FINALISER #define m_new_obj_with_finaliser(type) ((type*)(m_malloc_with_finaliser(sizeof(type)))) +#define m_new_obj_var_with_finaliser(type, var_type, var_num) ((type*)m_malloc_with_finaliser(sizeof(type) + sizeof(var_type) * (var_num))) #else #define m_new_obj_with_finaliser(type) m_new_obj(type) +#define m_new_obj_var_with_finaliser(type, var_type, var_num) m_new_obj_var(type, var_type, var_num) #endif #if MICROPY_MALLOC_USES_ALLOCATED_SIZE #define m_renew(type, ptr, old_num, new_num) ((type*)(m_realloc((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num)))) -- cgit v1.2.3 From 62b96147e6126961c5d1d6bb28b7ac7034fa9322 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 5 Dec 2017 00:38:41 +0200 Subject: py: mp_call_function_*_protected(): Pass-thru return value if possible. Return the result of called function. If exception happened, return MP_OBJ_NULL. Allows to use mp_call_function_*_protected() with callbacks returning values, etc. --- py/runtime.h | 5 +++-- py/runtime_utils.c | 12 ++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'py') diff --git a/py/runtime.h b/py/runtime.h index 9c1921cb5..3f0d1104e 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -106,8 +106,9 @@ mp_obj_t mp_call_method_n_kw(size_t n_args, size_t n_kw, const mp_obj_t *args); mp_obj_t mp_call_method_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_obj_t *args); mp_obj_t mp_call_method_self_n_kw(mp_obj_t meth, mp_obj_t self, size_t n_args, size_t n_kw, const mp_obj_t *args); // Call function and catch/dump exception - for Python callbacks from C code -void mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg); -void mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2); +// (return MP_OBJ_NULL in case of exception). +mp_obj_t mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg); +mp_obj_t mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2); typedef struct _mp_call_args_t { mp_obj_t fun; diff --git a/py/runtime_utils.c b/py/runtime_utils.c index a5c5403ba..b92c6bd76 100644 --- a/py/runtime_utils.c +++ b/py/runtime_utils.c @@ -27,22 +27,26 @@ #include "py/runtime.h" -void mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg) { +mp_obj_t mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_call_function_1(fun, arg); + mp_obj_t ret = mp_call_function_1(fun, arg); nlr_pop(); + return ret; } else { mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + return MP_OBJ_NULL; } } -void mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2) { +mp_obj_t mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_call_function_2(fun, arg1, arg2); + mp_obj_t ret = mp_call_function_2(fun, arg1, arg2); nlr_pop(); + return ret; } else { mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + return MP_OBJ_NULL; } } -- cgit v1.2.3 From 58f00d7c0e116e17ab141ae18275edea14b88431 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 5 Dec 2017 12:14:57 +1100 Subject: py/modbuiltins: Use standard arg-parsing helper func for builtin print. This allows the function to raise an exception when unknown keyword args are passed in. This patch also reduces code size by (in bytes): bare-arm: -24 minimal x86: -76 unix x64: -56 unix nanbox: -84 stm32: -40 esp8266: -68 cc3200: -48 Furthermore, this patch adds space (" ") to the set of ROM qstrs which means it doesn't need to be put in RAM if it's ever used. --- py/modbuiltins.c | 58 +++++++++++++++++++++++++++++++------------------------- py/qstrdefs.h | 1 + 2 files changed, 33 insertions(+), 26 deletions(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 0c78832ac..c8e3235f6 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -383,46 +383,52 @@ STATIC mp_obj_t mp_builtin_pow(size_t n_args, const mp_obj_t *args) { } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_pow_obj, 2, 3, mp_builtin_pow); -STATIC mp_obj_t mp_builtin_print(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) { - mp_map_elem_t *sep_elem = mp_map_lookup(kwargs, MP_OBJ_NEW_QSTR(MP_QSTR_sep), MP_MAP_LOOKUP); - mp_map_elem_t *end_elem = mp_map_lookup(kwargs, MP_OBJ_NEW_QSTR(MP_QSTR_end), MP_MAP_LOOKUP); - const char *sep_data = " "; - size_t sep_len = 1; - const char *end_data = "\n"; - size_t end_len = 1; - if (sep_elem != NULL && sep_elem->value != mp_const_none) { - sep_data = mp_obj_str_get_data(sep_elem->value, &sep_len); - } - if (end_elem != NULL && end_elem->value != mp_const_none) { - end_data = mp_obj_str_get_data(end_elem->value, &end_len); - } - #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - void *stream_obj = &mp_sys_stdout_obj; - mp_map_elem_t *file_elem = mp_map_lookup(kwargs, MP_OBJ_NEW_QSTR(MP_QSTR_file), MP_MAP_LOOKUP); - if (file_elem != NULL && file_elem->value != mp_const_none) { - stream_obj = MP_OBJ_TO_PTR(file_elem->value); // XXX may not be a concrete object - } +STATIC mp_obj_t mp_builtin_print(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + enum { ARG_sep, ARG_end, ARG_file }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_sep, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_QSTR(MP_QSTR__space_)} }, + { MP_QSTR_end, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_QSTR(MP_QSTR__0x0a_)} }, + #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES + { MP_QSTR_file, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_sys_stdout_obj)} }, + #endif + }; + + // parse args (a union is used to reduce the amount of C stack that is needed) + union { + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + size_t len[2]; + } u; + mp_arg_parse_all(0, NULL, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, u.args); - mp_print_t print = {stream_obj, mp_stream_write_adaptor}; + #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES + // TODO file may not be a concrete object (eg it could be a small-int) + mp_print_t print = {MP_OBJ_TO_PTR(u.args[ARG_file].u_obj), mp_stream_write_adaptor}; #endif + + // extract the objects first because we are going to use the other part of the union + mp_obj_t sep = u.args[ARG_sep].u_obj; + mp_obj_t end = u.args[ARG_end].u_obj; + const char *sep_data = mp_obj_str_get_data(sep, &u.len[0]); + const char *end_data = mp_obj_str_get_data(end, &u.len[1]); + for (size_t i = 0; i < n_args; i++) { if (i > 0) { #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - mp_stream_write_adaptor(stream_obj, sep_data, sep_len); + mp_stream_write_adaptor(print.data, sep_data, u.len[0]); #else - mp_print_strn(&mp_plat_print, sep_data, sep_len, 0, 0, 0); + mp_print_strn(&mp_plat_print, sep_data, u.len[0], 0, 0, 0); #endif } #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - mp_obj_print_helper(&print, args[i], PRINT_STR); + mp_obj_print_helper(&print, pos_args[i], PRINT_STR); #else - mp_obj_print_helper(&mp_plat_print, args[i], PRINT_STR); + mp_obj_print_helper(&mp_plat_print, pos_args[i], PRINT_STR); #endif } #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - mp_stream_write_adaptor(stream_obj, end_data, end_len); + mp_stream_write_adaptor(print.data, end_data, u.len[1]); #else - mp_print_strn(&mp_plat_print, end_data, end_len, 0, 0, 0); + mp_print_strn(&mp_plat_print, end_data, u.len[1], 0, 0, 0); #endif return mp_const_none; } diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 4ded5be08..1b480c9c7 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -40,6 +40,7 @@ Q(/) Q(%#o) Q(%#x) Q({:#b}) +Q( ) Q(\n) Q(maximum recursion depth exceeded) Q() -- cgit v1.2.3 From 5f8ad284f81c3f51be9b3d00ea9fee21ba727a97 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Thu, 7 Dec 2017 09:04:54 +0200 Subject: py/mpprint: Make "%p" format work properly on 64-bit systems. Before, the output was truncated to 32 bits. --- py/mpprint.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/mpprint.c b/py/mpprint.c index a569ef793..74912eb5f 100644 --- a/py/mpprint.c +++ b/py/mpprint.c @@ -512,7 +512,8 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { break; case 'p': case 'P': // don't bother to handle upcase for 'P' - chrs += mp_print_int(print, va_arg(args, unsigned int), 0, 16, 'a', flags, fill, width); + // Use unsigned long int to work on both ILP32 and LP64 systems + chrs += mp_print_int(print, va_arg(args, unsigned long int), 0, 16, 'a', flags, fill, width); break; #if MICROPY_PY_BUILTINS_FLOAT case 'e': -- cgit v1.2.3 From 5a10e63543cae424a71f93ea95b79d34df095832 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Thu, 7 Dec 2017 10:00:23 +0200 Subject: py/mpprint: Support "%lx" format on 64-bit systems. Before that, the output was truncated to 32 bits. Only "%x" format is handled, because a typical use is for addresses. This refactor actually decreased x86_64 code size by 30 bytes. --- py/mpprint.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'py') diff --git a/py/mpprint.c b/py/mpprint.c index 74912eb5f..d6d9cf963 100644 --- a/py/mpprint.c +++ b/py/mpprint.c @@ -446,11 +446,16 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { } } - // parse long specifiers (current not used) - //bool long_arg = false; + // parse long specifiers (only for LP64 model where they make a difference) + #ifndef __LP64__ + const + #endif + bool long_arg = false; if (*fmt == 'l') { ++fmt; - //long_arg = true; + #ifdef __LP64__ + long_arg = true; + #endif } if (*fmt == '\0') { @@ -505,11 +510,17 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { chrs += mp_print_int(print, va_arg(args, int), 1, 10, 'a', flags, fill, width); break; case 'x': - chrs += mp_print_int(print, va_arg(args, unsigned int), 0, 16, 'a', flags, fill, width); - break; - case 'X': - chrs += mp_print_int(print, va_arg(args, unsigned int), 0, 16, 'A', flags, fill, width); + case 'X': { + char fmt_c = 'x' - *fmt + 'A'; + mp_uint_t val; + if (long_arg) { + val = va_arg(args, unsigned long int); + } else { + val = va_arg(args, unsigned int); + } + chrs += mp_print_int(print, val, 0, 16, fmt_c, flags, fill, width); break; + } case 'p': case 'P': // don't bother to handle upcase for 'P' // Use unsigned long int to work on both ILP32 and LP64 systems -- cgit v1.2.3 From f5e097021ce95cf4398f89c4f4268efbc5717b89 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Thu, 7 Dec 2017 10:31:14 +0200 Subject: py/mpprint: Fix "%x" vs "%X" regression introduced in previous commit. --- py/mpprint.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/mpprint.c b/py/mpprint.c index d6d9cf963..c2e65301c 100644 --- a/py/mpprint.c +++ b/py/mpprint.c @@ -511,7 +511,7 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { break; case 'x': case 'X': { - char fmt_c = 'x' - *fmt + 'A'; + char fmt_c = *fmt - 'X' + 'A'; mp_uint_t val; if (long_arg) { val = va_arg(args, unsigned long int); -- cgit v1.2.3 From 88a8043a27c3f75c7e4e52e4e8b0d47005cd6bef Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Thu, 7 Dec 2017 10:52:40 +0200 Subject: py/malloc: MICROPY_MEM_STATS requires MICROPY_MALLOC_USES_ALLOCATED_SIZE. Error out if they're set incompatibly. --- py/malloc.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'py') diff --git a/py/malloc.c b/py/malloc.c index ea1d4c4b9..818a3e57a 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -39,6 +39,9 @@ #endif #if MICROPY_MEM_STATS +#if !MICROPY_MALLOC_USES_ALLOCATED_SIZE +#error MICROPY_MEM_STATS requires MICROPY_MALLOC_USES_ALLOCATED_SIZE +#endif #define UPDATE_PEAK() { if (MP_STATE_MEM(current_bytes_allocated) > MP_STATE_MEM(peak_bytes_allocated)) MP_STATE_MEM(peak_bytes_allocated) = MP_STATE_MEM(current_bytes_allocated); } #endif -- cgit v1.2.3 From 9ebc037eee575fd951dea92c82ed9704d9101924 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Thu, 7 Dec 2017 17:57:33 +0200 Subject: py/malloc: Allow to use debug logging if !MICROPY_MALLOC_USES_ALLOCATED_SIZE. This is mostly a workaround for forceful rebuilding of mpy-cross on every codebase change. If this file has debug logging enabled (by patching), mpy-cross build failed. --- py/malloc.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'py') diff --git a/py/malloc.c b/py/malloc.c index 818a3e57a..6835ed7c9 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -147,7 +147,11 @@ void *m_realloc(void *ptr, size_t new_num_bytes) { MP_STATE_MEM(current_bytes_allocated) += diff; UPDATE_PEAK(); #endif + #if MICROPY_MALLOC_USES_ALLOCATED_SIZE DEBUG_printf("realloc %p, %d, %d : %p\n", ptr, old_num_bytes, new_num_bytes, new_ptr); + #else + DEBUG_printf("realloc %p, %d : %p\n", ptr, new_num_bytes, new_ptr); + #endif return new_ptr; } @@ -171,7 +175,11 @@ void *m_realloc_maybe(void *ptr, size_t new_num_bytes, bool allow_move) { UPDATE_PEAK(); } #endif + #if MICROPY_MALLOC_USES_ALLOCATED_SIZE DEBUG_printf("realloc %p, %d, %d : %p\n", ptr, old_num_bytes, new_num_bytes, new_ptr); + #else + DEBUG_printf("realloc %p, %d, %d : %p\n", ptr, new_num_bytes, new_ptr); + #endif return new_ptr; } @@ -184,7 +192,11 @@ void m_free(void *ptr) { #if MICROPY_MEM_STATS MP_STATE_MEM(current_bytes_allocated) -= num_bytes; #endif + #if MICROPY_MALLOC_USES_ALLOCATED_SIZE DEBUG_printf("free %p, %d\n", ptr, num_bytes); + #else + DEBUG_printf("free %p\n", ptr); + #endif } #if MICROPY_MEM_STATS -- cgit v1.2.3 From 9ef4be8b41c7d256908ba319918c0f7d54346bf4 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 8 Dec 2017 00:10:44 +0200 Subject: py/gc: Add CLEAR_ON_SWEEP option to debug mis-traced objects. Accessing them will crash immediately instead still working for some time, until overwritten by some other data, leading to much less deterministic crashes. --- py/gc.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 172a5e8fc..9a3c57e61 100644 --- a/py/gc.c +++ b/py/gc.c @@ -44,6 +44,10 @@ // make this 1 to dump the heap each time it changes #define EXTENSIVE_HEAP_PROFILING (0) +// make this 1 to zero out swept memory to more eagerly +// detect untraced object still in use +#define CLEAR_ON_SWEEP (0) + #define WORDS_PER_BLOCK ((MICROPY_BYTES_PER_GC_BLOCK) / BYTES_PER_WORD) #define BYTES_PER_BLOCK (MICROPY_BYTES_PER_GC_BLOCK) @@ -286,6 +290,9 @@ STATIC void gc_sweep(void) { case AT_TAIL: if (free_tail) { ATB_ANY_TO_FREE(block); + #if CLEAR_ON_SWEEP + memset((void*)PTR_FROM_BLOCK(block), 0, BYTES_PER_BLOCK); + #endif } break; -- cgit v1.2.3 From f935bce3c52e8eb8f48f7f4947b1210074c359a6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 8 Dec 2017 18:23:23 +1100 Subject: py/{emitbc,asmbase}: Only clear emit labels to -1 when in debug mode. Clearing the labels to -1 is purely a debugging measure. For release builds there is no need to do it as the label offset table should always have the correct value assigned. --- py/asmbase.c | 4 +++- py/emitbc.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/asmbase.c b/py/asmbase.c index c941e917b..83a699cd4 100644 --- a/py/asmbase.c +++ b/py/asmbase.c @@ -47,8 +47,10 @@ void mp_asm_base_deinit(mp_asm_base_t *as, bool free_code) { void mp_asm_base_start_pass(mp_asm_base_t *as, int pass) { if (pass == MP_ASM_PASS_COMPUTE) { - // reset all labels + #ifndef NDEBUG + // With debugging enabled labels are checked for unique assignment memset(as->label_offsets, -1, as->max_num_labels * sizeof(size_t)); + #endif } else if (pass == MP_ASM_PASS_EMIT) { // allocating executable RAM is platform specific MP_PLAT_ALLOC_EXEC(as->code_offset, (void**)&as->code_base, &as->code_size); diff --git a/py/emitbc.c b/py/emitbc.c index 3f4dfc178..5e7fa623a 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -313,9 +313,12 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { emit->scope = scope; emit->last_source_line_offset = 0; emit->last_source_line = 1; + #ifndef NDEBUG + // With debugging enabled labels are checked for unique assignment if (pass < MP_PASS_EMIT) { memset(emit->label_offsets, -1, emit->max_num_labels * sizeof(mp_uint_t)); } + #endif emit->bytecode_offset = 0; emit->code_info_offset = 0; @@ -495,7 +498,6 @@ void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l) { emit->label_offsets[l] = emit->bytecode_offset; } else { // ensure label offset has not changed from MP_PASS_CODE_SIZE to MP_PASS_EMIT - //printf("l%d: (at %d vs %d)\n", l, emit->bytecode_offset, emit->label_offsets[l]); assert(emit->label_offsets[l] == emit->bytecode_offset); } } -- cgit v1.2.3 From 53e111800f9e53733042442eefea0ffc293a35df Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 8 Dec 2017 19:07:00 +1100 Subject: py/asmbase: Revert removal of clearing of label offsets for native emit. The assembler back-end for most architectures needs to know if a jump is backwards in order to emit optimised machine code, and they do this by checking if the destination label has been set or not. So always reset label offsets to -1 (this reverts partially the previous commit, with some minor optimisation for the if-logic with the pass variable). --- py/asmbase.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'py') diff --git a/py/asmbase.c b/py/asmbase.c index 83a699cd4..4c84c3b25 100644 --- a/py/asmbase.c +++ b/py/asmbase.c @@ -46,12 +46,10 @@ void mp_asm_base_deinit(mp_asm_base_t *as, bool free_code) { } void mp_asm_base_start_pass(mp_asm_base_t *as, int pass) { - if (pass == MP_ASM_PASS_COMPUTE) { - #ifndef NDEBUG - // With debugging enabled labels are checked for unique assignment + if (pass < MP_ASM_PASS_EMIT) { + // Reset labels so we can detect backwards jumps (and verify unique assignment) memset(as->label_offsets, -1, as->max_num_labels * sizeof(size_t)); - #endif - } else if (pass == MP_ASM_PASS_EMIT) { + } else { // allocating executable RAM is platform specific MP_PLAT_ALLOC_EXEC(as->code_offset, (void**)&as->code_base, &as->code_size); assert(as->code_base != NULL); -- cgit v1.2.3 From c0877cbb0d51f66fa828f6df5698751210391296 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 8 Dec 2017 20:40:55 +0200 Subject: py/objint_longlong: Check for zero division/modulo. --- py/objint_longlong.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'py') diff --git a/py/objint_longlong.c b/py/objint_longlong.c index 2e567c572..3e5ebadaf 100644 --- a/py/objint_longlong.c +++ b/py/objint_longlong.c @@ -151,9 +151,15 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i return mp_obj_new_int_from_ll(lhs_val * rhs_val); case MP_BINARY_OP_FLOOR_DIVIDE: case MP_BINARY_OP_INPLACE_FLOOR_DIVIDE: + if (rhs_val == 0) { + goto zero_division; + } return mp_obj_new_int_from_ll(lhs_val / rhs_val); case MP_BINARY_OP_MODULO: case MP_BINARY_OP_INPLACE_MODULO: + if (rhs_val == 0) { + goto zero_division; + } return mp_obj_new_int_from_ll(lhs_val % rhs_val); case MP_BINARY_OP_AND: @@ -210,6 +216,9 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i default: return MP_OBJ_NULL; // op not supported } + +zero_division: + mp_raise_msg(&mp_type_ZeroDivisionError, "division by zero"); } mp_obj_t mp_obj_new_int(mp_int_t value) { -- cgit v1.2.3 From 39dd89fe3142478b48d7282b8b1bdff933c25f32 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 9 Dec 2017 01:26:21 +0200 Subject: py/runtime: When tracing unary/binary ops, output op (method) name. E.g.: >>> 1+1 binary 26 __add__ 3 3 Output is similar to bytecode dump (numeric code, then op name). --- py/runtime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/runtime.c b/py/runtime.c index c7fe39367..457266c67 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -214,7 +214,7 @@ void mp_delete_global(qstr qst) { } mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { - DEBUG_OP_printf("unary " UINT_FMT " %p\n", op, arg); + DEBUG_OP_printf("unary " UINT_FMT " %q %p\n", op, mp_unary_op_method_name[op], arg); if (op == MP_UNARY_OP_NOT) { // "not x" is the negative of whether "x" is true per Python semantics @@ -275,7 +275,7 @@ mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { } mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { - DEBUG_OP_printf("binary " UINT_FMT " %p %p\n", op, lhs, rhs); + DEBUG_OP_printf("binary " UINT_FMT " %q %p %p\n", op, mp_binary_op_method_name[op], lhs, rhs); // TODO correctly distinguish inplace operators for mutable objects // lookup logic that CPython uses for +=: -- cgit v1.2.3 From 5453d88d5db94e686cf26930e88a5e20fd21d8f8 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 9 Dec 2017 01:48:26 +0200 Subject: py/gc: Factor out a macro to trace GC mark operations. To allow easier override it for custom tracing. --- py/gc.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 9a3c57e61..1b99ab4cf 100644 --- a/py/gc.c +++ b/py/gc.c @@ -195,6 +195,14 @@ bool gc_is_locked(void) { && ptr < (void*)MP_STATE_MEM(gc_pool_end) /* must be below end of pool */ \ ) +#ifndef TRACE_MARK +#if DEBUG_PRINT +#define TRACE_MARK(block, ptr) DEBUG_printf("gc_mark(%p)\n", ptr) +#else +#define TRACE_MARK(block, ptr) +#endif +#endif + // ptr should be of type void* #define VERIFY_MARK_AND_PUSH(ptr) \ do { \ @@ -202,7 +210,7 @@ bool gc_is_locked(void) { size_t _block = BLOCK_FROM_PTR(ptr); \ if (ATB_GET_KIND(_block) == AT_HEAD) { \ /* an unmarked head, mark it, and push it on gc stack */ \ - DEBUG_printf("gc_mark(%p)\n", ptr); \ + TRACE_MARK(_block, ptr); \ ATB_HEAD_TO_MARK(_block); \ if (MP_STATE_MEM(gc_sp) < &MP_STATE_MEM(gc_stack)[MICROPY_ALLOC_GC_STACK_SIZE]) { \ *MP_STATE_MEM(gc_sp)++ = _block; \ -- cgit v1.2.3 From dea3fb93c74ae61dc5168b62a780dc6ce7865e09 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 9 Dec 2017 01:54:01 +0200 Subject: py/gc: In sweep debug output, print pointer as a pointer. Or it will be truncated on a 64-bit platform. --- py/gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 1b99ab4cf..734f5c364 100644 --- a/py/gc.c +++ b/py/gc.c @@ -289,7 +289,7 @@ STATIC void gc_sweep(void) { } #endif free_tail = 1; - DEBUG_printf("gc_sweep(%x)\n", PTR_FROM_BLOCK(block)); + DEBUG_printf("gc_sweep(%p)\n", PTR_FROM_BLOCK(block)); #if MICROPY_PY_GC_COLLECT_RETVAL MP_STATE_MEM(gc_collected)++; #endif -- cgit v1.2.3 From fca1d1aa62306fc523d192c1e2dd2d20dccbe94f Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 9 Dec 2017 09:19:34 +0200 Subject: py/objfun: Factor out macro for decoding codestate size. fun_bc_call() starts with almost the same code as mp_obj_fun_bc_prepare_codestate(), the only difference is a way to allocate the codestate object (heap vs stack with heap fallback). Still, would be nice to avoid code duplication to make further refactoring easier. So, this commit factors out the common code before the allocation - decoding and calculating codestate size. It produces two values, so structured as a macro which writes to 2 variables passed as arguments. --- py/objfun.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) (limited to 'py') diff --git a/py/objfun.c b/py/objfun.c index 030b3f7cb..445f25d46 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -195,20 +195,30 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { // than this will try to use the heap, with fallback to stack allocation. #define VM_MAX_STATE_ON_STACK (11 * sizeof(mp_uint_t)) -// Set this to enable a simple stack overflow check. +// Set this to 1 to enable a simple stack overflow check. #define VM_DETECT_STACK_OVERFLOW (0) +#define DECODE_CODESTATE_SIZE(bytecode, n_state_out_var, state_size_out_var) \ + { \ + /* bytecode prelude: state size and exception stack size */ \ + n_state_out_var = mp_decode_uint_value(bytecode); \ + size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(bytecode)); \ + \ + n_state += VM_DETECT_STACK_OVERFLOW; \ + \ + /* state size in bytes */ \ + state_size_out_var = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); \ + } + #if MICROPY_STACKLESS mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { MP_STACK_CHECK(); mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); - // bytecode prelude: state size and exception stack size - size_t n_state = mp_decode_uint_value(self->bytecode); - size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(self->bytecode)); + size_t n_state, state_size; + DECODE_CODESTATE_SIZE(self->bytecode, n_state, state_size); // allocate state for locals and stack - size_t state_size = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); mp_code_state_t *code_state; code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); if (!code_state) { @@ -238,16 +248,10 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); DEBUG_printf("Func n_def_args: %d\n", self->n_def_args); - // bytecode prelude: state size and exception stack size - size_t n_state = mp_decode_uint_value(self->bytecode); - size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(self->bytecode)); - -#if VM_DETECT_STACK_OVERFLOW - n_state += 1; -#endif + size_t n_state, state_size; + DECODE_CODESTATE_SIZE(self->bytecode, n_state, state_size); // allocate state for locals and stack - size_t state_size = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); mp_code_state_t *code_state = NULL; if (state_size > VM_MAX_STATE_ON_STACK) { code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); -- cgit v1.2.3 From d72370def72c74ca98c1ec4eb7b58ba0fbcc9629 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 9 Dec 2017 10:57:40 +0200 Subject: py/objfun, vm: Add comments on codestate allocation in stackless mode. --- py/objfun.c | 6 +++++- py/vm.c | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/objfun.c b/py/objfun.c index 445f25d46..e27413e40 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -218,8 +218,12 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args size_t n_state, state_size; DECODE_CODESTATE_SIZE(self->bytecode, n_state, state_size); - // allocate state for locals and stack mp_code_state_t *code_state; + // If we use m_new_obj_var(), then on no memory, MemoryError will be + // raised. But this is not correct exception for a function call, + // RuntimeError should be raised instead. So, we use m_new_obj_var_maybe(), + // return NULL, then vm.c takes the needed action (either raise + // RuntimeError or fallback to stack allocation). code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); if (!code_state) { return NULL; diff --git a/py/vm.c b/py/vm.c index 564200037..e6679729b 100644 --- a/py/vm.c +++ b/py/vm.c @@ -937,6 +937,9 @@ unwind_jump:; deep_recursion_error: mp_exc_recursion_depth(); } + #else + // If we couldn't allocate codestate on heap, in + // non non-strict case fall thru to stack allocation. #endif } #endif @@ -974,6 +977,9 @@ unwind_jump:; else { goto deep_recursion_error; } + #else + // If we couldn't allocate codestate on heap, in + // non non-strict case fall thru to stack allocation. #endif } #endif @@ -1008,6 +1014,9 @@ unwind_jump:; else { goto deep_recursion_error; } + #else + // If we couldn't allocate codestate on heap, in + // non non-strict case fall thru to stack allocation. #endif } #endif @@ -1045,6 +1054,9 @@ unwind_jump:; else { goto deep_recursion_error; } + #else + // If we couldn't allocate codestate on heap, in + // non non-strict case fall thru to stack allocation. #endif } #endif -- cgit v1.2.3 From 2b00181592b23e5ca97464791c1ffa56e9495489 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 9 Dec 2017 12:45:28 +0200 Subject: py/objfun: Factor out macro for initializing codestate. This is second part of fun_bc_call() vs mp_obj_fun_bc_prepare_codestate() common code refactor. This factors out code to initialize codestate object. After this patch, mp_obj_fun_bc_prepare_codestate() is effectively DECODE_CODESTATE_SIZE() followed by allocation followed by INIT_CODESTATE(), and fun_bc_call() starts with that too. --- py/objfun.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'py') diff --git a/py/objfun.c b/py/objfun.c index e27413e40..8fb3ec6fa 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -210,6 +210,12 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { state_size_out_var = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); \ } +#define INIT_CODESTATE(code_state, _fun_bc, n_args, n_kw, args) \ + code_state->fun_bc = _fun_bc; \ + code_state->ip = 0; \ + mp_setup_code_state(code_state, n_args, n_kw, args); \ + code_state->old_globals = mp_globals_get(); + #if MICROPY_STACKLESS mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { MP_STACK_CHECK(); @@ -229,12 +235,9 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args return NULL; } - code_state->fun_bc = self; - code_state->ip = 0; - mp_setup_code_state(code_state, n_args, n_kw, args); + INIT_CODESTATE(code_state, self, n_args, n_kw, args); // execute the byte code with the correct globals context - code_state->old_globals = mp_globals_get(); mp_globals_set(self->globals); return code_state; @@ -265,12 +268,9 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const state_size = 0; // indicate that we allocated using alloca } - code_state->fun_bc = self; - code_state->ip = 0; - mp_setup_code_state(code_state, n_args, n_kw, args); + INIT_CODESTATE(code_state, self, n_args, n_kw, args); // execute the byte code with the correct globals context - code_state->old_globals = mp_globals_get(); mp_globals_set(self->globals); mp_vm_return_kind_t vm_return_kind = mp_execute_bytecode(code_state, MP_OBJ_NULL); mp_globals_set(code_state->old_globals); -- cgit v1.2.3 From a35d923cdf5a94ef9a29645f9dd461479dc819ce Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 9 Dec 2017 17:30:55 +0200 Subject: py/map: Allow to trace rehashing operations. --- py/map.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'py') diff --git a/py/map.c b/py/map.c index 4f76b9b16..696c7a2ea 100644 --- a/py/map.c +++ b/py/map.c @@ -33,6 +33,13 @@ #include "py/misc.h" #include "py/runtime.h" +#if MICROPY_DEBUG_VERBOSE // print debugging info +#define DEBUG_PRINT (1) +#else // don't print debugging info +#define DEBUG_PRINT (0) +#define DEBUG_printf(...) (void)0 +#endif + // Fixed empty map. Useful when need to call kw-receiving functions // without any keywords from C, etc. const mp_map_t mp_const_empty_map = { @@ -114,6 +121,7 @@ void mp_map_clear(mp_map_t *map) { STATIC void mp_map_rehash(mp_map_t *map) { size_t old_alloc = map->alloc; size_t new_alloc = get_hash_alloc_greater_or_equal_to(map->alloc + 1); + DEBUG_printf("mp_map_rehash(%p): " UINT_FMT " -> " UINT_FMT "\n", map, old_alloc, new_alloc); mp_map_elem_t *old_table = map->table; mp_map_elem_t *new_table = m_new0(mp_map_elem_t, new_alloc); // If we reach this point, table resizing succeeded, now we can edit the old map. -- cgit v1.2.3 From d21d029d55ddc026fcaf6673f2b712a645286dce Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 10 Dec 2017 01:05:29 +0200 Subject: py/mkrules.mk: Add "clean-frozen" target to clean frozen script/modules dir. This target removes any stray files (i.e. something not committed to git) from scripts/ and modules/ dirs (or whatever FROZEN_DIR and FROZEN_MPY_DIR is set to). The expected workflow is: 1. make clean-frozen 2. micropython -m upip -p modules 3. make As it can be expected that people may drop random thing in those dirs which they can miss later, the content is actually backed up before cleaning. --- py/mkrules.mk | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'py') diff --git a/py/mkrules.mk b/py/mkrules.mk index 72a5c3de0..96f6e35a6 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -160,6 +160,27 @@ clean: $(RM) -rf $(BUILD) $(CLEAN_EXTRA) .PHONY: clean +# Clean every non-git file from FROZEN_DIR/FROZEN_MPY_DIR, but making a backup. +# We run rmdir below to avoid empty backup dir (it will silently fail if backup +# is non-empty). +clean-frozen: + if [ -n "$(FROZEN_MPY_DIR)" ]; then \ + backup_dir=$(FROZEN_MPY_DIR).$$(date +%Y%m%dT%H%M%S); mkdir $$backup_dir; \ + cd $(FROZEN_MPY_DIR); git status --ignored -u all -s . | awk ' {print $$2}' \ + | xargs --no-run-if-empty cp --parents -t ../$$backup_dir; \ + rmdir ../$$backup_dir 2>/dev/null || true; \ + git clean -d -f .; \ + fi + + if [ -n "$(FROZEN_DIR)" ]; then \ + backup_dir=$(FROZEN_DIR).$$(date +%Y%m%dT%H%M%S); mkdir $$backup_dir; \ + cd $(FROZEN_DIR); git status --ignored -u all -s . | awk ' {print $$2}' \ + | xargs --no-run-if-empty cp --parents -t ../$$backup_dir; \ + rmdir ../$$backup_dir 2>/dev/null || true; \ + git clean -d -f .; \ + fi +.PHONY: clean-frozen + print-cfg: $(ECHO) "PY_SRC = $(PY_SRC)" $(ECHO) "BUILD = $(BUILD)" -- cgit v1.2.3 From e7fc765880ba6025ac5677db696669f43975483e Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 10 Dec 2017 02:15:43 +0200 Subject: unix/mpconfigport: Disable uio.resource_stream(). This function was implemented as an experiment, and was enabled only in unix port. To remind, it allows to access arbitrary files frozen as source modules (vs bytecode). However, further experimentation showed that the same functionality can be implemented with frozen bytecode. The process requires more steps, but with suitable toolset it doesn't matter patch. This process is: 1. Convert binary files into "Python resource module" with tools/mpy_bin2res.py. 2. Freeze as the bytecode. 3. Use micropython-lib's pkg_resources.resource_stream() to access it. In other words, the extra step is using tools/mpy_bin2res.py (because there would be wrapper for uio.resource_stream() anyway). Going frozen bytecode route allows more flexibility, and same/additional efficiency: 1. Frozen source support can be disabled altogether for additional code savings. 2. Resources could be also accessed as a buffer, not just as a stream. There're few caveats too: 1. It wasn't actually profiled the overhead of storing a resource in "Python resource module" vs storing it directly, but it's assumed that overhead is small. 2. The "efficiency" claim above applies to the case when resource file is frozen as the bytecode. If it's not, it actually will take a lot of RAM on loading. But in this case, the resource file should not be used (i.e. generated) in the first place, and micropython-lib's pkg_resources.resource_stream() implementation has the appropriate fallback to read the raw files instead. This still poses some distribution issues, e.g. to deployable to baremetal ports (which almost certainly would require freezeing as the bytecode), a distribution package should include the resource module. But for non-freezing deployment, presense of resource module will lead to memory inefficiency. All the discussion above reminds why uio.resource_stream() was implemented in the first place - to address some of the issues above. However, since then, frozen bytecode approach seems to prevail, so, while there're still some issues to address with it, this change is being made. This change saves 488 bytes for the unix x86_64 port. --- ports/unix/mpconfigport.h | 1 - py/mpconfig.h | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index a3d2bb7db..06ae1e234 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -103,7 +103,6 @@ #endif #define MICROPY_PY_CMATH (1) #define MICROPY_PY_IO_FILEIO (1) -#define MICROPY_PY_IO_RESOURCE_STREAM (1) #define MICROPY_PY_GC_COLLECT_RETVAL (1) #define MICROPY_MODULE_FROZEN_STR (1) diff --git a/py/mpconfig.h b/py/mpconfig.h index 095b7ef7b..778436708 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -938,7 +938,11 @@ typedef double mp_float_t; // Whether to provide "uio.resource_stream()" function with // the semantics of CPython's pkg_resources.resource_stream() -// (allows to access resources in frozen packages). +// (allows to access binary resources in frozen source packages). +// Note that the same functionality can be achieved in "pure +// Python" by prepocessing binary resources into Python source +// and bytecode-freezing it (with a simple helper module available +// e.g. in micropython-lib). #ifndef MICROPY_PY_IO_RESOURCE_STREAM #define MICROPY_PY_IO_RESOURCE_STREAM (0) #endif -- cgit v1.2.3 From 5b8998da6dfed6c8f54b8b34228f25d93dbb9d29 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 20 Nov 2017 17:29:58 +1100 Subject: py/runtime: Move mp_exc_recursion_depth to runtime and rename to raise. For consistency this helper function is renamed to match the other exception helpers, and moved to their location in runtime.c. --- py/runtime.c | 7 +++++++ py/runtime.h | 2 +- py/stackctrl.c | 7 +------ py/vm.c | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) (limited to 'py') diff --git a/py/runtime.c b/py/runtime.c index 457266c67..5fd053e1a 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1456,3 +1456,10 @@ NORETURN void mp_raise_OSError(int errno_) { NORETURN void mp_raise_NotImplementedError(const char *msg) { mp_raise_msg(&mp_type_NotImplementedError, msg); } + +#if MICROPY_STACK_CHECK +NORETURN void mp_raise_recursion_depth(void) { + nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError, + MP_OBJ_NEW_QSTR(MP_QSTR_maximum_space_recursion_space_depth_space_exceeded))); +} +#endif diff --git a/py/runtime.h b/py/runtime.h index 3f0d1104e..a19f64c06 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -151,7 +151,7 @@ NORETURN void mp_raise_ValueError(const char *msg); NORETURN void mp_raise_TypeError(const char *msg); NORETURN void mp_raise_NotImplementedError(const char *msg); NORETURN void mp_raise_OSError(int errno_); -NORETURN void mp_exc_recursion_depth(void); +NORETURN void mp_raise_recursion_depth(void); #if MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG #undef mp_check_self diff --git a/py/stackctrl.c b/py/stackctrl.c index 11165b9a6..5c07796bd 100644 --- a/py/stackctrl.c +++ b/py/stackctrl.c @@ -48,14 +48,9 @@ void mp_stack_set_limit(mp_uint_t limit) { MP_STATE_THREAD(stack_limit) = limit; } -NORETURN void mp_exc_recursion_depth(void) { - nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError, - MP_OBJ_NEW_QSTR(MP_QSTR_maximum_space_recursion_space_depth_space_exceeded))); -} - void mp_stack_check(void) { if (mp_stack_usage() >= MP_STATE_THREAD(stack_limit)) { - mp_exc_recursion_depth(); + mp_raise_recursion_depth(); } } diff --git a/py/vm.c b/py/vm.c index e6679729b..5011af5c3 100644 --- a/py/vm.c +++ b/py/vm.c @@ -935,7 +935,7 @@ unwind_jump:; #if MICROPY_STACKLESS_STRICT else { deep_recursion_error: - mp_exc_recursion_depth(); + mp_raise_recursion_depth(); } #else // If we couldn't allocate codestate on heap, in -- cgit v1.2.3 From 02d830c035aca166d70551e485ccd2d1658189c9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 26 Nov 2017 23:28:40 +1100 Subject: py: Introduce a Python stack for scoped allocation. This patch introduces the MICROPY_ENABLE_PYSTACK option (disabled by default) which enables a "Python stack" that allows to allocate and free memory in a scoped, or Last-In-First-Out (LIFO) way, similar to alloca(). A new memory allocation API is introduced along with this Py-stack. It includes both "local" and "nonlocal" LIFO allocation. Local allocation is intended to be equivalent to using alloca(), whereby the same function must free the memory. Nonlocal allocation is where another function may free the memory, so long as it's still LIFO. Follow-up patches will convert all uses of alloca() and VLA to the new scoped allocation API. The old behaviour (using alloca()) will still be available, but when MICROPY_ENABLE_PYSTACK is enabled then alloca() is no longer required or used. The benefits of enabling this option are (or will be once subsequent patches are made to convert alloca()/VLA): - Toolchains without alloca() can use this feature to obtain correct and efficient scoped memory allocation (compared to using the heap instead of alloca(), which is slower). - Even if alloca() is available, enabling the Py-stack gives slightly more efficient use of stack space when calling nested Python functions, due to the way that compilers implement alloca(). - Enabling the Py-stack with the stackless mode allows for even more efficient stack usage, as well as retaining high performance (because the heap is no longer used to build and destroy stackless code states). - With Py-stack and stackless enabled, Python-calling-Python is no longer recursive in the C mp_execute_bytecode function. The micropython.pystack_use() function is included to measure usage of the Python stack. --- py/gc.c | 7 +++ py/modmicropython.c | 10 +++++ py/modthread.c | 6 +++ py/mpconfig.h | 11 +++++ py/mpstate.h | 6 +++ py/nlr.h | 19 +++++++- py/nlrsetjmp.c | 1 + py/nlrthumb.c | 2 + py/nlrx64.c | 2 + py/nlrx86.c | 2 + py/nlrxtensa.c | 2 + py/py.mk | 1 + py/pystack.c | 56 ++++++++++++++++++++++++ py/pystack.h | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++++ py/runtime.c | 2 +- py/runtime.h | 1 + 16 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 py/pystack.c create mode 100644 py/pystack.h (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 734f5c364..5196954e2 100644 --- a/py/gc.c +++ b/py/gc.c @@ -320,11 +320,18 @@ void gc_collect_start(void) { #endif MP_STATE_MEM(gc_stack_overflow) = 0; MP_STATE_MEM(gc_sp) = MP_STATE_MEM(gc_stack); + // Trace root pointers. This relies on the root pointers being organised // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, // dict_globals, then the root pointer section of mp_state_vm. void **ptrs = (void**)(void*)&mp_state_ctx; gc_collect_root(ptrs, offsetof(mp_state_ctx_t, vm.qstr_last_chunk) / sizeof(void*)); + + #if MICROPY_ENABLE_PYSTACK + // Trace root pointers from the Python stack. + ptrs = (void**)(void*)MP_STATE_THREAD(pystack_start); + gc_collect_root(ptrs, (MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start)) / sizeof(void*)); + #endif } void gc_collect_root(void **ptrs, size_t len) { diff --git a/py/modmicropython.c b/py/modmicropython.c index 2aac53adc..c14a0177d 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -112,6 +112,13 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_stack_use_obj, mp_micropython_st #endif // MICROPY_PY_MICROPYTHON_MEM_INFO +#if MICROPY_ENABLE_PYSTACK +STATIC mp_obj_t mp_micropython_pystack_use(void) { + return MP_OBJ_NEW_SMALL_INT(mp_pystack_usage()); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_pystack_use_obj, mp_micropython_pystack_use); +#endif + #if MICROPY_ENABLE_GC STATIC mp_obj_t mp_micropython_heap_lock(void) { gc_lock(); @@ -167,6 +174,9 @@ STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0) { MP_ROM_QSTR(MP_QSTR_alloc_emergency_exception_buf), MP_ROM_PTR(&mp_alloc_emergency_exception_buf_obj) }, #endif + #if MICROPY_ENABLE_PYSTACK + { MP_ROM_QSTR(MP_QSTR_pystack_use), MP_ROM_PTR(&mp_micropython_pystack_use_obj) }, + #endif #if MICROPY_ENABLE_GC { MP_ROM_QSTR(MP_QSTR_heap_lock), MP_ROM_PTR(&mp_micropython_heap_lock_obj) }, { MP_ROM_QSTR(MP_QSTR_heap_unlock), MP_ROM_PTR(&mp_micropython_heap_unlock_obj) }, diff --git a/py/modthread.c b/py/modthread.c index cb071d0f8..61ada5035 100644 --- a/py/modthread.c +++ b/py/modthread.c @@ -165,6 +165,12 @@ STATIC void *thread_entry(void *args_in) { mp_stack_set_top(&ts + 1); // need to include ts in root-pointer scan mp_stack_set_limit(args->stack_size); + #if MICROPY_ENABLE_PYSTACK + // TODO threading and pystack is not fully supported, for now just make a small stack + mp_obj_t mini_pystack[128]; + mp_pystack_init(mini_pystack, &mini_pystack[128]); + #endif + // set locals and globals from the calling context mp_locals_set(args->dict_locals); mp_globals_set(args->dict_globals); diff --git a/py/mpconfig.h b/py/mpconfig.h index 778436708..96cd8c651 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -441,6 +441,17 @@ #define MICROPY_ENABLE_FINALISER (0) #endif +// Whether to enable a separate allocator for the Python stack. +// If enabled then the code must call mp_pystack_init before mp_init. +#ifndef MICROPY_ENABLE_PYSTACK +#define MICROPY_ENABLE_PYSTACK (0) +#endif + +// Number of bytes that memory returned by mp_pystack_alloc will be aligned by. +#ifndef MICROPY_PYSTACK_ALIGN +#define MICROPY_PYSTACK_ALIGN (8) +#endif + // Whether to check C stack usage. C stack used for calling Python functions, // etc. Not checking means segfault on overflow. #ifndef MICROPY_STACK_CHECK diff --git a/py/mpstate.h b/py/mpstate.h index 6a39ebdea..d3b53f915 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -224,6 +224,12 @@ typedef struct _mp_state_thread_t { #if MICROPY_STACK_CHECK size_t stack_limit; #endif + + #if MICROPY_ENABLE_PYSTACK + uint8_t *pystack_start; + uint8_t *pystack_end; + uint8_t *pystack_cur; + #endif } mp_state_thread_t; // This structure combines the above 3 structures. diff --git a/py/nlr.h b/py/nlr.h index 63fe392d9..1235f1460 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -62,15 +62,32 @@ struct _nlr_buf_t { #if MICROPY_NLR_SETJMP jmp_buf jmpbuf; #endif + + #if MICROPY_ENABLE_PYSTACK + void *pystack; + #endif }; +// Helper macros to save/restore the pystack state +#if MICROPY_ENABLE_PYSTACK +#define MP_NLR_SAVE_PYSTACK(nlr_buf) (nlr_buf)->pystack = MP_STATE_THREAD(pystack_cur) +#define MP_NLR_RESTORE_PYSTACK(nlr_buf) MP_STATE_THREAD(pystack_cur) = (nlr_buf)->pystack +#else +#define MP_NLR_SAVE_PYSTACK(nlr_buf) (void)nlr_buf +#define MP_NLR_RESTORE_PYSTACK(nlr_buf) (void)nlr_buf +#endif + #if MICROPY_NLR_SETJMP #include "py/mpstate.h" NORETURN void nlr_setjmp_jump(void *val); // nlr_push() must be defined as a macro, because "The stack context will be // invalidated if the function which called setjmp() returns." -#define nlr_push(buf) ((buf)->prev = MP_STATE_THREAD(nlr_top), MP_STATE_THREAD(nlr_top) = (buf), setjmp((buf)->jmpbuf)) +#define nlr_push(buf) ( \ + (buf)->prev = MP_STATE_THREAD(nlr_top), \ + MP_NLR_SAVE_PYSTACK(buf), \ + MP_STATE_THREAD(nlr_top) = (buf), \ + setjmp((buf)->jmpbuf)) #define nlr_pop() { MP_STATE_THREAD(nlr_top) = MP_STATE_THREAD(nlr_top)->prev; } #define nlr_jump(val) nlr_setjmp_jump(val) #else diff --git a/py/nlrsetjmp.c b/py/nlrsetjmp.c index 1fb459440..63376a553 100644 --- a/py/nlrsetjmp.c +++ b/py/nlrsetjmp.c @@ -35,6 +35,7 @@ void nlr_setjmp_jump(void *val) { nlr_jump_fail(val); } top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); *top_ptr = top->prev; longjmp(top->jmpbuf, 1); } diff --git a/py/nlrthumb.c b/py/nlrthumb.c index 6e7d71766..eab5759f2 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -82,6 +82,7 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); *top = nlr; return 0; // normal return } @@ -99,6 +100,7 @@ NORETURN __attribute__((naked)) void nlr_jump(void *val) { } top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); *top_ptr = top->prev; __asm volatile ( diff --git a/py/nlrx64.c b/py/nlrx64.c index 847d10398..ddcd76166 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -91,6 +91,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); *top = nlr; return 0; // normal return } @@ -108,6 +109,7 @@ NORETURN void nlr_jump(void *val) { } top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); *top_ptr = top->prev; __asm volatile ( diff --git a/py/nlrx86.c b/py/nlrx86.c index 094dea3cc..3a27460eb 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -73,6 +73,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); *top = nlr; return 0; // normal return } @@ -90,6 +91,7 @@ NORETURN void nlr_jump(void *val) { } top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); *top_ptr = top->prev; __asm volatile ( diff --git a/py/nlrxtensa.c b/py/nlrxtensa.c index 4520e7e7a..5a969fc87 100644 --- a/py/nlrxtensa.c +++ b/py/nlrxtensa.c @@ -58,6 +58,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); *top = nlr; return 0; // normal return } @@ -75,6 +76,7 @@ NORETURN void nlr_jump(void *val) { } top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); *top_ptr = top->prev; __asm volatile ( diff --git a/py/py.mk b/py/py.mk index f5faad182..0b5d5f8c4 100644 --- a/py/py.mk +++ b/py/py.mk @@ -110,6 +110,7 @@ PY_O_BASENAME = \ nlrsetjmp.o \ malloc.o \ gc.o \ + pystack.o \ qstr.o \ vstr.o \ mpprint.o \ diff --git a/py/pystack.c b/py/pystack.c new file mode 100644 index 000000000..767c307e9 --- /dev/null +++ b/py/pystack.c @@ -0,0 +1,56 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/runtime.h" + +#if MICROPY_ENABLE_PYSTACK + +void mp_pystack_init(void *start, void *end) { + MP_STATE_THREAD(pystack_start) = start; + MP_STATE_THREAD(pystack_end) = end; + MP_STATE_THREAD(pystack_cur) = start; +} + +void *mp_pystack_alloc(size_t n_bytes) { + n_bytes = (n_bytes + (MICROPY_PYSTACK_ALIGN - 1)) & ~(MICROPY_PYSTACK_ALIGN - 1); + #if MP_PYSTACK_DEBUG + n_bytes += MICROPY_PYSTACK_ALIGN; + #endif + if (MP_STATE_THREAD(pystack_cur) + n_bytes > MP_STATE_THREAD(pystack_end)) { + // out of memory in the pystack + mp_raise_recursion_depth(); + } + void *ptr = MP_STATE_THREAD(pystack_cur); + MP_STATE_THREAD(pystack_cur) += n_bytes; + #if MP_PYSTACK_DEBUG + *(size_t*)(MP_STATE_THREAD(pystack_cur) - MICROPY_PYSTACK_ALIGN) = n_bytes; + #endif + return ptr; +} + +#endif diff --git a/py/pystack.h b/py/pystack.h new file mode 100644 index 000000000..82ac3743d --- /dev/null +++ b/py/pystack.h @@ -0,0 +1,123 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_PY_PYSTACK_H +#define MICROPY_INCLUDED_PY_PYSTACK_H + +#include "py/mpstate.h" + +// Enable this debugging option to check that the amount of memory freed is +// consistent with amounts that were previously allocated. +#define MP_PYSTACK_DEBUG (0) + +#if MICROPY_ENABLE_PYSTACK + +void mp_pystack_init(void *start, void *end); +void *mp_pystack_alloc(size_t n_bytes); + +// This function can free multiple continuous blocks at once: just pass the +// pointer to the block that was allocated first and it and all subsequently +// allocated blocks will be freed. +static inline void mp_pystack_free(void *ptr) { + assert((uint8_t*)ptr >= MP_STATE_THREAD(pystack_start)); + assert((uint8_t*)ptr <= MP_STATE_THREAD(pystack_cur)); + #if MP_PYSTACK_DEBUG + size_t n_bytes_to_free = MP_STATE_THREAD(pystack_cur) - (uint8_t*)ptr; + size_t n_bytes = *(size_t*)(MP_STATE_THREAD(pystack_cur) - MICROPY_PYSTACK_ALIGN); + while (n_bytes < n_bytes_to_free) { + n_bytes += *(size_t*)(MP_STATE_THREAD(pystack_cur) - n_bytes - MICROPY_PYSTACK_ALIGN); + } + if (n_bytes != n_bytes_to_free) { + mp_printf(&mp_plat_print, "mp_pystack_free() failed: %u != %u\n", (uint)n_bytes_to_free, + (uint)*(size_t*)(MP_STATE_THREAD(pystack_cur) - MICROPY_PYSTACK_ALIGN)); + assert(0); + } + #endif + MP_STATE_THREAD(pystack_cur) = (uint8_t*)ptr; +} + +static inline void mp_pystack_realloc(void *ptr, size_t n_bytes) { + mp_pystack_free(ptr); + mp_pystack_alloc(n_bytes); +} + +static inline size_t mp_pystack_usage(void) { + return MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start); +} + +static inline size_t mp_pystack_limit(void) { + return MP_STATE_THREAD(pystack_end) - MP_STATE_THREAD(pystack_start); +} + +#endif + +#if !MICROPY_ENABLE_PYSTACK + +#define mp_local_alloc(n_bytes) alloca(n_bytes) + +static inline void mp_local_free(void *ptr) { + (void)ptr; +} + +static inline void *mp_nonlocal_alloc(size_t n_bytes) { + return m_new(uint8_t, n_bytes); +} + +static inline void *mp_nonlocal_realloc(void *ptr, size_t old_n_bytes, size_t new_n_bytes) { + return m_renew(uint8_t, ptr, old_n_bytes, new_n_bytes); +} + +static inline void mp_nonlocal_free(void *ptr, size_t n_bytes) { + m_del(uint8_t, ptr, n_bytes); +} + +#else + +static inline void *mp_local_alloc(size_t n_bytes) { + return mp_pystack_alloc(n_bytes); +} + +static inline void mp_local_free(void *ptr) { + mp_pystack_free(ptr); +} + +static inline void *mp_nonlocal_alloc(size_t n_bytes) { + return mp_pystack_alloc(n_bytes); +} + +static inline void *mp_nonlocal_realloc(void *ptr, size_t old_n_bytes, size_t new_n_bytes) { + (void)old_n_bytes; + mp_pystack_realloc(ptr, new_n_bytes); + return ptr; +} + +static inline void mp_nonlocal_free(void *ptr, size_t n_bytes) { + (void)n_bytes; + mp_pystack_free(ptr); +} + +#endif + +#endif // MICROPY_INCLUDED_PY_PYSTACK_H diff --git a/py/runtime.c b/py/runtime.c index 5fd053e1a..3a4a8a93b 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1457,7 +1457,7 @@ NORETURN void mp_raise_NotImplementedError(const char *msg) { mp_raise_msg(&mp_type_NotImplementedError, msg); } -#if MICROPY_STACK_CHECK +#if MICROPY_STACK_CHECK || MICROPY_ENABLE_PYSTACK NORETURN void mp_raise_recursion_depth(void) { nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError, MP_OBJ_NEW_QSTR(MP_QSTR_maximum_space_recursion_space_depth_space_exceeded))); diff --git a/py/runtime.h b/py/runtime.h index a19f64c06..6288e8836 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -27,6 +27,7 @@ #define MICROPY_INCLUDED_PY_RUNTIME_H #include "py/mpstate.h" +#include "py/pystack.h" typedef enum { MP_VM_RETURN_NORMAL, -- cgit v1.2.3 From 1e5a33df413bbd8a8aa5bd880be445c684fc5506 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 26 Nov 2017 23:37:19 +1100 Subject: py: Convert all uses of alloca() to use new scoped allocation API. --- py/builtinimport.c | 5 +++-- py/compile.c | 3 ++- py/objboundmeth.c | 8 ++++++++ py/objfun.c | 12 ++++++++++++ py/runtime.c | 3 ++- py/vm.c | 16 ++++++++++++++-- 6 files changed, 41 insertions(+), 6 deletions(-) (limited to 'py') diff --git a/py/builtinimport.c b/py/builtinimport.c index 9235e946c..2157902c9 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -318,7 +318,7 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { } uint new_mod_l = (mod_len == 0 ? (size_t)(p - this_name) : (size_t)(p - this_name) + 1 + mod_len); - char *new_mod = alloca(new_mod_l); + char *new_mod = mp_local_alloc(new_mod_l); memcpy(new_mod, this_name, p - this_name); if (mod_len != 0) { new_mod[p - this_name] = '.'; @@ -326,9 +326,10 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { } qstr new_mod_q = qstr_from_strn(new_mod, new_mod_l); + mp_local_free(new_mod); DEBUG_printf("Resolved base name for relative import: '%s'\n", qstr_str(new_mod_q)); module_name = MP_OBJ_NEW_QSTR(new_mod_q); - mod_str = new_mod; + mod_str = qstr_str(new_mod_q); mod_len = new_mod_l; } diff --git a/py/compile.c b/py/compile.c index ee017498a..52d10ee5e 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1050,7 +1050,7 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { for (int i = 0; i < n; i++) { len += qstr_len(MP_PARSE_NODE_LEAF_ARG(pns->nodes[i])); } - char *q_ptr = alloca(len); + char *q_ptr = mp_local_alloc(len); char *str_dest = q_ptr; for (int i = 0; i < n; i++) { if (i > 0) { @@ -1062,6 +1062,7 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { str_dest += str_src_len; } qstr q_full = qstr_from_strn(q_ptr, len); + mp_local_free(q_ptr); EMIT_ARG(import_name, q_full); if (is_as) { for (int i = 1; i < n; i++) { diff --git a/py/objboundmeth.c b/py/objboundmeth.c index 890f8b15b..b0df6a68a 100644 --- a/py/objboundmeth.c +++ b/py/objboundmeth.c @@ -51,6 +51,9 @@ mp_obj_t mp_call_method_self_n_kw(mp_obj_t meth, mp_obj_t self, size_t n_args, s // need to insert self before all other args and then call meth size_t n_total = n_args + 2 * n_kw; mp_obj_t *args2 = NULL; + #if MICROPY_ENABLE_PYSTACK + args2 = mp_pystack_alloc(sizeof(mp_obj_t) * (1 + n_total)); + #else mp_obj_t *free_args2 = NULL; if (n_total > 4) { // try to use heap to allocate temporary args array @@ -61,12 +64,17 @@ mp_obj_t mp_call_method_self_n_kw(mp_obj_t meth, mp_obj_t self, size_t n_args, s // (fallback to) use stack to allocate temporary args array args2 = alloca(sizeof(mp_obj_t) * (1 + n_total)); } + #endif args2[0] = self; memcpy(args2 + 1, args, n_total * sizeof(mp_obj_t)); mp_obj_t res = mp_call_function_n_kw(meth, n_args + 1, n_kw, args2); + #if MICROPY_ENABLE_PYSTACK + mp_pystack_free(args2); + #else if (free_args2 != NULL) { m_del(mp_obj_t, free_args2, 1 + n_total); } + #endif return res; } diff --git a/py/objfun.c b/py/objfun.c index 8fb3ec6fa..e6d33d287 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -225,6 +225,9 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args DECODE_CODESTATE_SIZE(self->bytecode, n_state, state_size); mp_code_state_t *code_state; + #if MICROPY_ENABLE_PYSTACK + code_state = mp_pystack_alloc(sizeof(mp_code_state_t) + state_size); + #else // If we use m_new_obj_var(), then on no memory, MemoryError will be // raised. But this is not correct exception for a function call, // RuntimeError should be raised instead. So, we use m_new_obj_var_maybe(), @@ -234,6 +237,7 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args if (!code_state) { return NULL; } + #endif INIT_CODESTATE(code_state, self, n_args, n_kw, args); @@ -260,6 +264,9 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const // allocate state for locals and stack mp_code_state_t *code_state = NULL; + #if MICROPY_ENABLE_PYSTACK + code_state = mp_pystack_alloc(sizeof(mp_code_state_t) + state_size); + #else if (state_size > VM_MAX_STATE_ON_STACK) { code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); } @@ -267,6 +274,7 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const code_state = alloca(sizeof(mp_code_state_t) + state_size); state_size = 0; // indicate that we allocated using alloca } + #endif INIT_CODESTATE(code_state, self, n_args, n_kw, args); @@ -312,10 +320,14 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const result = code_state->state[n_state - 1]; } + #if MICROPY_ENABLE_PYSTACK + mp_pystack_free(code_state); + #else // free the state if it was allocated on the heap if (state_size != 0) { m_del_var(mp_code_state_t, byte, state_size, code_state); } + #endif if (vm_return_kind == MP_VM_RETURN_NORMAL) { return result; diff --git a/py/runtime.c b/py/runtime.c index 3a4a8a93b..8df0c0a08 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1348,11 +1348,12 @@ import_error: const char *pkg_name = mp_obj_str_get_data(dest[0], &pkg_name_len); const uint dot_name_len = pkg_name_len + 1 + qstr_len(name); - char *dot_name = alloca(dot_name_len); + char *dot_name = mp_local_alloc(dot_name_len); memcpy(dot_name, pkg_name, pkg_name_len); dot_name[pkg_name_len] = '.'; memcpy(dot_name + pkg_name_len + 1, qstr_str(name), qstr_len(name)); qstr dot_name_q = qstr_from_strn(dot_name, dot_name_len); + mp_local_free(dot_name); mp_obj_t args[5]; args[0] = MP_OBJ_NEW_QSTR(dot_name_q); diff --git a/py/vm.c b/py/vm.c index 5011af5c3..a8a73f323 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1108,7 +1108,13 @@ unwind_return: if (code_state->prev != NULL) { mp_obj_t res = *sp; mp_globals_set(code_state->old_globals); - code_state = code_state->prev; + mp_code_state_t *new_code_state = code_state->prev; + #if MICROPY_ENABLE_PYSTACK + // The sizeof in the following statement does not include the size of the variable + // part of the struct. This arg is anyway not used if pystack is enabled. + mp_nonlocal_free(code_state, sizeof(mp_code_state_t)); + #endif + code_state = new_code_state; *code_state->sp = res; goto run_code_state; } @@ -1450,7 +1456,13 @@ unwind_loop: #if MICROPY_STACKLESS } else if (code_state->prev != NULL) { mp_globals_set(code_state->old_globals); - code_state = code_state->prev; + mp_code_state_t *new_code_state = code_state->prev; + #if MICROPY_ENABLE_PYSTACK + // The sizeof in the following statement does not include the size of the variable + // part of the struct. This arg is anyway not used if pystack is enabled. + mp_nonlocal_free(code_state, sizeof(mp_code_state_t)); + #endif + code_state = new_code_state; size_t n_state = mp_decode_uint_value(code_state->fun_bc->bytecode); fastn = &code_state->state[n_state - 1]; exc_stack = (mp_exc_stack_t*)(code_state->state + n_state); -- cgit v1.2.3 From 30fd8484ebe41faad467fbc8dd4a6f72250f203c Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 26 Nov 2017 23:48:23 +1100 Subject: py/runtime: Use the Python stack when building *arg and **kwarg state. With MICROPY_ENABLE_PYSTACK enabled the following language constructs no longer allocate on the heap: f(*arg), f(**kwarg). --- py/runtime.c | 12 ++++++------ py/vm.c | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 8 deletions(-) (limited to 'py') diff --git a/py/runtime.c b/py/runtime.c index 8df0c0a08..41f2f6976 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -669,7 +669,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ // allocate memory for the new array of args args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len); - args2 = m_new(mp_obj_t, args2_alloc); + args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t)); // copy the self if (self != MP_OBJ_NULL) { @@ -690,7 +690,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ // allocate memory for the new array of args args2_alloc = 1 + n_args + len + 2 * (n_kw + kw_dict_len); - args2 = m_new(mp_obj_t, args2_alloc); + args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t)); // copy the self if (self != MP_OBJ_NULL) { @@ -706,7 +706,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ // allocate memory for the new array of args args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len) + 3; - args2 = m_new(mp_obj_t, args2_alloc); + args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t)); // copy the self if (self != MP_OBJ_NULL) { @@ -723,7 +723,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { if (args2_len >= args2_alloc) { - args2 = m_renew(mp_obj_t, args2, args2_alloc, args2_alloc * 2); + args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), args2_alloc * 2 * sizeof(mp_obj_t)); args2_alloc *= 2; } args2[args2_len++] = item; @@ -774,7 +774,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ if (new_alloc < 4) { new_alloc = 4; } - args2 = m_renew(mp_obj_t, args2, args2_alloc, new_alloc); + args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), new_alloc * sizeof(mp_obj_t)); args2_alloc = new_alloc; } @@ -806,7 +806,7 @@ mp_obj_t mp_call_method_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ob mp_call_prepare_args_n_kw_var(have_self, n_args_n_kw, args, &out_args); mp_obj_t res = mp_call_function_n_kw(out_args.fun, out_args.n_args, out_args.n_kw, out_args.args); - m_del(mp_obj_t, out_args.args, out_args.n_alloc); + mp_nonlocal_free(out_args.args, out_args.n_alloc * sizeof(mp_obj_t)); return res; } diff --git a/py/vm.c b/py/vm.c index a8a73f323..b18a2b5e3 100644 --- a/py/vm.c +++ b/py/vm.c @@ -966,7 +966,11 @@ unwind_jump:; mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(out_args.fun, out_args.n_args, out_args.n_kw, out_args.args); - m_del(mp_obj_t, out_args.args, out_args.n_alloc); + #if !MICROPY_ENABLE_PYSTACK + // Freeing args at this point does not follow a LIFO order so only do it if + // pystack is not enabled. For pystack, they are freed when code_state is. + mp_nonlocal_free(out_args.args, out_args.n_alloc * sizeof(mp_obj_t)); + #endif if (new_state) { new_state->prev = code_state; code_state = new_state; @@ -1043,7 +1047,11 @@ unwind_jump:; mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(out_args.fun, out_args.n_args, out_args.n_kw, out_args.args); - m_del(mp_obj_t, out_args.args, out_args.n_alloc); + #if !MICROPY_ENABLE_PYSTACK + // Freeing args at this point does not follow a LIFO order so only do it if + // pystack is not enabled. For pystack, they are freed when code_state is. + mp_nonlocal_free(out_args.args, out_args.n_alloc * sizeof(mp_obj_t)); + #endif if (new_state) { new_state->prev = code_state; code_state = new_state; @@ -1110,6 +1118,8 @@ unwind_return: mp_globals_set(code_state->old_globals); mp_code_state_t *new_code_state = code_state->prev; #if MICROPY_ENABLE_PYSTACK + // Free code_state, and args allocated by mp_call_prepare_args_n_kw_var + // (The latter is implicitly freed when using pystack due to its LIFO nature.) // The sizeof in the following statement does not include the size of the variable // part of the struct. This arg is anyway not used if pystack is enabled. mp_nonlocal_free(code_state, sizeof(mp_code_state_t)); @@ -1458,6 +1468,8 @@ unwind_loop: mp_globals_set(code_state->old_globals); mp_code_state_t *new_code_state = code_state->prev; #if MICROPY_ENABLE_PYSTACK + // Free code_state, and args allocated by mp_call_prepare_args_n_kw_var + // (The latter is implicitly freed when using pystack due to its LIFO nature.) // The sizeof in the following statement does not include the size of the variable // part of the struct. This arg is anyway not used if pystack is enabled. mp_nonlocal_free(code_state, sizeof(mp_code_state_t)); -- cgit v1.2.3 From 9c027073568d9be43bd1aabcac60a6ff208c4299 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 11 Dec 2017 22:38:30 +1100 Subject: py/objexcept: Use INT_FMT when printing errno value. --- py/objexcept.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/objexcept.c b/py/objexcept.c index b87609a6b..380bc3534 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -118,7 +118,7 @@ STATIC void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_pr if (o->base.type == &mp_type_OSError && MP_OBJ_IS_SMALL_INT(o->args->items[0])) { qstr qst = mp_errno_to_str(o->args->items[0]); if (qst != MP_QSTR_NULL) { - mp_printf(print, "[Errno %d] %q", MP_OBJ_SMALL_INT_VALUE(o->args->items[0]), qst); + mp_printf(print, "[Errno " INT_FMT "] %q", MP_OBJ_SMALL_INT_VALUE(o->args->items[0]), qst); return; } } -- cgit v1.2.3 From 2759bec8587b0c0b7da1fa3a013e4f1b1530ad27 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 11 Dec 2017 22:39:12 +1100 Subject: py: Extend nan-boxing config to have 47-bit small integers. The nan-boxing representation has an extra 16-bits of space to store small-int values, and making use of it allows to create and manipulate full 32-bit positive integers (ie up to 0xffffffff) without using the heap. --- py/mpconfig.h | 2 +- py/obj.h | 4 ++-- py/parse.c | 18 +++++++++++++++--- py/smallint.h | 6 +++--- 4 files changed, 21 insertions(+), 9 deletions(-) (limited to 'py') diff --git a/py/mpconfig.h b/py/mpconfig.h index 96cd8c651..19bae4f88 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -79,7 +79,7 @@ // - seeeeeee eeeeffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff 64-bit fp, e != 0x7ff // - s1111111 11110000 00000000 00000000 00000000 00000000 00000000 00000000 +/- inf // - 01111111 11111000 00000000 00000000 00000000 00000000 00000000 00000000 normalised nan -// - 01111111 11111101 00000000 00000000 iiiiiiii iiiiiiii iiiiiiii iiiiiii1 small int +// - 01111111 11111101 iiiiiiii iiiiiiii iiiiiiii iiiiiiii iiiiiiii iiiiiii1 small int // - 01111111 11111110 00000000 00000000 qqqqqqqq qqqqqqqq qqqqqqqq qqqqqqq1 str // - 01111111 11111100 00000000 00000000 pppppppp pppppppp pppppppp pppppp00 ptr (4 byte alignment) // Stored as O = R + 0x8004000000000000, retrieved as R = O - 0x8004000000000000. diff --git a/py/obj.h b/py/obj.h index 31c3ce95c..10cb48961 100644 --- a/py/obj.h +++ b/py/obj.h @@ -170,8 +170,8 @@ static inline bool MP_OBJ_IS_OBJ(mp_const_obj_t o) static inline bool MP_OBJ_IS_SMALL_INT(mp_const_obj_t o) { return ((((mp_int_t)(o)) & 0xffff000000000000) == 0x0001000000000000); } -#define MP_OBJ_SMALL_INT_VALUE(o) (((intptr_t)(o)) >> 1) -#define MP_OBJ_NEW_SMALL_INT(small_int) ((mp_obj_t)(((uintptr_t)(small_int)) << 1) | 0x0001000000000001) +#define MP_OBJ_SMALL_INT_VALUE(o) (((mp_int_t)((o) << 16)) >> 17) +#define MP_OBJ_NEW_SMALL_INT(small_int) (((((uint64_t)(small_int)) & 0x7fffffffffff) << 1) | 0x0001000000000001) static inline bool MP_OBJ_IS_QSTR(mp_const_obj_t o) { return ((((mp_int_t)(o)) & 0xffff000000000000) == 0x0002000000000000); } diff --git a/py/parse.c b/py/parse.c index f7fe30418..b80cc8fb1 100644 --- a/py/parse.c +++ b/py/parse.c @@ -369,6 +369,18 @@ STATIC mp_parse_node_t make_node_const_object(parser_t *parser, size_t src_line, return (mp_parse_node_t)pn; } +STATIC mp_parse_node_t mp_parse_node_new_small_int_checked(parser_t *parser, mp_obj_t o_val) { + (void)parser; + mp_int_t val = MP_OBJ_SMALL_INT_VALUE(o_val); + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D + // A parse node is only 32-bits and the small-int value must fit in 31-bits + if (((val ^ (val << 1)) & 0xffffffff80000000) != 0) { + return make_node_const_object(parser, 0, o_val); + } + #endif + return mp_parse_node_new_small_int(val); +} + STATIC void push_result_token(parser_t *parser, const rule_t *rule) { mp_parse_node_t pn; mp_lexer_t *lex = parser->lexer; @@ -380,7 +392,7 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) { if (rule->rule_id == RULE_atom && (elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP)) != NULL) { if (MP_OBJ_IS_SMALL_INT(elem->value)) { - pn = mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(elem->value)); + pn = mp_parse_node_new_small_int_checked(parser, elem->value); } else { pn = make_node_const_object(parser, lex->tok_line, elem->value); } @@ -394,7 +406,7 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) { } else if (lex->tok_kind == MP_TOKEN_INTEGER) { mp_obj_t o = mp_parse_num_integer(lex->vstr.buf, lex->vstr.len, 0, lex); if (MP_OBJ_IS_SMALL_INT(o)) { - pn = mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(o)); + pn = mp_parse_node_new_small_int_checked(parser, o); } else { pn = make_node_const_object(parser, lex->tok_line, o); } @@ -686,7 +698,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args pop_result(parser); } if (MP_OBJ_IS_SMALL_INT(arg0)) { - push_result_node(parser, mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(arg0))); + push_result_node(parser, mp_parse_node_new_small_int_checked(parser, arg0)); } else { // TODO reuse memory for parse node struct? push_result_node(parser, make_node_const_object(parser, 0, arg0)); diff --git a/py/smallint.h b/py/smallint.h index 42679a78f..6a3c75236 100644 --- a/py/smallint.h +++ b/py/smallint.h @@ -50,10 +50,10 @@ #elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D -#define MP_SMALL_INT_MIN ((mp_int_t)(((mp_int_t)0xffffffff80000000) >> 1)) -#define MP_SMALL_INT_FITS(n) ((((n) ^ ((n) << 1)) & 0xffffffff80000000) == 0) +#define MP_SMALL_INT_MIN ((mp_int_t)(((mp_int_t)0xffff800000000000) >> 1)) +#define MP_SMALL_INT_FITS(n) ((((n) ^ ((n) << 1)) & 0xffff800000000000) == 0) // Mask to truncate mp_int_t to positive value -#define MP_SMALL_INT_POSITIVE_MASK ~(0xffffffff80000000 | (0xffffffff80000000 >> 1)) +#define MP_SMALL_INT_POSITIVE_MASK ~(0xffff800000000000 | (0xffff800000000000 >> 1)) #endif -- cgit v1.2.3 From d3f82bc42576ccd80206a17a4941fd5c28c56530 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 11 Dec 2017 22:51:52 +1100 Subject: py/mpstate.h: Remove obsolete comment about nlr_top being coded in asm. --- py/mpstate.h | 1 - 1 file changed, 1 deletion(-) (limited to 'py') diff --git a/py/mpstate.h b/py/mpstate.h index d3b53f915..45e89590f 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -215,7 +215,6 @@ typedef struct _mp_state_thread_t { mp_obj_dict_t *dict_locals; mp_obj_dict_t *dict_globals; - // Note: nlr asm code has the offset of this hard-coded nlr_buf_t *nlr_top; // ROOT POINTER // Stack top at the start of program -- cgit v1.2.3 From d32d22dfd79439e07739166eaa3f7e41466c7ec8 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 30 Nov 2017 10:31:42 +1100 Subject: py/objtype: Implement better support for overriding native's __init__. This patch cleans up and generalises part of the code which handles overriding and calling a native base-class's __init__ method. It defers the call to the native make_new() function until after the user (Python) __init__() method has run. That user method now has the chance to call the native __init__/make_new and pass it different arguments. If the user doesn't call the super().__init__ method then it will be called automatically after the user code finishes, to finalise construction of the instance. --- py/objexcept.c | 15 ------------ py/objtype.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 63 insertions(+), 28 deletions(-) (limited to 'py') diff --git a/py/objexcept.c b/py/objexcept.c index 380bc3534..524f105ce 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -212,27 +212,12 @@ STATIC void exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { } } -STATIC mp_obj_t exc___init__(size_t n_args, const mp_obj_t *args) { - mp_obj_exception_t *self = MP_OBJ_TO_PTR(args[0]); - mp_obj_t argst = mp_obj_new_tuple(n_args - 1, args + 1); - self->args = MP_OBJ_TO_PTR(argst); - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(exc___init___obj, 1, MP_OBJ_FUN_ARGS_MAX, exc___init__); - -STATIC const mp_rom_map_elem_t exc_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR___init__), MP_ROM_PTR(&exc___init___obj) }, -}; - -STATIC MP_DEFINE_CONST_DICT(exc_locals_dict, exc_locals_dict_table); - const mp_obj_type_t mp_type_BaseException = { { &mp_type_type }, .name = MP_QSTR_BaseException, .print = mp_obj_exception_print, .make_new = mp_obj_exception_make_new, .attr = exception_attr, - .locals_dict = (mp_obj_dict_t*)&exc_locals_dict, }; #define MP_DEFINE_EXCEPTION(exc_name, base_name) \ diff --git a/py/objtype.c b/py/objtype.c index 267cae815..22c8f01bc 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -46,14 +46,6 @@ STATIC mp_obj_t static_class_method_make_new(const mp_obj_type_t *self_in, size_ /******************************************************************************/ // instance object -STATIC mp_obj_t mp_obj_new_instance(const mp_obj_type_t *class, size_t subobjs) { - mp_obj_instance_t *o = m_new_obj_var(mp_obj_instance_t, mp_obj_t, subobjs); - o->base.type = class; - mp_map_init(&o->members, 0); - mp_seq_clear(o->subobj, 0, subobjs, sizeof(*o->subobj)); - return MP_OBJ_FROM_PTR(o); -} - STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_type_t **last_native_base) { int count = 0; for (;;) { @@ -87,6 +79,30 @@ STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_t } } +// This wrapper function is allows a subclass of a native type to call the +// __init__() method (corresponding to type->make_new) of the native type. +STATIC mp_obj_t native_base_init_wrapper(size_t n_args, const mp_obj_t *args) { + mp_obj_instance_t *self = MP_OBJ_TO_PTR(args[0]); + const mp_obj_type_t *native_base = NULL; + instance_count_native_bases(self->base.type, &native_base); + self->subobj[0] = native_base->make_new(native_base, n_args - 1, 0, args + 1); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(native_base_init_wrapper_obj, 1, MP_OBJ_FUN_ARGS_MAX, native_base_init_wrapper); + +STATIC mp_obj_t mp_obj_new_instance(const mp_obj_type_t *class, size_t subobjs) { + mp_obj_instance_t *o = m_new_obj_var(mp_obj_instance_t, mp_obj_t, subobjs); + o->base.type = class; + mp_map_init(&o->members, 0); + // Initialise the native base-class slot (should be 1 at most) with a valid + // object. It doesn't matter which object, so long as it can be uniquely + // distinguished from a native class that is initialised. + if (subobjs != 0) { + o->subobj[0] = MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj); + } + return MP_OBJ_FROM_PTR(o); +} + // TODO // This implements depth-first left-to-right MRO, which is not compliant with Python3 MRO // http://python-history.blogspot.com/2010/06/method-resolution-order.html @@ -280,7 +296,13 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size if (init_fn[0] == MP_OBJ_SENTINEL) { // Native type's constructor is what wins - it gets all our arguments, // and none Python classes are initialized at all. - o->subobj[0] = native_base->make_new(native_base, n_args, n_kw, args); + + // Since type->make_new() implements both __new__() and __init__() in + // one go, of which the latter may be overridden by the Python subclass, + // we defer (see the end of this function) the call of the native + // constructor to give a chance for the Python __init__() method to call + // said native constructor. + } else if (init_fn[0] != MP_OBJ_NULL) { // now call Python class __new__ function with all args if (n_args == 0 && n_kw == 0) { @@ -305,6 +327,8 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size o = MP_OBJ_TO_PTR(new_ret); // now call Python class __init__ function with all args + // This method has a chance to call super().__init__() to construct a + // possible native base class. init_fn[0] = init_fn[1] = MP_OBJ_NULL; lookup.obj = o; lookup.attr = MP_QSTR___init__; @@ -333,6 +357,12 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size } + // If the type had a native base that was not explicitly initialised + // (constructed) by the Python __init__() method then construct it now. + if (native_base != NULL && o->subobj[0] == MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj)) { + o->subobj[0] = native_base->make_new(native_base, n_args, n_kw, args); + } + return MP_OBJ_FROM_PTR(o); } @@ -1107,6 +1137,11 @@ STATIC void super_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { .is_type = false, }; + // Allow a call super().__init__() to reach any native base classes + if (attr == MP_QSTR___init__) { + lookup.meth_offset = offsetof(mp_obj_type_t, make_new); + } + if (type->parent == NULL) { // no parents, do nothing #if MICROPY_MULTIPLE_INHERITANCE @@ -1116,19 +1151,34 @@ STATIC void super_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { const mp_obj_t *items = parent_tuple->items; for (size_t i = 0; i < len; i++) { assert(MP_OBJ_IS_TYPE(items[i], &mp_type_type)); + if (MP_OBJ_TO_PTR(items[i]) == &mp_type_object) { + // The "object" type will be searched at the end of this function, + // and we don't want to lookup native methods in object. + continue; + } mp_obj_class_lookup(&lookup, (mp_obj_type_t*)MP_OBJ_TO_PTR(items[i])); if (dest[0] != MP_OBJ_NULL) { - return; + break; } } #endif - } else { + } else if (type->parent != &mp_type_object) { mp_obj_class_lookup(&lookup, type->parent); - if (dest[0] != MP_OBJ_NULL) { - return; + } + + if (dest[0] != MP_OBJ_NULL) { + if (dest[0] == MP_OBJ_SENTINEL) { + // Looked up native __init__ so defer to it + dest[0] = MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj); + dest[1] = self->obj; } + return; } + // Reset meth_offset so we don't look up any native methods in object, + // because object never takes up the native base-class slot. + lookup.meth_offset = 0; + mp_obj_class_lookup(&lookup, &mp_type_object); } -- cgit v1.2.3 From c78ef92d787d7bab8acbec69e978037ec2b20d21 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 12 Dec 2017 15:22:03 +1100 Subject: py/objtype: Refactor object's handling of __new__ to not create 2 objs. Before this patch, if a user defined the __new__() function for a class then two instances of that class would be created: once before __new__ is called and once during the __new__ call (assuming the user creates some instance, eg using super().__new__, which is most of the time). The first one was then discarded. This refactor makes it so that a new instance is only created if the user __new__ function doesn't exist. --- py/objobject.c | 9 ++++++--- py/objtype.c | 57 ++++++++++++++++++++++++++------------------------------- py/objtype.h | 5 +++++ 3 files changed, 37 insertions(+), 34 deletions(-) (limited to 'py') diff --git a/py/objobject.c b/py/objobject.c index 49d2ec62e..265fcfbf2 100644 --- a/py/objobject.c +++ b/py/objobject.c @@ -52,9 +52,12 @@ STATIC mp_obj_t object___new__(mp_obj_t cls) { if (!MP_OBJ_IS_TYPE(cls, &mp_type_type) || !mp_obj_is_instance_type((mp_obj_type_t*)MP_OBJ_TO_PTR(cls))) { mp_raise_TypeError("__new__ arg must be a user-type"); } - mp_obj_t o = MP_OBJ_SENTINEL; - mp_obj_t res = mp_obj_instance_make_new(MP_OBJ_TO_PTR(cls), 1, 0, &o); - return res; + // This executes only "__new__" part of instance creation. + // TODO: This won't work well for classes with native bases. + // TODO: This is a hack, should be resolved along the lines of + // https://github.com/micropython/micropython/issues/606#issuecomment-43685883 + const mp_obj_type_t *native_base; + return MP_OBJ_FROM_PTR(mp_obj_new_instance(MP_OBJ_TO_PTR(cls), &native_base)); } STATIC MP_DEFINE_CONST_FUN_OBJ_1(object___new___fun_obj, object___new__); STATIC MP_DEFINE_CONST_STATICMETHOD_OBJ(object___new___obj, MP_ROM_PTR(&object___new___fun_obj)); diff --git a/py/objtype.c b/py/objtype.c index 22c8f01bc..33757287a 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -90,17 +90,22 @@ STATIC mp_obj_t native_base_init_wrapper(size_t n_args, const mp_obj_t *args) { } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(native_base_init_wrapper_obj, 1, MP_OBJ_FUN_ARGS_MAX, native_base_init_wrapper); -STATIC mp_obj_t mp_obj_new_instance(const mp_obj_type_t *class, size_t subobjs) { - mp_obj_instance_t *o = m_new_obj_var(mp_obj_instance_t, mp_obj_t, subobjs); +#if !MICROPY_CPYTHON_COMPAT +STATIC +#endif +mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *class, const mp_obj_type_t **native_base) { + size_t num_native_bases = instance_count_native_bases(class, native_base); + assert(num_native_bases < 2); + mp_obj_instance_t *o = m_new_obj_var(mp_obj_instance_t, mp_obj_t, num_native_bases); o->base.type = class; mp_map_init(&o->members, 0); // Initialise the native base-class slot (should be 1 at most) with a valid // object. It doesn't matter which object, so long as it can be uniquely // distinguished from a native class that is initialised. - if (subobjs != 0) { + if (num_native_bases != 0) { o->subobj[0] = MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj); } - return MP_OBJ_FROM_PTR(o); + return o; } // TODO @@ -267,20 +272,6 @@ STATIC void instance_print(const mp_print_t *print, mp_obj_t self_in, mp_print_k mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size_t n_kw, const mp_obj_t *args) { assert(mp_obj_is_instance_type(self)); - const mp_obj_type_t *native_base = NULL; - size_t num_native_bases = instance_count_native_bases(self, &native_base); - assert(num_native_bases < 2); - - mp_obj_instance_t *o = MP_OBJ_TO_PTR(mp_obj_new_instance(self, num_native_bases)); - - // This executes only "__new__" part of instance creation. - // TODO: This won't work well for classes with native bases. - // TODO: This is a hack, should be resolved along the lines of - // https://github.com/micropython/micropython/issues/606#issuecomment-43685883 - if (n_args == 1 && *args == MP_OBJ_SENTINEL) { - return MP_OBJ_FROM_PTR(o); - } - // look for __new__ function mp_obj_t init_fn[2] = {MP_OBJ_NULL}; struct class_lookup_data lookup = { @@ -292,10 +283,12 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size }; mp_obj_class_lookup(&lookup, self); - mp_obj_t new_ret = MP_OBJ_FROM_PTR(o); - if (init_fn[0] == MP_OBJ_SENTINEL) { - // Native type's constructor is what wins - it gets all our arguments, - // and none Python classes are initialized at all. + const mp_obj_type_t *native_base = NULL; + mp_obj_instance_t *o; + if (init_fn[0] == MP_OBJ_NULL || init_fn[0] == MP_OBJ_SENTINEL) { + // Either there is no __new__() method defined or there is a native + // constructor. In both cases create a blank instance. + o = mp_obj_new_instance(self, &native_base); // Since type->make_new() implements both __new__() and __init__() in // one go, of which the latter may be overridden by the Python subclass, @@ -303,8 +296,9 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size // constructor to give a chance for the Python __init__() method to call // said native constructor. - } else if (init_fn[0] != MP_OBJ_NULL) { - // now call Python class __new__ function with all args + } else { + // Call Python class __new__ function with all args to create an instance + mp_obj_t new_ret; if (n_args == 0 && n_kw == 0) { mp_obj_t args2[1] = {MP_OBJ_FROM_PTR(self)}; new_ret = mp_call_function_n_kw(init_fn[0], 1, 0, args2); @@ -316,16 +310,17 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size m_del(mp_obj_t, args2, 1 + n_args + 2 * n_kw); } - } + // https://docs.python.org/3.4/reference/datamodel.html#object.__new__ + // "If __new__() does not return an instance of cls, then the new + // instance's __init__() method will not be invoked." + if (mp_obj_get_type(new_ret) != self) { + return new_ret; + } - // https://docs.python.org/3.4/reference/datamodel.html#object.__new__ - // "If __new__() does not return an instance of cls, then the new instance's __init__() method will not be invoked." - if (mp_obj_get_type(new_ret) != self) { - return new_ret; + // The instance returned by __new__() becomes the new object + o = MP_OBJ_TO_PTR(new_ret); } - o = MP_OBJ_TO_PTR(new_ret); - // now call Python class __init__ function with all args // This method has a chance to call super().__init__() to construct a // possible native base class. diff --git a/py/objtype.h b/py/objtype.h index 52419f3cd..1f4313084 100644 --- a/py/objtype.h +++ b/py/objtype.h @@ -37,6 +37,11 @@ typedef struct _mp_obj_instance_t { // TODO maybe cache __getattr__ and __setattr__ for efficient lookup of them } mp_obj_instance_t; +#if MICROPY_CPYTHON_COMPAT +// this is needed for object.__new__ +mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *cls, const mp_obj_type_t **native_base); +#endif + // this needs to be exposed for MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE to work void mp_obj_instance_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); -- cgit v1.2.3 From f1c9e7760d91cb141b0f03ee348b3fe36d2cf450 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 6 Mar 2017 16:46:34 +1100 Subject: py/builtinimport: Call __init__ for modules imported via a weak link. This is a bit of a clumsy way of doing it but solves the issue of __init__ not running when a module is imported via its weak-link name. Ideally a better solution would be found. --- py/builtinimport.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'py') diff --git a/py/builtinimport.c b/py/builtinimport.c index 2157902c9..97c1789f8 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -389,6 +389,19 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { } // found weak linked module module_obj = el->value; + if (MICROPY_MODULE_BUILTIN_INIT) { + // look for __init__ and call it if it exists + // Note: this code doesn't work fully correctly because it allows the + // __init__ function to be called twice if the module is imported by its + // non-weak-link name. Also, this code is duplicated in objmodule.c. + mp_obj_t dest[2]; + mp_load_method_maybe(el->value, MP_QSTR___init__, dest); + if (dest[0] != MP_OBJ_NULL) { + mp_call_method_n_kw(0, 0, dest); + // register module so __init__ is not called again + mp_module_register(mod_name, el->value); + } + } } else { no_exist: #else -- cgit v1.2.3 From cf8e8c29e72ef4871b9d5ab3de32bdaf429c5dbb Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 15 Dec 2017 10:21:10 +1100 Subject: py/emitglue: Change type of bit-field to explicitly unsigned mp_uint_t. Some compilers can treat enum types as signed, in which case 3 bits is not enough to encode all mp_raw_code_kind_t values. So change the type to mp_uint_t. --- py/emitglue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/emitglue.h b/py/emitglue.h index 43930333d..f2a48c5e5 100644 --- a/py/emitglue.h +++ b/py/emitglue.h @@ -40,7 +40,7 @@ typedef enum { } mp_raw_code_kind_t; typedef struct _mp_raw_code_t { - mp_raw_code_kind_t kind : 3; + mp_uint_t kind : 3; // of type mp_raw_code_kind_t mp_uint_t scope_flags : 7; mp_uint_t n_pos_args : 11; union { -- cgit v1.2.3 From 63644016669c173190c71d39fb33ad9502fe2b0f Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 21 Oct 2017 12:13:44 +0300 Subject: py/objgenerator: Allow to pend an exception for next execution. This implements .pend_throw(exc) method, which sets up an exception to be triggered on the next call to generator's .__next__() or .send() method. This is unlike .throw(), which immediately starts to execute the generator to process the exception. This effectively adds Future-like capabilities to generator protocol (exception will be raised in the future). The need for such a method arised to implement uasyncio wait_for() function efficiently (its behavior is clearly "Future" like, and normally would require to introduce an expensive Future wrapper around all native couroutines, like upstream asyncio does). py/objgenerator: pend_throw: Return previous pended value. This effectively allows to store an additional value (not necessary an exception) in a coroutine while it's not being executed. uasyncio has exactly this usecase: to mark a coro waiting in I/O queue (and thus not executed in the normal scheduling queue), for the purpose of implementing wait_for() function (cancellation of such waiting coro by a timeout). --- py/mpconfig.h | 9 +++++++++ py/objgenerator.c | 30 ++++++++++++++++++++++++++++-- tests/basics/generator_pend_throw.py | 26 ++++++++++++++++++++++++++ tests/basics/generator_pend_throw.py.exp | 4 ++++ tests/run-tests | 2 +- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 tests/basics/generator_pend_throw.py create mode 100644 tests/basics/generator_pend_throw.py.exp (limited to 'py') diff --git a/py/mpconfig.h b/py/mpconfig.h index 19bae4f88..763bb378e 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -706,6 +706,15 @@ typedef double mp_float_t; #define MICROPY_PY_ASYNC_AWAIT (1) #endif +// Non-standard .pend_throw() method for generators, allowing for +// Future-like behavior with respect to exception handling: an +// exception set with .pend_throw() will activate on the next call +// to generator's .send() or .__next__(). (This is useful to implement +// async schedulers.) +#ifndef MICROPY_PY_GENERATOR_PEND_THROW +#define MICROPY_PY_GENERATOR_PEND_THROW (1) +#endif + // Issue a warning when comparing str and bytes objects #ifndef MICROPY_PY_STR_BYTES_CMP_WARN #define MICROPY_PY_STR_BYTES_CMP_WARN (0) diff --git a/py/objgenerator.c b/py/objgenerator.c index 9a294debb..8c1260b60 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -4,7 +4,7 @@ * The MIT License (MIT) * * Copyright (c) 2013, 2014 Damien P. George - * Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2014-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -104,7 +104,16 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ mp_raise_TypeError("can't send non-None value to a just-started generator"); } } else { - *self->code_state.sp = send_value; + #if MICROPY_PY_GENERATOR_PEND_THROW + // If exception is pending (set using .pend_throw()), process it now. + if (*self->code_state.sp != mp_const_none) { + throw_value = *self->code_state.sp; + *self->code_state.sp = MP_OBJ_NULL; + } else + #endif + { + *self->code_state.sp = send_value; + } } mp_obj_dict_t *old_globals = mp_globals_get(); mp_globals_set(self->globals); @@ -125,6 +134,9 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ case MP_VM_RETURN_YIELD: *ret_val = *self->code_state.sp; + #if MICROPY_PY_GENERATOR_PEND_THROW + *self->code_state.sp = mp_const_none; + #endif break; case MP_VM_RETURN_EXCEPTION: { @@ -219,10 +231,24 @@ STATIC mp_obj_t gen_instance_close(mp_obj_t self_in) { STATIC MP_DEFINE_CONST_FUN_OBJ_1(gen_instance_close_obj, gen_instance_close); +STATIC mp_obj_t gen_instance_pend_throw(mp_obj_t self_in, mp_obj_t exc_in) { + mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in); + if (self->code_state.sp == self->code_state.state - 1) { + mp_raise_TypeError("can't pend throw to just-started generator"); + } + mp_obj_t prev = *self->code_state.sp; + *self->code_state.sp = exc_in; + return prev; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(gen_instance_pend_throw_obj, gen_instance_pend_throw); + STATIC const mp_rom_map_elem_t gen_instance_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&gen_instance_close_obj) }, { MP_ROM_QSTR(MP_QSTR_send), MP_ROM_PTR(&gen_instance_send_obj) }, { MP_ROM_QSTR(MP_QSTR_throw), MP_ROM_PTR(&gen_instance_throw_obj) }, + #if MICROPY_PY_GENERATOR_PEND_THROW + { MP_ROM_QSTR(MP_QSTR_pend_throw), MP_ROM_PTR(&gen_instance_pend_throw_obj) }, + #endif }; STATIC MP_DEFINE_CONST_DICT(gen_instance_locals_dict, gen_instance_locals_dict_table); diff --git a/tests/basics/generator_pend_throw.py b/tests/basics/generator_pend_throw.py new file mode 100644 index 000000000..949655612 --- /dev/null +++ b/tests/basics/generator_pend_throw.py @@ -0,0 +1,26 @@ +def gen(): + i = 0 + while 1: + yield i + i += 1 + +g = gen() + +try: + g.pend_throw +except AttributeError: + print("SKIP") + raise SystemExit + + +print(next(g)) +print(next(g)) +g.pend_throw(ValueError()) + +v = None +try: + v = next(g) +except Exception as e: + print("raised", repr(e)) + +print("ret was:", v) diff --git a/tests/basics/generator_pend_throw.py.exp b/tests/basics/generator_pend_throw.py.exp new file mode 100644 index 000000000..f9894a089 --- /dev/null +++ b/tests/basics/generator_pend_throw.py.exp @@ -0,0 +1,4 @@ +0 +1 +raised ValueError() +ret was: None diff --git a/tests/run-tests b/tests/run-tests index 45cb1a746..8719befbd 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -337,7 +337,7 @@ def run_tests(pyb, tests, args, base_path="."): # Some tests are known to fail with native emitter # Remove them from the below when they work if args.emit == 'native': - skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_return generator_send'.split()}) # require yield + skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_pend_throw generator_return generator_send'.split()}) # require yield skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class globals_del string_join'.split()}) # require yield skip_tests.update({'basics/async_%s.py' % t for t in 'def await await2 for for2 with with2'.split()}) # require yield skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs -- cgit v1.2.3 From f5fb68e94fe72a7d8c158a47c2374ee0cea0227c Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Dec 2017 13:13:21 +1100 Subject: py/runtime: Remove unnecessary break statements from switch. --- py/runtime.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'py') diff --git a/py/runtime.c b/py/runtime.c index 41f2f6976..9dff9847a 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -413,7 +413,6 @@ mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { // use standard precision return MP_OBJ_NEW_SMALL_INT(lhs_val * rhs_val); } - break; } case MP_BINARY_OP_FLOOR_DIVIDE: case MP_BINARY_OP_INPLACE_FLOOR_DIVIDE: @@ -488,10 +487,10 @@ mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { return MP_OBJ_FROM_PTR(tuple); } - case MP_BINARY_OP_LESS: return mp_obj_new_bool(lhs_val < rhs_val); break; - case MP_BINARY_OP_MORE: return mp_obj_new_bool(lhs_val > rhs_val); break; - case MP_BINARY_OP_LESS_EQUAL: return mp_obj_new_bool(lhs_val <= rhs_val); break; - case MP_BINARY_OP_MORE_EQUAL: return mp_obj_new_bool(lhs_val >= rhs_val); break; + case MP_BINARY_OP_LESS: return mp_obj_new_bool(lhs_val < rhs_val); + case MP_BINARY_OP_MORE: return mp_obj_new_bool(lhs_val > rhs_val); + case MP_BINARY_OP_LESS_EQUAL: return mp_obj_new_bool(lhs_val <= rhs_val); + case MP_BINARY_OP_MORE_EQUAL: return mp_obj_new_bool(lhs_val >= rhs_val); default: goto unsupported_op; -- cgit v1.2.3 From 136cb7f27c27ee128d3adab9850b74f6129c8732 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Dec 2017 13:37:15 +1100 Subject: py/map: Don't include ordered-dict mutating code when not needed. --- py/map.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'py') diff --git a/py/map.c b/py/map.c index 696c7a2ea..6abf4853f 100644 --- a/py/map.c +++ b/py/map.c @@ -170,6 +170,7 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t if (map->is_ordered) { for (mp_map_elem_t *elem = &map->table[0], *top = &map->table[map->used]; elem < top; elem++) { if (elem->key == index || (!compare_only_ptrs && mp_obj_equal(elem->key, index))) { + #if MICROPY_PY_COLLECTIONS_ORDEREDDICT if (MP_UNLIKELY(lookup_kind == MP_MAP_LOOKUP_REMOVE_IF_FOUND)) { // remove the found element by moving the rest of the array down mp_obj_t value = elem->value; @@ -180,9 +181,11 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t elem->key = MP_OBJ_NULL; elem->value = value; } + #endif return elem; } } + #if MICROPY_PY_COLLECTIONS_ORDEREDDICT if (MP_LIKELY(lookup_kind != MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)) { return NULL; } @@ -198,6 +201,9 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t map->all_keys_are_qstrs = 0; } return elem; + #else + return NULL; + #endif } // map is a hash table (not an ordered array), so do a hash lookup -- cgit v1.2.3 From 7db79d8b031e69392432ace687c9fcfdd2c56d4c Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Dec 2017 14:01:19 +1100 Subject: py/objset: Remove unneeded check from set_equal. set_equal is called only from set_binary_op, and this guarantees that the second arg to set_equal is always a set or frozenset. So there is no need to do a further check. --- py/objset.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'py') diff --git a/py/objset.c b/py/objset.c index 3e98c30e8..799ba9df0 100644 --- a/py/objset.c +++ b/py/objset.c @@ -351,11 +351,9 @@ STATIC mp_obj_t set_issuperset_proper(mp_obj_t self_in, mp_obj_t other_in) { } STATIC mp_obj_t set_equal(mp_obj_t self_in, mp_obj_t other_in) { + assert(is_set_or_frozenset(other_in)); check_set_or_frozenset(self_in); mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in); - if (!is_set_or_frozenset(other_in)) { - return mp_const_false; - } mp_obj_set_t *other = MP_OBJ_TO_PTR(other_in); if (self->set.used != other->set.used) { return mp_const_false; -- cgit v1.2.3 From 374eaf5271c9bfc63b5aa1139de55753ad67cc9a Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Dec 2017 15:42:58 +1100 Subject: py/mpz: Fix pow3 function so it handles the case when 3rd arg is 1. In this case the result should always be 0, even if 2nd arg is 0. --- py/mpz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/mpz.c b/py/mpz.c index d300a8e5d..16112c201 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -1390,7 +1390,7 @@ void mpz_pow_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs) { can have dest, lhs, rhs the same; mod can't be the same as dest */ void mpz_pow3_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs, const mpz_t *mod) { - if (lhs->len == 0 || rhs->neg != 0) { + if (lhs->len == 0 || rhs->neg != 0 || (mod->len == 1 && mod->dig[0] == 1)) { mpz_set_from_int(dest, 0); return; } -- cgit v1.2.3 From ae1be76d4063997b2703ba1ac9d009b31bc06eca Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Dec 2017 15:45:56 +1100 Subject: py/mpz: Apply a small code-size optimisation. --- py/mpz.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/mpz.c b/py/mpz.c index 16112c201..018a5454f 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -1395,8 +1395,9 @@ void mpz_pow3_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs, const mpz_t return; } + mpz_set_from_int(dest, 1); + if (rhs->len == 0) { - mpz_set_from_int(dest, 1); return; } @@ -1404,8 +1405,6 @@ void mpz_pow3_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs, const mpz_t mpz_t *n = mpz_clone(rhs); mpz_t quo; mpz_init_zero(&quo); - mpz_set_from_int(dest, 1); - while (n->len > 0) { if ((n->dig[0] & 1) != 0) { mpz_mul_inpl(dest, dest, x); -- cgit v1.2.3 From 304a3bcc1cf4f272b62a6cd8f14583db84d84189 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Dec 2017 16:59:08 +1100 Subject: py/modio: Use correct config macro to enable resource_stream function. --- py/modio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/modio.c b/py/modio.c index 828bcec46..3a5c69c4c 100644 --- a/py/modio.c +++ b/py/modio.c @@ -131,7 +131,7 @@ STATIC const mp_obj_type_t bufwriter_type = { }; #endif // MICROPY_PY_IO_BUFFEREDWRITER -#if MICROPY_MODULE_FROZEN_STR +#if MICROPY_PY_IO_RESOURCE_STREAM STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) { VSTR_FIXED(path_buf, MICROPY_ALLOC_PATH_MAX); size_t len; @@ -179,7 +179,7 @@ STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) { mp_obj_t path_out = mp_obj_new_str(path_buf.buf, path_buf.len); return mp_builtin_open(1, &path_out, (mp_map_t*)&mp_const_empty_map); } -MP_DEFINE_CONST_FUN_OBJ_2(resource_stream_obj, resource_stream); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(resource_stream_obj, resource_stream); #endif STATIC const mp_rom_map_elem_t mp_module_io_globals_table[] = { -- cgit v1.2.3 From 6a3a742a6c9caaa2be0fd0aac7a5df4ac816081c Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 18 Dec 2017 18:57:15 +1100 Subject: py/nlr: Factor out common NLR code to generic functions. Each NLR implementation (Thumb, x86, x64, xtensa, setjmp) duplicates a lot of the NLR code, specifically that dealing with pushing and popping the NLR pointer to maintain the linked-list of NLR buffers. This patch factors all of that code out of the specific implementations into generic functions in nlr.c. This eliminates duplicated code. The factoring also allows to make the machine-specific NLR code pure assembler code, thus allowing nlrthumb.c to use naked function attributes in the correct way (naked functions can only have basic inline assembler code in them). There is a small overhead introduced (typically 1 machine instruction) because now the generic nlr_jump() must call nlr_jump_tail() rather than them being one combined function. --- py/nlr.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ py/nlr.h | 71 +++++++++++++++++++++++++++---------------------------- py/nlrsetjmp.c | 10 +------- py/nlrthumb.c | 38 +++--------------------------- py/nlrx64.c | 70 +++++++++++++++++++++--------------------------------- py/nlrx86.c | 63 +++++++++++-------------------------------------- py/nlrxtensa.c | 34 ++++----------------------- py/py.mk | 1 + 8 files changed, 157 insertions(+), 204 deletions(-) create mode 100644 py/nlr.c (limited to 'py') diff --git a/py/nlr.c b/py/nlr.c new file mode 100644 index 000000000..e4d6d10c0 --- /dev/null +++ b/py/nlr.c @@ -0,0 +1,74 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/mpstate.h" + +// Helper macros to save/restore the pystack state +#if MICROPY_ENABLE_PYSTACK +#define MP_NLR_SAVE_PYSTACK(nlr_buf) (nlr_buf)->pystack = MP_STATE_THREAD(pystack_cur) +#define MP_NLR_RESTORE_PYSTACK(nlr_buf) MP_STATE_THREAD(pystack_cur) = (nlr_buf)->pystack +#else +#define MP_NLR_SAVE_PYSTACK(nlr_buf) (void)nlr_buf +#define MP_NLR_RESTORE_PYSTACK(nlr_buf) (void)nlr_buf +#endif + +#if !MICROPY_NLR_SETJMP +// When not using setjmp, nlr_push_tail is called from inline asm so needs special care +#if MICROPY_NLR_X86 && (defined(_WIN32) || defined(__CYGWIN__)) +// On these 32-bit platforms make sure nlr_push_tail doesn't have a leading underscore +unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); +#else +// LTO can't see inside inline asm functions so explicitly mark nlr_push_tail as used +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); +#endif +#endif + +unsigned int nlr_push_tail(nlr_buf_t *nlr) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); + *top = nlr; + return 0; // normal return +} + +void nlr_pop(void) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + *top = (*top)->prev; +} + +NORETURN void nlr_jump(void *val) { + nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); + nlr_buf_t *top = *top_ptr; + if (top == NULL) { + nlr_jump_fail(val); + } + + top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); + *top_ptr = top->prev; + + nlr_jump_tail(top); +} diff --git a/py/nlr.h b/py/nlr.h index 1235f1460..012a73c31 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -30,29 +30,29 @@ // exception handling, basically a stack of setjmp/longjmp buffers #include -#include #include #include "py/mpconfig.h" -typedef struct _nlr_buf_t nlr_buf_t; -struct _nlr_buf_t { - // the entries here must all be machine word size - nlr_buf_t *prev; - void *ret_val; // always a concrete object (an exception instance) -#if !defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP +// If MICROPY_NLR_SETJMP is not enabled then auto-detect the machine arch +// Allow a port to set MICROPY_NLR_NUM_REGS to define their own implementation +#if !MICROPY_NLR_SETJMP && !defined(MICROPY_NLR_NUM_REGS) #if defined(__i386__) - void *regs[6]; + #define MICROPY_NLR_X86 (1) + #define MICROPY_NLR_NUM_REGS (6) #elif defined(__x86_64__) - #if defined(__CYGWIN__) - void *regs[12]; - #else - void *regs[8]; - #endif + #define MICROPY_NLR_X64 (1) + #if defined(__CYGWIN__) + #define MICROPY_NLR_NUM_REGS (12) + #else + #define MICROPY_NLR_NUM_REGS (8) + #endif #elif defined(__thumb2__) || defined(__thumb__) || defined(__arm__) - void *regs[10]; + #define MICROPY_NLR_THUMB (1) + #define MICROPY_NLR_NUM_REGS (10) #elif defined(__xtensa__) - void *regs[10]; + #define MICROPY_NLR_XTENSA (1) + #define MICROPY_NLR_NUM_REGS (10) #else #define MICROPY_NLR_SETJMP (1) //#warning "No native NLR support for this arch, using setjmp implementation" @@ -60,41 +60,39 @@ struct _nlr_buf_t { #endif #if MICROPY_NLR_SETJMP - jmp_buf jmpbuf; +#include #endif +typedef struct _nlr_buf_t nlr_buf_t; +struct _nlr_buf_t { + // the entries here must all be machine word size + nlr_buf_t *prev; + void *ret_val; // always a concrete object (an exception instance) + + #if MICROPY_NLR_SETJMP + jmp_buf jmpbuf; + #else + void *regs[MICROPY_NLR_NUM_REGS]; + #endif + #if MICROPY_ENABLE_PYSTACK void *pystack; #endif }; -// Helper macros to save/restore the pystack state -#if MICROPY_ENABLE_PYSTACK -#define MP_NLR_SAVE_PYSTACK(nlr_buf) (nlr_buf)->pystack = MP_STATE_THREAD(pystack_cur) -#define MP_NLR_RESTORE_PYSTACK(nlr_buf) MP_STATE_THREAD(pystack_cur) = (nlr_buf)->pystack -#else -#define MP_NLR_SAVE_PYSTACK(nlr_buf) (void)nlr_buf -#define MP_NLR_RESTORE_PYSTACK(nlr_buf) (void)nlr_buf -#endif - #if MICROPY_NLR_SETJMP -#include "py/mpstate.h" - -NORETURN void nlr_setjmp_jump(void *val); // nlr_push() must be defined as a macro, because "The stack context will be // invalidated if the function which called setjmp() returns." -#define nlr_push(buf) ( \ - (buf)->prev = MP_STATE_THREAD(nlr_top), \ - MP_NLR_SAVE_PYSTACK(buf), \ - MP_STATE_THREAD(nlr_top) = (buf), \ - setjmp((buf)->jmpbuf)) -#define nlr_pop() { MP_STATE_THREAD(nlr_top) = MP_STATE_THREAD(nlr_top)->prev; } -#define nlr_jump(val) nlr_setjmp_jump(val) +// For this case it is safe to call nlr_push_tail() first. +#define nlr_push(buf) (nlr_push_tail(buf), setjmp((buf)->jmpbuf)) #else unsigned int nlr_push(nlr_buf_t *); +#endif + +unsigned int nlr_push_tail(nlr_buf_t *top); void nlr_pop(void); NORETURN void nlr_jump(void *val); -#endif +NORETURN void nlr_jump_tail(nlr_buf_t *top); // This must be implemented by a port. It's called by nlr_jump // if no nlr buf has been pushed. It must not return, but rather @@ -123,7 +121,6 @@ NORETURN void nlr_jump_fail(void *val); /* #define nlr_push(val) \ printf("nlr_push: before: nlr_top=%p, val=%p\n", MP_STATE_THREAD(nlr_top), val),assert(MP_STATE_THREAD(nlr_top) != val),nlr_push(val) -#endif */ #endif diff --git a/py/nlrsetjmp.c b/py/nlrsetjmp.c index 63376a553..6ddad3793 100644 --- a/py/nlrsetjmp.c +++ b/py/nlrsetjmp.c @@ -28,15 +28,7 @@ #if MICROPY_NLR_SETJMP -void nlr_setjmp_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; +NORETURN void nlr_jump_tail(nlr_buf_t *top) { longjmp(top->jmpbuf, 1); } diff --git a/py/nlrthumb.c b/py/nlrthumb.c index eab5759f2..4edd1456d 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) +#if MICROPY_NLR_THUMB #undef nlr_push @@ -37,7 +37,6 @@ // r4-r11, r13=sp __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { - __asm volatile ( "str r4, [r0, #12] \n" // store r4 into nlr_buf "str r5, [r0, #16] \n" // store r5 into nlr_buf @@ -75,36 +74,10 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { "b nlr_push_tail \n" // do the rest in C #endif ); - - return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - -NORETURN __attribute__((naked)) void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; - +NORETURN __attribute__((naked)) void nlr_jump_tail(nlr_buf_t *top) { __asm volatile ( - "mov r0, %0 \n" // r0 points to nlr_buf "ldr r4, [r0, #12] \n" // load r4 from nlr_buf "ldr r5, [r0, #16] \n" // load r5 from nlr_buf "ldr r6, [r0, #20] \n" // load r6 from nlr_buf @@ -133,12 +106,7 @@ NORETURN __attribute__((naked)) void nlr_jump(void *val) { #endif "movs r0, #1 \n" // return 1, non-local return "bx lr \n" // return - : // output operands - : "r"(top) // input operands - : // clobbered registers ); - - for (;;); // needed to silence compiler warning } -#endif // (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) +#endif // MICROPY_NLR_THUMB diff --git a/py/nlrx64.c b/py/nlrx64.c index ddcd76166..e7e2e1474 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__x86_64__) +#if MICROPY_NLR_X64 #undef nlr_push @@ -39,8 +39,6 @@ #define NLR_OS_WINDOWS 0 #endif -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); - unsigned int nlr_push(nlr_buf_t *nlr) { (void)nlr; @@ -88,54 +86,38 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - -NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; +NORETURN void nlr_jump_tail(nlr_buf_t *top) { + (void)top; __asm volatile ( - "movq %0, %%rcx \n" // %rcx points to nlr_buf #if NLR_OS_WINDOWS - "movq 88(%%rcx), %%rsi \n" // load saved %rsi - "movq 80(%%rcx), %%rdi \n" // load saved %rdr + "movq 88(%rcx), %rsi \n" // load saved %rsi + "movq 80(%rcx), %rdi \n" // load saved %rdr + "movq 72(%rcx), %r15 \n" // load saved %r15 + "movq 64(%rcx), %r14 \n" // load saved %r14 + "movq 56(%rcx), %r13 \n" // load saved %r13 + "movq 48(%rcx), %r12 \n" // load saved %r12 + "movq 40(%rcx), %rbx \n" // load saved %rbx + "movq 32(%rcx), %rsp \n" // load saved %rsp + "movq 24(%rcx), %rbp \n" // load saved %rbp + "movq 16(%rcx), %rax \n" // load saved %rip + #else + "movq 72(%rdi), %r15 \n" // load saved %r15 + "movq 64(%rdi), %r14 \n" // load saved %r14 + "movq 56(%rdi), %r13 \n" // load saved %r13 + "movq 48(%rdi), %r12 \n" // load saved %r12 + "movq 40(%rdi), %rbx \n" // load saved %rbx + "movq 32(%rdi), %rsp \n" // load saved %rsp + "movq 24(%rdi), %rbp \n" // load saved %rbp + "movq 16(%rdi), %rax \n" // load saved %rip #endif - "movq 72(%%rcx), %%r15 \n" // load saved %r15 - "movq 64(%%rcx), %%r14 \n" // load saved %r14 - "movq 56(%%rcx), %%r13 \n" // load saved %r13 - "movq 48(%%rcx), %%r12 \n" // load saved %r12 - "movq 40(%%rcx), %%rbx \n" // load saved %rbx - "movq 32(%%rcx), %%rsp \n" // load saved %rsp - "movq 24(%%rcx), %%rbp \n" // load saved %rbp - "movq 16(%%rcx), %%rax \n" // load saved %rip - "movq %%rax, (%%rsp) \n" // store saved %rip to stack - "xorq %%rax, %%rax \n" // clear return register - "inc %%al \n" // increase to make 1, non-local return + "movq %rax, (%rsp) \n" // store saved %rip to stack + "xorq %rax, %rax \n" // clear return register + "inc %al \n" // increase to make 1, non-local return "ret \n" // return - : // output operands - : "r"(top) // input operands - : // clobbered registers ); for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__x86_64__) +#endif // MICROPY_NLR_X64 diff --git a/py/nlrx86.c b/py/nlrx86.c index 3a27460eb..cc37f72af 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -26,25 +26,13 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__i386__) +#if MICROPY_NLR_X86 #undef nlr_push // For reference, x86 callee save regs are: // ebx, esi, edi, ebp, esp, eip -#if defined(_WIN32) || defined(__CYGWIN__) -#define NLR_OS_WINDOWS 1 -#else -#define NLR_OS_WINDOWS 0 -#endif - -#if NLR_OS_WINDOWS -unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); -#else -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); -#endif - unsigned int nlr_push(nlr_buf_t *nlr) { (void)nlr; @@ -70,48 +58,23 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - -NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; +NORETURN void nlr_jump_tail(nlr_buf_t *top) { + (void)top; __asm volatile ( - "mov %0, %%edx \n" // %edx points to nlr_buf - "mov 28(%%edx), %%esi \n" // load saved %esi - "mov 24(%%edx), %%edi \n" // load saved %edi - "mov 20(%%edx), %%ebx \n" // load saved %ebx - "mov 16(%%edx), %%esp \n" // load saved %esp - "mov 12(%%edx), %%ebp \n" // load saved %ebp - "mov 8(%%edx), %%eax \n" // load saved %eip - "mov %%eax, (%%esp) \n" // store saved %eip to stack - "xor %%eax, %%eax \n" // clear return register - "inc %%al \n" // increase to make 1, non-local return + "mov 28(%edx), %esi \n" // load saved %esi + "mov 24(%edx), %edi \n" // load saved %edi + "mov 20(%edx), %ebx \n" // load saved %ebx + "mov 16(%edx), %esp \n" // load saved %esp + "mov 12(%edx), %ebp \n" // load saved %ebp + "mov 8(%edx), %eax \n" // load saved %eip + "mov %eax, (%esp) \n" // store saved %eip to stack + "xor %eax, %eax \n" // clear return register + "inc %al \n" // increase to make 1, non-local return "ret \n" // return - : // output operands - : "r"(top) // input operands - : // clobbered registers ); for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__i386__) +#endif // MICROPY_NLR_X86 diff --git a/py/nlrxtensa.c b/py/nlrxtensa.c index 5a969fc87..6b9918227 100644 --- a/py/nlrxtensa.c +++ b/py/nlrxtensa.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__xtensa__) +#if MICROPY_NLR_XTENSA #undef nlr_push @@ -37,6 +37,7 @@ // a3-a7 = rest of args unsigned int nlr_push(nlr_buf_t *nlr) { + (void)nlr; __asm volatile ( "s32i.n a0, a2, 8 \n" // save regs... @@ -55,32 +56,10 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - -NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; +NORETURN void nlr_jump_tail(nlr_buf_t *top) { + (void)top; __asm volatile ( - "mov.n a2, %0 \n" // a2 points to nlr_buf "l32i.n a0, a2, 8 \n" // restore regs... "l32i.n a1, a2, 12 \n" "l32i.n a8, a2, 16 \n" @@ -93,12 +72,9 @@ NORETURN void nlr_jump(void *val) { "l32i.n a15, a2, 44 \n" "movi.n a2, 1 \n" // return 1, non-local return "ret.n \n" // return - : // output operands - : "r"(top) // input operands - : // clobbered registers ); for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__xtensa__) +#endif // MICROPY_NLR_XTENSA diff --git a/py/py.mk b/py/py.mk index 0b5d5f8c4..de82a971b 100644 --- a/py/py.mk +++ b/py/py.mk @@ -103,6 +103,7 @@ endif # py object files PY_O_BASENAME = \ mpstate.o \ + nlr.o \ nlrx86.o \ nlrx64.o \ nlrthumb.o \ -- cgit v1.2.3 From 26d4a6fa45526fa7c267cd9228259642701708f6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 20 Dec 2017 16:54:31 +1100 Subject: py/malloc: Remove unneeded code checking m_malloc return value. m_malloc already checks for a failed allocation so there's no need to check for it in m_malloc0. --- py/malloc.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'py') diff --git a/py/malloc.c b/py/malloc.c index 6835ed7c9..ba5c952f3 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -117,9 +117,6 @@ void *m_malloc_with_finaliser(size_t num_bytes) { void *m_malloc0(size_t num_bytes) { void *ptr = m_malloc(num_bytes); - if (ptr == NULL && num_bytes != 0) { - m_malloc_fail(num_bytes); - } // If this config is set then the GC clears all memory, so we don't need to. #if !MICROPY_GC_CONSERVATIVE_CLEAR memset(ptr, 0, num_bytes); -- cgit v1.2.3 From 096e967aad6df760c16de4878b8b2eea02330011 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 26 Dec 2017 18:39:51 +0200 Subject: Revert "py/nlr: Factor out common NLR code to generic functions." This reverts commit 6a3a742a6c9caaa2be0fd0aac7a5df4ac816081c. The above commit has number of faults starting from the motivation down to the actual implementation. 1. Faulty implementation. The original code contained functions like: NORETURN void nlr_jump(void *val) { nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); nlr_buf_t *top = *top_ptr; ... __asm volatile ( "mov %0, %%edx \n" // %edx points to nlr_buf "mov 28(%%edx), %%esi \n" // load saved %esi "mov 24(%%edx), %%edi \n" // load saved %edi "mov 20(%%edx), %%ebx \n" // load saved %ebx "mov 16(%%edx), %%esp \n" // load saved %esp "mov 12(%%edx), %%ebp \n" // load saved %ebp "mov 8(%%edx), %%eax \n" // load saved %eip "mov %%eax, (%%esp) \n" // store saved %eip to stack "xor %%eax, %%eax \n" // clear return register "inc %%al \n" // increase to make 1, non-local return "ret \n" // return : // output operands : "r"(top) // input operands : // clobbered registers ); } Which clearly stated that C-level variable should be a parameter of the assembly, whcih then moved it into correct register. Whereas now it's: NORETURN void nlr_jump_tail(nlr_buf_t *top) { (void)top; __asm volatile ( "mov 28(%edx), %esi \n" // load saved %esi "mov 24(%edx), %edi \n" // load saved %edi "mov 20(%edx), %ebx \n" // load saved %ebx "mov 16(%edx), %esp \n" // load saved %esp "mov 12(%edx), %ebp \n" // load saved %ebp "mov 8(%edx), %eax \n" // load saved %eip "mov %eax, (%esp) \n" // store saved %eip to stack "xor %eax, %eax \n" // clear return register "inc %al \n" // increase to make 1, non-local return "ret \n" // return ); for (;;); // needed to silence compiler warning } Which just tries to perform operations on a completely random register (edx in this case). The outcome is the expected: saving the pure random luck of the compiler putting the right value in the random register above, there's a crash. 2. Non-critical assessment. The original commit message says "There is a small overhead introduced (typically 1 machine instruction)". That machine instruction is a call if a compiler doesn't perform tail optimization (happens regularly), and it's 1 instruction only with the broken code shown above, fixing it requires adding more. With inefficiencies already presented in the NLR code, the overhead becomes "considerable" (several times more than 1%), not "small". The commit message also says "This eliminates duplicated code.". An obvious way to eliminate duplication would be to factor out common code to macros, not introduce overhead and breakage like above. 3. Faulty motivation. All this started with a report of warnings/errors happening for a niche compiler. It could have been solved in one the direct ways: a) fixing it just for affected compiler(s); b) rewriting it in proper assembly (like it was before BTW); c) by not doing anything at all, MICROPY_NLR_SETJMP exists exactly to address minor-impact cases like thar (where a) or b) are not applicable). Instead, a backwards "solution" was put forward, leading to all the issues above. The best action thus appears to be revert and rework, not trying to work around what went haywire in the first place. --- py/nlr.c | 74 ---------------------------------------------------------- py/nlr.h | 71 ++++++++++++++++++++++++++++--------------------------- py/nlrsetjmp.c | 10 +++++++- py/nlrthumb.c | 38 +++++++++++++++++++++++++++--- py/nlrx64.c | 70 +++++++++++++++++++++++++++++++++--------------------- py/nlrx86.c | 63 ++++++++++++++++++++++++++++++++++++++----------- py/nlrxtensa.c | 34 +++++++++++++++++++++++---- py/py.mk | 1 - 8 files changed, 204 insertions(+), 157 deletions(-) delete mode 100644 py/nlr.c (limited to 'py') diff --git a/py/nlr.c b/py/nlr.c deleted file mode 100644 index e4d6d10c0..000000000 --- a/py/nlr.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2013-2017 Damien P. George - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "py/mpstate.h" - -// Helper macros to save/restore the pystack state -#if MICROPY_ENABLE_PYSTACK -#define MP_NLR_SAVE_PYSTACK(nlr_buf) (nlr_buf)->pystack = MP_STATE_THREAD(pystack_cur) -#define MP_NLR_RESTORE_PYSTACK(nlr_buf) MP_STATE_THREAD(pystack_cur) = (nlr_buf)->pystack -#else -#define MP_NLR_SAVE_PYSTACK(nlr_buf) (void)nlr_buf -#define MP_NLR_RESTORE_PYSTACK(nlr_buf) (void)nlr_buf -#endif - -#if !MICROPY_NLR_SETJMP -// When not using setjmp, nlr_push_tail is called from inline asm so needs special care -#if MICROPY_NLR_X86 && (defined(_WIN32) || defined(__CYGWIN__)) -// On these 32-bit platforms make sure nlr_push_tail doesn't have a leading underscore -unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); -#else -// LTO can't see inside inline asm functions so explicitly mark nlr_push_tail as used -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); -#endif -#endif - -unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - -NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; - - nlr_jump_tail(top); -} diff --git a/py/nlr.h b/py/nlr.h index 012a73c31..1235f1460 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -30,29 +30,29 @@ // exception handling, basically a stack of setjmp/longjmp buffers #include +#include #include #include "py/mpconfig.h" -// If MICROPY_NLR_SETJMP is not enabled then auto-detect the machine arch -// Allow a port to set MICROPY_NLR_NUM_REGS to define their own implementation -#if !MICROPY_NLR_SETJMP && !defined(MICROPY_NLR_NUM_REGS) +typedef struct _nlr_buf_t nlr_buf_t; +struct _nlr_buf_t { + // the entries here must all be machine word size + nlr_buf_t *prev; + void *ret_val; // always a concrete object (an exception instance) +#if !defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP #if defined(__i386__) - #define MICROPY_NLR_X86 (1) - #define MICROPY_NLR_NUM_REGS (6) + void *regs[6]; #elif defined(__x86_64__) - #define MICROPY_NLR_X64 (1) - #if defined(__CYGWIN__) - #define MICROPY_NLR_NUM_REGS (12) - #else - #define MICROPY_NLR_NUM_REGS (8) - #endif + #if defined(__CYGWIN__) + void *regs[12]; + #else + void *regs[8]; + #endif #elif defined(__thumb2__) || defined(__thumb__) || defined(__arm__) - #define MICROPY_NLR_THUMB (1) - #define MICROPY_NLR_NUM_REGS (10) + void *regs[10]; #elif defined(__xtensa__) - #define MICROPY_NLR_XTENSA (1) - #define MICROPY_NLR_NUM_REGS (10) + void *regs[10]; #else #define MICROPY_NLR_SETJMP (1) //#warning "No native NLR support for this arch, using setjmp implementation" @@ -60,39 +60,41 @@ #endif #if MICROPY_NLR_SETJMP -#include -#endif - -typedef struct _nlr_buf_t nlr_buf_t; -struct _nlr_buf_t { - // the entries here must all be machine word size - nlr_buf_t *prev; - void *ret_val; // always a concrete object (an exception instance) - - #if MICROPY_NLR_SETJMP jmp_buf jmpbuf; - #else - void *regs[MICROPY_NLR_NUM_REGS]; - #endif +#endif #if MICROPY_ENABLE_PYSTACK void *pystack; #endif }; +// Helper macros to save/restore the pystack state +#if MICROPY_ENABLE_PYSTACK +#define MP_NLR_SAVE_PYSTACK(nlr_buf) (nlr_buf)->pystack = MP_STATE_THREAD(pystack_cur) +#define MP_NLR_RESTORE_PYSTACK(nlr_buf) MP_STATE_THREAD(pystack_cur) = (nlr_buf)->pystack +#else +#define MP_NLR_SAVE_PYSTACK(nlr_buf) (void)nlr_buf +#define MP_NLR_RESTORE_PYSTACK(nlr_buf) (void)nlr_buf +#endif + #if MICROPY_NLR_SETJMP +#include "py/mpstate.h" + +NORETURN void nlr_setjmp_jump(void *val); // nlr_push() must be defined as a macro, because "The stack context will be // invalidated if the function which called setjmp() returns." -// For this case it is safe to call nlr_push_tail() first. -#define nlr_push(buf) (nlr_push_tail(buf), setjmp((buf)->jmpbuf)) +#define nlr_push(buf) ( \ + (buf)->prev = MP_STATE_THREAD(nlr_top), \ + MP_NLR_SAVE_PYSTACK(buf), \ + MP_STATE_THREAD(nlr_top) = (buf), \ + setjmp((buf)->jmpbuf)) +#define nlr_pop() { MP_STATE_THREAD(nlr_top) = MP_STATE_THREAD(nlr_top)->prev; } +#define nlr_jump(val) nlr_setjmp_jump(val) #else unsigned int nlr_push(nlr_buf_t *); -#endif - -unsigned int nlr_push_tail(nlr_buf_t *top); void nlr_pop(void); NORETURN void nlr_jump(void *val); -NORETURN void nlr_jump_tail(nlr_buf_t *top); +#endif // This must be implemented by a port. It's called by nlr_jump // if no nlr buf has been pushed. It must not return, but rather @@ -121,6 +123,7 @@ NORETURN void nlr_jump_fail(void *val); /* #define nlr_push(val) \ printf("nlr_push: before: nlr_top=%p, val=%p\n", MP_STATE_THREAD(nlr_top), val),assert(MP_STATE_THREAD(nlr_top) != val),nlr_push(val) +#endif */ #endif diff --git a/py/nlrsetjmp.c b/py/nlrsetjmp.c index 6ddad3793..63376a553 100644 --- a/py/nlrsetjmp.c +++ b/py/nlrsetjmp.c @@ -28,7 +28,15 @@ #if MICROPY_NLR_SETJMP -NORETURN void nlr_jump_tail(nlr_buf_t *top) { +void nlr_setjmp_jump(void *val) { + nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); + nlr_buf_t *top = *top_ptr; + if (top == NULL) { + nlr_jump_fail(val); + } + top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); + *top_ptr = top->prev; longjmp(top->jmpbuf, 1); } diff --git a/py/nlrthumb.c b/py/nlrthumb.c index 4edd1456d..eab5759f2 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if MICROPY_NLR_THUMB +#if (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) #undef nlr_push @@ -37,6 +37,7 @@ // r4-r11, r13=sp __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { + __asm volatile ( "str r4, [r0, #12] \n" // store r4 into nlr_buf "str r5, [r0, #16] \n" // store r5 into nlr_buf @@ -74,10 +75,36 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { "b nlr_push_tail \n" // do the rest in C #endif ); + + return 0; // needed to silence compiler warning } -NORETURN __attribute__((naked)) void nlr_jump_tail(nlr_buf_t *top) { +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); + *top = nlr; + return 0; // normal return +} + +void nlr_pop(void) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + *top = (*top)->prev; +} + +NORETURN __attribute__((naked)) void nlr_jump(void *val) { + nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); + nlr_buf_t *top = *top_ptr; + if (top == NULL) { + nlr_jump_fail(val); + } + + top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); + *top_ptr = top->prev; + __asm volatile ( + "mov r0, %0 \n" // r0 points to nlr_buf "ldr r4, [r0, #12] \n" // load r4 from nlr_buf "ldr r5, [r0, #16] \n" // load r5 from nlr_buf "ldr r6, [r0, #20] \n" // load r6 from nlr_buf @@ -106,7 +133,12 @@ NORETURN __attribute__((naked)) void nlr_jump_tail(nlr_buf_t *top) { #endif "movs r0, #1 \n" // return 1, non-local return "bx lr \n" // return + : // output operands + : "r"(top) // input operands + : // clobbered registers ); + + for (;;); // needed to silence compiler warning } -#endif // MICROPY_NLR_THUMB +#endif // (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) diff --git a/py/nlrx64.c b/py/nlrx64.c index e7e2e1474..ddcd76166 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if MICROPY_NLR_X64 +#if !MICROPY_NLR_SETJMP && defined(__x86_64__) #undef nlr_push @@ -39,6 +39,8 @@ #define NLR_OS_WINDOWS 0 #endif +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); + unsigned int nlr_push(nlr_buf_t *nlr) { (void)nlr; @@ -86,38 +88,54 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -NORETURN void nlr_jump_tail(nlr_buf_t *top) { - (void)top; +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); + *top = nlr; + return 0; // normal return +} + +void nlr_pop(void) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + *top = (*top)->prev; +} + +NORETURN void nlr_jump(void *val) { + nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); + nlr_buf_t *top = *top_ptr; + if (top == NULL) { + nlr_jump_fail(val); + } + + top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); + *top_ptr = top->prev; __asm volatile ( + "movq %0, %%rcx \n" // %rcx points to nlr_buf #if NLR_OS_WINDOWS - "movq 88(%rcx), %rsi \n" // load saved %rsi - "movq 80(%rcx), %rdi \n" // load saved %rdr - "movq 72(%rcx), %r15 \n" // load saved %r15 - "movq 64(%rcx), %r14 \n" // load saved %r14 - "movq 56(%rcx), %r13 \n" // load saved %r13 - "movq 48(%rcx), %r12 \n" // load saved %r12 - "movq 40(%rcx), %rbx \n" // load saved %rbx - "movq 32(%rcx), %rsp \n" // load saved %rsp - "movq 24(%rcx), %rbp \n" // load saved %rbp - "movq 16(%rcx), %rax \n" // load saved %rip - #else - "movq 72(%rdi), %r15 \n" // load saved %r15 - "movq 64(%rdi), %r14 \n" // load saved %r14 - "movq 56(%rdi), %r13 \n" // load saved %r13 - "movq 48(%rdi), %r12 \n" // load saved %r12 - "movq 40(%rdi), %rbx \n" // load saved %rbx - "movq 32(%rdi), %rsp \n" // load saved %rsp - "movq 24(%rdi), %rbp \n" // load saved %rbp - "movq 16(%rdi), %rax \n" // load saved %rip + "movq 88(%%rcx), %%rsi \n" // load saved %rsi + "movq 80(%%rcx), %%rdi \n" // load saved %rdr #endif - "movq %rax, (%rsp) \n" // store saved %rip to stack - "xorq %rax, %rax \n" // clear return register - "inc %al \n" // increase to make 1, non-local return + "movq 72(%%rcx), %%r15 \n" // load saved %r15 + "movq 64(%%rcx), %%r14 \n" // load saved %r14 + "movq 56(%%rcx), %%r13 \n" // load saved %r13 + "movq 48(%%rcx), %%r12 \n" // load saved %r12 + "movq 40(%%rcx), %%rbx \n" // load saved %rbx + "movq 32(%%rcx), %%rsp \n" // load saved %rsp + "movq 24(%%rcx), %%rbp \n" // load saved %rbp + "movq 16(%%rcx), %%rax \n" // load saved %rip + "movq %%rax, (%%rsp) \n" // store saved %rip to stack + "xorq %%rax, %%rax \n" // clear return register + "inc %%al \n" // increase to make 1, non-local return "ret \n" // return + : // output operands + : "r"(top) // input operands + : // clobbered registers ); for (;;); // needed to silence compiler warning } -#endif // MICROPY_NLR_X64 +#endif // !MICROPY_NLR_SETJMP && defined(__x86_64__) diff --git a/py/nlrx86.c b/py/nlrx86.c index cc37f72af..3a27460eb 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -26,13 +26,25 @@ #include "py/mpstate.h" -#if MICROPY_NLR_X86 +#if !MICROPY_NLR_SETJMP && defined(__i386__) #undef nlr_push // For reference, x86 callee save regs are: // ebx, esi, edi, ebp, esp, eip +#if defined(_WIN32) || defined(__CYGWIN__) +#define NLR_OS_WINDOWS 1 +#else +#define NLR_OS_WINDOWS 0 +#endif + +#if NLR_OS_WINDOWS +unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); +#else +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); +#endif + unsigned int nlr_push(nlr_buf_t *nlr) { (void)nlr; @@ -58,23 +70,48 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -NORETURN void nlr_jump_tail(nlr_buf_t *top) { - (void)top; +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); + *top = nlr; + return 0; // normal return +} + +void nlr_pop(void) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + *top = (*top)->prev; +} + +NORETURN void nlr_jump(void *val) { + nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); + nlr_buf_t *top = *top_ptr; + if (top == NULL) { + nlr_jump_fail(val); + } + + top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); + *top_ptr = top->prev; __asm volatile ( - "mov 28(%edx), %esi \n" // load saved %esi - "mov 24(%edx), %edi \n" // load saved %edi - "mov 20(%edx), %ebx \n" // load saved %ebx - "mov 16(%edx), %esp \n" // load saved %esp - "mov 12(%edx), %ebp \n" // load saved %ebp - "mov 8(%edx), %eax \n" // load saved %eip - "mov %eax, (%esp) \n" // store saved %eip to stack - "xor %eax, %eax \n" // clear return register - "inc %al \n" // increase to make 1, non-local return + "mov %0, %%edx \n" // %edx points to nlr_buf + "mov 28(%%edx), %%esi \n" // load saved %esi + "mov 24(%%edx), %%edi \n" // load saved %edi + "mov 20(%%edx), %%ebx \n" // load saved %ebx + "mov 16(%%edx), %%esp \n" // load saved %esp + "mov 12(%%edx), %%ebp \n" // load saved %ebp + "mov 8(%%edx), %%eax \n" // load saved %eip + "mov %%eax, (%%esp) \n" // store saved %eip to stack + "xor %%eax, %%eax \n" // clear return register + "inc %%al \n" // increase to make 1, non-local return "ret \n" // return + : // output operands + : "r"(top) // input operands + : // clobbered registers ); for (;;); // needed to silence compiler warning } -#endif // MICROPY_NLR_X86 +#endif // !MICROPY_NLR_SETJMP && defined(__i386__) diff --git a/py/nlrxtensa.c b/py/nlrxtensa.c index 6b9918227..5a969fc87 100644 --- a/py/nlrxtensa.c +++ b/py/nlrxtensa.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if MICROPY_NLR_XTENSA +#if !MICROPY_NLR_SETJMP && defined(__xtensa__) #undef nlr_push @@ -37,7 +37,6 @@ // a3-a7 = rest of args unsigned int nlr_push(nlr_buf_t *nlr) { - (void)nlr; __asm volatile ( "s32i.n a0, a2, 8 \n" // save regs... @@ -56,10 +55,32 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -NORETURN void nlr_jump_tail(nlr_buf_t *top) { - (void)top; +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); + *top = nlr; + return 0; // normal return +} + +void nlr_pop(void) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + *top = (*top)->prev; +} + +NORETURN void nlr_jump(void *val) { + nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); + nlr_buf_t *top = *top_ptr; + if (top == NULL) { + nlr_jump_fail(val); + } + + top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); + *top_ptr = top->prev; __asm volatile ( + "mov.n a2, %0 \n" // a2 points to nlr_buf "l32i.n a0, a2, 8 \n" // restore regs... "l32i.n a1, a2, 12 \n" "l32i.n a8, a2, 16 \n" @@ -72,9 +93,12 @@ NORETURN void nlr_jump_tail(nlr_buf_t *top) { "l32i.n a15, a2, 44 \n" "movi.n a2, 1 \n" // return 1, non-local return "ret.n \n" // return + : // output operands + : "r"(top) // input operands + : // clobbered registers ); for (;;); // needed to silence compiler warning } -#endif // MICROPY_NLR_XTENSA +#endif // !MICROPY_NLR_SETJMP && defined(__xtensa__) diff --git a/py/py.mk b/py/py.mk index de82a971b..0b5d5f8c4 100644 --- a/py/py.mk +++ b/py/py.mk @@ -103,7 +103,6 @@ endif # py object files PY_O_BASENAME = \ mpstate.o \ - nlr.o \ nlrx86.o \ nlrx64.o \ nlrthumb.o \ -- cgit v1.2.3 From 97cc48553828ed091325d0d3922eb4cd6c377221 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 28 Dec 2017 15:59:09 +1100 Subject: py/nlrthumb: Fix use of naked funcs, must only contain basic asm code. A function with a naked attribute must only contain basic inline asm statements and no C code. For nlr_push this means removing the "return 0" statement. But for some gcc versions this induces a compiler warning so the __builtin_unreachable() line needs to be added. For nlr_jump, this function contains a combination of C code and inline asm so cannot be naked. --- py/nlrthumb.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/nlrthumb.c b/py/nlrthumb.c index eab5759f2..18d31eb70 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -76,7 +76,10 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { #endif ); - return 0; // needed to silence compiler warning + #if defined(__GNUC__) + // Older versions of gcc give an error when naked functions don't return a value + __builtin_unreachable(); + #endif } __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { @@ -92,7 +95,7 @@ void nlr_pop(void) { *top = (*top)->prev; } -NORETURN __attribute__((naked)) void nlr_jump(void *val) { +NORETURN void nlr_jump(void *val) { nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); nlr_buf_t *top = *top_ptr; if (top == NULL) { @@ -138,7 +141,11 @@ NORETURN __attribute__((naked)) void nlr_jump(void *val) { : // clobbered registers ); + #if defined(__GNUC__) + __builtin_unreachable(); + #else for (;;); // needed to silence compiler warning + #endif } #endif // (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) -- cgit v1.2.3 From 5bf8e85fc828974199d469db711aa2f9649c467b Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 28 Dec 2017 16:18:39 +1100 Subject: py/nlr: Clean up selection and config of NLR implementation. If MICROPY_NLR_SETJMP is not enabled and the machine is auto-detected then nlr.h now defines some convenience macros for the individual NLR implementations to use (eg MICROPY_NLR_THUMB). This keeps nlr.h and the implementation in sync, and also makes the nlr_buf_t struct easier to read. --- py/nlr.h | 44 +++++++++++++++++++++++++++----------------- py/nlrthumb.c | 4 ++-- py/nlrx64.c | 4 ++-- py/nlrx86.c | 4 ++-- py/nlrxtensa.c | 4 ++-- 5 files changed, 35 insertions(+), 25 deletions(-) (limited to 'py') diff --git a/py/nlr.h b/py/nlr.h index 1235f1460..bd9fcc884 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -30,29 +30,28 @@ // exception handling, basically a stack of setjmp/longjmp buffers #include -#include #include #include "py/mpconfig.h" -typedef struct _nlr_buf_t nlr_buf_t; -struct _nlr_buf_t { - // the entries here must all be machine word size - nlr_buf_t *prev; - void *ret_val; // always a concrete object (an exception instance) -#if !defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP +// If MICROPY_NLR_SETJMP is not enabled then auto-detect the machine arch +#if !MICROPY_NLR_SETJMP #if defined(__i386__) - void *regs[6]; + #define MICROPY_NLR_X86 (1) + #define MICROPY_NLR_NUM_REGS (6) #elif defined(__x86_64__) - #if defined(__CYGWIN__) - void *regs[12]; - #else - void *regs[8]; - #endif + #define MICROPY_NLR_X64 (1) + #if defined(__CYGWIN__) + #define MICROPY_NLR_NUM_REGS (12) + #else + #define MICROPY_NLR_NUM_REGS (8) + #endif #elif defined(__thumb2__) || defined(__thumb__) || defined(__arm__) - void *regs[10]; + #define MICROPY_NLR_THUMB (1) + #define MICROPY_NLR_NUM_REGS (10) #elif defined(__xtensa__) - void *regs[10]; + #define MICROPY_NLR_XTENSA (1) + #define MICROPY_NLR_NUM_REGS (10) #else #define MICROPY_NLR_SETJMP (1) //#warning "No native NLR support for this arch, using setjmp implementation" @@ -60,9 +59,21 @@ struct _nlr_buf_t { #endif #if MICROPY_NLR_SETJMP - jmp_buf jmpbuf; +#include #endif +typedef struct _nlr_buf_t nlr_buf_t; +struct _nlr_buf_t { + // the entries here must all be machine word size + nlr_buf_t *prev; + void *ret_val; // always a concrete object (an exception instance) + + #if MICROPY_NLR_SETJMP + jmp_buf jmpbuf; + #else + void *regs[MICROPY_NLR_NUM_REGS]; + #endif + #if MICROPY_ENABLE_PYSTACK void *pystack; #endif @@ -123,7 +134,6 @@ NORETURN void nlr_jump_fail(void *val); /* #define nlr_push(val) \ printf("nlr_push: before: nlr_top=%p, val=%p\n", MP_STATE_THREAD(nlr_top), val),assert(MP_STATE_THREAD(nlr_top) != val),nlr_push(val) -#endif */ #endif diff --git a/py/nlrthumb.c b/py/nlrthumb.c index 18d31eb70..cc081e3ff 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) +#if MICROPY_NLR_THUMB #undef nlr_push @@ -148,4 +148,4 @@ NORETURN void nlr_jump(void *val) { #endif } -#endif // (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) +#endif // MICROPY_NLR_THUMB diff --git a/py/nlrx64.c b/py/nlrx64.c index ddcd76166..927b21591 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__x86_64__) +#if MICROPY_NLR_X64 #undef nlr_push @@ -138,4 +138,4 @@ NORETURN void nlr_jump(void *val) { for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__x86_64__) +#endif // MICROPY_NLR_X64 diff --git a/py/nlrx86.c b/py/nlrx86.c index 3a27460eb..0e03eef6f 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__i386__) +#if MICROPY_NLR_X86 #undef nlr_push @@ -114,4 +114,4 @@ NORETURN void nlr_jump(void *val) { for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__i386__) +#endif // MICROPY_NLR_X86 diff --git a/py/nlrxtensa.c b/py/nlrxtensa.c index 5a969fc87..73f14385d 100644 --- a/py/nlrxtensa.c +++ b/py/nlrxtensa.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__xtensa__) +#if MICROPY_NLR_XTENSA #undef nlr_push @@ -101,4 +101,4 @@ NORETURN void nlr_jump(void *val) { for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__xtensa__) +#endif // MICROPY_NLR_XTENSA -- cgit v1.2.3 From b25f92160b318a096c516c430afde5472a944c19 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 28 Dec 2017 16:46:30 +1100 Subject: py/nlr: Factor out common NLR code to macro and generic funcs in nlr.c. Each NLR implementation (Thumb, x86, x64, xtensa, setjmp) duplicates a lot of the NLR code, specifically that dealing with pushing and popping the NLR pointer to maintain the linked-list of NLR buffers. This patch factors all of that code out of the specific implementations into generic functions in nlr.c, along with a helper macro in nlr.h. This eliminates duplicated code. --- py/nlr.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ py/nlr.h | 27 ++++++++++++++++----------- py/nlrsetjmp.c | 6 +++--- py/nlrthumb.c | 25 ++----------------------- py/nlrx64.c | 23 +---------------------- py/nlrx86.c | 23 +---------------------- py/nlrxtensa.c | 23 +---------------------- py/py.mk | 1 + 8 files changed, 76 insertions(+), 103 deletions(-) create mode 100644 py/nlr.c (limited to 'py') diff --git a/py/nlr.c b/py/nlr.c new file mode 100644 index 000000000..52d56afb8 --- /dev/null +++ b/py/nlr.c @@ -0,0 +1,51 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/mpstate.h" + +#if !MICROPY_NLR_SETJMP +// When not using setjmp, nlr_push_tail is called from inline asm so needs special c +#if MICROPY_NLR_X86 && (defined(_WIN32) || defined(__CYGWIN__)) +// On these 32-bit platforms make sure nlr_push_tail doesn't have a leading undersco +unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); +#else +// LTO can't see inside inline asm functions so explicitly mark nlr_push_tail as use +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); +#endif +#endif + +unsigned int nlr_push_tail(nlr_buf_t *nlr) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); + *top = nlr; + return 0; // normal return +} + +void nlr_pop(void) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + *top = (*top)->prev; +} diff --git a/py/nlr.h b/py/nlr.h index bd9fcc884..e4dfa6896 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -88,24 +88,29 @@ struct _nlr_buf_t { #define MP_NLR_RESTORE_PYSTACK(nlr_buf) (void)nlr_buf #endif -#if MICROPY_NLR_SETJMP -#include "py/mpstate.h" +// Helper macro to use at the start of a specific nlr_jump implementation +#define MP_NLR_JUMP_HEAD(val, top) \ + nlr_buf_t **_top_ptr = &MP_STATE_THREAD(nlr_top); \ + nlr_buf_t *top = *_top_ptr; \ + if (top == NULL) { \ + nlr_jump_fail(val); \ + } \ + top->ret_val = val; \ + MP_NLR_RESTORE_PYSTACK(top); \ + *_top_ptr = top->prev; \ -NORETURN void nlr_setjmp_jump(void *val); +#if MICROPY_NLR_SETJMP // nlr_push() must be defined as a macro, because "The stack context will be // invalidated if the function which called setjmp() returns." -#define nlr_push(buf) ( \ - (buf)->prev = MP_STATE_THREAD(nlr_top), \ - MP_NLR_SAVE_PYSTACK(buf), \ - MP_STATE_THREAD(nlr_top) = (buf), \ - setjmp((buf)->jmpbuf)) -#define nlr_pop() { MP_STATE_THREAD(nlr_top) = MP_STATE_THREAD(nlr_top)->prev; } -#define nlr_jump(val) nlr_setjmp_jump(val) +// For this case it is safe to call nlr_push_tail() first. +#define nlr_push(buf) (nlr_push_tail(buf), setjmp((buf)->jmpbuf)) #else unsigned int nlr_push(nlr_buf_t *); +#endif + +unsigned int nlr_push_tail(nlr_buf_t *top); void nlr_pop(void); NORETURN void nlr_jump(void *val); -#endif // This must be implemented by a port. It's called by nlr_jump // if no nlr buf has been pushed. It must not return, but rather diff --git a/py/nlrsetjmp.c b/py/nlrsetjmp.c index 63376a553..960dd86f5 100644 --- a/py/nlrsetjmp.c +++ b/py/nlrsetjmp.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,11 +24,11 @@ * THE SOFTWARE. */ -#include "py/nlr.h" +#include "py/mpstate.h" #if MICROPY_NLR_SETJMP -void nlr_setjmp_jump(void *val) { +void nlr_jump(void *val) { nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); nlr_buf_t *top = *top_ptr; if (top == NULL) { diff --git a/py/nlrthumb.c b/py/nlrthumb.c index cc081e3ff..fb0a92236 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -82,29 +82,8 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { #endif } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; + MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "mov r0, %0 \n" // r0 points to nlr_buf diff --git a/py/nlrx64.c b/py/nlrx64.c index 927b21591..663a457b7 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -88,29 +88,8 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; + MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "movq %0, %%rcx \n" // %rcx points to nlr_buf diff --git a/py/nlrx86.c b/py/nlrx86.c index 0e03eef6f..9490c4f42 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -70,29 +70,8 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; + MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "mov %0, %%edx \n" // %edx points to nlr_buf diff --git a/py/nlrxtensa.c b/py/nlrxtensa.c index 73f14385d..cd3dee364 100644 --- a/py/nlrxtensa.c +++ b/py/nlrxtensa.c @@ -55,29 +55,8 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - MP_NLR_SAVE_PYSTACK(nlr); - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - MP_NLR_RESTORE_PYSTACK(top); - *top_ptr = top->prev; + MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "mov.n a2, %0 \n" // a2 points to nlr_buf diff --git a/py/py.mk b/py/py.mk index 0b5d5f8c4..de82a971b 100644 --- a/py/py.mk +++ b/py/py.mk @@ -103,6 +103,7 @@ endif # py object files PY_O_BASENAME = \ mpstate.o \ + nlr.o \ nlrx86.o \ nlrx64.o \ nlrthumb.o \ -- cgit v1.2.3 From 1039c5e6998561c452cfc23a836eec374bef39bd Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 23 Dec 2017 23:04:02 +1100 Subject: py/parse: Split out rule name from rule struct into separate array. The rule name is only used for debugging, and this patch makes things a bit cleaner by completely separating out the rule name from the rest of the rule data. --- py/parse.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) (limited to 'py') diff --git a/py/parse.c b/py/parse.c index b80cc8fb1..7fcdf2c43 100644 --- a/py/parse.c +++ b/py/parse.c @@ -61,9 +61,6 @@ typedef struct _rule_t { byte rule_id; byte act; -#ifdef USE_RULE_NAME - const char *rule_name; -#endif uint16_t arg[]; } rule_t; @@ -94,13 +91,8 @@ enum { #define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t) #define rule(r) (RULE_ARG_RULE | RULE_##r) #define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) -#ifdef USE_RULE_NAME -#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; -#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; -#else #define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; #define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; -#endif #include "py/grammar.h" #undef or #undef and @@ -130,6 +122,23 @@ STATIC const rule_t *const rules[] = { #undef DEF_RULE_NC }; +#if USE_RULE_NAME +// Define an array of rule names corresponding to each rule +STATIC const char *const rule_name_table[] = { +#define DEF_RULE(rule, comp, kind, ...) #rule, +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + "", // RULE_const_object +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) #rule, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +}; +#endif + typedef struct _rule_stack_t { size_t src_line : 8 * sizeof(size_t) - 8; // maximum bits storing source line number size_t rule_id : 8; // this must be large enough to fit largest rule number @@ -313,11 +322,11 @@ void mp_parse_node_print(mp_parse_node_t pn, size_t indent) { #endif } else { size_t n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); -#ifdef USE_RULE_NAME - printf("%s(%u) (n=%u)\n", rules[MP_PARSE_NODE_STRUCT_KIND(pns)]->rule_name, (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n); -#else + #if USE_RULE_NAME + printf("%s(%u) (n=%u)\n", rule_name_table[MP_PARSE_NODE_STRUCT_KIND(pns)], (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n); + #else printf("rule(%u) (n=%u)\n", (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n); -#endif + #endif for (size_t i = 0; i < n; i++) { mp_parse_node_print(pns->nodes[i], indent + 2); } @@ -797,7 +806,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { for (int j = 0; j < parser.rule_stack_top; ++j) { printf(" "); } - printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack); + printf("%s n=%d i=%d bt=%d\n", rule_name_table[rule->rule_id], n, i, backtrack); */ switch (rule->act & RULE_ACT_KIND_MASK) { -- cgit v1.2.3 From 845511af257cd33f1397c5dd05b1c9198d2be96a Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 23 Dec 2017 23:36:08 +1100 Subject: py/parse: Break rule data into separate act and arg arrays. Instead of each rule being stored in ROM as a struct with rule_id, act and arg, the act and arg parts are now in separate arrays and the rule_id part is removed because it's not needed. This reduces code size, by roughly one byte per grammar rule, around 150 bytes. --- py/parse.c | 84 ++++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 27 deletions(-) (limited to 'py') diff --git a/py/parse.c b/py/parse.c index 7fcdf2c43..bc2afb494 100644 --- a/py/parse.c +++ b/py/parse.c @@ -61,7 +61,7 @@ typedef struct _rule_t { byte rule_id; byte act; - uint16_t arg[]; + const uint16_t *arg; } rule_t; enum { @@ -81,6 +81,8 @@ enum { #undef DEF_RULE_NC }; +// Define an array of actions corresponding to each rule +STATIC const uint8_t rule_act_table[] = { #define or(n) (RULE_ACT_OR | n) #define and(n) (RULE_ACT_AND | n) #define and_ident(n) (RULE_ACT_AND | n | RULE_ACT_ALLOW_IDENT) @@ -88,35 +90,55 @@ enum { #define one_or_more (RULE_ACT_LIST | 2) #define list (RULE_ACT_LIST | 1) #define list_with_end (RULE_ACT_LIST | 3) -#define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t) -#define rule(r) (RULE_ARG_RULE | RULE_##r) -#define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) -#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; -#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; + +#define DEF_RULE(rule, comp, kind, ...) kind, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + + 0, // RULE_const_object + +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) kind, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + #undef or #undef and +#undef and_ident +#undef and_blank +#undef one_or_more #undef list #undef list_with_end +}; + +// Define the argument data for each rule +#define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t) +#define rule(r) (RULE_ARG_RULE | RULE_##r) +#define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) + +#define DEF_RULE(rule, comp, kind, ...) static const uint16_t const rule_arg_##rule[] = { __VA_ARGS__ }; +#define DEF_RULE_NC(rule, kind, ...) static const uint16_t const rule_arg_##rule[] = { __VA_ARGS__ }; +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + #undef tok #undef rule #undef opt_rule -#undef one_or_more -#undef DEF_RULE -#undef DEF_RULE_NC -STATIC const rule_t *const rules[] = { -// define rules with a compile function -#define DEF_RULE(rule, comp, kind, ...) &rule_##rule, +// Define an array of pointers to corresponding rule data +STATIC const uint16_t *const rule_arg_table[] = { +#define DEF_RULE(rule, comp, kind, ...) &rule_arg_##rule[0], #define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC NULL, // RULE_const_object - -// define rules without a compile function #define DEF_RULE(rule, comp, kind, ...) -#define DEF_RULE_NC(rule, kind, ...) &rule_##rule, +#define DEF_RULE_NC(rule, kind, ...) &rule_arg_##rule[0], #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC @@ -139,6 +161,10 @@ STATIC const char *const rule_name_table[] = { }; #endif +#if (MICROPY_COMP_CONST_FOLDING && MICROPY_COMP_CONST) || !MICROPY_ENABLE_DOC_STRING +static const rule_t rule_pass_stmt = {RULE_pass_stmt, (RULE_ACT_AND | 1), &rule_arg_pass_stmt[0]}; +#endif + typedef struct _rule_stack_t { size_t src_line : 8 * sizeof(size_t) - 8; // maximum bits storing source line number size_t rule_id : 8; // this must be large enough to fit largest rule number @@ -214,7 +240,7 @@ STATIC void *parser_alloc(parser_t *parser, size_t num_bytes) { return ret; } -STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t arg_i) { +STATIC void push_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t arg_i) { if (parser->rule_stack_top >= parser->rule_stack_alloc) { rule_stack_t *rs = m_renew(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc, parser->rule_stack_alloc + MICROPY_ALLOC_PARSE_RULE_INC); parser->rule_stack = rs; @@ -222,19 +248,21 @@ STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, siz } rule_stack_t *rs = &parser->rule_stack[parser->rule_stack_top++]; rs->src_line = src_line; - rs->rule_id = rule->rule_id; + rs->rule_id = rule_id; rs->arg_i = arg_i; } STATIC void push_rule_from_arg(parser_t *parser, size_t arg) { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE); size_t rule_id = arg & RULE_ARG_ARG_MASK; - push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0); + push_rule(parser, parser->lexer->tok_line, rule_id, 0); } -STATIC void pop_rule(parser_t *parser, const rule_t **rule, size_t *arg_i, size_t *src_line) { +STATIC void pop_rule(parser_t *parser, rule_t *rule, size_t *arg_i, size_t *src_line) { parser->rule_stack_top -= 1; - *rule = rules[parser->rule_stack[parser->rule_stack_top].rule_id]; + rule->rule_id = parser->rule_stack[parser->rule_stack_top].rule_id; + rule->act = rule_act_table[rule->rule_id]; + rule->arg = rule_arg_table[rule->rule_id]; *arg_i = parser->rule_stack[parser->rule_stack_top].arg_i; *src_line = parser->rule_stack[parser->rule_stack_top].src_line; } @@ -656,7 +684,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args if (qstr_str(id)[0] == '_') { pop_result(parser); // pop const(value) pop_result(parser); // pop id - push_result_rule(parser, 0, rules[RULE_pass_stmt], 0); // replace with "pass" + push_result_rule(parser, 0, &rule_pass_stmt, 0); // replace with "pass" return true; } @@ -782,7 +810,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break; default: top_level_rule = RULE_file_input; } - push_rule(&parser, lex->tok_line, rules[top_level_rule], 0); + push_rule(&parser, lex->tok_line, top_level_rule, 0); // parse! @@ -797,7 +825,9 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { break; } - pop_rule(&parser, &rule, &i, &rule_src_line); + rule_t rule_data; + pop_rule(&parser, &rule_data, &i, &rule_src_line); + rule = &rule_data; n = rule->act & RULE_ACT_ARG_MASK; /* @@ -827,7 +857,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } else { assert(kind == RULE_ARG_RULE); if (i + 1 < n) { - push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule + push_rule(&parser, rule_src_line, rule->rule_id, i + 1); // save this or-rule } push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule goto next_rule; @@ -879,7 +909,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } } else { - push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule + push_rule(&parser, rule_src_line, rule->rule_id, i + 1); // save this and-rule push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule goto next_rule; } @@ -900,7 +930,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // Pushing the "pass" rule here will overwrite any RULE_const_object // entry that was on the result stack, allowing the GC to reclaim // the memory from the const object when needed. - push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0); + push_result_rule(&parser, rule_src_line, &rule_pass_stmt, 0); break; } } @@ -1009,7 +1039,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } else { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE); - push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule + push_rule(&parser, rule_src_line, rule->rule_id, i + 1); // save this list-rule push_rule_from_arg(&parser, arg); // push child of list-rule goto next_rule; } -- cgit v1.2.3 From 815a8cd1ae3a0a451b0b0b687277093f3392cdab Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 23 Dec 2017 23:49:48 +1100 Subject: py/parse: Pass rule_id to push_result_rule, instead of passing rule_t*. Reduces code size by eliminating quite a few pointer dereferences. --- py/parse.c | 60 ++++++++++++++++++++++++++++-------------------------------- 1 file changed, 28 insertions(+), 32 deletions(-) (limited to 'py') diff --git a/py/parse.c b/py/parse.c index bc2afb494..fffa98953 100644 --- a/py/parse.c +++ b/py/parse.c @@ -161,10 +161,6 @@ STATIC const char *const rule_name_table[] = { }; #endif -#if (MICROPY_COMP_CONST_FOLDING && MICROPY_COMP_CONST) || !MICROPY_ENABLE_DOC_STRING -static const rule_t rule_pass_stmt = {RULE_pass_stmt, (RULE_ACT_AND | 1), &rule_arg_pass_stmt[0]}; -#endif - typedef struct _rule_stack_t { size_t src_line : 8 * sizeof(size_t) - 8; // maximum bits storing source line number size_t rule_id : 8; // this must be large enough to fit largest rule number @@ -491,12 +487,12 @@ STATIC const mp_rom_map_elem_t mp_constants_table[] = { STATIC MP_DEFINE_CONST_MAP(mp_constants_map, mp_constants_table); #endif -STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t num_args); +STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args); #if MICROPY_COMP_CONST_FOLDING -STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t *num_args) { - if (rule->rule_id == RULE_or_test - || rule->rule_id == RULE_and_test) { +STATIC bool fold_logical_constants(parser_t *parser, uint8_t rule_id, size_t *num_args) { + if (rule_id == RULE_or_test + || rule_id == RULE_and_test) { // folding for binary logical ops: or and size_t copy_to = *num_args; for (size_t i = copy_to; i > 0;) { @@ -506,7 +502,7 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t // always need to keep the last value break; } - if (rule->rule_id == RULE_or_test) { + if (rule_id == RULE_or_test) { if (mp_parse_node_is_const_true(pn)) { // break; @@ -533,7 +529,7 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t // we did a complete folding if there's only 1 arg left return *num_args == 1; - } else if (rule->rule_id == RULE_not_test_2) { + } else if (rule_id == RULE_not_test_2) { // folding for unary logical op: not mp_parse_node_t pn = peek_result(parser, 0); if (mp_parse_node_is_const_false(pn)) { @@ -551,23 +547,23 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t return false; } -STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args) { +STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { // this code does folding of arbitrary integer expressions, eg 1 + 2 * 3 + 4 // it does not do partial folding, eg 1 + 2 + x -> 3 + x mp_obj_t arg0; - if (rule->rule_id == RULE_expr - || rule->rule_id == RULE_xor_expr - || rule->rule_id == RULE_and_expr) { + if (rule_id == RULE_expr + || rule_id == RULE_xor_expr + || rule_id == RULE_and_expr) { // folding for binary ops: | ^ & mp_parse_node_t pn = peek_result(parser, num_args - 1); if (!mp_parse_node_get_int_maybe(pn, &arg0)) { return false; } mp_binary_op_t op; - if (rule->rule_id == RULE_expr) { + if (rule_id == RULE_expr) { op = MP_BINARY_OP_OR; - } else if (rule->rule_id == RULE_xor_expr) { + } else if (rule_id == RULE_xor_expr) { op = MP_BINARY_OP_XOR; } else { op = MP_BINARY_OP_AND; @@ -580,9 +576,9 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args } arg0 = mp_binary_op(op, arg0, arg1); } - } else if (rule->rule_id == RULE_shift_expr - || rule->rule_id == RULE_arith_expr - || rule->rule_id == RULE_term) { + } else if (rule_id == RULE_shift_expr + || rule_id == RULE_arith_expr + || rule_id == RULE_term) { // folding for binary ops: << >> + - * / % // mp_parse_node_t pn = peek_result(parser, num_args - 1); if (!mp_parse_node_get_int_maybe(pn, &arg0)) { @@ -626,7 +622,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args } arg0 = mp_binary_op(op, arg0, arg1); } - } else if (rule->rule_id == RULE_factor_2) { + } else if (rule_id == RULE_factor_2) { // folding for unary ops: + - ~ mp_parse_node_t pn = peek_result(parser, 0); if (!mp_parse_node_get_int_maybe(pn, &arg0)) { @@ -645,7 +641,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args arg0 = mp_unary_op(op, arg0); #if MICROPY_COMP_CONST - } else if (rule->rule_id == RULE_expr_stmt) { + } else if (rule_id == RULE_expr_stmt) { mp_parse_node_t pn1 = peek_result(parser, 0); if (!MP_PARSE_NODE_IS_NULL(pn1) && !(MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_augassign) @@ -684,7 +680,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args if (qstr_str(id)[0] == '_') { pop_result(parser); // pop const(value) pop_result(parser); // pop id - push_result_rule(parser, 0, &rule_pass_stmt, 0); // replace with "pass" + push_result_rule(parser, 0, RULE_pass_stmt, 0); // replace with "pass" return true; } @@ -700,7 +696,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args #endif #if MICROPY_COMP_MODULE_CONST - } else if (rule->rule_id == RULE_atom_expr_normal) { + } else if (rule_id == RULE_atom_expr_normal) { mp_parse_node_t pn0 = peek_result(parser, 1); mp_parse_node_t pn1 = peek_result(parser, 0); if (!(MP_PARSE_NODE_IS_ID(pn0) @@ -745,9 +741,9 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args } #endif -STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t num_args) { +STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args) { // optimise away parenthesis around an expression if possible - if (rule->rule_id == RULE_atom_paren) { + if (rule_id == RULE_atom_paren) { // there should be just 1 arg for this rule mp_parse_node_t pn = peek_result(parser, 0); if (MP_PARSE_NODE_IS_NULL(pn)) { @@ -761,11 +757,11 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *ru } #if MICROPY_COMP_CONST_FOLDING - if (fold_logical_constants(parser, rule, &num_args)) { + if (fold_logical_constants(parser, rule_id, &num_args)) { // we folded this rule so return straight away return; } - if (fold_constants(parser, rule, num_args)) { + if (fold_constants(parser, rule_id, num_args)) { // we folded this rule so return straight away return; } @@ -773,7 +769,7 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *ru mp_parse_node_struct_t *pn = parser_alloc(parser, sizeof(mp_parse_node_struct_t) + sizeof(mp_parse_node_t) * num_args); pn->source_line = src_line; - pn->kind_num_nodes = (rule->rule_id & 0xff) | (num_args << 8); + pn->kind_num_nodes = (rule_id & 0xff) | (num_args << 8); for (size_t i = num_args; i > 0; i--) { pn->nodes[i - 1] = pop_result(parser); } @@ -930,7 +926,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // Pushing the "pass" rule here will overwrite any RULE_const_object // entry that was on the result stack, allowing the GC to reclaim // the memory from the const object when needed. - push_result_rule(&parser, rule_src_line, &rule_pass_stmt, 0); + push_result_rule(&parser, rule_src_line, RULE_pass_stmt, 0); break; } } @@ -976,7 +972,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { i += 1; } - push_result_rule(&parser, rule_src_line, rule, i); + push_result_rule(&parser, rule_src_line, rule->rule_id, i); } break; } @@ -1058,12 +1054,12 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // list matched single item if (had_trailing_sep) { // if there was a trailing separator, make a list of a single item - push_result_rule(&parser, rule_src_line, rule, i); + push_result_rule(&parser, rule_src_line, rule->rule_id, i); } else { // just leave single item on stack (ie don't wrap in a list) } } else { - push_result_rule(&parser, rule_src_line, rule, i); + push_result_rule(&parser, rule_src_line, rule->rule_id, i); } break; } -- cgit v1.2.3 From 66d8885d85bb591810b25e763c0921372ab74ef4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 23 Dec 2017 23:53:01 +1100 Subject: py/parse: Pass rule_id to push_result_token, instead of passing rule_t*. --- py/parse.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'py') diff --git a/py/parse.c b/py/parse.c index fffa98953..6244af432 100644 --- a/py/parse.c +++ b/py/parse.c @@ -414,7 +414,7 @@ STATIC mp_parse_node_t mp_parse_node_new_small_int_checked(parser_t *parser, mp_ return mp_parse_node_new_small_int(val); } -STATIC void push_result_token(parser_t *parser, const rule_t *rule) { +STATIC void push_result_token(parser_t *parser, uint8_t rule_id) { mp_parse_node_t pn; mp_lexer_t *lex = parser->lexer; if (lex->tok_kind == MP_TOKEN_NAME) { @@ -422,7 +422,7 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) { #if MICROPY_COMP_CONST // if name is a standalone identifier, look it up in the table of dynamic constants mp_map_elem_t *elem; - if (rule->rule_id == RULE_atom + if (rule_id == RULE_atom && (elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP)) != NULL) { if (MP_OBJ_IS_SMALL_INT(elem->value)) { pn = mp_parse_node_new_small_int_checked(parser, elem->value); @@ -433,7 +433,7 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) { pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, id); } #else - (void)rule; + (void)rule_id; pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, id); #endif } else if (lex->tok_kind == MP_TOKEN_INTEGER) { @@ -846,7 +846,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK; if (kind == RULE_ARG_TOK) { if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) { - push_result_token(&parser, rule); + push_result_token(&parser, rule->rule_id); mp_lexer_to_next(lex); goto next_rule; } @@ -890,7 +890,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { if (lex->tok_kind == tok_kind) { // matched token if (tok_kind == MP_TOKEN_NAME) { - push_result_token(&parser, rule); + push_result_token(&parser, rule->rule_id); } mp_lexer_to_next(lex); } else { @@ -1022,7 +1022,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { if (i & 1 & n) { // separators which are tokens are not pushed to result stack } else { - push_result_token(&parser, rule); + push_result_token(&parser, rule->rule_id); } mp_lexer_to_next(lex); // got element of list, so continue parsing list -- cgit v1.2.3 From c2c92ceefc86d7e5cdc358c4ea3226844391ae44 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 24 Dec 2017 00:01:02 +1100 Subject: py/parse: Remove rule_t struct because it's no longer needed. --- py/parse.c | 79 ++++++++++++++++++++++++++++---------------------------------- 1 file changed, 36 insertions(+), 43 deletions(-) (limited to 'py') diff --git a/py/parse.c b/py/parse.c index 6244af432..1af0e9c5f 100644 --- a/py/parse.c +++ b/py/parse.c @@ -58,12 +58,6 @@ // (un)comment to use rule names; for debugging //#define USE_RULE_NAME (1) -typedef struct _rule_t { - byte rule_id; - byte act; - const uint16_t *arg; -} rule_t; - enum { // define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) RULE_##rule, @@ -254,13 +248,12 @@ STATIC void push_rule_from_arg(parser_t *parser, size_t arg) { push_rule(parser, parser->lexer->tok_line, rule_id, 0); } -STATIC void pop_rule(parser_t *parser, rule_t *rule, size_t *arg_i, size_t *src_line) { +STATIC uint8_t pop_rule(parser_t *parser, size_t *arg_i, size_t *src_line) { parser->rule_stack_top -= 1; - rule->rule_id = parser->rule_stack[parser->rule_stack_top].rule_id; - rule->act = rule_act_table[rule->rule_id]; - rule->arg = rule_arg_table[rule->rule_id]; + uint8_t rule_id = parser->rule_stack[parser->rule_stack_top].rule_id; *arg_i = parser->rule_stack[parser->rule_stack_top].arg_i; *src_line = parser->rule_stack[parser->rule_stack_top].src_line; + return rule_id; } bool mp_parse_node_is_const_false(mp_parse_node_t pn) { @@ -810,10 +803,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // parse! - size_t n, i; // state for the current rule - size_t rule_src_line; // source line for the first token matched by the current rule bool backtrack = false; - const rule_t *rule = NULL; for (;;) { next_rule: @@ -821,10 +811,13 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { break; } - rule_t rule_data; - pop_rule(&parser, &rule_data, &i, &rule_src_line); - rule = &rule_data; - n = rule->act & RULE_ACT_ARG_MASK; + // Pop the next rule to process it + size_t i; // state for the current rule + size_t rule_src_line; // source line for the first token matched by the current rule + uint8_t rule_id = pop_rule(&parser, &i, &rule_src_line); + uint8_t rule_act = rule_act_table[rule_id]; + const uint16_t *rule_arg = rule_arg_table[rule_id]; + size_t n = rule_act & RULE_ACT_ARG_MASK; /* // debugging @@ -832,10 +825,10 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { for (int j = 0; j < parser.rule_stack_top; ++j) { printf(" "); } - printf("%s n=%d i=%d bt=%d\n", rule_name_table[rule->rule_id], n, i, backtrack); + printf("%s n=%d i=%d bt=%d\n", rule_name_table[rule_id], n, i, backtrack); */ - switch (rule->act & RULE_ACT_KIND_MASK) { + switch (rule_act & RULE_ACT_KIND_MASK) { case RULE_ACT_OR: if (i > 0 && !backtrack) { goto next_rule; @@ -843,19 +836,19 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { backtrack = false; } for (; i < n; ++i) { - uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK; + uint16_t kind = rule_arg[i] & RULE_ARG_KIND_MASK; if (kind == RULE_ARG_TOK) { - if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) { - push_result_token(&parser, rule->rule_id); + if (lex->tok_kind == (rule_arg[i] & RULE_ARG_ARG_MASK)) { + push_result_token(&parser, rule_id); mp_lexer_to_next(lex); goto next_rule; } } else { assert(kind == RULE_ARG_RULE); if (i + 1 < n) { - push_rule(&parser, rule_src_line, rule->rule_id, i + 1); // save this or-rule + push_rule(&parser, rule_src_line, rule_id, i + 1); // save this or-rule } - push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule + push_rule_from_arg(&parser, rule_arg[i]); // push child of or-rule goto next_rule; } } @@ -867,7 +860,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // failed, backtrack if we can, else syntax error if (backtrack) { assert(i > 0); - if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) { + if ((rule_arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) { // an optional rule that failed, so continue with next arg push_result_node(&parser, MP_PARSE_NODE_NULL); backtrack = false; @@ -884,13 +877,13 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // progress through the rule for (; i < n; ++i) { - if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + if ((rule_arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { // need to match a token - mp_token_kind_t tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK; + mp_token_kind_t tok_kind = rule_arg[i] & RULE_ARG_ARG_MASK; if (lex->tok_kind == tok_kind) { // matched token if (tok_kind == MP_TOKEN_NAME) { - push_result_token(&parser, rule->rule_id); + push_result_token(&parser, rule_id); } mp_lexer_to_next(lex); } else { @@ -905,8 +898,8 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } } else { - push_rule(&parser, rule_src_line, rule->rule_id, i + 1); // save this and-rule - push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule + push_rule(&parser, rule_src_line, rule_id, i + 1); // save this and-rule + push_rule_from_arg(&parser, rule_arg[i]); // push child of and-rule goto next_rule; } } @@ -917,7 +910,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { #if !MICROPY_ENABLE_DOC_STRING // this code discards lonely statements, such as doc strings - if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) { + if (input_kind != MP_PARSE_SINGLE_INPUT && rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) { mp_parse_node_t p = peek_result(&parser, 1); if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) || MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_const_object)) { @@ -937,8 +930,8 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { size_t num_not_nil = 0; for (size_t x = n; x > 0;) { --x; - if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { - mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; + if ((rule_arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + mp_token_kind_t tok_kind = rule_arg[x] & RULE_ARG_ARG_MASK; if (tok_kind == MP_TOKEN_NAME) { // only tokens which were names are pushed to stack i += 1; @@ -953,7 +946,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } - if (num_not_nil == 1 && (rule->act & RULE_ACT_ALLOW_IDENT)) { + if (num_not_nil == 1 && (rule_act & RULE_ACT_ALLOW_IDENT)) { // this rule has only 1 argument and should not be emitted mp_parse_node_t pn = MP_PARSE_NODE_NULL; for (size_t x = 0; x < i; ++x) { @@ -966,19 +959,19 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } else { // this rule must be emitted - if (rule->act & RULE_ACT_ADD_BLANK) { + if (rule_act & RULE_ACT_ADD_BLANK) { // and add an extra blank node at the end (used by the compiler to store data) push_result_node(&parser, MP_PARSE_NODE_NULL); i += 1; } - push_result_rule(&parser, rule_src_line, rule->rule_id, i); + push_result_rule(&parser, rule_src_line, rule_id, i); } break; } default: { - assert((rule->act & RULE_ACT_KIND_MASK) == RULE_ACT_LIST); + assert((rule_act & RULE_ACT_KIND_MASK) == RULE_ACT_LIST); // n=2 is: item item* // n=1 is: item (sep item)* @@ -1016,13 +1009,13 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } else { for (;;) { - size_t arg = rule->arg[i & 1 & n]; + size_t arg = rule_arg[i & 1 & n]; if ((arg & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) { if (i & 1 & n) { // separators which are tokens are not pushed to result stack } else { - push_result_token(&parser, rule->rule_id); + push_result_token(&parser, rule_id); } mp_lexer_to_next(lex); // got element of list, so continue parsing list @@ -1035,7 +1028,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } else { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE); - push_rule(&parser, rule_src_line, rule->rule_id, i + 1); // save this list-rule + push_rule(&parser, rule_src_line, rule_id, i + 1); // save this list-rule push_rule_from_arg(&parser, arg); // push child of list-rule goto next_rule; } @@ -1045,7 +1038,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // compute number of elements in list, result in i i -= 1; - if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + if ((n & 1) && (rule_arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { // don't count separators when they are tokens i = (i + 1) / 2; } @@ -1054,12 +1047,12 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // list matched single item if (had_trailing_sep) { // if there was a trailing separator, make a list of a single item - push_result_rule(&parser, rule_src_line, rule->rule_id, i); + push_result_rule(&parser, rule_src_line, rule_id, i); } else { // just leave single item on stack (ie don't wrap in a list) } } else { - push_result_rule(&parser, rule_src_line, rule->rule_id, i); + push_result_rule(&parser, rule_src_line, rule_id, i); } break; } -- cgit v1.2.3 From 0016a45368d8e7272d1c1f891994d6de4effcb40 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 26 Dec 2017 13:39:02 +1100 Subject: py/parse: Compress rule pointer table to table of offsets. This is the sixth and final patch in a series of patches to the parser that aims to reduce code size by compressing the data corresponding to the rules of the grammar. Prior to this set of patches the rules were stored as rule_t structs with rule_id, act and arg members. And then there was a big table of pointers which allowed to lookup the address of a rule_t struct given the id of that rule. The changes that have been made are: - Breaking up of the rule_t struct into individual components, with each component in a separate array. - Removal of the rule_id part of the struct because it's not needed. - Put all the rule arg data in a big array. - Change the table of pointers to rules to a table of offsets within the array of rule arg data. The last point is what is done in this patch here and brings about the biggest decreases in code size, because an array of pointers is now an array of bytes. Code size changes for the six patches combined is: bare-arm: -644 minimal x86: -1856 unix x64: -5408 unix nanbox: -2080 stm32: -720 esp8266: -812 cc3200: -712 For the change in parser performance: it was measured on pyboard that these six patches combined gave an increase in script parse time of about 0.4%. This is due to the slightly more complicated way of looking up the data for a rule (since the 9th bit of the offset into the rule arg data table is calculated with an if statement). This is an acceptable increase in parse time considering that parsing is only done once per script (if compiled on the target). --- py/parse.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 10 deletions(-) (limited to 'py') diff --git a/py/parse.c b/py/parse.c index 1af0e9c5f..8e0793185 100644 --- a/py/parse.c +++ b/py/parse.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013-2015 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -108,13 +108,20 @@ STATIC const uint8_t rule_act_table[] = { #undef list_with_end }; -// Define the argument data for each rule +// Define the argument data for each rule, as a combined array +STATIC const uint16_t rule_arg_combined_table[] = { #define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t) #define rule(r) (RULE_ARG_RULE | RULE_##r) #define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) -#define DEF_RULE(rule, comp, kind, ...) static const uint16_t const rule_arg_##rule[] = { __VA_ARGS__ }; -#define DEF_RULE_NC(rule, kind, ...) static const uint16_t const rule_arg_##rule[] = { __VA_ARGS__ }; +#define DEF_RULE(rule, comp, kind, ...) __VA_ARGS__, +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) __VA_ARGS__, #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC @@ -122,22 +129,60 @@ STATIC const uint8_t rule_act_table[] = { #undef tok #undef rule #undef opt_rule +}; + +// Macro to create a list of N-1 identifiers where N is the number of variable arguments to the macro +#define RULE_PADDING(rule, ...) RULE_PADDING2(rule, __VA_ARGS__, RULE_PADDING_IDS(rule)) +#define RULE_PADDING2(rule, ...) RULE_PADDING3(rule, __VA_ARGS__) +#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) __VA_ARGS__ +#define RULE_PADDING_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, + +// Use an enum to create constants that specify where in rule_arg_combined_table a given rule starts +enum { +#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET_##rule, RULE_PADDING(rule, __VA_ARGS__) +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET_##rule, RULE_PADDING(rule, __VA_ARGS__) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +}; -// Define an array of pointers to corresponding rule data -STATIC const uint16_t *const rule_arg_table[] = { -#define DEF_RULE(rule, comp, kind, ...) &rule_arg_##rule[0], +// Use the above enum values to create a table of offsets for each rule's arg +// data, which indexes rule_arg_combined_table. The offsets require 9 bits of +// storage but only the lower 8 bits are stored here. The 9th bit is computed +// in get_rule_arg using the FIRST_RULE_WITH_OFFSET_ABOVE_255 constant. +STATIC const uint8_t rule_arg_offset_table[] = { +#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET_##rule & 0xff, #define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC - NULL, // RULE_const_object + 0, // RULE_const_object #define DEF_RULE(rule, comp, kind, ...) -#define DEF_RULE_NC(rule, kind, ...) &rule_arg_##rule[0], +#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET_##rule & 0xff, #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC }; +// Define a constant that's used to determine the 9th bit of the values in rule_arg_offset_table +static const size_t FIRST_RULE_WITH_OFFSET_ABOVE_255 = +#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET_##rule >= 0x100 ? RULE_##rule : +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET_##rule >= 0x100 ? RULE_##rule : +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +0; + #if USE_RULE_NAME // Define an array of rule names corresponding to each rule STATIC const char *const rule_name_table[] = { @@ -189,6 +234,14 @@ typedef struct _parser_t { #endif } parser_t; +STATIC const uint16_t *get_rule_arg(uint8_t r_id) { + size_t off = rule_arg_offset_table[r_id]; + if (r_id >= FIRST_RULE_WITH_OFFSET_ABOVE_255) { + off |= 0x100; + } + return &rule_arg_combined_table[off]; +} + STATIC void *parser_alloc(parser_t *parser, size_t num_bytes) { // use a custom memory allocator to store parse nodes sequentially in large chunks @@ -816,7 +869,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { size_t rule_src_line; // source line for the first token matched by the current rule uint8_t rule_id = pop_rule(&parser, &i, &rule_src_line); uint8_t rule_act = rule_act_table[rule_id]; - const uint16_t *rule_arg = rule_arg_table[rule_id]; + const uint16_t *rule_arg = get_rule_arg(rule_id); size_t n = rule_act & RULE_ACT_ARG_MASK; /* -- cgit v1.2.3 From d3fbfa491f46ec7e3f69d12e037cb3da7b3ae984 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 26 Dec 2017 13:39:26 +1100 Subject: py/parse: Update debugging code to compile on 64-bit arch. --- py/parse.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'py') diff --git a/py/parse.c b/py/parse.c index 8e0793185..238c94695 100644 --- a/py/parse.c +++ b/py/parse.c @@ -872,14 +872,14 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { const uint16_t *rule_arg = get_rule_arg(rule_id); size_t n = rule_act & RULE_ACT_ARG_MASK; - /* + #if 0 // debugging - printf("depth=%d ", parser.rule_stack_top); + printf("depth=" UINT_FMT " ", parser.rule_stack_top); for (int j = 0; j < parser.rule_stack_top; ++j) { printf(" "); } - printf("%s n=%d i=%d bt=%d\n", rule_name_table[rule_id], n, i, backtrack); - */ + printf("%s n=" UINT_FMT " i=" UINT_FMT " bt=%d\n", rule_name_table[rule_id], n, i, backtrack); + #endif switch (rule_act & RULE_ACT_KIND_MASK) { case RULE_ACT_OR: -- cgit v1.2.3 From c7cb1dfcb9f89eb0a2d2a90a5e496e919c8a0b94 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 29 Dec 2017 00:53:57 +1100 Subject: py/parse: Fix macro evaluation by avoiding empty __VA_ARGS__. Empty __VA_ARGS__ are not allowed in the C preprocessor so adjust the rule arg offset calculation to not use them. Also, some compilers (eg MSVC) require an extra layer of macro expansion. --- py/parse.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'py') diff --git a/py/parse.c b/py/parse.c index 238c94695..8c1286492 100644 --- a/py/parse.c +++ b/py/parse.c @@ -131,39 +131,46 @@ STATIC const uint16_t rule_arg_combined_table[] = { #undef opt_rule }; -// Macro to create a list of N-1 identifiers where N is the number of variable arguments to the macro +// Macro to create a list of N identifiers where N is the number of variable arguments to the macro +#define RULE_EXPAND(x) x #define RULE_PADDING(rule, ...) RULE_PADDING2(rule, __VA_ARGS__, RULE_PADDING_IDS(rule)) -#define RULE_PADDING2(rule, ...) RULE_PADDING3(rule, __VA_ARGS__) -#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) __VA_ARGS__ +#define RULE_PADDING2(rule, ...) RULE_EXPAND(RULE_PADDING3(rule, __VA_ARGS__)) +#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, ...) __VA_ARGS__ #define RULE_PADDING_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, -// Use an enum to create constants that specify where in rule_arg_combined_table a given rule starts +// Use an enum to create constants specifying how much room a rule takes in rule_arg_combined_table enum { -#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET_##rule, RULE_PADDING(rule, __VA_ARGS__) +#define DEF_RULE(rule, comp, kind, ...) RULE_PADDING(rule, __VA_ARGS__) #define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC #define DEF_RULE(rule, comp, kind, ...) -#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET_##rule, RULE_PADDING(rule, __VA_ARGS__) +#define DEF_RULE_NC(rule, kind, ...) RULE_PADDING(rule, __VA_ARGS__) #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC }; +// Macro to compute the start of a rule in rule_arg_combined_table +#define RULE_ARG_OFFSET(rule, ...) RULE_ARG_OFFSET2(rule, __VA_ARGS__, RULE_ARG_OFFSET_IDS(rule)) +#define RULE_ARG_OFFSET2(rule, ...) RULE_EXPAND(RULE_ARG_OFFSET3(rule, __VA_ARGS__)) +#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) _13 +#define RULE_ARG_OFFSET_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r, + // Use the above enum values to create a table of offsets for each rule's arg // data, which indexes rule_arg_combined_table. The offsets require 9 bits of // storage but only the lower 8 bits are stored here. The 9th bit is computed // in get_rule_arg using the FIRST_RULE_WITH_OFFSET_ABOVE_255 constant. STATIC const uint8_t rule_arg_offset_table[] = { -#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET_##rule & 0xff, +#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) & 0xff, #define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC 0, // RULE_const_object #define DEF_RULE(rule, comp, kind, ...) -#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET_##rule & 0xff, +#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) & 0xff, #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC @@ -171,13 +178,13 @@ STATIC const uint8_t rule_arg_offset_table[] = { // Define a constant that's used to determine the 9th bit of the values in rule_arg_offset_table static const size_t FIRST_RULE_WITH_OFFSET_ABOVE_255 = -#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET_##rule >= 0x100 ? RULE_##rule : +#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) >= 0x100 ? RULE_##rule : #define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC #define DEF_RULE(rule, comp, kind, ...) -#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET_##rule >= 0x100 ? RULE_##rule : +#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) >= 0x100 ? RULE_##rule : #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC -- cgit v1.2.3 From 9766fddcdc29cb2b5a8657389b870703580a6e57 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 28 Dec 2017 14:02:36 +1100 Subject: py/mpz: Simplify handling of borrow and quo adjustment in mpn_div. The motivation behind this patch is to remove unreachable code in mpn_div. This unreachable code was added some time ago in 9a21d2e070c9ee0ef2c003f3a668e635c6ae4401, when a loop in mpn_div was copied and adjusted to work when mpz_dig_t was exactly half of the size of mpz_dbl_dig_t (a common case). The loop was copied correctly but it wasn't noticed at the time that the final part of the calculation of num-quo*den could be optimised, and hence unreachable code was left for a case that never occurred. The observation for the optimisation is that the initial value of quo in mpn_div is either exact or too large (never too small), and therefore the subtraction of quo*den from num may subtract exactly enough or too much (but never too little). Using this observation the part of the algorithm that handles the borrow value can be simplified, and most importantly this eliminates the unreachable code. The new code has been tested with DIG_SIZE=3 and DIG_SIZE=4 by dividing all possible combinations of non-negative integers with between 0 and 3 (inclusive) mpz digits. --- py/mpz.c | 114 ++++++++++++++++++++++++--------------------------------------- py/mpz.h | 4 +++ 2 files changed, 48 insertions(+), 70 deletions(-) (limited to 'py') diff --git a/py/mpz.c b/py/mpz.c index 018a5454f..78dee3132 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -537,83 +537,57 @@ STATIC void mpn_div(mpz_dig_t *num_dig, size_t *num_len, const mpz_dig_t *den_di // not to overflow the borrow variable. And the shifting of // borrow needs some special logic (it's a shift right with // round up). - - if (DIG_SIZE < 8 * sizeof(mpz_dbl_dig_t) / 2) { - const mpz_dig_t *d = den_dig; - mpz_dbl_dig_t d_norm = 0; - mpz_dbl_dig_signed_t borrow = 0; - - for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { - d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); - borrow += (mpz_dbl_dig_t)*n - (mpz_dbl_dig_t)quo * (d_norm & DIG_MASK); // will overflow if DIG_SIZE >= 8*sizeof(mpz_dbl_dig_t)/2 - *n = borrow & DIG_MASK; - borrow >>= DIG_SIZE; - } - borrow += *num_dig; // will overflow if DIG_SIZE >= 8*sizeof(mpz_dbl_dig_t)/2 - *num_dig = borrow & DIG_MASK; - borrow >>= DIG_SIZE; - - // adjust quotient if it is too big - for (; borrow != 0; --quo) { - d = den_dig; - d_norm = 0; - mpz_dbl_dig_t carry = 0; - for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { - d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); - carry += (mpz_dbl_dig_t)*n + (d_norm & DIG_MASK); - *n = carry & DIG_MASK; - carry >>= DIG_SIZE; - } - carry += *num_dig; - *num_dig = carry & DIG_MASK; - carry >>= DIG_SIZE; - - borrow += carry; - } - } else { // DIG_SIZE == 8 * sizeof(mpz_dbl_dig_t) / 2 - const mpz_dig_t *d = den_dig; - mpz_dbl_dig_t d_norm = 0; - mpz_dbl_dig_t borrow = 0; - - for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { - d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); - mpz_dbl_dig_t x = (mpz_dbl_dig_t)quo * (d_norm & DIG_MASK); - if (x >= *n || *n - x <= borrow) { - borrow += (mpz_dbl_dig_t)x - (mpz_dbl_dig_t)*n; - *n = (-borrow) & DIG_MASK; - borrow = (borrow >> DIG_SIZE) + ((borrow & DIG_MASK) == 0 ? 0 : 1); // shift-right with round-up - } else { - *n = ((mpz_dbl_dig_t)*n - (mpz_dbl_dig_t)x - (mpz_dbl_dig_t)borrow) & DIG_MASK; - borrow = 0; - } - } - if (borrow >= *num_dig) { - borrow -= (mpz_dbl_dig_t)*num_dig; - *num_dig = (-borrow) & DIG_MASK; + // + const mpz_dig_t *d = den_dig; + mpz_dbl_dig_t d_norm = 0; + mpz_dbl_dig_t borrow = 0; + for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { + d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); + mpz_dbl_dig_t x = (mpz_dbl_dig_t)quo * (d_norm & DIG_MASK); + #if DIG_SIZE < MPZ_DBL_DIG_SIZE / 2 + borrow += (mpz_dbl_dig_t)*n - x; // will overflow if DIG_SIZE >= MPZ_DBL_DIG_SIZE/2 + *n = borrow & DIG_MASK; + borrow = (mpz_dbl_dig_signed_t)borrow >> DIG_SIZE; + #else // DIG_SIZE == MPZ_DBL_DIG_SIZE / 2 + if (x >= *n || *n - x <= borrow) { + borrow += x - (mpz_dbl_dig_t)*n; + *n = (-borrow) & DIG_MASK; borrow = (borrow >> DIG_SIZE) + ((borrow & DIG_MASK) == 0 ? 0 : 1); // shift-right with round-up } else { - *num_dig = (*num_dig - borrow) & DIG_MASK; + *n = ((mpz_dbl_dig_t)*n - x - borrow) & DIG_MASK; borrow = 0; } + #endif + } - // adjust quotient if it is too big - for (; borrow != 0; --quo) { - d = den_dig; - d_norm = 0; - mpz_dbl_dig_t carry = 0; - for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { - d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); - carry += (mpz_dbl_dig_t)*n + (d_norm & DIG_MASK); - *n = carry & DIG_MASK; - carry >>= DIG_SIZE; - } - carry += (mpz_dbl_dig_t)*num_dig; - *num_dig = carry & DIG_MASK; - carry >>= DIG_SIZE; + #if DIG_SIZE < MPZ_DBL_DIG_SIZE / 2 + // Borrow was negative in the above for-loop, make it positive for next if-block. + borrow = -borrow; + #endif - //assert(borrow >= carry); // enable this to check the logic - borrow -= carry; + // At this point we have either: + // + // 1. quo was the correct value and the most-sig-digit of num is exactly + // cancelled by borrow (borrow == *num_dig). In this case there is + // nothing more to do. + // + // 2. quo was too large, we subtracted too many den from num, and the + // most-sig-digit of num is 1 less than borrow (borrow == *num_dig + 1). + // In this case we must reduce quo and add back den to num until the + // carry from this operation cancels out the borrow. + // + borrow -= *num_dig; + for (; borrow != 0; --quo) { + d = den_dig; + d_norm = 0; + mpz_dbl_dig_t carry = 0; + for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { + d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); + carry += (mpz_dbl_dig_t)*n + (d_norm & DIG_MASK); + *n = carry & DIG_MASK; + carry >>= DIG_SIZE; } + borrow -= carry; } // store this digit of the quotient diff --git a/py/mpz.h b/py/mpz.h index e2d0c30aa..3c36cac66 100644 --- a/py/mpz.h +++ b/py/mpz.h @@ -55,18 +55,22 @@ #endif #if MPZ_DIG_SIZE > 16 +#define MPZ_DBL_DIG_SIZE (64) typedef uint32_t mpz_dig_t; typedef uint64_t mpz_dbl_dig_t; typedef int64_t mpz_dbl_dig_signed_t; #elif MPZ_DIG_SIZE > 8 +#define MPZ_DBL_DIG_SIZE (32) typedef uint16_t mpz_dig_t; typedef uint32_t mpz_dbl_dig_t; typedef int32_t mpz_dbl_dig_signed_t; #elif MPZ_DIG_SIZE > 4 +#define MPZ_DBL_DIG_SIZE (16) typedef uint8_t mpz_dig_t; typedef uint16_t mpz_dbl_dig_t; typedef int16_t mpz_dbl_dig_signed_t; #else +#define MPZ_DBL_DIG_SIZE (8) typedef uint8_t mpz_dig_t; typedef uint8_t mpz_dbl_dig_t; typedef int8_t mpz_dbl_dig_signed_t; -- cgit v1.2.3 From e78427443094163d1839387a2470fd22612ca8d1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 28 Dec 2017 14:14:06 +1100 Subject: py/mpz: In mpz_as_str_inpl, convert always-false checks to assertions. There are two checks that are always false so can be converted to (negated) assertions to save code space and execution time. They are: 1. The check of the str parameter, which is required to be non-NULL as per the original comment that it has enough space in it as calculated by mp_int_format_size. And for all uses of this function str is indeed non-NULL. 2. The check of the base parameter, which is already required to be between 2 and 16 (inclusive) via the assertion in mp_int_format_size. --- py/mpz.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'py') diff --git a/py/mpz.c b/py/mpz.c index 78dee3132..380e8968e 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -1647,16 +1647,12 @@ char *mpz_as_str(const mpz_t *i, unsigned int base) { } #endif -// assumes enough space as calculated by mp_int_format_size +// assumes enough space in str as calculated by mp_int_format_size +// base must be between 2 and 32 inclusive // returns length of string, not including null byte size_t mpz_as_str_inpl(const mpz_t *i, unsigned int base, const char *prefix, char base_char, char comma, char *str) { - if (str == NULL) { - return 0; - } - if (base < 2 || base > 32) { - str[0] = 0; - return 0; - } + assert(str != NULL); + assert(2 <= base && base <= 32); size_t ilen = i->len; -- cgit v1.2.3 From b184b6ae5334b87472897d0034a7388acafe41cc Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 26 Dec 2017 10:52:09 +0100 Subject: py/nlr: Fix nlr functions for 64bit ports built with gcc on Windows The number of registers used should be 10, not 12, to match the assembly code in nlrx64.c. With this change the 64bit mingw builds don't need to use the setjmp implementation, and this fixes miscellaneous crashes and assertion failures as reported in #1751 for instance. To avoid mistakes in the future where something gcc-related for Windows only gets fixed for one particular compiler/environment combination, make use of a MICROPY_NLR_OS_WINDOWS macro. To make sure everything nlr-related is now ok when built with gcc this has been verified with: - unix port built with gcc on Cygwin (i686-pc-cygwin-gcc and x86_64-pc-cygwin-gcc, version 6.4.0) - windows port built with mingw-w64's gcc from Cygwin (i686-w64-mingw32-gcc and x86_64-w64-mingw32-gcc, version 6.4.0) and MSYS2 (like the ones on Cygwin but version 7.2.0) --- ports/windows/Makefile | 4 ---- py/nlr.c | 2 +- py/nlr.h | 10 ++++++++-- py/nlrx64.c | 10 ++-------- py/nlrx86.c | 8 +------- 5 files changed, 12 insertions(+), 22 deletions(-) (limited to 'py') diff --git a/ports/windows/Makefile b/ports/windows/Makefile index b6433300f..725cb686e 100644 --- a/ports/windows/Makefile +++ b/ports/windows/Makefile @@ -50,10 +50,6 @@ CFLAGS_MOD += -DMICROPY_USE_READLINE=2 LDFLAGS_MOD += -lreadline endif -ifeq ($(CROSS_COMPILE),x86_64-w64-mingw32-) -CFLAGS_MOD += -DMICROPY_NLR_SETJMP=1 -endif - LIB += -lws2_32 # List of sources for qstr extraction diff --git a/py/nlr.c b/py/nlr.c index 52d56afb8..7114d4997 100644 --- a/py/nlr.c +++ b/py/nlr.c @@ -28,7 +28,7 @@ #if !MICROPY_NLR_SETJMP // When not using setjmp, nlr_push_tail is called from inline asm so needs special c -#if MICROPY_NLR_X86 && (defined(_WIN32) || defined(__CYGWIN__)) +#if MICROPY_NLR_X86 && MICROPY_NLR_OS_WINDOWS // On these 32-bit platforms make sure nlr_push_tail doesn't have a leading undersco unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); #else diff --git a/py/nlr.h b/py/nlr.h index e4dfa6896..90595a12d 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -36,13 +36,19 @@ // If MICROPY_NLR_SETJMP is not enabled then auto-detect the machine arch #if !MICROPY_NLR_SETJMP +// A lot of nlr-related things need different treatment on Windows +#if defined(_WIN32) || defined(__CYGWIN__) +#define MICROPY_NLR_OS_WINDOWS 1 +#else +#define MICROPY_NLR_OS_WINDOWS 0 +#endif #if defined(__i386__) #define MICROPY_NLR_X86 (1) #define MICROPY_NLR_NUM_REGS (6) #elif defined(__x86_64__) #define MICROPY_NLR_X64 (1) - #if defined(__CYGWIN__) - #define MICROPY_NLR_NUM_REGS (12) + #if MICROPY_NLR_OS_WINDOWS + #define MICROPY_NLR_NUM_REGS (10) #else #define MICROPY_NLR_NUM_REGS (8) #endif diff --git a/py/nlrx64.c b/py/nlrx64.c index 663a457b7..a3a1cf341 100644 --- a/py/nlrx64.c +++ b/py/nlrx64.c @@ -33,18 +33,12 @@ // x86-64 callee-save registers are: // rbx, rbp, rsp, r12, r13, r14, r15 -#if defined(_WIN32) || defined(__CYGWIN__) -#define NLR_OS_WINDOWS 1 -#else -#define NLR_OS_WINDOWS 0 -#endif - __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); unsigned int nlr_push(nlr_buf_t *nlr) { (void)nlr; - #if NLR_OS_WINDOWS + #if MICROPY_NLR_OS_WINDOWS __asm volatile ( "movq (%rsp), %rax \n" // load return %rip @@ -93,7 +87,7 @@ NORETURN void nlr_jump(void *val) { __asm volatile ( "movq %0, %%rcx \n" // %rcx points to nlr_buf - #if NLR_OS_WINDOWS + #if MICROPY_NLR_OS_WINDOWS "movq 88(%%rcx), %%rsi \n" // load saved %rsi "movq 80(%%rcx), %%rdi \n" // load saved %rdr #endif diff --git a/py/nlrx86.c b/py/nlrx86.c index 9490c4f42..23882cc30 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -33,13 +33,7 @@ // For reference, x86 callee save regs are: // ebx, esi, edi, ebp, esp, eip -#if defined(_WIN32) || defined(__CYGWIN__) -#define NLR_OS_WINDOWS 1 -#else -#define NLR_OS_WINDOWS 0 -#endif - -#if NLR_OS_WINDOWS +#if MICROPY_NLR_OS_WINDOWS unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); #else __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); -- cgit v1.2.3 From 42c4dd09a19ac864825786ece39d759bca34abd2 Mon Sep 17 00:00:00 2001 From: stijn Date: Thu, 28 Dec 2017 11:02:29 +0100 Subject: py/nlr: Fix missing trailing characters in comments in nlr.c --- py/nlr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/nlr.c b/py/nlr.c index 7114d4997..03d01577e 100644 --- a/py/nlr.c +++ b/py/nlr.c @@ -27,12 +27,12 @@ #include "py/mpstate.h" #if !MICROPY_NLR_SETJMP -// When not using setjmp, nlr_push_tail is called from inline asm so needs special c +// When not using setjmp, nlr_push_tail is called from inline asm so needs special care #if MICROPY_NLR_X86 && MICROPY_NLR_OS_WINDOWS -// On these 32-bit platforms make sure nlr_push_tail doesn't have a leading undersco +// On these 32-bit platforms make sure nlr_push_tail doesn't have a leading underscore unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); #else -// LTO can't see inside inline asm functions so explicitly mark nlr_push_tail as use +// LTO can't see inside inline asm functions so explicitly mark nlr_push_tail as used __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); #endif #endif -- cgit v1.2.3 From 253f2bd7be39e201db78960be456ac0f8c61b05f Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 4 Feb 2018 13:35:21 +1100 Subject: py/compile: Combine compiler-opt of 2 and 3 tuple-to-tuple assignment. This patch combines the compiler optimisation code for double and triple tuple-to-tuple assignment, taking it from two separate if-blocks to one combined if-block. This can be done because the code for both of these optimisations has a lot in common. Combining them together reduces code size for ports that have the triple-tuple optimisation enabled (and doesn't change code size for ports that have it disabled). --- py/compile.c | 85 ++++++++++++++++++++++++++++++----------------------------- py/mpconfig.h | 2 +- 2 files changed, 44 insertions(+), 43 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 52d10ee5e..42ae8a829 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1941,51 +1941,52 @@ STATIC void compile_expr_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { } } else { plain_assign: - if (MICROPY_COMP_DOUBLE_TUPLE_ASSIGN - && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_testlist_star_expr) - && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr) - && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[1]) == 2 - && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[0]) == 2) { - // optimisation for a, b = c, d - mp_parse_node_struct_t *pns10 = (mp_parse_node_struct_t*)pns->nodes[1]; + #if MICROPY_COMP_DOUBLE_TUPLE_ASSIGN + if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_testlist_star_expr) + && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr)) { mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns->nodes[0]; - if (MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) - || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[1], PN_star_expr)) { - // can't optimise when it's a star expression on the lhs - goto no_optimisation; - } - compile_node(comp, pns10->nodes[0]); // rhs - compile_node(comp, pns10->nodes[1]); // rhs - EMIT(rot_two); - c_assign(comp, pns0->nodes[0], ASSIGN_STORE); // lhs store - c_assign(comp, pns0->nodes[1], ASSIGN_STORE); // lhs store - } else if (MICROPY_COMP_TRIPLE_TUPLE_ASSIGN - && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_testlist_star_expr) - && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr) - && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[1]) == 3 - && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[0]) == 3) { - // optimisation for a, b, c = d, e, f - mp_parse_node_struct_t *pns10 = (mp_parse_node_struct_t*)pns->nodes[1]; - mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns->nodes[0]; - if (MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) - || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[1], PN_star_expr) - || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[2], PN_star_expr)) { - // can't optimise when it's a star expression on the lhs - goto no_optimisation; + pns1 = (mp_parse_node_struct_t*)pns->nodes[1]; + uint32_t n_pns0 = MP_PARSE_NODE_STRUCT_NUM_NODES(pns0); + // Can only optimise a tuple-to-tuple assignment when all of the following hold: + // - equal number of items in LHS and RHS tuples + // - 2 or 3 items in the tuples + // - there are no star expressions in the LHS tuple + if (n_pns0 == MP_PARSE_NODE_STRUCT_NUM_NODES(pns1) + && (n_pns0 == 2 + #if MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + || n_pns0 == 3 + #endif + ) + && !MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) + && !MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[1], PN_star_expr) + #if MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + && (n_pns0 == 2 || !MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[2], PN_star_expr)) + #endif + ) { + // Optimisation for a, b = c, d or a, b, c = d, e, f + compile_node(comp, pns1->nodes[0]); // rhs + compile_node(comp, pns1->nodes[1]); // rhs + #if MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + if (n_pns0 == 3) { + compile_node(comp, pns1->nodes[2]); // rhs + EMIT(rot_three); + } + #endif + EMIT(rot_two); + c_assign(comp, pns0->nodes[0], ASSIGN_STORE); // lhs store + c_assign(comp, pns0->nodes[1], ASSIGN_STORE); // lhs store + #if MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + if (n_pns0 == 3) { + c_assign(comp, pns0->nodes[2], ASSIGN_STORE); // lhs store + } + #endif + return; } - compile_node(comp, pns10->nodes[0]); // rhs - compile_node(comp, pns10->nodes[1]); // rhs - compile_node(comp, pns10->nodes[2]); // rhs - EMIT(rot_three); - EMIT(rot_two); - c_assign(comp, pns0->nodes[0], ASSIGN_STORE); // lhs store - c_assign(comp, pns0->nodes[1], ASSIGN_STORE); // lhs store - c_assign(comp, pns0->nodes[2], ASSIGN_STORE); // lhs store - } else { - no_optimisation: - compile_node(comp, pns->nodes[1]); // rhs - c_assign(comp, pns->nodes[0], ASSIGN_STORE); // lhs store } + #endif + + compile_node(comp, pns->nodes[1]); // rhs + c_assign(comp, pns->nodes[0], ASSIGN_STORE); // lhs store } } else { goto plain_assign; diff --git a/py/mpconfig.h b/py/mpconfig.h index 763bb378e..f2a8c98cb 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -347,7 +347,7 @@ #endif // Whether to enable optimisation of: a, b, c = d, e, f -// Cost 156 bytes (Thumb2) +// Requires MICROPY_COMP_DOUBLE_TUPLE_ASSIGN and costs 68 bytes (Thumb2) #ifndef MICROPY_COMP_TRIPLE_TUPLE_ASSIGN #define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (0) #endif -- cgit v1.2.3 From b45c8c17f0f295e919a90659d4c1880a47b228c1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 7 Feb 2018 15:44:29 +1100 Subject: py/objtype: Check and prevent delete/store on a fixed locals map. Note that the check for elem!=NULL is removed for the MP_MAP_LOOKUP_ADD_IF_NOT_FOUND case because mp_map_lookup will always return non-NULL for such a case. --- py/objtype.c | 12 ++++++------ tests/basics/del_attr.py | 7 +++++++ tests/basics/setattr1.py | 7 +++++++ 3 files changed, 20 insertions(+), 6 deletions(-) (limited to 'py') diff --git a/py/objtype.c b/py/objtype.c index 33757287a..7df349ce9 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -975,21 +975,21 @@ STATIC void type_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (self->locals_dict != NULL) { assert(self->locals_dict->base.type == &mp_type_dict); // MicroPython restriction, for now mp_map_t *locals_map = &self->locals_dict->map; + if (locals_map->is_fixed) { + // can't apply delete/store to a fixed map + return; + } if (dest[1] == MP_OBJ_NULL) { // delete attribute mp_map_elem_t *elem = mp_map_lookup(locals_map, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_REMOVE_IF_FOUND); - // note that locals_map may be in ROM, so remove will fail in that case if (elem != NULL) { dest[0] = MP_OBJ_NULL; // indicate success } } else { // store attribute mp_map_elem_t *elem = mp_map_lookup(locals_map, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); - // note that locals_map may be in ROM, so add will fail in that case - if (elem != NULL) { - elem->value = dest[1]; - dest[0] = MP_OBJ_NULL; // indicate success - } + elem->value = dest[1]; + dest[0] = MP_OBJ_NULL; // indicate success } } } diff --git a/tests/basics/del_attr.py b/tests/basics/del_attr.py index bec7afb84..8487b97e3 100644 --- a/tests/basics/del_attr.py +++ b/tests/basics/del_attr.py @@ -30,3 +30,10 @@ try: del c.x except AttributeError: print("AttributeError") + +# try to del an attribute of a built-in class +try: + del int.to_bytes +except (AttributeError, TypeError): + # uPy raises AttributeError, CPython raises TypeError + print('AttributeError/TypeError') diff --git a/tests/basics/setattr1.py b/tests/basics/setattr1.py index 9693aca81..e4acb14ca 100644 --- a/tests/basics/setattr1.py +++ b/tests/basics/setattr1.py @@ -16,3 +16,10 @@ try: setattr(a, b'var3', 1) except TypeError: print('TypeError') + +# try setattr on a built-in function +try: + setattr(int, 'to_bytes', 1) +except (AttributeError, TypeError): + # uPy raises AttributeError, CPython raises TypeError + print('AttributeError/TypeError') -- cgit v1.2.3 From 771dfb0826f7416848b4302ce8ce49a7a556a27f Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 7 Feb 2018 16:10:42 +1100 Subject: py/modbuiltins: For builtin_chr, use uint8_t instead of char for array. The array should be of type unsigned byte because that is the type of the values being stored. And changing to uint8_t helps to prevent warnings from some static analysers. --- py/modbuiltins.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index c8e3235f6..e6f82df6f 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -137,7 +137,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_callable_obj, mp_builtin_callable); STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { #if MICROPY_PY_BUILTINS_STR_UNICODE mp_uint_t c = mp_obj_get_int(o_in); - char str[4]; + uint8_t str[4]; int len = 0; if (c < 0x80) { *str = c; len = 1; @@ -159,12 +159,12 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { } else { mp_raise_ValueError("chr() arg not in range(0x110000)"); } - return mp_obj_new_str_via_qstr(str, len); + return mp_obj_new_str_via_qstr((char*)str, len); #else mp_int_t ord = mp_obj_get_int(o_in); if (0 <= ord && ord <= 0xff) { - char str[1] = {ord}; - return mp_obj_new_str_via_qstr(str, 1); + uint8_t str[1] = {ord}; + return mp_obj_new_str_via_qstr((char*)str, 1); } else { mp_raise_ValueError("chr() arg not in range(256)"); } -- cgit v1.2.3 From 0c650d427601d5c84bd623d58abc9be5451c576c Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 8 Feb 2018 13:30:33 +1100 Subject: py/vm: Simplify stack sentinel values for unwind return and jump. This patch simplifies how sentinel values are stored on the stack when doing an unwind return or jump. Instead of storing two values on the stack for an unwind jump it now stores only one: a negative small integer means unwind-return and a non-negative small integer means unwind-jump with the value being the number of exceptions to unwind. The savings in code size are: bare-arm: -56 minimal x86: -68 unix x64: -80 unix nanbox: -4 stm32: -56 cc3200: -64 esp8266: -76 esp32: -156 --- py/vm.c | 61 +++++++++++++++++++++++-------------------------------------- 1 file changed, 23 insertions(+), 38 deletions(-) (limited to 'py') diff --git a/py/vm.c b/py/vm.c index b18a2b5e3..c629a61e2 100644 --- a/py/vm.c +++ b/py/vm.c @@ -48,14 +48,6 @@ // top element. // Exception stack also grows up, top element is also pointed at. -// Exception stack unwind reasons (WHY_* in CPython-speak) -// TODO perhaps compress this to RETURN=0, JUMP>0, with number of unwinds -// left to do encoded in the JUMP number -typedef enum { - UNWIND_RETURN = 1, - UNWIND_JUMP, -} mp_unwind_reason_t; - #define DECODE_UINT \ mp_uint_t unum = 0; \ do { \ @@ -613,29 +605,18 @@ dispatch_loop: mp_call_method_n_kw(3, 0, sp); SET_TOP(mp_const_none); } else if (MP_OBJ_IS_SMALL_INT(TOP())) { - mp_int_t cause_val = MP_OBJ_SMALL_INT_VALUE(TOP()); - if (cause_val == UNWIND_RETURN) { - // stack: (..., __exit__, ctx_mgr, ret_val, UNWIND_RETURN) - mp_obj_t ret_val = sp[-1]; - sp[-1] = mp_const_none; - sp[0] = mp_const_none; - sp[1] = mp_const_none; - mp_call_method_n_kw(3, 0, sp - 3); - sp[-3] = ret_val; - sp[-2] = MP_OBJ_NEW_SMALL_INT(UNWIND_RETURN); - } else { - assert(cause_val == UNWIND_JUMP); - // stack: (..., __exit__, ctx_mgr, dest_ip, num_exc, UNWIND_JUMP) - mp_obj_t dest_ip = sp[-2]; - mp_obj_t num_exc = sp[-1]; - sp[-2] = mp_const_none; - sp[-1] = mp_const_none; - sp[0] = mp_const_none; - mp_call_method_n_kw(3, 0, sp - 4); - sp[-4] = dest_ip; - sp[-3] = num_exc; - sp[-2] = MP_OBJ_NEW_SMALL_INT(UNWIND_JUMP); - } + // Getting here there are two distinct cases: + // - unwind return, stack: (..., __exit__, ctx_mgr, ret_val, SMALL_INT(-1)) + // - unwind jump, stack: (..., __exit__, ctx_mgr, dest_ip, SMALL_INT(num_exc)) + // For both cases we do exactly the same thing. + mp_obj_t data = sp[-1]; + mp_obj_t cause = sp[0]; + sp[-1] = mp_const_none; + sp[0] = mp_const_none; + sp[1] = mp_const_none; + mp_call_method_n_kw(3, 0, sp - 3); + sp[-3] = data; + sp[-2] = cause; sp -= 2; // we removed (__exit__, ctx_mgr) } else { assert(mp_obj_is_exception_instance(TOP())); @@ -680,10 +661,11 @@ unwind_jump:; // of a "with" block contains the context manager info. // We're going to run "finally" code as a coroutine // (not calling it recursively). Set up a sentinel - // on a stack so it can return back to us when it is + // on the stack so it can return back to us when it is // done (when WITH_CLEANUP or END_FINALLY reached). - PUSH((mp_obj_t)unum); // push number of exception handlers left to unwind - PUSH(MP_OBJ_NEW_SMALL_INT(UNWIND_JUMP)); // push sentinel + // The sentinel is the number of exception handlers left to + // unwind, which is a non-negative integer. + PUSH(MP_OBJ_NEW_SMALL_INT(unum)); ip = exc_sp->handler; // get exception handler byte code address exc_sp--; // pop exception handler goto dispatch_loop; // run the exception handler @@ -720,11 +702,14 @@ unwind_jump:; } else if (MP_OBJ_IS_SMALL_INT(TOP())) { // We finished "finally" coroutine and now dispatch back // to our caller, based on TOS value - mp_unwind_reason_t reason = MP_OBJ_SMALL_INT_VALUE(POP()); - if (reason == UNWIND_RETURN) { + mp_int_t cause = MP_OBJ_SMALL_INT_VALUE(POP()); + if (cause < 0) { + // A negative cause indicates unwind return goto unwind_return; } else { - assert(reason == UNWIND_JUMP); + // Otherwise it's an unwind jump and we must push as a raw + // number the number of exception handlers to unwind + PUSH((mp_obj_t)cause); goto unwind_jump; } } else { @@ -1101,7 +1086,7 @@ unwind_return: // (not calling it recursively). Set up a sentinel // on a stack so it can return back to us when it is // done (when WITH_CLEANUP or END_FINALLY reached). - PUSH(MP_OBJ_NEW_SMALL_INT(UNWIND_RETURN)); + PUSH(MP_OBJ_NEW_SMALL_INT(-1)); ip = exc_sp->handler; exc_sp--; goto dispatch_loop; -- cgit v1.2.3 From b75cb8392bdf5f8c12072eac3adbdaad53d1e8d2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 8 Feb 2018 14:02:50 +1100 Subject: py/parsenum: Fix parsing of floats that are close to subnormal. Prior to this patch, a float literal that was close to subnormal would have a loss of precision when parsed. The worst case was something like float('10000000000000000000e-326') which returned 0.0. --- py/parsenum.c | 14 ++++++++++++-- tests/float/float_parse.py | 5 +++++ tests/float/float_parse_doubleprec.py | 5 +++++ 3 files changed, 22 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/parsenum.c b/py/parsenum.c index 98e773685..124489c66 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -172,10 +172,15 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool #if MICROPY_PY_BUILTINS_FLOAT // DEC_VAL_MAX only needs to be rough and is used to retain precision while not overflowing +// SMALL_NORMAL_VAL is the smallest power of 10 that is still a normal float #if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT #define DEC_VAL_MAX 1e20F +#define SMALL_NORMAL_VAL (1e-37F) +#define SMALL_NORMAL_EXP (-37) #elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE #define DEC_VAL_MAX 1e200 +#define SMALL_NORMAL_VAL (1e-307) +#define SMALL_NORMAL_EXP (-307) #endif const char *top = str + len; @@ -275,8 +280,13 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool exp_val = -exp_val; } - // apply the exponent - dec_val *= MICROPY_FLOAT_C_FUN(pow)(10, exp_val + exp_extra); + // apply the exponent, making sure it's not a subnormal value + exp_val += exp_extra; + if (exp_val < SMALL_NORMAL_EXP) { + exp_val -= SMALL_NORMAL_EXP; + dec_val *= SMALL_NORMAL_VAL; + } + dec_val *= MICROPY_FLOAT_C_FUN(pow)(10, exp_val); } // negate value if needed diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py index 448eff3bc..de4ea455f 100644 --- a/tests/float/float_parse.py +++ b/tests/float/float_parse.py @@ -20,3 +20,8 @@ print(float('.' + '9' * 70 + 'e-50') == float('1e-50')) print(float('.' + '0' * 60 + '1e10') == float('1e-51')) print(float('.' + '0' * 60 + '9e25')) print(float('.' + '0' * 60 + '9e40')) + +# ensure that accuracy is retained when value is close to a subnormal +print(float('1.00000000000000000000e-37')) +print(float('10.0000000000000000000e-38')) +print(float('100.000000000000000000e-39')) diff --git a/tests/float/float_parse_doubleprec.py b/tests/float/float_parse_doubleprec.py index 356601130..2ea7842f3 100644 --- a/tests/float/float_parse_doubleprec.py +++ b/tests/float/float_parse_doubleprec.py @@ -14,3 +14,8 @@ print(float('.' + '9' * 400 + 'e-100')) print(float('.' + '0' * 400 + '9e100')) print(float('.' + '0' * 400 + '9e200')) print(float('.' + '0' * 400 + '9e400')) + +# ensure that accuracy is retained when value is close to a subnormal +print(float('1.00000000000000000000e-307')) +print(float('10.0000000000000000000e-308')) +print(float('100.000000000000000000e-309')) -- cgit v1.2.3 From bbb08431f3c45d94c1e6aa0762b8f13ccce1fe8e Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 8 Feb 2018 14:35:43 +1100 Subject: py/objfloat: Fix case of raising 0 to -infinity. It was raising an exception but it should return infinity. --- py/objfloat.c | 2 +- tests/float/builtin_float_pow.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/float/builtin_float_pow.py (limited to 'py') diff --git a/py/objfloat.c b/py/objfloat.c index 75212a4d2..e4d5a6570 100644 --- a/py/objfloat.c +++ b/py/objfloat.c @@ -293,7 +293,7 @@ mp_obj_t mp_obj_float_binary_op(mp_binary_op_t op, mp_float_t lhs_val, mp_obj_t break; case MP_BINARY_OP_POWER: case MP_BINARY_OP_INPLACE_POWER: - if (lhs_val == 0 && rhs_val < 0) { + if (lhs_val == 0 && rhs_val < 0 && !isinf(rhs_val)) { goto zero_division_error; } if (lhs_val < 0 && rhs_val != MICROPY_FLOAT_C_FUN(floor)(rhs_val)) { diff --git a/tests/float/builtin_float_pow.py b/tests/float/builtin_float_pow.py new file mode 100644 index 000000000..2de1b4817 --- /dev/null +++ b/tests/float/builtin_float_pow.py @@ -0,0 +1,11 @@ +# test builtin pow function with float args + +print(pow(0.0, 0.0)) +print(pow(0, 1.0)) +print(pow(1.0, 1)) +print(pow(2.0, 3.0)) +print(pow(2.0, -4.0)) + +print(pow(0.0, float('inf'))) +print(pow(0.0, float('-inf'))) +print(pow(0.0, float('nan'))) -- cgit v1.2.3 From 19aee9438a7a8cf8539536dab5147aedb6b16bb3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 14 Feb 2018 18:19:22 +1100 Subject: py/unicode: Clean up utf8 funcs and provide non-utf8 inline versions. This patch provides inline versions of the utf8 helper functions for the case when unicode is disabled (MICROPY_PY_BUILTINS_STR_UNICODE set to 0). This saves code size. The unichar_charlen function is also renamed to utf8_charlen to match the other utf8 helper functions, and the signature of this function is adjusted for consistency (const char* -> const byte*, mp_uint_t -> size_t). --- py/misc.h | 8 +++++++- py/modbuiltins.c | 2 +- py/objstr.c | 2 +- py/objstrunicode.c | 2 +- py/unicode.c | 29 +++++++++++------------------ 5 files changed, 21 insertions(+), 22 deletions(-) (limited to 'py') diff --git a/py/misc.h b/py/misc.h index 5d557db6f..a14bef7fe 100644 --- a/py/misc.h +++ b/py/misc.h @@ -121,8 +121,15 @@ typedef uint32_t unichar; typedef uint unichar; #endif +#if MICROPY_PY_BUILTINS_STR_UNICODE unichar utf8_get_char(const byte *s); const byte *utf8_next_char(const byte *s); +size_t utf8_charlen(const byte *str, size_t len); +#else +static inline unichar utf8_get_char(const byte *s) { return *s; } +static inline const byte *utf8_next_char(const byte *s) { return s + 1; } +static inline size_t utf8_charlen(const byte *str, size_t len) { (void)str; return len; } +#endif bool unichar_isspace(unichar c); bool unichar_isalpha(unichar c); @@ -135,7 +142,6 @@ bool unichar_islower(unichar c); unichar unichar_tolower(unichar c); unichar unichar_toupper(unichar c); mp_uint_t unichar_xdigit_value(unichar c); -mp_uint_t unichar_charlen(const char *str, mp_uint_t len); #define UTF8_IS_NONASCII(ch) ((ch) & 0x80) #define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80) diff --git a/py/modbuiltins.c b/py/modbuiltins.c index e6f82df6f..6b8886804 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -346,7 +346,7 @@ STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) { const char *str = mp_obj_str_get_data(o_in, &len); #if MICROPY_PY_BUILTINS_STR_UNICODE if (MP_OBJ_IS_STR(o_in)) { - len = unichar_charlen(str, len); + len = utf8_charlen((const byte*)str, len); if (len == 1) { return mp_obj_new_int(utf8_get_char((const byte*)str)); } diff --git a/py/objstr.c b/py/objstr.c index 30153813d..ed9ab4e45 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -1704,7 +1704,7 @@ STATIC mp_obj_t str_count(size_t n_args, const mp_obj_t *args) { // if needle_len is zero then we count each gap between characters as an occurrence if (needle_len == 0) { - return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char*)start, end - start) + 1); + return MP_OBJ_NEW_SMALL_INT(utf8_charlen(start, end - start) + 1); } // count the occurrences diff --git a/py/objstrunicode.c b/py/objstrunicode.c index a1f54b8a2..badb569d7 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -104,7 +104,7 @@ STATIC mp_obj_t uni_unary_op(mp_unary_op_t op, mp_obj_t self_in) { case MP_UNARY_OP_BOOL: return mp_obj_new_bool(str_len != 0); case MP_UNARY_OP_LEN: - return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char *)str_data, str_len)); + return MP_OBJ_NEW_SMALL_INT(utf8_charlen(str_data, str_len)); default: return MP_OBJ_NULL; // op not supported } diff --git a/py/unicode.c b/py/unicode.c index 140b7ba71..935dc9012 100644 --- a/py/unicode.c +++ b/py/unicode.c @@ -67,9 +67,9 @@ STATIC const uint8_t attr[] = { AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0 }; -// TODO: Rename to str_get_char -unichar utf8_get_char(const byte *s) { #if MICROPY_PY_BUILTINS_STR_UNICODE + +unichar utf8_get_char(const byte *s) { unichar ord = *s++; if (!UTF8_IS_NONASCII(ord)) return ord; ord &= 0x7F; @@ -80,22 +80,14 @@ unichar utf8_get_char(const byte *s) { ord = (ord << 6) | (*s++ & 0x3F); } return ord; -#else - return *s; -#endif } -// TODO: Rename to str_next_char const byte *utf8_next_char(const byte *s) { -#if MICROPY_PY_BUILTINS_STR_UNICODE ++s; while (UTF8_IS_CONT(*s)) { ++s; } return s; -#else - return s + 1; -#endif } mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) { @@ -109,21 +101,18 @@ mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) { return i; } -// TODO: Rename to str_charlen -mp_uint_t unichar_charlen(const char *str, mp_uint_t len) { -#if MICROPY_PY_BUILTINS_STR_UNICODE - mp_uint_t charlen = 0; - for (const char *top = str + len; str < top; ++str) { +size_t utf8_charlen(const byte *str, size_t len) { + size_t charlen = 0; + for (const byte *top = str + len; str < top; ++str) { if (!UTF8_IS_CONT(*str)) { ++charlen; } } return charlen; -#else - return len; -#endif } +#endif + // Be aware: These unichar_is* functions are actually ASCII-only! bool unichar_isspace(unichar c) { return c < 128 && (attr[c] & FL_SPACE) != 0; @@ -183,6 +172,8 @@ mp_uint_t unichar_xdigit_value(unichar c) { return n; } +#if MICROPY_PY_BUILTINS_STR_UNICODE + bool utf8_check(const byte *p, size_t len) { uint8_t need = 0; const byte *end = p + len; @@ -210,3 +201,5 @@ bool utf8_check(const byte *p, size_t len) { } return need == 0; // no pending fragments allowed } + +#endif -- cgit v1.2.3 From e98ff40604170eb231225a4285d9ef740b8b9501 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 14 Feb 2018 18:27:14 +1100 Subject: py/modbuiltins: Simplify casts from char to byte ptr in builtin ord. --- py/modbuiltins.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 6b8886804..5461816af 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -343,19 +343,19 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_oct_obj, mp_builtin_oct); STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) { size_t len; - const char *str = mp_obj_str_get_data(o_in, &len); + const byte *str = (const byte*)mp_obj_str_get_data(o_in, &len); #if MICROPY_PY_BUILTINS_STR_UNICODE if (MP_OBJ_IS_STR(o_in)) { - len = utf8_charlen((const byte*)str, len); + len = utf8_charlen(str, len); if (len == 1) { - return mp_obj_new_int(utf8_get_char((const byte*)str)); + return mp_obj_new_int(utf8_get_char(str)); } } else #endif { // a bytes object, or a str without unicode support (don't sign extend the char) if (len == 1) { - return MP_OBJ_NEW_SMALL_INT(((const byte*)str)[0]); + return MP_OBJ_NEW_SMALL_INT(str[0]); } } -- cgit v1.2.3 From 5604b710c2490e2a7cfd1bac6cd60fc2c8527a37 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 14 Feb 2018 18:41:17 +1100 Subject: py/emitglue: When assigning bytecode only pass bytecode len if needed. Most embedded targets will have this bit of the code disabled, saving a small amount of code space. --- py/emitbc.c | 2 ++ py/emitglue.c | 5 ++++- py/emitglue.h | 5 ++++- py/persistentcode.c | 6 +++++- 4 files changed, 15 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/emitbc.c b/py/emitbc.c index 5e7fa623a..32e833000 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -437,7 +437,9 @@ void mp_emit_bc_end_pass(emit_t *emit) { } else if (emit->pass == MP_PASS_EMIT) { mp_emit_glue_assign_bytecode(emit->scope->raw_code, emit->code_base, + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS emit->code_info_size + emit->bytecode_size, + #endif emit->const_table, #if MICROPY_PERSISTENT_CODE_SAVE emit->ct_cur_obj, emit->ct_cur_raw_code, diff --git a/py/emitglue.c b/py/emitglue.c index d2add988f..74bf8ddca 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -55,7 +55,10 @@ mp_raw_code_t *mp_emit_glue_new_raw_code(void) { return rc; } -void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, mp_uint_t len, +void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + size_t len, + #endif const mp_uint_t *const_table, #if MICROPY_PERSISTENT_CODE_SAVE uint16_t n_obj, uint16_t n_raw_code, diff --git a/py/emitglue.h b/py/emitglue.h index f2a48c5e5..0830a0d5c 100644 --- a/py/emitglue.h +++ b/py/emitglue.h @@ -63,7 +63,10 @@ typedef struct _mp_raw_code_t { mp_raw_code_t *mp_emit_glue_new_raw_code(void); -void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, mp_uint_t len, +void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + size_t len, + #endif const mp_uint_t *const_table, #if MICROPY_PERSISTENT_CODE_SAVE uint16_t n_obj, uint16_t n_raw_code, diff --git a/py/persistentcode.c b/py/persistentcode.c index e0bb8f1d6..7113b0dc2 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -200,7 +200,11 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { // create raw_code and return it mp_raw_code_t *rc = mp_emit_glue_new_raw_code(); - mp_emit_glue_assign_bytecode(rc, bytecode, bc_len, const_table, + mp_emit_glue_assign_bytecode(rc, bytecode, + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + bc_len, + #endif + const_table, #if MICROPY_PERSISTENT_CODE_SAVE n_obj, n_raw_code, #endif -- cgit v1.2.3 From d77da83d55c5de4768720fa578b9ffa5ec502dc6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 14 Feb 2018 23:17:06 +1100 Subject: py/objrange: Implement (in)equality comparison between range objects. This feature is not often used so is guarded by the config option MICROPY_PY_BUILTINS_RANGE_BINOP which is disabled by default. With this option disabled MicroPython will always return false when comparing two range objects for equality (unless they are exactly the same object instance). This does not match CPython so if (in)equality between range objects is needed then this option should be enabled. Enabling this option costs between 100 and 200 bytes of code space depending on the machine architecture. --- py/mpconfig.h | 8 ++++++++ py/objrange.c | 21 +++++++++++++++++++++ tests/basics/builtin_range_binop.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 tests/basics/builtin_range_binop.py (limited to 'py') diff --git a/py/mpconfig.h b/py/mpconfig.h index f2a8c98cb..b8a96f0b0 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -787,6 +787,14 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_RANGE_ATTRS (1) #endif +// Whether to support binary ops [only (in)equality is defined] between range +// objects. With this option disabled all range objects that are not exactly +// the same object will compare as not-equal. With it enabled the semantics +// match CPython and ranges are equal if they yield the same sequence of items. +#ifndef MICROPY_PY_BUILTINS_RANGE_BINOP +#define MICROPY_PY_BUILTINS_RANGE_BINOP (0) +#endif + // Whether to support timeout exceptions (like socket.timeout) #ifndef MICROPY_PY_BUILTINS_TIMEOUTERROR #define MICROPY_PY_BUILTINS_TIMEOUTERROR (0) diff --git a/py/objrange.c b/py/objrange.c index 3874adb11..86aa0ccfe 100644 --- a/py/objrange.c +++ b/py/objrange.c @@ -138,6 +138,24 @@ STATIC mp_obj_t range_unary_op(mp_unary_op_t op, mp_obj_t self_in) { } } +#if MICROPY_PY_BUILTINS_RANGE_BINOP +STATIC mp_obj_t range_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { + if (!MP_OBJ_IS_TYPE(rhs_in, &mp_type_range) || op != MP_BINARY_OP_EQUAL) { + return MP_OBJ_NULL; // op not supported + } + mp_obj_range_t *lhs = MP_OBJ_TO_PTR(lhs_in); + mp_obj_range_t *rhs = MP_OBJ_TO_PTR(rhs_in); + mp_int_t lhs_len = range_len(lhs); + mp_int_t rhs_len = range_len(rhs); + return mp_obj_new_bool( + lhs_len == rhs_len + && (lhs_len == 0 + || (lhs->start == rhs->start + && (lhs_len == 1 || lhs->step == rhs->step))) + ); +} +#endif + STATIC mp_obj_t range_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { if (value == MP_OBJ_SENTINEL) { // load @@ -195,6 +213,9 @@ const mp_obj_type_t mp_type_range = { .print = range_print, .make_new = range_make_new, .unary_op = range_unary_op, + #if MICROPY_PY_BUILTINS_RANGE_BINOP + .binary_op = range_binary_op, + #endif .subscr = range_subscr, .getiter = range_getiter, #if MICROPY_PY_BUILTINS_RANGE_ATTRS diff --git a/tests/basics/builtin_range_binop.py b/tests/basics/builtin_range_binop.py new file mode 100644 index 000000000..e4e054c27 --- /dev/null +++ b/tests/basics/builtin_range_binop.py @@ -0,0 +1,32 @@ +# test binary operations on range objects; (in)equality only + +# this "feature test" actually tests the implementation but is the best we can do +if range(1) != range(1): + print("SKIP") + raise SystemExit + +# basic (in)equality +print(range(1) == range(1)) +print(range(1) != range(1)) +print(range(1) != range(2)) + +# empty range +print(range(0) == range(0)) +print(range(1, 0) == range(0)) +print(range(1, 4, -1) == range(6, 3)) + +# 1 element range +print(range(1, 4, 10) == range(1, 4, 10)) +print(range(1, 4, 10) == range(1, 4, 20)) +print(range(1, 4, 10) == range(1, 8, 20)) + +# more than 1 element +print(range(0, 3, 2) == range(0, 3, 2)) +print(range(0, 3, 2) == range(0, 4, 2)) +print(range(0, 3, 2) == range(0, 5, 2)) + +# unsupported binary op +try: + range(1) + 10 +except TypeError: + print('TypeError') -- cgit v1.2.3 From 73d1d20b46dda54340ad2819b865f47ca25f4850 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 15 Feb 2018 16:50:02 +1100 Subject: py/objexcept: Remove long-obsolete mp_const_MemoryError_obj. This constant exception instance was once used by m_malloc_fail() to raise a MemoryError without allocating memory, but it was made obsolete long ago by 3556e45711c3b7ec712748d013e678d035185bdd. The functionality is now replaced by the use of mp_emergency_exception_obj which lives in the global uPy state, and which can handle any exception type, not just MemoryError. --- py/obj.h | 1 - py/objexcept.c | 3 --- py/vm.c | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) (limited to 'py') diff --git a/py/obj.h b/py/obj.h index 10cb48961..2e715253c 100644 --- a/py/obj.h +++ b/py/obj.h @@ -624,7 +624,6 @@ extern const struct _mp_obj_str_t mp_const_empty_bytes_obj; extern const struct _mp_obj_tuple_t mp_const_empty_tuple_obj; extern const struct _mp_obj_singleton_t mp_const_ellipsis_obj; extern const struct _mp_obj_singleton_t mp_const_notimplemented_obj; -extern const struct _mp_obj_exception_t mp_const_MemoryError_obj; extern const struct _mp_obj_exception_t mp_const_GeneratorExit_obj; // General API for objects diff --git a/py/objexcept.c b/py/objexcept.c index 524f105ce..ccb0bad0d 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -43,9 +43,6 @@ // Number of traceback entries to reserve in the emergency exception buffer #define EMG_TRACEBACK_ALLOC (2 * TRACEBACK_ENTRY_LEN) -// Instance of MemoryError exception - needed by mp_malloc_fail -const mp_obj_exception_t mp_const_MemoryError_obj = {{&mp_type_MemoryError}, 0, 0, NULL, (mp_obj_tuple_t*)&mp_const_empty_tuple_obj}; - // Optionally allocated buffer for storing the first argument of an exception // allocated when the heap is locked. #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF diff --git a/py/vm.c b/py/vm.c index c629a61e2..ceb2060f9 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1374,8 +1374,7 @@ unwind_loop: // set file and line number that the exception occurred at // TODO: don't set traceback for exceptions re-raised by END_FINALLY. // But consider how to handle nested exceptions. - // TODO need a better way of not adding traceback to constant objects (right now, just GeneratorExit_obj and MemoryError_obj) - if (nlr.ret_val != &mp_const_GeneratorExit_obj && nlr.ret_val != &mp_const_MemoryError_obj) { + if (nlr.ret_val != &mp_const_GeneratorExit_obj) { const byte *ip = code_state->fun_bc->bytecode; ip = mp_decode_uint_skip(ip); // skip n_state ip = mp_decode_uint_skip(ip); // skip n_exc_stack -- cgit v1.2.3 From 5591bd237a69a3a1a6ac03cb1dc9edcde835f708 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Tue, 13 Feb 2018 22:00:20 +0100 Subject: py/nlrthumb: Do not mark nlr_push as not returning anything. By adding __builtin_unreachable() at the end of nlr_push, we're essentially telling the compiler that this function will never return. When GCC LTO is in use, this means that any time nlr_push() is called (which is often), the compiler thinks this function will never return and thus eliminates all code following the call. Note: I've added a 'return 0' for older GCC versions like 4.6 which complain about not returning anything (which doesn't make sense in a naked function). Newer GCC versions (tested 4.8, 5.4 and some others) don't complain about this. --- py/nlrthumb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/nlrthumb.c b/py/nlrthumb.c index fb0a92236..7dbeac1b6 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -76,9 +76,9 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { #endif ); - #if defined(__GNUC__) + #if defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)) // Older versions of gcc give an error when naked functions don't return a value - __builtin_unreachable(); + return 0; #endif } -- cgit v1.2.3 From 7b2a9b059a4c60252b37f61cdbc2a28e375438a5 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 19 Feb 2018 00:26:14 +1100 Subject: py/pystack: Use "pystack exhausted" as error msg for out of pystack mem. Using the message "maximum recursion depth exceeded" for when the pystack runs out of memory can be misleading because the pystack can run out for reasons other than deep recursion (although in most cases pystack exhaustion is probably indirectly related to deep recursion). And it's important to give the user more precise feedback as to the reason for the error: if they know precisely that the pystack was exhausted then they have a chance to increase the amount of memory available to the pystack (as opposed to not knowing if it was the C stack or pystack that ran out). Also, C stack exhaustion is more serious than pystack exhaustion because it could have been that the C stack overflowed and overwrote/corrupted some data and so the system must be restarted. The pystack can never corrupt data in this way so pystack exhaustion does not require a system restart. Knowing the difference between these two cases is therefore important. The actual exception type for pystack exhaustion remains as RuntimeError so that programatically it behaves the same as a C stack exhaustion. --- py/pystack.c | 3 ++- py/qstrdefs.h | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/pystack.c b/py/pystack.c index 767c307e9..552e59d53 100644 --- a/py/pystack.c +++ b/py/pystack.c @@ -43,7 +43,8 @@ void *mp_pystack_alloc(size_t n_bytes) { #endif if (MP_STATE_THREAD(pystack_cur) + n_bytes > MP_STATE_THREAD(pystack_end)) { // out of memory in the pystack - mp_raise_recursion_depth(); + nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError, + MP_OBJ_NEW_QSTR(MP_QSTR_pystack_space_exhausted))); } void *ptr = MP_STATE_THREAD(pystack_cur); MP_STATE_THREAD(pystack_cur) += n_bytes; diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 1b480c9c7..a60905812 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -52,3 +52,7 @@ Q() Q() Q() Q(utf-8) + +#if MICROPY_ENABLE_PYSTACK +Q(pystack exhausted) +#endif -- cgit v1.2.3 From a3e01d3642d6c287bf2d0c14b3d75bf305327819 Mon Sep 17 00:00:00 2001 From: Mike Wadsten Date: Sun, 18 Feb 2018 21:51:04 -0600 Subject: py/objdict: Disallow possible modifications to fixed dicts. --- py/objdict.c | 13 ++++++++++++ tests/basics/dict_fixed.py | 48 ++++++++++++++++++++++++++++++++++++++++++ tests/basics/dict_fixed.py.exp | 7 ++++++ 3 files changed, 68 insertions(+) create mode 100644 tests/basics/dict_fixed.py create mode 100644 tests/basics/dict_fixed.py.exp (limited to 'py') diff --git a/py/objdict.c b/py/objdict.c index f6fb594b3..c0647067a 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -195,9 +195,16 @@ STATIC mp_obj_t dict_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { /******************************************************************************/ /* dict methods */ +STATIC void mp_ensure_not_fixed(const mp_obj_dict_t *dict) { + if (dict->map.is_fixed) { + mp_raise_TypeError(NULL); + } +} + STATIC mp_obj_t dict_clear(mp_obj_t self_in) { mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); mp_obj_dict_t *self = MP_OBJ_TO_PTR(self_in); + mp_ensure_not_fixed(self); mp_map_clear(&self->map); @@ -253,6 +260,9 @@ STATIC MP_DEFINE_CONST_CLASSMETHOD_OBJ(dict_fromkeys_obj, MP_ROM_PTR(&dict_fromk STATIC mp_obj_t dict_get_helper(size_t n_args, const mp_obj_t *args, mp_map_lookup_kind_t lookup_kind) { mp_check_self(MP_OBJ_IS_DICT_TYPE(args[0])); mp_obj_dict_t *self = MP_OBJ_TO_PTR(args[0]); + if (lookup_kind != MP_MAP_LOOKUP) { + mp_ensure_not_fixed(self); + } mp_map_elem_t *elem = mp_map_lookup(&self->map, args[1], lookup_kind); mp_obj_t value; if (elem == NULL || elem->value == MP_OBJ_NULL) { @@ -295,6 +305,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(dict_setdefault_obj, 2, 3, dict_setde STATIC mp_obj_t dict_popitem(mp_obj_t self_in) { mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); mp_obj_dict_t *self = MP_OBJ_TO_PTR(self_in); + mp_ensure_not_fixed(self); size_t cur = 0; mp_map_elem_t *next = dict_iter_next(self, &cur); if (next == NULL) { @@ -313,6 +324,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(dict_popitem_obj, dict_popitem); STATIC mp_obj_t dict_update(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) { mp_check_self(MP_OBJ_IS_DICT_TYPE(args[0])); mp_obj_dict_t *self = MP_OBJ_TO_PTR(args[0]); + mp_ensure_not_fixed(self); mp_arg_check_num(n_args, kwargs->used, 1, 2, true); @@ -578,6 +590,7 @@ size_t mp_obj_dict_len(mp_obj_t self_in) { mp_obj_t mp_obj_dict_store(mp_obj_t self_in, mp_obj_t key, mp_obj_t value) { mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); mp_obj_dict_t *self = MP_OBJ_TO_PTR(self_in); + mp_ensure_not_fixed(self); mp_map_lookup(&self->map, key, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = value; return self_in; } diff --git a/tests/basics/dict_fixed.py b/tests/basics/dict_fixed.py new file mode 100644 index 000000000..4261a0655 --- /dev/null +++ b/tests/basics/dict_fixed.py @@ -0,0 +1,48 @@ +# test that fixed dictionaries cannot be modified + +try: + import uerrno +except ImportError: + print("SKIP") + raise SystemExit + +# Save a copy of uerrno.errorcode, so we can check later +# that it hasn't been modified. +errorcode_copy = uerrno.errorcode.copy() + +try: + uerrno.errorcode.popitem() +except TypeError: + print("TypeError") + +try: + uerrno.errorcode.pop(0) +except TypeError: + print("TypeError") + +try: + uerrno.errorcode.setdefault(0, 0) +except TypeError: + print("TypeError") + +try: + uerrno.errorcode.update([(1, 2)]) +except TypeError: + print("TypeError") + +try: + del uerrno.errorcode[1] +except TypeError: + print("TypeError") + +try: + uerrno.errorcode[1] = 'foo' +except TypeError: + print("TypeError") + +try: + uerrno.errorcode.clear() +except TypeError: + print("TypeError") + +assert uerrno.errorcode == errorcode_copy diff --git a/tests/basics/dict_fixed.py.exp b/tests/basics/dict_fixed.py.exp new file mode 100644 index 000000000..ffaeb4085 --- /dev/null +++ b/tests/basics/dict_fixed.py.exp @@ -0,0 +1,7 @@ +TypeError +TypeError +TypeError +TypeError +TypeError +TypeError +TypeError -- cgit v1.2.3 From ea7cf2b738c3b103b472547500fb0107b956fc0c Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Sat, 27 Jan 2018 18:37:38 +0100 Subject: py/gc: Reduce code size by specialising VERIFY_MARK_AND_PUSH macro. This macro is written out explicitly in the two locations that it is used and then the code is optimised, opening possibilities for further optimisations and reducing code size: unix: -48 minimal CROSS=1: -32 stm32: -32 --- py/gc.c | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 5196954e2..9089a26be 100644 --- a/py/gc.c +++ b/py/gc.c @@ -203,24 +203,6 @@ bool gc_is_locked(void) { #endif #endif -// ptr should be of type void* -#define VERIFY_MARK_AND_PUSH(ptr) \ - do { \ - if (VERIFY_PTR(ptr)) { \ - size_t _block = BLOCK_FROM_PTR(ptr); \ - if (ATB_GET_KIND(_block) == AT_HEAD) { \ - /* an unmarked head, mark it, and push it on gc stack */ \ - TRACE_MARK(_block, ptr); \ - ATB_HEAD_TO_MARK(_block); \ - if (MP_STATE_MEM(gc_sp) < &MP_STATE_MEM(gc_stack)[MICROPY_ALLOC_GC_STACK_SIZE]) { \ - *MP_STATE_MEM(gc_sp)++ = _block; \ - } else { \ - MP_STATE_MEM(gc_stack_overflow) = 1; \ - } \ - } \ - } \ - } while (0) - STATIC void gc_drain_stack(void) { while (MP_STATE_MEM(gc_sp) > MP_STATE_MEM(gc_stack)) { // pop the next block off the stack @@ -236,7 +218,20 @@ STATIC void gc_drain_stack(void) { void **ptrs = (void**)PTR_FROM_BLOCK(block); for (size_t i = n_blocks * BYTES_PER_BLOCK / sizeof(void*); i > 0; i--, ptrs++) { void *ptr = *ptrs; - VERIFY_MARK_AND_PUSH(ptr); + if (VERIFY_PTR(ptr)) { + // Mark and push this pointer + size_t childblock = BLOCK_FROM_PTR(ptr); + if (ATB_GET_KIND(childblock) == AT_HEAD) { + // an unmarked head, mark it, and push it on gc stack + TRACE_MARK(childblock, ptr); + ATB_HEAD_TO_MARK(childblock); + if (MP_STATE_MEM(gc_sp) < &MP_STATE_MEM(gc_stack)[MICROPY_ALLOC_GC_STACK_SIZE]) { + *MP_STATE_MEM(gc_sp)++ = childblock; + } else { + MP_STATE_MEM(gc_stack_overflow) = 1; + } + } + } } } } @@ -337,8 +332,17 @@ void gc_collect_start(void) { void gc_collect_root(void **ptrs, size_t len) { for (size_t i = 0; i < len; i++) { void *ptr = ptrs[i]; - VERIFY_MARK_AND_PUSH(ptr); - gc_drain_stack(); + if (VERIFY_PTR(ptr)) { + // Mark and push this pointer + size_t block = BLOCK_FROM_PTR(ptr); + if (ATB_GET_KIND(block) == AT_HEAD) { + // an unmarked head, mark it, and push it on gc stack + TRACE_MARK(block, ptr); + ATB_HEAD_TO_MARK(block); + *MP_STATE_MEM(gc_sp)++ = block; + gc_drain_stack(); + } + } } } -- cgit v1.2.3 From 5c9e5618e088ddc233cd3bd1b1f48572ab403983 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Thu, 15 Feb 2018 17:10:13 +0100 Subject: py/gc: Rename gc_drain_stack to gc_mark_subtree and pass it first block. This saves a bit in code size: minimal CROSS=1: -44 unix: -96 --- py/gc.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 9089a26be..616ff783f 100644 --- a/py/gc.c +++ b/py/gc.c @@ -203,11 +203,13 @@ bool gc_is_locked(void) { #endif #endif -STATIC void gc_drain_stack(void) { - while (MP_STATE_MEM(gc_sp) > MP_STATE_MEM(gc_stack)) { - // pop the next block off the stack - size_t block = *--MP_STATE_MEM(gc_sp); - +// Take the given block as the topmost block on the stack. Check all it's +// children: mark the unmarked child blocks and put those newly marked +// blocks on the stack. When all children have been checked, pop off the +// topmost block on the stack and repeat with that one. +STATIC void gc_mark_subtree(size_t block) { + // Start with the block passed in the argument. + for (;;) { // work out number of consecutive blocks in the chain starting with this one size_t n_blocks = 0; do { @@ -233,6 +235,14 @@ STATIC void gc_drain_stack(void) { } } } + + // Are there any blocks on the stack? + if (MP_STATE_MEM(gc_sp) <= MP_STATE_MEM(gc_stack)) { + break; // No, stack is empty, we're done. + } + + // pop the next block off the stack + block = *--MP_STATE_MEM(gc_sp); } } @@ -245,8 +255,7 @@ STATIC void gc_deal_with_stack_overflow(void) { for (size_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; block++) { // trace (again) if mark bit set if (ATB_GET_KIND(block) == AT_MARK) { - *MP_STATE_MEM(gc_sp)++ = block; - gc_drain_stack(); + gc_mark_subtree(block); } } } @@ -339,8 +348,7 @@ void gc_collect_root(void **ptrs, size_t len) { // an unmarked head, mark it, and push it on gc stack TRACE_MARK(block, ptr); ATB_HEAD_TO_MARK(block); - *MP_STATE_MEM(gc_sp)++ = block; - gc_drain_stack(); + gc_mark_subtree(block); } } } -- cgit v1.2.3 From 736faef2233fe9c721b940a108b28808d4c7f7a0 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Thu, 15 Feb 2018 17:22:34 +0100 Subject: py/gc: Make GC stack pointer a local variable. This saves a bit in code size, and saves some precious .bss RAM: .text .bss minimal CROSS=1: -28 -4 unix (64-bit): -64 -8 --- py/gc.c | 11 +++++------ py/mpstate.h | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 616ff783f..92f6348cb 100644 --- a/py/gc.c +++ b/py/gc.c @@ -209,6 +209,7 @@ bool gc_is_locked(void) { // topmost block on the stack and repeat with that one. STATIC void gc_mark_subtree(size_t block) { // Start with the block passed in the argument. + size_t sp = 0; for (;;) { // work out number of consecutive blocks in the chain starting with this one size_t n_blocks = 0; @@ -227,8 +228,8 @@ STATIC void gc_mark_subtree(size_t block) { // an unmarked head, mark it, and push it on gc stack TRACE_MARK(childblock, ptr); ATB_HEAD_TO_MARK(childblock); - if (MP_STATE_MEM(gc_sp) < &MP_STATE_MEM(gc_stack)[MICROPY_ALLOC_GC_STACK_SIZE]) { - *MP_STATE_MEM(gc_sp)++ = childblock; + if (sp < MICROPY_ALLOC_GC_STACK_SIZE) { + MP_STATE_MEM(gc_stack)[sp++] = childblock; } else { MP_STATE_MEM(gc_stack_overflow) = 1; } @@ -237,19 +238,18 @@ STATIC void gc_mark_subtree(size_t block) { } // Are there any blocks on the stack? - if (MP_STATE_MEM(gc_sp) <= MP_STATE_MEM(gc_stack)) { + if (sp == 0) { break; // No, stack is empty, we're done. } // pop the next block off the stack - block = *--MP_STATE_MEM(gc_sp); + block = MP_STATE_MEM(gc_stack)[--sp]; } } STATIC void gc_deal_with_stack_overflow(void) { while (MP_STATE_MEM(gc_stack_overflow)) { MP_STATE_MEM(gc_stack_overflow) = 0; - MP_STATE_MEM(gc_sp) = MP_STATE_MEM(gc_stack); // scan entire memory looking for blocks which have been marked but not their children for (size_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; block++) { @@ -323,7 +323,6 @@ void gc_collect_start(void) { MP_STATE_MEM(gc_alloc_amount) = 0; #endif MP_STATE_MEM(gc_stack_overflow) = 0; - MP_STATE_MEM(gc_sp) = MP_STATE_MEM(gc_stack); // Trace root pointers. This relies on the root pointers being organised // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, diff --git a/py/mpstate.h b/py/mpstate.h index 45e89590f..a7ffbbf3c 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -78,7 +78,6 @@ typedef struct _mp_state_mem_t { int gc_stack_overflow; size_t gc_stack[MICROPY_ALLOC_GC_STACK_SIZE]; - size_t *gc_sp; uint16_t gc_lock_depth; // This variable controls auto garbage collection. If set to 0 then the -- cgit v1.2.3 From 2a0cbc0d38e8e342c9a54c66a0cca10cae371c6a Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 19 Feb 2018 16:08:20 +1100 Subject: py/gc: Update comment now that gc_drain_stack is called gc_mark_subtree. --- py/gc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 92f6348cb..84c9918fd 100644 --- a/py/gc.c +++ b/py/gc.c @@ -341,10 +341,9 @@ void gc_collect_root(void **ptrs, size_t len) { for (size_t i = 0; i < len; i++) { void *ptr = ptrs[i]; if (VERIFY_PTR(ptr)) { - // Mark and push this pointer size_t block = BLOCK_FROM_PTR(ptr); if (ATB_GET_KIND(block) == AT_HEAD) { - // an unmarked head, mark it, and push it on gc stack + // An unmarked head: mark it, and mark all its children TRACE_MARK(block, ptr); ATB_HEAD_TO_MARK(block); gc_mark_subtree(block); -- cgit v1.2.3 From a8775aaeb053b268cddd629105f02b014df47f64 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 15 Feb 2018 18:12:01 +1100 Subject: py/qstr: Add QSTR_TOTAL() macro to get number of qstrs. --- py/qstr.h | 1 + 1 file changed, 1 insertion(+) (limited to 'py') diff --git a/py/qstr.h b/py/qstr.h index 63fd0c369..f4375ee0e 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -56,6 +56,7 @@ typedef struct _qstr_pool_t { } qstr_pool_t; #define QSTR_FROM_STR_STATIC(s) (qstr_from_strn((s), strlen(s))) +#define QSTR_TOTAL() (MP_STATE_VM(last_pool)->total_prev_len + MP_STATE_VM(last_pool)->len) void qstr_init(void); -- cgit v1.2.3 From 98647e83c733a8a8d36522f71e677367ca8ddd03 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 15 Feb 2018 18:12:27 +1100 Subject: py/modbuiltins: Simplify and generalise dir() by probing qstrs. This patch improves the builtin dir() function by probing the target object with all possible qstrs via mp_load_method_maybe. This is very simple (in terms of implementation), doesn't require recursion, and allows to list all methods of user-defined classes (without duplicates) even if they have multiple inheritance with a common parent. The downside is that it can be slow because it has to iterate through all the qstrs in the system, but the "dir()" function is anyway mostly used for testing frameworks and user introspection of types, so speed is not considered a priority. In addition to providing a more complete implementation of dir(), this patch is simpler than the previous implementation and saves some code space: bare-arm: -80 minimal x86: -80 unix x64: -56 unix nanbox: -48 stm32: -80 cc3200: -80 esp8266: -104 esp32: -64 --- py/modbuiltins.c | 46 ++++++++++++--------------------------------- tests/basics/builtin_dir.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 34 deletions(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 5461816af..5d4ec8ffe 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -173,46 +173,24 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_chr_obj, mp_builtin_chr); STATIC mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { - // TODO make this function more general and less of a hack - - mp_obj_dict_t *dict = NULL; - mp_map_t *members = NULL; - if (n_args == 0) { - // make a list of names in the local name space - dict = mp_locals_get(); - } else { // n_args == 1 - // make a list of names in the given object - if (MP_OBJ_IS_TYPE(args[0], &mp_type_module)) { - dict = mp_obj_module_get_globals(args[0]); - } else { - mp_obj_type_t *type; - if (MP_OBJ_IS_TYPE(args[0], &mp_type_type)) { - type = MP_OBJ_TO_PTR(args[0]); - } else { - type = mp_obj_get_type(args[0]); - } - if (type->locals_dict != NULL && type->locals_dict->base.type == &mp_type_dict) { - dict = type->locals_dict; - } - } - if (mp_obj_is_instance_type(mp_obj_get_type(args[0]))) { - mp_obj_instance_t *inst = MP_OBJ_TO_PTR(args[0]); - members = &inst->members; - } - } - mp_obj_t dir = mp_obj_new_list(0, NULL); - if (dict != NULL) { + if (n_args == 0) { + // Make a list of names in the local namespace + mp_obj_dict_t *dict = mp_locals_get(); for (size_t i = 0; i < dict->map.alloc; i++) { if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) { mp_obj_list_append(dir, dict->map.table[i].key); } } - } - if (members != NULL) { - for (size_t i = 0; i < members->alloc; i++) { - if (MP_MAP_SLOT_IS_FILLED(members, i)) { - mp_obj_list_append(dir, members->table[i].key); + } else { // n_args == 1 + // Make a list of names in the given object + // Implemented by probing all possible qstrs with mp_load_method_maybe + size_t nqstr = QSTR_TOTAL(); + for (size_t i = 1; i < nqstr; ++i) { + mp_obj_t dest[2]; + mp_load_method_maybe(args[0], i, dest); + if (dest[0] != MP_OBJ_NULL) { + mp_obj_list_append(dir, MP_OBJ_NEW_QSTR(i)); } } } diff --git a/tests/basics/builtin_dir.py b/tests/basics/builtin_dir.py index 16e7e669e..c15a76f4c 100644 --- a/tests/basics/builtin_dir.py +++ b/tests/basics/builtin_dir.py @@ -17,3 +17,22 @@ foo = Foo() print('__init__' in dir(foo)) print('x' in dir(foo)) +# dir of subclass +class A: + def a(): + pass +class B(A): + def b(): + pass +d = dir(B()) +print(d.count('a'), d.count('b')) + +# dir of class with multiple bases and a common parent +class C(A): + def c(): + pass +class D(B, C): + def d(): + pass +d = dir(D()) +print(d.count('a'), d.count('b'), d.count('c'), d.count('d')) -- cgit v1.2.3 From 165aab12a3918004325238c794e27e7f4adbb401 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 15 Feb 2018 18:12:31 +1100 Subject: py/repl: Generalise REPL autocomplete to use qstr probing. This patch changes the way REPL autocomplete finds matches. It now probes the target object for all qstrs via mp_load_method_maybe to look for a match with the given input string. Similar to how the builtin dir() function works, this new algorithm now find all methods and instances of user-defined classes including attributes of their parent classes. This helps a lot at the REPL prompt for user-discovery and to autocomplete names even for classes that are derived. The downside is that this new algorithm is slower than the previous one, and in particular will be slower the more qstrs there are in the system. But because REPL autocomplete is primarily used in an interactive way it is not that important to make it fast, as long as it is "fast enough" compared to human reaction. On a slow microcontroller (CPU running at 16MHz) the autocomplete time for a list of 35 names in the outer namespace (pressing tab at a bare prompt) takes about 160ms with this algorithm, compared to about 40ms for the previous implementation (this time includes the actual printing of the names as well). This time of 160ms is very reasonable especially given the new functionality of listing all the names. This patch also decreases code size by: bare-arm: +0 minimal x86: -128 unix x64: -128 unix nanbox: -224 stm32: -88 cc3200: -80 esp8266: -92 esp32: -84 --- py/repl.c | 78 ++++++++++++++-------------------- tests/cmdline/repl_autocomplete.py | 3 +- tests/cmdline/repl_autocomplete.py.exp | 3 +- tests/unix/extra_coverage.py.exp | 10 ++--- 4 files changed, 39 insertions(+), 55 deletions(-) (limited to 'py') diff --git a/py/repl.c b/py/repl.c index 7e8922e19..61ae2c1fe 100644 --- a/py/repl.c +++ b/py/repl.c @@ -27,6 +27,7 @@ #include #include "py/obj.h" #include "py/runtime.h" +#include "py/builtin.h" #include "py/repl.h" #if MICROPY_HELPER_REPL @@ -136,8 +137,11 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print } } - // begin search in locals dict - mp_obj_dict_t *dict = mp_locals_get(); + size_t nqstr = QSTR_TOTAL(); + + // begin search in outer global dict which is accessed from __main__ + mp_obj_t obj = MP_OBJ_FROM_PTR(&mp_module___main__); + mp_obj_t dest[2]; for (;;) { // get next word in string to complete @@ -148,43 +152,20 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print size_t s_len = str - s_start; if (str < top) { - // a complete word, lookup in current dict - - mp_obj_t obj = MP_OBJ_NULL; - for (size_t i = 0; i < dict->map.alloc; i++) { - if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) { - size_t d_len; - const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len); - if (s_len == d_len && strncmp(s_start, d_str, d_len) == 0) { - obj = dict->map.table[i].value; - break; - } - } + // a complete word, lookup in current object + qstr q = qstr_find_strn(s_start, s_len); + if (q == MP_QSTR_NULL) { + // lookup will fail + return 0; } + mp_load_method_maybe(obj, q, dest); + obj = dest[0]; // attribute, method, or MP_OBJ_NULL if nothing found if (obj == MP_OBJ_NULL) { // lookup failed return 0; } - // found an object of this name; try to get its dict - if (MP_OBJ_IS_TYPE(obj, &mp_type_module)) { - dict = mp_obj_module_get_globals(obj); - } else { - mp_obj_type_t *type; - if (MP_OBJ_IS_TYPE(obj, &mp_type_type)) { - type = MP_OBJ_TO_PTR(obj); - } else { - type = mp_obj_get_type(obj); - } - if (type->locals_dict != NULL && type->locals_dict->base.type == &mp_type_dict) { - dict = type->locals_dict; - } else { - // obj has no dict - return 0; - } - } - // skip '.' to move to next word ++str; @@ -192,14 +173,15 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print // end of string, do completion on this partial name // look for matches - int n_found = 0; const char *match_str = NULL; size_t match_len = 0; - for (size_t i = 0; i < dict->map.alloc; i++) { - if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) { - size_t d_len; - const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len); - if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { + qstr q_first = 0, q_last; + for (qstr q = 1; q < nqstr; ++q) { + size_t d_len; + const char *d_str = (const char*)qstr_data(q, &d_len); + if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { + mp_load_method_maybe(obj, q, dest); + if (dest[0] != MP_OBJ_NULL) { if (match_str == NULL) { match_str = d_str; match_len = d_len; @@ -213,13 +195,16 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print } } } - ++n_found; + if (q_first == 0) { + q_first = q; + } + q_last = q; } } } // nothing found - if (n_found == 0) { + if (q_first == 0) { // If there're no better alternatives, and if it's first word // in the line, try to complete "import". if (s_start == org_str) { @@ -234,7 +219,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print } // 1 match found, or multiple matches with a common prefix - if (n_found == 1 || match_len > s_len) { + if (q_first == q_last || match_len > s_len) { *compl_str = match_str + s_len; return match_len - s_len; } @@ -245,11 +230,12 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print #define MAX_LINE_LEN (4 * WORD_SLOT_LEN) int line_len = MAX_LINE_LEN; // force a newline for first word - for (size_t i = 0; i < dict->map.alloc; i++) { - if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) { - size_t d_len; - const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len); - if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { + for (qstr q = q_first; q <= q_last; ++q) { + size_t d_len; + const char *d_str = (const char*)qstr_data(q, &d_len); + if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { + mp_load_method_maybe(obj, q, dest); + if (dest[0] != MP_OBJ_NULL) { int gap = (line_len + WORD_SLOT_LEN - 1) / WORD_SLOT_LEN * WORD_SLOT_LEN - line_len; if (gap < 2) { gap += WORD_SLOT_LEN; diff --git a/tests/cmdline/repl_autocomplete.py b/tests/cmdline/repl_autocomplete.py index a848cab0f..27cad428f 100644 --- a/tests/cmdline/repl_autocomplete.py +++ b/tests/cmdline/repl_autocomplete.py @@ -6,5 +6,4 @@ x = '123' 1, x.isdi () i = str i.lowe ('ABC') -j = None -j.  +None.  diff --git a/tests/cmdline/repl_autocomplete.py.exp b/tests/cmdline/repl_autocomplete.py.exp index dfb998ff6..75002985e 100644 --- a/tests/cmdline/repl_autocomplete.py.exp +++ b/tests/cmdline/repl_autocomplete.py.exp @@ -10,6 +10,5 @@ Use \.\+ >>> i = str >>> i.lower('ABC') 'abc' ->>> j = None ->>> j. +>>> None. >>> diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index 26fd2e332..8a2c6aea9 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -24,11 +24,11 @@ RuntimeError: # repl ame__ -__name__ path argv version -version_info implementation platform byteorder -maxsize exit stdin stdout -stderr modules exc_info getsizeof -print_exception +__class__ __name__ argv byteorder +exc_info exit getsizeof implementation +maxsize modules path platform +print_exception stderr stdin +stdout version version_info ementation # attrtuple (start=1, stop=2, step=3) -- cgit v1.2.3 From 4e469085c192017c5244bbc115bac90f4bb667cb Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 19 Feb 2018 16:25:30 +1100 Subject: py/objstr: Protect against creating bytes(n) with n negative. Prior to this patch uPy (on a 32-bit arch) would have severe issues when calling bytes(-1): such a call would call vstr_init_len(vstr, -1) which would then +1 on the len and call vstr_init(vstr, 0), which would then round this up and allocate a small amount of memory for the vstr. The bytes constructor would then attempt to zero out all this memory, thinking it had allocated 2^32-1 bytes. --- py/objstr.c | 5 ++++- tests/basics/bytes.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/objstr.c b/py/objstr.c index ed9ab4e45..13d957105 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -223,7 +223,10 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size } if (MP_OBJ_IS_SMALL_INT(args[0])) { - uint len = MP_OBJ_SMALL_INT_VALUE(args[0]); + mp_int_t len = MP_OBJ_SMALL_INT_VALUE(args[0]); + if (len < 0) { + mp_raise_ValueError(NULL); + } vstr_t vstr; vstr_init_len(&vstr, len); memset(vstr.buf, 0, len); diff --git a/tests/basics/bytes.py b/tests/basics/bytes.py index 1d97e6b16..0b6b14fa5 100644 --- a/tests/basics/bytes.py +++ b/tests/basics/bytes.py @@ -56,3 +56,9 @@ print(x[0], x[1], x[2], x[3]) print(bytes([128, 255])) # For sequence of unknown len print(bytes(iter([128, 255]))) + +# Shouldn't be able to make bytes with negative length +try: + bytes(-1) +except ValueError: + print('ValueError') -- cgit v1.2.3 From 6e7819ee2ee20ec9f09feb40b68be5973797f874 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 20 Feb 2018 17:56:58 +1100 Subject: py/objmodule: Factor common code for calling __init__ on builtin module. --- py/builtinimport.c | 14 +------------- py/objmodule.c | 28 +++++++++++++++++----------- py/objmodule.h | 9 +++++++++ 3 files changed, 27 insertions(+), 24 deletions(-) (limited to 'py') diff --git a/py/builtinimport.c b/py/builtinimport.c index 97c1789f8..19bc9fc95 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -389,19 +389,7 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { } // found weak linked module module_obj = el->value; - if (MICROPY_MODULE_BUILTIN_INIT) { - // look for __init__ and call it if it exists - // Note: this code doesn't work fully correctly because it allows the - // __init__ function to be called twice if the module is imported by its - // non-weak-link name. Also, this code is duplicated in objmodule.c. - mp_obj_t dest[2]; - mp_load_method_maybe(el->value, MP_QSTR___init__, dest); - if (dest[0] != MP_OBJ_NULL) { - mp_call_method_n_kw(0, 0, dest); - // register module so __init__ is not called again - mp_module_register(mod_name, el->value); - } - } + mp_module_call_init(mod_name, module_obj); } else { no_exist: #else diff --git a/py/objmodule.c b/py/objmodule.c index f9363e379..c4aba3a7b 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -247,17 +247,7 @@ mp_obj_t mp_module_get(qstr module_name) { if (el == NULL) { return MP_OBJ_NULL; } - - if (MICROPY_MODULE_BUILTIN_INIT) { - // look for __init__ and call it if it exists - mp_obj_t dest[2]; - mp_load_method_maybe(el->value, MP_QSTR___init__, dest); - if (dest[0] != MP_OBJ_NULL) { - mp_call_method_n_kw(0, 0, dest); - // register module so __init__ is not called again - mp_module_register(module_name, el->value); - } - } + mp_module_call_init(module_name, el->value); } // module found, return it @@ -268,3 +258,19 @@ void mp_module_register(qstr qst, mp_obj_t module) { mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = module; } + +#if MICROPY_MODULE_BUILTIN_INIT +void mp_module_call_init(qstr module_name, mp_obj_t module_obj) { + // Look for __init__ and call it if it exists + mp_obj_t dest[2]; + mp_load_method_maybe(module_obj, MP_QSTR___init__, dest); + if (dest[0] != MP_OBJ_NULL) { + mp_call_method_n_kw(0, 0, dest); + // Register module so __init__ is not called again. + // If a module can be referenced by more than one name (eg due to weak links) + // then __init__ will still be called for each distinct import, and it's then + // up to the particular module to make sure it's __init__ code only runs once. + mp_module_register(module_name, module_obj); + } +} +#endif diff --git a/py/objmodule.h b/py/objmodule.h index b5c07dc33..b7702ec50 100644 --- a/py/objmodule.h +++ b/py/objmodule.h @@ -34,4 +34,13 @@ extern const mp_map_t mp_builtin_module_weak_links_map; mp_obj_t mp_module_get(qstr module_name); void mp_module_register(qstr qstr, mp_obj_t module); +#if MICROPY_MODULE_BUILTIN_INIT +void mp_module_call_init(qstr module_name, mp_obj_t module_obj); +#else +static inline void mp_module_call_init(qstr module_name, mp_obj_t module_obj) { + (void)module_name; + (void)module_obj; +} +#endif + #endif // MICROPY_INCLUDED_PY_OBJMODULE_H -- cgit v1.2.3 From 209936880df4d63730ea77cf4b6f84e5f2f56d7c Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 20 Feb 2018 18:00:44 +1100 Subject: py/builtinimport: Add compile-time option to disable external imports. The new option is MICROPY_ENABLE_EXTERNAL_IMPORT and is enabled by default so that the default behaviour is the same as before. With it disabled import is only supported for built-in modules, not for external files nor frozen modules. This allows to support targets that have no filesystem of any kind and that only have access to pre-supplied built-in modules implemented natively. --- py/builtinimport.c | 39 +++++++++++++++++++++++++++++++++++++++ py/mpconfig.h | 7 +++++++ py/runtime.c | 9 +++++++++ 3 files changed, 55 insertions(+) (limited to 'py') diff --git a/py/builtinimport.c b/py/builtinimport.c index 19bc9fc95..b8ed096ca 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -44,6 +44,8 @@ #define DEBUG_printf(...) (void)0 #endif +#if MICROPY_ENABLE_EXTERNAL_IMPORT + #define PATH_SEP_CHAR '/' bool mp_obj_is_package(mp_obj_t module) { @@ -473,4 +475,41 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { // Otherwise, we need to return top-level package return top_module_obj; } + +#else // MICROPY_ENABLE_EXTERNAL_IMPORT + +mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { + // Check that it's not a relative import + if (n_args >= 5 && MP_OBJ_SMALL_INT_VALUE(args[4]) != 0) { + mp_raise_NotImplementedError("relative import"); + } + + // Check if module already exists, and return it if it does + qstr module_name_qstr = mp_obj_str_get_qstr(args[0]); + mp_obj_t module_obj = mp_module_get(module_name_qstr); + if (module_obj != MP_OBJ_NULL) { + return module_obj; + } + + #if MICROPY_MODULE_WEAK_LINKS + // Check if there is a weak link to this module + mp_map_elem_t *el = mp_map_lookup((mp_map_t*)&mp_builtin_module_weak_links_map, MP_OBJ_NEW_QSTR(module_name_qstr), MP_MAP_LOOKUP); + if (el != NULL) { + // Found weak-linked module + mp_module_call_init(module_name_qstr, el->value); + return el->value; + } + #endif + + // Couldn't find the module, so fail + if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { + mp_raise_msg(&mp_type_ImportError, "module not found"); + } else { + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ImportError, + "no module named '%q'", module_name_qstr)); + } +} + +#endif // MICROPY_ENABLE_EXTERNAL_IMPORT + MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin___import___obj, 1, 5, mp_builtin___import__); diff --git a/py/mpconfig.h b/py/mpconfig.h index b8a96f0b0..e9071df2f 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -405,6 +405,13 @@ /*****************************************************************************/ /* Python internal features */ +// Whether to enable import of external modules +// When disabled, only importing of built-in modules is supported +// When enabled, a port must implement mp_import_stat (among other things) +#ifndef MICROPY_ENABLE_EXTERNAL_IMPORT +#define MICROPY_ENABLE_EXTERNAL_IMPORT (1) +#endif + // Whether to use the POSIX reader for importing files #ifndef MICROPY_READER_POSIX #define MICROPY_READER_POSIX (0) diff --git a/py/runtime.c b/py/runtime.c index 9dff9847a..65d0df639 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1337,6 +1337,8 @@ import_error: return dest[0]; } + #if MICROPY_ENABLE_EXTERNAL_IMPORT + // See if it's a package, then can try FS import if (!mp_obj_is_package(module)) { goto import_error; @@ -1363,6 +1365,13 @@ import_error: // TODO lookup __import__ and call that instead of going straight to builtin implementation return mp_builtin___import__(5, args); + + #else + + // Package import not supported with external imports disabled + goto import_error; + + #endif } void mp_import_all(mp_obj_t module) { -- cgit v1.2.3 From 7e2a48858cec549879ae87384398008d82887766 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 20 Feb 2018 18:30:22 +1100 Subject: py/modmicropython: Allow to have stack_use() func without mem_info(). The micropython.stack_use() function is useful to query the current C stack usage, and it's inclusion in the micropython module doesn't need to be tied to the inclusion of mem_info()/qstr_info() because it doesn't rely on any of the code from these functions. So this patch introduces the config option MICROPY_PY_MICROPYTHON_STACK_USE which can be used to independently control the inclusion of stack_use(). By default it is enabled if MICROPY_PY_MICROPYTHON_MEM_INFO is enabled (thus not changing any of the existing ports). --- py/modmicropython.c | 10 +++++----- py/mpconfig.h | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'py') diff --git a/py/modmicropython.c b/py/modmicropython.c index c14a0177d..5cc7bdbe2 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -103,15 +103,15 @@ STATIC mp_obj_t mp_micropython_qstr_info(size_t n_args, const mp_obj_t *args) { } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_micropython_qstr_info_obj, 0, 1, mp_micropython_qstr_info); -#if MICROPY_STACK_CHECK +#endif // MICROPY_PY_MICROPYTHON_MEM_INFO + +#if MICROPY_PY_MICROPYTHON_STACK_USE STATIC mp_obj_t mp_micropython_stack_use(void) { return MP_OBJ_NEW_SMALL_INT(mp_stack_usage()); } STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_stack_use_obj, mp_micropython_stack_use); #endif -#endif // MICROPY_PY_MICROPYTHON_MEM_INFO - #if MICROPY_ENABLE_PYSTACK STATIC mp_obj_t mp_micropython_pystack_use(void) { return MP_OBJ_NEW_SMALL_INT(mp_pystack_usage()); @@ -167,10 +167,10 @@ STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { #endif { MP_ROM_QSTR(MP_QSTR_mem_info), MP_ROM_PTR(&mp_micropython_mem_info_obj) }, { MP_ROM_QSTR(MP_QSTR_qstr_info), MP_ROM_PTR(&mp_micropython_qstr_info_obj) }, - #if MICROPY_STACK_CHECK +#endif + #if MICROPY_PY_MICROPYTHON_STACK_USE { MP_ROM_QSTR(MP_QSTR_stack_use), MP_ROM_PTR(&mp_micropython_stack_use_obj) }, #endif -#endif #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0) { MP_ROM_QSTR(MP_QSTR_alloc_emergency_exception_buf), MP_ROM_PTR(&mp_alloc_emergency_exception_buf_obj) }, #endif diff --git a/py/mpconfig.h b/py/mpconfig.h index e9071df2f..d38536f97 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -907,6 +907,11 @@ typedef double mp_float_t; #define MICROPY_PY_MICROPYTHON_MEM_INFO (0) #endif +// Whether to provide "micropython.stack_use" function +#ifndef MICROPY_PY_MICROPYTHON_STACK_USE +#define MICROPY_PY_MICROPYTHON_STACK_USE (MICROPY_PY_MICROPYTHON_MEM_INFO) +#endif + // Whether to provide "array" module. Note that large chunk of the // underlying code is shared with "bytearray" builtin type, so to // get real savings, it should be disabled too. -- cgit v1.2.3 From 8769049e935a89daa51883755457a083a5589d4e Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 20 Feb 2018 19:19:02 +1100 Subject: py/objstr: Remove unnecessary check for positive splits variable. At this point in the code the variable "splits" is guaranteed to be positive due to the check for "splits == 0" above it. --- py/objstr.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'py') diff --git a/py/objstr.c b/py/objstr.c index 13d957105..c42f38e75 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -663,9 +663,7 @@ STATIC mp_obj_t str_rsplit(size_t n_args, const mp_obj_t *args) { } res->items[idx--] = mp_obj_new_str_of_type(self_type, s + sep_len, last - s - sep_len); last = s; - if (splits > 0) { - splits--; - } + splits--; } if (idx != 0) { // We split less parts than split limit, now go cleanup surplus -- cgit v1.2.3 From fe3e17b026595f3386114e784b26a4a07810055c Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 21 Feb 2018 00:20:46 +1100 Subject: py/objint: Use MP_OBJ_IS_STR_OR_BYTES macro instead of 2 separate ones. --- py/objint.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/objint.c b/py/objint.c index 4f2e610a5..59c58f2a6 100644 --- a/py/objint.c +++ b/py/objint.c @@ -378,7 +378,7 @@ mp_obj_t mp_obj_int_binary_op_extra_cases(mp_binary_op_t op, mp_obj_t lhs_in, mp // true acts as 0 return mp_binary_op(op, lhs_in, MP_OBJ_NEW_SMALL_INT(1)); } else if (op == MP_BINARY_OP_MULTIPLY) { - if (MP_OBJ_IS_STR(rhs_in) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_bytes) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_tuple) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_list)) { + if (MP_OBJ_IS_STR_OR_BYTES(rhs_in) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_tuple) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_list)) { // multiply is commutative for these types, so delegate to them return mp_binary_op(op, rhs_in, lhs_in); } -- cgit v1.2.3 From 970eedce8f37af46cb2b67fa0e91d76c82057541 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 6 Feb 2018 00:06:42 +0200 Subject: py/objdeque: Implement ucollections.deque type with fixed size. So far, implements just append() and popleft() methods, required for a normal queue. Constructor doesn't accept an arbitarry sequence to initialize from (am empty deque is always created), so an empty tuple must be passed as such. Only fixed-size deques are supported, so 2nd argument (size) is required. There's also an extension to CPython - if True is passed as 3rd argument, append(), instead of silently overwriting the oldest item on queue overflow, will throw IndexError. This behavior is desired in many cases, where queues should store information reliably, instead of silently losing some items. --- py/modcollections.c | 3 + py/mpconfig.h | 5 ++ py/obj.h | 1 + py/objdeque.c | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++++ py/py.mk | 1 + 5 files changed, 169 insertions(+) create mode 100644 py/objdeque.c (limited to 'py') diff --git a/py/modcollections.c b/py/modcollections.c index 1a1560387..bb6488471 100644 --- a/py/modcollections.c +++ b/py/modcollections.c @@ -30,6 +30,9 @@ STATIC const mp_rom_map_elem_t mp_module_collections_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ucollections) }, + #if MICROPY_PY_COLLECTIONS_DEQUE + { MP_ROM_QSTR(MP_QSTR_deque), MP_ROM_PTR(&mp_type_deque) }, + #endif { MP_ROM_QSTR(MP_QSTR_namedtuple), MP_ROM_PTR(&mp_namedtuple_obj) }, #if MICROPY_PY_COLLECTIONS_ORDEREDDICT { MP_ROM_QSTR(MP_QSTR_OrderedDict), MP_ROM_PTR(&mp_type_ordereddict) }, diff --git a/py/mpconfig.h b/py/mpconfig.h index d38536f97..532b54ab0 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -936,6 +936,11 @@ typedef double mp_float_t; #define MICROPY_PY_COLLECTIONS (1) #endif +// Whether to provide "ucollections.deque" type +#ifndef MICROPY_PY_COLLECTIONS_DEQUE +#define MICROPY_PY_COLLECTIONS_DEQUE (0) +#endif + // Whether to provide "collections.OrderedDict" type #ifndef MICROPY_PY_COLLECTIONS_ORDEREDDICT #define MICROPY_PY_COLLECTIONS_ORDEREDDICT (0) diff --git a/py/obj.h b/py/obj.h index 2e715253c..32b430154 100644 --- a/py/obj.h +++ b/py/obj.h @@ -553,6 +553,7 @@ extern const mp_obj_type_t mp_type_list; extern const mp_obj_type_t mp_type_map; // map (the python builtin, not the dict implementation detail) extern const mp_obj_type_t mp_type_enumerate; extern const mp_obj_type_t mp_type_filter; +extern const mp_obj_type_t mp_type_deque; extern const mp_obj_type_t mp_type_dict; extern const mp_obj_type_t mp_type_ordereddict; extern const mp_obj_type_t mp_type_range; diff --git a/py/objdeque.c b/py/objdeque.c new file mode 100644 index 000000000..15936f9ab --- /dev/null +++ b/py/objdeque.c @@ -0,0 +1,159 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2018 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/mpconfig.h" +#if MICROPY_PY_COLLECTIONS_DEQUE + +#include "py/runtime.h" + +typedef struct _mp_obj_deque_t { + mp_obj_base_t base; + size_t alloc; + size_t i_get; + size_t i_put; + mp_obj_t *items; + uint32_t flags; + #define FLAG_CHECK_OVERFLOW 1 +} mp_obj_deque_t; + +STATIC mp_obj_t deque_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 2, 3, false); + + /* Initialization from existing sequence is not supported, so an empty + tuple must be passed as such. */ + if (args[0] != mp_const_empty_tuple) { + mp_raise_ValueError(NULL); + } + + mp_obj_deque_t *o = m_new_obj(mp_obj_deque_t); + o->base.type = type; + o->alloc = mp_obj_get_int(args[1]) + 1; + o->i_get = o->i_put = 0; + o->items = m_new(mp_obj_t, o->alloc); + mp_seq_clear(o->items, 0, o->alloc, sizeof(*o->items)); + + if (n_args > 2) { + o->flags = mp_obj_get_int(args[2]); + } + + return MP_OBJ_FROM_PTR(o); +} + +STATIC mp_obj_t deque_unary_op(mp_unary_op_t op, mp_obj_t self_in) { + mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); + switch (op) { + case MP_UNARY_OP_BOOL: + return mp_obj_new_bool(self->i_get != self->i_put); + case MP_UNARY_OP_LEN: { + mp_int_t len = self->i_put - self->i_get; + if (len < 0) { + len += self->alloc; + } + return MP_OBJ_NEW_SMALL_INT(len); + } + #if MICROPY_PY_SYS_GETSIZEOF + case MP_UNARY_OP_SIZEOF: { + size_t sz = sizeof(*self) + sizeof(mp_obj_t) * self->alloc; + return MP_OBJ_NEW_SMALL_INT(sz); + } + #endif + default: + return MP_OBJ_NULL; // op not supported + } +} + +STATIC mp_obj_t mp_obj_deque_append(mp_obj_t self_in, mp_obj_t arg) { + mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); + + size_t new_i_put = self->i_put + 1; + if (new_i_put == self->alloc) { + new_i_put = 0; + } + + if (self->flags & FLAG_CHECK_OVERFLOW && new_i_put == self->i_get) { + mp_raise_msg(&mp_type_IndexError, "full"); + } + + self->items[self->i_put] = arg; + self->i_put = new_i_put; + + if (self->i_get == new_i_put) { + if (++self->i_get == self->alloc) { + self->i_get = 0; + } + } + + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(deque_append_obj, mp_obj_deque_append); + +STATIC mp_obj_t deque_popleft(mp_obj_t self_in) { + mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); + + if (self->i_get == self->i_put) { + mp_raise_msg(&mp_type_IndexError, "empty"); + } + + mp_obj_t ret = self->items[self->i_get]; + self->items[self->i_get] = MP_OBJ_NULL; + + if (++self->i_get == self->alloc) { + self->i_get = 0; + } + + return ret; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(deque_popleft_obj, deque_popleft); + +STATIC mp_obj_t deque_clear(mp_obj_t self_in) { + mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); + self->i_get = self->i_put = 0; + mp_seq_clear(self->items, 0, self->alloc, sizeof(*self->items)); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(deque_clear_obj, deque_clear); + +STATIC const mp_rom_map_elem_t deque_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_append), MP_ROM_PTR(&deque_append_obj) }, + #if 0 + { MP_ROM_QSTR(MP_QSTR_clear), MP_ROM_PTR(&deque_clear_obj) }, + #endif + { MP_ROM_QSTR(MP_QSTR_popleft), MP_ROM_PTR(&deque_popleft_obj) }, +}; + +STATIC MP_DEFINE_CONST_DICT(deque_locals_dict, deque_locals_dict_table); + +const mp_obj_type_t mp_type_deque = { + { &mp_type_type }, + .name = MP_QSTR_deque, + .make_new = deque_make_new, + .unary_op = deque_unary_op, + .locals_dict = (mp_obj_dict_t*)&deque_locals_dict, +}; + +#endif // MICROPY_PY_COLLECTIONS_DEQUE diff --git a/py/py.mk b/py/py.mk index de82a971b..6cfe21ca9 100644 --- a/py/py.mk +++ b/py/py.mk @@ -158,6 +158,7 @@ PY_O_BASENAME = \ objcell.o \ objclosure.o \ objcomplex.o \ + objdeque.o \ objdict.o \ objenumerate.o \ objexcept.o \ -- cgit v1.2.3 From 6c3faf6c17521940eb3ec48a8af3d28b513e2ba2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 21 Feb 2018 22:52:58 +1100 Subject: py/objdeque: Allow to compile without warnings by disabling deque_clear. --- py/objdeque.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'py') diff --git a/py/objdeque.c b/py/objdeque.c index 15936f9ab..a4c42c31f 100644 --- a/py/objdeque.c +++ b/py/objdeque.c @@ -130,6 +130,7 @@ STATIC mp_obj_t deque_popleft(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(deque_popleft_obj, deque_popleft); +#if 0 STATIC mp_obj_t deque_clear(mp_obj_t self_in) { mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); self->i_get = self->i_put = 0; @@ -137,6 +138,7 @@ STATIC mp_obj_t deque_clear(mp_obj_t self_in) { return mp_const_none; } STATIC MP_DEFINE_CONST_FUN_OBJ_1(deque_clear_obj, deque_clear); +#endif STATIC const mp_rom_map_elem_t deque_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_append), MP_ROM_PTR(&deque_append_obj) }, -- cgit v1.2.3 From 160d6708684de8c71895f90bc699a5879fb6ed29 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 21 Feb 2018 23:34:17 +1100 Subject: py/objdeque: Protect against negative maxlen in deque constructor. Otherwise passing -1 as maxlen will lead to a zero allocation and subsequent unbound buffer overflow in deque.append() because i_put is allowed to grow without bound. --- py/objdeque.c | 8 +++++++- tests/basics/deque1.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/objdeque.c b/py/objdeque.c index a4c42c31f..573a48baf 100644 --- a/py/objdeque.c +++ b/py/objdeque.c @@ -50,9 +50,15 @@ STATIC mp_obj_t deque_make_new(const mp_obj_type_t *type, size_t n_args, size_t mp_raise_ValueError(NULL); } + // Protect against -1 leading to zero-length allocation and bad array access + mp_int_t maxlen = mp_obj_get_int(args[1]); + if (maxlen < 0) { + mp_raise_ValueError(NULL); + } + mp_obj_deque_t *o = m_new_obj(mp_obj_deque_t); o->base.type = type; - o->alloc = mp_obj_get_int(args[1]) + 1; + o->alloc = maxlen + 1; o->i_get = o->i_put = 0; o->items = m_new(mp_obj_t, o->alloc); mp_seq_clear(o->items, 0, o->alloc, sizeof(*o->items)); diff --git a/tests/basics/deque1.py b/tests/basics/deque1.py index 6b5669c45..19966fcb0 100644 --- a/tests/basics/deque1.py +++ b/tests/basics/deque1.py @@ -55,6 +55,12 @@ d.append(4) d.append(5) print(d.popleft(), d.popleft()) +# Negative maxlen is not allowed +try: + deque((), -1) +except ValueError: + print("ValueError") + # Unsupported unary op try: ~d -- cgit v1.2.3 From 6e675c1baaf0de0d56a2345376d6b5600bfab3aa Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 21 Feb 2018 23:36:46 +1100 Subject: py/objdeque: Use m_new0 when allocating items to avoid need to clear. Saves a few bytes of code space, and is more efficient because with MICROPY_GC_CONSERVATIVE_CLEAR enabled by default all memory is already cleared when allocated. --- py/objdeque.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'py') diff --git a/py/objdeque.c b/py/objdeque.c index 573a48baf..bbb078103 100644 --- a/py/objdeque.c +++ b/py/objdeque.c @@ -60,8 +60,7 @@ STATIC mp_obj_t deque_make_new(const mp_obj_type_t *type, size_t n_args, size_t o->base.type = type; o->alloc = maxlen + 1; o->i_get = o->i_put = 0; - o->items = m_new(mp_obj_t, o->alloc); - mp_seq_clear(o->items, 0, o->alloc, sizeof(*o->items)); + o->items = m_new0(mp_obj_t, o->alloc); if (n_args > 2) { o->flags = mp_obj_get_int(args[2]); -- cgit v1.2.3 From 8ca469cae299f8e93dfe721bb915adb217dc4cfe Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Feb 2018 11:13:04 +1100 Subject: py/py.mk: Split list of uPy sources into core and extmod files. If a port only needs the core files then it can now use the $(PY_CORE_O) variable instead of $(PY_O). $(PY_EXTMOD_O) contains the list of extmod files (including some files from lib/). $(PY_O) retains its original definition as the list of all object file (including those for frozen code) and is a convenience variable for ports that want everything. --- py/py.mk | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/py.mk b/py/py.mk index 6cfe21ca9..20574d2c5 100644 --- a/py/py.mk +++ b/py/py.mk @@ -101,7 +101,7 @@ $(BUILD)/extmod/modbtree.o: CFLAGS += $(BTREE_DEFS) endif # py object files -PY_O_BASENAME = \ +PY_CORE_O_BASENAME = \ mpstate.o \ nlr.o \ nlrx86.o \ @@ -214,6 +214,8 @@ PY_O_BASENAME = \ repl.o \ smallint.o \ frozenmod.o \ + +PY_EXTMOD_O_BASENAME = \ ../extmod/moductypes.o \ ../extmod/modujson.o \ ../extmod/modure.o \ @@ -248,7 +250,11 @@ PY_O_BASENAME = \ ../lib/utils/printf.o \ # prepend the build destination prefix to the py object files -PY_O = $(addprefix $(PY_BUILD)/, $(PY_O_BASENAME)) +PY_CORE_O = $(addprefix $(PY_BUILD)/, $(PY_CORE_O_BASENAME)) +PY_EXTMOD_O = $(addprefix $(PY_BUILD)/, $(PY_EXTMOD_O_BASENAME)) + +# this is a convenience variable for ports that want core, extmod and frozen code +PY_O = $(PY_CORE_O) $(PY_EXTMOD_O) # object file for frozen files ifneq ($(FROZEN_DIR),) @@ -262,7 +268,7 @@ endif # Sources that may contain qstrings SRC_QSTR_IGNORE = nlr% emitnx86% emitnx64% emitnthumb% emitnarm% emitnxtensa% -SRC_QSTR = $(SRC_MOD) $(addprefix py/,$(filter-out $(SRC_QSTR_IGNORE),$(PY_O_BASENAME:.o=.c)) emitnative.c) +SRC_QSTR = $(SRC_MOD) $(addprefix py/,$(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) emitnative.c $(PY_EXTMOD_O_BASENAME:.o=.c)) # Anything that depends on FORCE will be considered out-of-date FORCE: -- cgit v1.2.3 From 65ef59a9b550bf89ae5b14130f0f8e6ff6d357ca Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Feb 2018 11:32:31 +1100 Subject: py/py.mk: Remove .. path component from list of extmod files. This just makes it a bit cleaner in the output of the build process: instead of "CC ../../py/../extmod/" there is now "CC ../../extmod/". --- py/py.mk | 75 ++++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 38 insertions(+), 37 deletions(-) (limited to 'py') diff --git a/py/py.mk b/py/py.mk index 20574d2c5..879c63248 100644 --- a/py/py.mk +++ b/py/py.mk @@ -101,7 +101,7 @@ $(BUILD)/extmod/modbtree.o: CFLAGS += $(BTREE_DEFS) endif # py object files -PY_CORE_O_BASENAME = \ +PY_CORE_O_BASENAME = $(addprefix py/,\ mpstate.o \ nlr.o \ nlrx86.o \ @@ -214,44 +214,45 @@ PY_CORE_O_BASENAME = \ repl.o \ smallint.o \ frozenmod.o \ + ) PY_EXTMOD_O_BASENAME = \ - ../extmod/moductypes.o \ - ../extmod/modujson.o \ - ../extmod/modure.o \ - ../extmod/moduzlib.o \ - ../extmod/moduheapq.o \ - ../extmod/modutimeq.o \ - ../extmod/moduhashlib.o \ - ../extmod/modubinascii.o \ - ../extmod/virtpin.o \ - ../extmod/machine_mem.o \ - ../extmod/machine_pinbase.o \ - ../extmod/machine_signal.o \ - ../extmod/machine_pulse.o \ - ../extmod/machine_i2c.o \ - ../extmod/machine_spi.o \ - ../extmod/modussl_axtls.o \ - ../extmod/modussl_mbedtls.o \ - ../extmod/modurandom.o \ - ../extmod/moduselect.o \ - ../extmod/modwebsocket.o \ - ../extmod/modwebrepl.o \ - ../extmod/modframebuf.o \ - ../extmod/vfs.o \ - ../extmod/vfs_reader.o \ - ../extmod/vfs_fat.o \ - ../extmod/vfs_fat_diskio.o \ - ../extmod/vfs_fat_file.o \ - ../extmod/vfs_fat_misc.o \ - ../extmod/utime_mphal.o \ - ../extmod/uos_dupterm.o \ - ../lib/embed/abort_.o \ - ../lib/utils/printf.o \ + extmod/moductypes.o \ + extmod/modujson.o \ + extmod/modure.o \ + extmod/moduzlib.o \ + extmod/moduheapq.o \ + extmod/modutimeq.o \ + extmod/moduhashlib.o \ + extmod/modubinascii.o \ + extmod/virtpin.o \ + extmod/machine_mem.o \ + extmod/machine_pinbase.o \ + extmod/machine_signal.o \ + extmod/machine_pulse.o \ + extmod/machine_i2c.o \ + extmod/machine_spi.o \ + extmod/modussl_axtls.o \ + extmod/modussl_mbedtls.o \ + extmod/modurandom.o \ + extmod/moduselect.o \ + extmod/modwebsocket.o \ + extmod/modwebrepl.o \ + extmod/modframebuf.o \ + extmod/vfs.o \ + extmod/vfs_reader.o \ + extmod/vfs_fat.o \ + extmod/vfs_fat_diskio.o \ + extmod/vfs_fat_file.o \ + extmod/vfs_fat_misc.o \ + extmod/utime_mphal.o \ + extmod/uos_dupterm.o \ + lib/embed/abort_.o \ + lib/utils/printf.o \ # prepend the build destination prefix to the py object files -PY_CORE_O = $(addprefix $(PY_BUILD)/, $(PY_CORE_O_BASENAME)) -PY_EXTMOD_O = $(addprefix $(PY_BUILD)/, $(PY_EXTMOD_O_BASENAME)) +PY_CORE_O = $(addprefix $(BUILD)/, $(PY_CORE_O_BASENAME)) +PY_EXTMOD_O = $(addprefix $(BUILD)/, $(PY_EXTMOD_O_BASENAME)) # this is a convenience variable for ports that want core, extmod and frozen code PY_O = $(PY_CORE_O) $(PY_EXTMOD_O) @@ -267,8 +268,8 @@ PY_O += $(BUILD)/$(BUILD)/frozen_mpy.o endif # Sources that may contain qstrings -SRC_QSTR_IGNORE = nlr% emitnx86% emitnx64% emitnthumb% emitnarm% emitnxtensa% -SRC_QSTR = $(SRC_MOD) $(addprefix py/,$(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) emitnative.c $(PY_EXTMOD_O_BASENAME:.o=.c)) +SRC_QSTR_IGNORE = py/nlr% py/emitnx86% py/emitnx64% py/emitnthumb% py/emitnarm% py/emitnxtensa% +SRC_QSTR = $(SRC_MOD) $(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) py/emitnative.c $(PY_EXTMOD_O_BASENAME:.o=.c) # Anything that depends on FORCE will be considered out-of-date FORCE: -- cgit v1.2.3 From 6af4515969045701a44fc282f72f68b7faaeb538 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Feb 2018 11:35:53 +1100 Subject: py: Use "GEN" consistently for describing files generated in the build. --- py/makeversionhdr.py | 2 +- py/mkrules.mk | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/makeversionhdr.py b/py/makeversionhdr.py index 749160b4d..aedc292e4 100644 --- a/py/makeversionhdr.py +++ b/py/makeversionhdr.py @@ -99,7 +99,7 @@ def make_version_header(filename): # Only write the file if we need to if write_file: - print("Generating %s" % filename) + print("GEN %s" % filename) with open(filename, 'w') as f: f.write(file_data) diff --git a/py/mkrules.mk b/py/mkrules.mk index 96f6e35a6..fa7138695 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -97,7 +97,7 @@ $(HEADER_BUILD): ifneq ($(FROZEN_DIR),) $(BUILD)/frozen.c: $(wildcard $(FROZEN_DIR)/*) $(HEADER_BUILD) $(FROZEN_EXTRA_DEPS) - $(ECHO) "Generating $@" + $(ECHO) "GEN $@" $(Q)$(MAKE_FROZEN) $(FROZEN_DIR) > $@ endif @@ -118,7 +118,7 @@ $(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py $(TOP)/mpy-cross/mpy-cross # to build frozen_mpy.c from all .mpy files $(BUILD)/frozen_mpy.c: $(FROZEN_MPY_MPY_FILES) $(BUILD)/genhdr/qstrdefs.generated.h - @$(ECHO) "Creating $@" + @$(ECHO) "GEN $@" $(Q)$(MPY_TOOL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(FROZEN_MPY_MPY_FILES) > $@ endif -- cgit v1.2.3 From 638b860066ddf6a684ce2d573917b3c8a5817ba2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 23 Feb 2018 17:24:57 +1100 Subject: extmod/vfs_fat: Merge remaining vfs_fat_misc.c code into vfs_fat.c. The only function left in vfs_fat_misc.c is fat_vfs_import_stat() which can logically go into vfs_fat.c, allowing to remove vfs_fat_misc.c. --- extmod/vfs_fat.c | 14 ++++++++++++++ extmod/vfs_fat_misc.c | 49 ------------------------------------------------- py/py.mk | 1 - 3 files changed, 14 insertions(+), 50 deletions(-) delete mode 100644 extmod/vfs_fat_misc.c (limited to 'py') diff --git a/extmod/vfs_fat.c b/extmod/vfs_fat.c index 58af0a8e8..e696e0fa8 100644 --- a/extmod/vfs_fat.c +++ b/extmod/vfs_fat.c @@ -47,6 +47,20 @@ #define mp_obj_fat_vfs_t fs_user_mount_t +mp_import_stat_t fat_vfs_import_stat(fs_user_mount_t *vfs, const char *path) { + FILINFO fno; + assert(vfs != NULL); + FRESULT res = f_stat(&vfs->fatfs, path, &fno); + if (res == FR_OK) { + if ((fno.fattrib & AM_DIR) != 0) { + return MP_IMPORT_STAT_DIR; + } else { + return MP_IMPORT_STAT_FILE; + } + } + return MP_IMPORT_STAT_NO_EXIST; +} + STATIC mp_obj_t fat_vfs_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_arg_check_num(n_args, n_kw, 1, 1, false); diff --git a/extmod/vfs_fat_misc.c b/extmod/vfs_fat_misc.c deleted file mode 100644 index d72d9c69c..000000000 --- a/extmod/vfs_fat_misc.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2014 Damien P. George - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "py/mpconfig.h" -#if MICROPY_VFS_FAT - -#include -#include "py/runtime.h" -#include "lib/oofatfs/ff.h" -#include "extmod/vfs_fat.h" - -mp_import_stat_t fat_vfs_import_stat(fs_user_mount_t *vfs, const char *path) { - FILINFO fno; - assert(vfs != NULL); - FRESULT res = f_stat(&vfs->fatfs, path, &fno); - if (res == FR_OK) { - if ((fno.fattrib & AM_DIR) != 0) { - return MP_IMPORT_STAT_DIR; - } else { - return MP_IMPORT_STAT_FILE; - } - } - return MP_IMPORT_STAT_NO_EXIST; -} - -#endif // MICROPY_VFS_FAT diff --git a/py/py.mk b/py/py.mk index 879c63248..7c4cf82d8 100644 --- a/py/py.mk +++ b/py/py.mk @@ -244,7 +244,6 @@ PY_EXTMOD_O_BASENAME = \ extmod/vfs_fat.o \ extmod/vfs_fat_diskio.o \ extmod/vfs_fat_file.o \ - extmod/vfs_fat_misc.o \ extmod/utime_mphal.o \ extmod/uos_dupterm.o \ lib/embed/abort_.o \ -- cgit v1.2.3 From 7dfa56e40e9c343cbf4a1726a4babecc69a6b732 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 24 Feb 2018 23:03:17 +1100 Subject: py/compile: Adjust c_assign_atom_expr() to use return instead of goto. Makes the flow of the function a little more obvious, and allows to reach 100% coverage of compile.c when using gcov. --- py/compile.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 42ae8a829..9200b346b 100644 --- a/py/compile.c +++ b/py/compile.c @@ -376,6 +376,7 @@ STATIC void c_assign_atom_expr(compiler_t *comp, mp_parse_node_struct_t *pns, as EMIT(store_subscr); } } + return; } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_period) { assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0])); if (assign_kind == ASSIGN_AUG_LOAD) { @@ -387,16 +388,10 @@ STATIC void c_assign_atom_expr(compiler_t *comp, mp_parse_node_struct_t *pns, as } EMIT_ARG(store_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0])); } - } else { - goto cannot_assign; + return; } - } else { - goto cannot_assign; } - return; - -cannot_assign: compile_syntax_error(comp, (mp_parse_node_t)pns, "can't assign to expression"); } -- cgit v1.2.3 From c0bcf00ed100181a532240d904395de11addcd33 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 24 Feb 2018 23:10:20 +1100 Subject: py/asm*.c: Remove unnecessary check for num_locals<0 in asm entry func. All callers of the asm entry function guarantee that num_locals>=0, so no need to add an explicit check for it. Use an assertion instead. Also, the signature of asm_x86_entry is changed to match the other asm entry functions. --- py/asmarm.c | 5 +---- py/asmthumb.c | 5 ++--- py/asmx64.c | 4 +--- py/asmx86.c | 3 ++- py/asmx86.h | 2 +- 5 files changed, 7 insertions(+), 12 deletions(-) (limited to 'py') diff --git a/py/asmarm.c b/py/asmarm.c index 552fdfb34..1a8923bc2 100644 --- a/py/asmarm.c +++ b/py/asmarm.c @@ -150,10 +150,7 @@ void asm_arm_bkpt(asm_arm_t *as) { // | low address | high address in RAM void asm_arm_entry(asm_arm_t *as, int num_locals) { - - if (num_locals < 0) { - num_locals = 0; - } + assert(num_locals >= 0); as->stack_adjust = 0; as->push_reglist = 1 << ASM_ARM_REG_R1 diff --git a/py/asmthumb.c b/py/asmthumb.c index 5316a7efb..c5b45f2f5 100644 --- a/py/asmthumb.c +++ b/py/asmthumb.c @@ -104,6 +104,8 @@ STATIC void asm_thumb_write_word32(asm_thumb_t *as, int w32) { // | low address | high address in RAM void asm_thumb_entry(asm_thumb_t *as, int num_locals) { + assert(num_locals >= 0); + // work out what to push and how many extra spaces to reserve on stack // so that we have enough for all locals and it's aligned an 8-byte boundary // we push extra regs (r1, r2, r3) to help do the stack adjustment @@ -111,9 +113,6 @@ void asm_thumb_entry(asm_thumb_t *as, int num_locals) { // for push rlist, lowest numbered register at the lowest address uint reglist; uint stack_adjust; - if (num_locals < 0) { - num_locals = 0; - } // don't pop r0 because it's used for return value switch (num_locals) { case 0: diff --git a/py/asmx64.c b/py/asmx64.c index aa2a8ec7c..c900a08d1 100644 --- a/py/asmx64.c +++ b/py/asmx64.c @@ -526,11 +526,9 @@ void asm_x64_jcc_label(asm_x64_t *as, int jcc_type, mp_uint_t label) { } void asm_x64_entry(asm_x64_t *as, int num_locals) { + assert(num_locals >= 0); asm_x64_push_r64(as, ASM_X64_REG_RBP); asm_x64_mov_r64_r64(as, ASM_X64_REG_RBP, ASM_X64_REG_RSP); - if (num_locals < 0) { - num_locals = 0; - } num_locals |= 1; // make it odd so stack is aligned on 16 byte boundary asm_x64_sub_r64_i32(as, ASM_X64_REG_RSP, num_locals * WORD_SIZE); asm_x64_push_r64(as, ASM_X64_REG_RBX); diff --git a/py/asmx86.c b/py/asmx86.c index 6a78fbd5e..3938baaac 100644 --- a/py/asmx86.c +++ b/py/asmx86.c @@ -387,7 +387,8 @@ void asm_x86_jcc_label(asm_x86_t *as, mp_uint_t jcc_type, mp_uint_t label) { } } -void asm_x86_entry(asm_x86_t *as, mp_uint_t num_locals) { +void asm_x86_entry(asm_x86_t *as, int num_locals) { + assert(num_locals >= 0); asm_x86_push_r32(as, ASM_X86_REG_EBP); asm_x86_mov_r32_r32(as, ASM_X86_REG_EBP, ASM_X86_REG_ESP); if (num_locals > 0) { diff --git a/py/asmx86.h b/py/asmx86.h index bd5895453..09559850c 100644 --- a/py/asmx86.h +++ b/py/asmx86.h @@ -104,7 +104,7 @@ void asm_x86_test_r8_with_r8(asm_x86_t* as, int src_r32_a, int src_r32_b); void asm_x86_setcc_r8(asm_x86_t* as, mp_uint_t jcc_type, int dest_r8); void asm_x86_jmp_label(asm_x86_t* as, mp_uint_t label); void asm_x86_jcc_label(asm_x86_t* as, mp_uint_t jcc_type, mp_uint_t label); -void asm_x86_entry(asm_x86_t* as, mp_uint_t num_locals); +void asm_x86_entry(asm_x86_t* as, int num_locals); void asm_x86_exit(asm_x86_t* as); void asm_x86_mov_arg_to_r32(asm_x86_t *as, int src_arg_num, int dest_r32); void asm_x86_mov_local_to_r32(asm_x86_t* as, int src_local_num, int dest_r32); -- cgit v1.2.3 From f75c7ad1a9dcfa2cfe5ccd12dda7c396a6bd973c Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 25 Feb 2018 22:59:19 +1100 Subject: py/mpz: In mpz_clone, remove unused check for NULL dig. This path for src->deg==NULL is never used because mpz_clone() is always called with an argument that has a non-zero integer value, and hence has some digits allocated to it (mpz_clone() is a static function private to mpz.c all callers of this function first check if the integer value is zero and if so take a special-case path, bypassing the call to mpz_clone()). There is some unused and commented-out functions that may actually pass a zero-valued mpz to mpz_clone(), so some TODOs are added to these function in case they are needed in the future. --- py/mpz.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'py') diff --git a/py/mpz.c b/py/mpz.c index 380e8968e..fa5086862 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -705,17 +705,14 @@ STATIC void mpz_need_dig(mpz_t *z, size_t need) { } STATIC mpz_t *mpz_clone(const mpz_t *src) { + assert(src->alloc != 0); mpz_t *z = m_new_obj(mpz_t); z->neg = src->neg; z->fixed_dig = 0; z->alloc = src->alloc; z->len = src->len; - if (src->dig == NULL) { - z->dig = NULL; - } else { - z->dig = m_new(mpz_dig_t, z->alloc); - memcpy(z->dig, src->dig, src->alloc * sizeof(mpz_dig_t)); - } + z->dig = m_new(mpz_dig_t, z->alloc); + memcpy(z->dig, src->dig, src->alloc * sizeof(mpz_dig_t)); return z; } @@ -983,6 +980,7 @@ these functions are unused /* returns abs(z) */ mpz_t *mpz_abs(const mpz_t *z) { + // TODO: handle case of z->alloc=0 mpz_t *z2 = mpz_clone(z); z2->neg = 0; return z2; @@ -991,6 +989,7 @@ mpz_t *mpz_abs(const mpz_t *z) { /* returns -z */ mpz_t *mpz_neg(const mpz_t *z) { + // TODO: handle case of z->alloc=0 mpz_t *z2 = mpz_clone(z); z2->neg = 1 - z2->neg; return z2; @@ -1408,6 +1407,7 @@ these functions are unused */ mpz_t *mpz_gcd(const mpz_t *z1, const mpz_t *z2) { if (z1->len == 0) { + // TODO: handle case of z2->alloc=0 mpz_t *a = mpz_clone(z2); a->neg = 0; return a; -- cgit v1.2.3 From 9d8347a9aac40f8cc168b0226c2e74f776a7d4bf Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 26 Feb 2018 16:08:58 +1100 Subject: py/mpstate.h: Add repl_line state for MICROPY_REPL_EVENT_DRIVEN. --- py/mpstate.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'py') diff --git a/py/mpstate.h b/py/mpstate.h index a7ffbbf3c..816698f4e 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -166,6 +166,10 @@ typedef struct _mp_state_vm_t { // root pointers for extmod + #if MICROPY_REPL_EVENT_DRIVEN + vstr_t *repl_line; + #endif + #if MICROPY_PY_OS_DUPTERM mp_obj_t dupterm_objs[MICROPY_PY_OS_DUPTERM]; mp_obj_t dupterm_arr_obj; -- cgit v1.2.3 From 22ade2f5c4ac88c90a013cbf4b81c8d795487f33 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 27 Feb 2018 15:39:31 +1100 Subject: py/vm: Fix case of handling raised StopIteration within yield from. This patch concerns the handling of an NLR-raised StopIteration, raised during a call to mp_resume() which is handling the yield from opcode. Previously, commit 6738c1dded8e436686f85008ec0a4fc47406ab7a introduced code to handle this case, along with a test. It seems that it was lucky that the test worked because the code did not correctly handle the stack pointer (sp). Furthermore, commit 79d996a57b351e0ef354eb1e2f644b194433cc73 improved the way mp_resume() propagated certain exceptions: it changed raising an NLR value to returning MP_VM_RETURN_EXCEPTION. This change meant that the test introduced in gen_yield_from_ducktype.py was no longer hitting the code introduced in 6738c1dded8e436686f85008ec0a4fc47406ab7a. The patch here does two things: 1. Fixes the handling of sp in the VM for the case that yield from is interrupted by a StopIteration raised via NLR. 2. Introduces a new test to check this handling of sp and re-covers the code in the VM. --- py/vm.c | 4 +++- tests/basics/gen_yield_from.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/vm.c b/py/vm.c index ceb2060f9..416de6b1a 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1162,6 +1162,7 @@ yield: mp_obj_t send_value = POP(); mp_obj_t t_exc = MP_OBJ_NULL; mp_obj_t ret_value; + code_state->sp = sp; // Save sp because it's needed if mp_resume raises StopIteration if (inject_exc != MP_OBJ_NULL) { t_exc = inject_exc; inject_exc = MP_OBJ_NULL; @@ -1361,7 +1362,8 @@ exception_handler: } else if (*code_state->ip == MP_BC_YIELD_FROM) { // StopIteration inside yield from call means return a value of // yield from, so inject exception's value as yield from's result - *++code_state->sp = mp_obj_exception_get_value(MP_OBJ_FROM_PTR(nlr.ret_val)); + // (Instead of stack pop then push we just replace exhausted gen with value) + *code_state->sp = mp_obj_exception_get_value(MP_OBJ_FROM_PTR(nlr.ret_val)); code_state->ip++; // yield from is over, move to next instruction goto outer_dispatch_loop; // continue with dispatch loop } diff --git a/tests/basics/gen_yield_from.py b/tests/basics/gen_yield_from.py index 5196b48d2..4e68aec63 100644 --- a/tests/basics/gen_yield_from.py +++ b/tests/basics/gen_yield_from.py @@ -40,3 +40,16 @@ def gen6(): g = gen6() print(list(g)) + +# StopIteration from within a Python function, within a native iterator (map), within a yield from +def gen7(x): + if x < 3: + return x + else: + raise StopIteration(444) + +def gen8(): + print((yield from map(gen7, range(100)))) + +g = gen8() +print(list(g)) -- cgit v1.2.3 From a9f6d4921829195d7310aa8b07a4a705577161a5 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 27 Feb 2018 15:48:09 +1100 Subject: py/vm: Simplify handling of special-case STOP_ITERATION in yield from. There's no need to have MP_OBJ_NULL a special case, the code can re-use the MP_OBJ_STOP_ITERATION value to signal the special case and the VM can detect this with only one check (for MP_OBJ_STOP_ITERATION). --- py/runtime.c | 3 +-- py/vm.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'py') diff --git a/py/runtime.c b/py/runtime.c index 65d0df639..54ec0d70b 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1215,13 +1215,12 @@ mp_vm_return_kind_t mp_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t th if (type->iternext != NULL && send_value == mp_const_none) { mp_obj_t ret = type->iternext(self_in); + *ret_val = ret; if (ret != MP_OBJ_STOP_ITERATION) { - *ret_val = ret; return MP_VM_RETURN_YIELD; } else { // Emulate raise StopIteration() // Special case, handled in vm.c - *ret_val = MP_OBJ_NULL; return MP_VM_RETURN_NORMAL; } } diff --git a/py/vm.c b/py/vm.c index 416de6b1a..2a8e3c990 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1178,8 +1178,7 @@ yield: } else if (ret_kind == MP_VM_RETURN_NORMAL) { // Pop exhausted gen sp--; - // TODO: When ret_value can be MP_OBJ_NULL here?? - if (ret_value == MP_OBJ_NULL || ret_value == MP_OBJ_STOP_ITERATION) { + if (ret_value == MP_OBJ_STOP_ITERATION) { // Optimize StopIteration // TODO: get StopIteration's value PUSH(mp_const_none); -- cgit v1.2.3 From bc12eca461a317df842ce2e616afa97670cd0ce3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 1 Mar 2018 15:47:17 +1100 Subject: py/formatfloat: Fix rounding of %f format with edge-case FP values. Prior to this patch the %f formatting of some FP values could be off by up to 1, eg '%.0f' % 123 would return "122" (unix x64). Depending on the FP precision (single vs double) certain numbers would format correctly, but others wolud not. This patch should fix all cases of rounding for %f. --- py/formatfloat.c | 2 +- tests/float/float_format.py | 11 +++++++++++ tests/unix/extra_coverage.py.exp | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tests/float/float_format.py (limited to 'py') diff --git a/py/formatfloat.c b/py/formatfloat.c index 4228f99ff..22dd8aaac 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -341,7 +341,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch // Round // If we print non-exponential format (i.e. 'f'), but a digit we're going // to round by (e) is too far away, then there's nothing to round. - if ((org_fmt != 'f' || e <= 1) && f >= FPCONST(5.0)) { + if ((org_fmt != 'f' || e <= num_digits) && f >= FPCONST(5.0)) { char *rs = s; rs--; while (1) { diff --git a/tests/float/float_format.py b/tests/float/float_format.py new file mode 100644 index 000000000..4d5ad1d69 --- /dev/null +++ b/tests/float/float_format.py @@ -0,0 +1,11 @@ +# test float formatting + +# general rounding +for val in (116, 1111, 1234, 5010, 11111): + print('%.0f' % val) + print('%.1f' % val) + print('%.3f' % val) + +# make sure rounding is done at the correct precision +for prec in range(8): + print(('%%.%df' % prec) % 6e-5) diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index a030155ba..54e19d14a 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -56,7 +56,7 @@ Warning: test +1e+00 +1e+00 # binary -122 +123 456 # VM 2 1 -- cgit v1.2.3 From 7b050fa76c6a763043739d40c82dde839d7f8fd9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 1 Mar 2018 16:02:59 +1100 Subject: py/formatfloat: Fix case where floats could render with a ":" character. Prior to this patch, some architectures (eg unix x86) could render floats with a ":" character in them, eg 1e+39 would come out as ":e+38" (":" is just after "9" in ASCII so this is like 10e+38). This patch fixes some of these cases. --- py/formatfloat.c | 2 +- tests/float/float_format.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/formatfloat.c b/py/formatfloat.c index 22dd8aaac..60dcee6f5 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -258,7 +258,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch } // It can be that f was right on the edge of an entry in pos_pow needs to be reduced - if (f >= FPCONST(10.0)) { + if ((int)f >= 10) { e += 1; f *= FPCONST(0.1); } diff --git a/tests/float/float_format.py b/tests/float/float_format.py index 4d5ad1d69..cda395ce0 100644 --- a/tests/float/float_format.py +++ b/tests/float/float_format.py @@ -9,3 +9,7 @@ for val in (116, 1111, 1234, 5010, 11111): # make sure rounding is done at the correct precision for prec in range(8): print(('%%.%df' % prec) % 6e-5) + +# check certain cases that had a digit value of 10 render as a ":" character +print('%.2e' % float('9' * 51 + 'e-39')) +print('%.2e' % float('9' * 40 + 'e-21')) -- cgit v1.2.3 From 955ee6477f4b1d3a70bfe97a1e7727848bf2d06d Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 1 Mar 2018 17:00:02 +1100 Subject: py/formatfloat: Fix case where floats could render with negative digits. Prior to this patch, some architectures (eg unix x86) could render floats with "negative" digits, like ")". For example, '%.23e' % 1e-80 would come out as "1.0000000000000000/)/(,*0e-80". This patch fixes the known cases. --- py/formatfloat.c | 6 +++++- tests/float/float_format.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/formatfloat.c b/py/formatfloat.c index 60dcee6f5..dc7fc1d1f 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -330,7 +330,11 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch // Print the digits of the mantissa for (int i = 0; i < num_digits; ++i, --dec) { int32_t d = (int32_t)f; - *s++ = '0' + d; + if (d < 0) { + *s++ = '0'; + } else { + *s++ = '0' + d; + } if (dec == 0 && prec > 0) { *s++ = '.'; } diff --git a/tests/float/float_format.py b/tests/float/float_format.py index cda395ce0..d43535cf2 100644 --- a/tests/float/float_format.py +++ b/tests/float/float_format.py @@ -13,3 +13,7 @@ for prec in range(8): # check certain cases that had a digit value of 10 render as a ":" character print('%.2e' % float('9' * 51 + 'e-39')) print('%.2e' % float('9' * 40 + 'e-21')) + +# check a case that would render negative digit values, eg ")" characters +# the string is converted back to a float to check for no illegal characters +float('%.23e' % 1e-80) -- cgit v1.2.3 From 9884a2c712988de8c69a03a5d5be8add043f141e Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 2 Mar 2018 11:01:24 +1100 Subject: py/objint: Remove unreachable code checking for int type in format func. All callers of mp_obj_int_formatted() are expected to pass in a valid int object, and they do: - mp_obj_int_print() should always pass through an int object because it is the print special method for int instances. - mp_print_mp_int() checks that the argument is an int, and if not converts it to a small int. This patch saves around 20-50 bytes of code space. --- py/objint.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'py') diff --git a/py/objint.c b/py/objint.c index 59c58f2a6..270e16969 100644 --- a/py/objint.c +++ b/py/objint.c @@ -222,27 +222,26 @@ size_t mp_int_format_size(size_t num_bits, int base, const char *prefix, char co char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_const_obj_t self_in, int base, const char *prefix, char base_char, char comma) { fmt_int_t num; + #if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_NONE + // Only have small ints; get the integer value to format. + num = MP_OBJ_SMALL_INT_VALUE(self_in); + #else if (MP_OBJ_IS_SMALL_INT(self_in)) { // A small int; get the integer value to format. num = MP_OBJ_SMALL_INT_VALUE(self_in); -#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE - } else if (MP_OBJ_IS_TYPE(self_in, &mp_type_int)) { + } else { + assert(MP_OBJ_IS_TYPE(self_in, &mp_type_int)); // Not a small int. -#if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_LONGLONG + #if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_LONGLONG const mp_obj_int_t *self = self_in; // Get the value to format; mp_obj_get_int truncates to mp_int_t. num = self->val; -#else + #else // Delegate to the implementation for the long int. return mp_obj_int_formatted_impl(buf, buf_size, fmt_size, self_in, base, prefix, base_char, comma); -#endif -#endif - } else { - // Not an int. - **buf = '\0'; - *fmt_size = 0; - return *buf; + #endif } + #endif char sign = '\0'; if (num < 0) { -- cgit v1.2.3 From d4b55eff44da38e51616124883508c0d6b6678d3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 13 Mar 2018 13:23:30 +1100 Subject: py/misc.h: Remove unused count_lead_ones() inline function. This function was never used for unicode/utf8 handling code, or anything else, so remove it to keep things clean. --- py/misc.h | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'py') diff --git a/py/misc.h b/py/misc.h index a14bef7fe..72560da1e 100644 --- a/py/misc.h +++ b/py/misc.h @@ -204,20 +204,6 @@ int DEBUG_printf(const char *fmt, ...); extern mp_uint_t mp_verbose_flag; -// This is useful for unicode handling. Some CPU archs has -// special instructions for efficient implementation of this -// function (e.g. CLZ on ARM). -// NOTE: this function is unused at the moment -#ifndef count_lead_ones -static inline mp_uint_t count_lead_ones(byte val) { - mp_uint_t c = 0; - for (byte mask = 0x80; val & mask; mask >>= 1) { - c++; - } - return c; -} -#endif - /** float internals *************/ #if MICROPY_PY_BUILTINS_FLOAT -- cgit v1.2.3 From 9f811e9096819230905a5eb47812934531005403 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 13 Mar 2018 14:01:55 +1100 Subject: py/obj.h: Clean up by removing commented-out inline versions of macros. --- py/obj.h | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) (limited to 'py') diff --git a/py/obj.h b/py/obj.h index 32b430154..717e33bbc 100644 --- a/py/obj.h +++ b/py/obj.h @@ -250,6 +250,8 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; // The macros below are derived from the ones above and are used to // check for more specific object types. +// Note: these are kept as macros because inline functions sometimes use much +// more code space than the equivalent macros, depending on the compiler. #define MP_OBJ_IS_TYPE(o, t) (MP_OBJ_IS_OBJ(o) && (((mp_obj_base_t*)MP_OBJ_TO_PTR(o))->type == (t))) // this does not work for checking int, str or fun; use below macros for that #define MP_OBJ_IS_INT(o) (MP_OBJ_IS_SMALL_INT(o) || MP_OBJ_IS_TYPE(o, &mp_type_int)) @@ -257,17 +259,6 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; #define MP_OBJ_IS_STR_OR_BYTES(o) (MP_OBJ_IS_QSTR(o) || (MP_OBJ_IS_OBJ(o) && ((mp_obj_base_t*)MP_OBJ_TO_PTR(o))->type->binary_op == mp_obj_str_binary_op)) #define MP_OBJ_IS_FUN(o) (MP_OBJ_IS_OBJ(o) && (((mp_obj_base_t*)MP_OBJ_TO_PTR(o))->type->name == MP_QSTR_function)) -// Note: inline functions sometimes use much more code space than the -// equivalent macros, depending on the compiler. -//static inline bool MP_OBJ_IS_TYPE(mp_const_obj_t o, const mp_obj_type_t *t) { return (MP_OBJ_IS_OBJ(o) && (((mp_obj_base_t*)(o))->type == (t))); } // this does not work for checking a string, use below macro for that -//static inline bool MP_OBJ_IS_INT(mp_const_obj_t o) { return (MP_OBJ_IS_SMALL_INT(o) || MP_OBJ_IS_TYPE(o, &mp_type_int)); } // returns true if o is a small int or long int -// Need to forward declare these for the inline function to compile. -extern const mp_obj_type_t mp_type_int; -extern const mp_obj_type_t mp_type_bool; -static inline bool mp_obj_is_integer(mp_const_obj_t o) { return MP_OBJ_IS_INT(o) || MP_OBJ_IS_TYPE(o, &mp_type_bool); } // returns true if o is bool, small int or long int -//static inline bool MP_OBJ_IS_STR(mp_const_obj_t o) { return (MP_OBJ_IS_QSTR(o) || MP_OBJ_IS_TYPE(o, &mp_type_str)); } - - // These macros are used to declare and define constant function objects // You can put "static" in front of the definitions to make them local @@ -681,6 +672,7 @@ bool mp_obj_is_true(mp_obj_t arg); bool mp_obj_is_callable(mp_obj_t o_in); bool mp_obj_equal(mp_obj_t o1, mp_obj_t o2); +static inline bool mp_obj_is_integer(mp_const_obj_t o) { return MP_OBJ_IS_INT(o) || MP_OBJ_IS_TYPE(o, &mp_type_bool); } // returns true if o is bool, small int or long int mp_int_t mp_obj_get_int(mp_const_obj_t arg); mp_int_t mp_obj_get_int_truncated(mp_const_obj_t arg); bool mp_obj_get_int_maybe(mp_const_obj_t arg, mp_int_t *value); -- cgit v1.2.3 From e0bc438e4badb8f2d356479a3bee2b9c45fd69ca Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 13 Mar 2018 14:03:15 +1100 Subject: py/obj.h: Move declaration of mp_obj_list_init to objlist.h. If this function is used then objlist.h is already included to get the definition of mp_obj_list_t. --- py/obj.h | 2 -- py/objlist.h | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/obj.h b/py/obj.h index 717e33bbc..2779b3f86 100644 --- a/py/obj.h +++ b/py/obj.h @@ -744,8 +744,6 @@ void mp_obj_tuple_del(mp_obj_t self_in); mp_int_t mp_obj_tuple_hash(mp_obj_t self_in); // list -struct _mp_obj_list_t; -void mp_obj_list_init(struct _mp_obj_list_t *o, size_t n); mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg); mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value); void mp_obj_list_get(mp_obj_t self_in, size_t *len, mp_obj_t **items); diff --git a/py/objlist.h b/py/objlist.h index 28b5495a9..a43663db7 100644 --- a/py/objlist.h +++ b/py/objlist.h @@ -35,4 +35,6 @@ typedef struct _mp_obj_list_t { mp_obj_t *items; } mp_obj_list_t; +void mp_obj_list_init(mp_obj_list_t *o, size_t n); + #endif // MICROPY_INCLUDED_PY_OBJLIST_H -- cgit v1.2.3 From f6a1f18603de5c4d2321bcf4f967df298850e3f6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 16 Mar 2018 23:54:06 +1100 Subject: py/makeqstrdefs.py: Optimise by using compiled re's so it runs faster. By using pre-compiled regexs, using startswith(), and explicitly checking for empty lines (of which around 30% of the input lines are), automatic qstr extraction is speed up by about 10%. --- py/makeqstrdefs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index 525dec197..176440136 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -24,12 +24,16 @@ def write_out(fname, output): f.write("\n".join(output) + "\n") def process_file(f): + re_line = re.compile(r"#[line]*\s\d+\s\"([^\"]+)\"") + re_qstr = re.compile(r'MP_QSTR_[_a-zA-Z0-9]+') output = [] last_fname = None for line in f: + if line.isspace(): + continue # match gcc-like output (# n "file") and msvc-like output (#line n "file") - if line and (line[0:2] == "# " or line[0:5] == "#line"): - m = re.match(r"#[line]*\s\d+\s\"([^\"]+)\"", line) + if line.startswith(('# ', '#line')): + m = re_line.match(line) assert m is not None fname = m.group(1) if not fname.endswith(".c"): @@ -39,7 +43,7 @@ def process_file(f): output = [] last_fname = fname continue - for match in re.findall(r'MP_QSTR_[_a-zA-Z0-9]+', line): + for match in re_qstr.findall(line): name = match.replace('MP_QSTR_', '') if name not in QSTRING_BLACK_LIST: output.append('Q(' + name + ')') -- cgit v1.2.3 From 5edce4539b239eab9b045bb2fde18456f6fdbfe4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 17 Mar 2018 00:31:40 +1100 Subject: py/objexcept: Make MP_DEFINE_EXCEPTION public so ports can define excs. --- py/objexcept.c | 16 +++------------- py/objexcept.h | 13 +++++++++++++ 2 files changed, 16 insertions(+), 13 deletions(-) (limited to 'py') diff --git a/py/objexcept.c b/py/objexcept.c index ccb0bad0d..1e746bc81 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -93,7 +93,7 @@ mp_obj_t mp_alloc_emergency_exception_buf(mp_obj_t size_in) { // definition module-private so far, have it here. const mp_obj_exception_t mp_const_GeneratorExit_obj = {{&mp_type_GeneratorExit}, 0, 0, NULL, (mp_obj_tuple_t*)&mp_const_empty_tuple_obj}; -STATIC void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { +void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { mp_obj_exception_t *o = MP_OBJ_TO_PTR(o_in); mp_print_kind_t k = kind & ~PRINT_EXC_SUBCLASS; bool is_subclass = kind & PRINT_EXC_SUBCLASS; @@ -186,7 +186,7 @@ mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in) { } } -STATIC void exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { +void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_exception_t *self = MP_OBJ_TO_PTR(self_in); if (dest[0] != MP_OBJ_NULL) { // store/delete attribute @@ -214,17 +214,7 @@ const mp_obj_type_t mp_type_BaseException = { .name = MP_QSTR_BaseException, .print = mp_obj_exception_print, .make_new = mp_obj_exception_make_new, - .attr = exception_attr, -}; - -#define MP_DEFINE_EXCEPTION(exc_name, base_name) \ -const mp_obj_type_t mp_type_ ## exc_name = { \ - { &mp_type_type }, \ - .name = MP_QSTR_ ## exc_name, \ - .print = mp_obj_exception_print, \ - .make_new = mp_obj_exception_make_new, \ - .attr = exception_attr, \ - .parent = &mp_type_ ## base_name, \ + .attr = mp_obj_exception_attr, }; // List of all exceptions, arranged as in the table at: diff --git a/py/objexcept.h b/py/objexcept.h index f67651a7e..7c3076224 100644 --- a/py/objexcept.h +++ b/py/objexcept.h @@ -37,4 +37,17 @@ typedef struct _mp_obj_exception_t { mp_obj_tuple_t *args; } mp_obj_exception_t; +void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind); +void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); + +#define MP_DEFINE_EXCEPTION(exc_name, base_name) \ +const mp_obj_type_t mp_type_ ## exc_name = { \ + { &mp_type_type }, \ + .name = MP_QSTR_ ## exc_name, \ + .print = mp_obj_exception_print, \ + .make_new = mp_obj_exception_make_new, \ + .attr = mp_obj_exception_attr, \ + .parent = &mp_type_ ## base_name, \ +}; + #endif // MICROPY_INCLUDED_PY_OBJEXCEPT_H -- cgit v1.2.3 From 32807881954f106b9735de74fe984062a0815b81 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 30 Mar 2018 11:09:00 +1100 Subject: py/runtime: Check that keys in dicts passed as ** args are strings. Prior to this patch the code would crash if a key in a ** dict was anything other than a str or qstr. This is because mp_setup_code_state() assumes that keys in kwargs are qstrs (for efficiency). Thanks to @jepler for finding the bug. --- py/obj.h | 1 + py/objstr.c | 6 ++++++ py/runtime.c | 8 ++++---- tests/basics/fun_calldblstar.py | 5 +++++ 4 files changed, 16 insertions(+), 4 deletions(-) (limited to 'py') diff --git a/py/obj.h b/py/obj.h index 2779b3f86..80599966e 100644 --- a/py/obj.h +++ b/py/obj.h @@ -720,6 +720,7 @@ qstr mp_obj_str_get_qstr(mp_obj_t self_in); // use this if you will anyway conve const char *mp_obj_str_get_str(mp_obj_t self_in); // use this only if you need the string to be null terminated const char *mp_obj_str_get_data(mp_obj_t self_in, size_t *len); mp_obj_t mp_obj_str_intern(mp_obj_t str); +mp_obj_t mp_obj_str_intern_checked(mp_obj_t obj); void mp_str_print_quoted(const mp_print_t *print, const byte *str_data, size_t str_len, bool is_bytes); #if MICROPY_PY_BUILTINS_FLOAT diff --git a/py/objstr.c b/py/objstr.c index c42f38e75..0b11533f8 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -2062,6 +2062,12 @@ mp_obj_t mp_obj_str_intern(mp_obj_t str) { return mp_obj_new_str_via_qstr((const char*)data, len); } +mp_obj_t mp_obj_str_intern_checked(mp_obj_t obj) { + size_t len; + const char *data = mp_obj_str_get_data(obj, &len); + return mp_obj_new_str_via_qstr((const char*)data, len); +} + mp_obj_t mp_obj_new_bytes(const byte* data, size_t len) { return mp_obj_new_str_copy(&mp_type_bytes, data, len); } diff --git a/py/runtime.c b/py/runtime.c index 54ec0d70b..ca68fe982 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -748,8 +748,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ if (MP_MAP_SLOT_IS_FILLED(map, i)) { // the key must be a qstr, so intern it if it's a string mp_obj_t key = map->table[i].key; - if (MP_OBJ_IS_TYPE(key, &mp_type_str)) { - key = mp_obj_str_intern(key); + if (!MP_OBJ_IS_QSTR(key)) { + key = mp_obj_str_intern_checked(key); } args2[args2_len++] = key; args2[args2_len++] = map->table[i].value; @@ -778,8 +778,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ } // the key must be a qstr, so intern it if it's a string - if (MP_OBJ_IS_TYPE(key, &mp_type_str)) { - key = mp_obj_str_intern(key); + if (!MP_OBJ_IS_QSTR(key)) { + key = mp_obj_str_intern_checked(key); } // get the value corresponding to the key diff --git a/tests/basics/fun_calldblstar.py b/tests/basics/fun_calldblstar.py index aae9828cf..4a503698f 100644 --- a/tests/basics/fun_calldblstar.py +++ b/tests/basics/fun_calldblstar.py @@ -6,6 +6,11 @@ def f(a, b): f(1, **{'b':2}) f(1, **{'b':val for val in range(1)}) +try: + f(1, **{len:2}) +except TypeError: + print('TypeError') + # test calling a method with keywords given by **dict class A: -- cgit v1.2.3 From f50b64cab58025e080f994147b75a8ffc55d2b35 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 30 Mar 2018 12:37:04 +1100 Subject: py/runtime: Be sure that non-intercepted thrown object is an exception. The VM expects that, if mp_resume() returns MP_VM_RETURN_EXCEPTION, then the returned value is an exception instance (eg to add a traceback to it). It's possible that a value passed to a generator's throw() is not an exception so must be explicitly checked for if the thrown value is not intercepted by the generator. Thanks to @jepler for finding the bug. --- py/runtime.c | 2 +- tests/basics/gen_yield_from_throw3.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/runtime.c b/py/runtime.c index ca68fe982..219ec22de 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1282,7 +1282,7 @@ mp_vm_return_kind_t mp_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t th // will be propagated up. This behavior is approved by test_pep380.py // test_delegation_of_close_to_non_generator(), // test_delegating_throw_to_non_generator() - *ret_val = throw_value; + *ret_val = mp_make_raise_obj(throw_value); return MP_VM_RETURN_EXCEPTION; } } diff --git a/tests/basics/gen_yield_from_throw3.py b/tests/basics/gen_yield_from_throw3.py index 0f6c7c842..85b6f71f9 100644 --- a/tests/basics/gen_yield_from_throw3.py +++ b/tests/basics/gen_yield_from_throw3.py @@ -28,3 +28,30 @@ print(g.throw(123)) g = gen() print(next(g)) print(g.throw(ZeroDivisionError)) + +# this user-defined generator doesn't have a throw() method +class Iter2: + def __iter__(self): + return self + + def __next__(self): + return 1 + +def gen2(): + yield from Iter2() + +# the thrown ValueError is not intercepted by the user class +g = gen2() +print(next(g)) +try: + g.throw(ValueError) +except: + print('ValueError') + +# the thrown 123 is not an exception so raises a TypeError +g = gen2() +print(next(g)) +try: + g.throw(123) +except TypeError: + print('TypeError') -- cgit v1.2.3 From c7f880eda38783817bd490b8f2f0b60412c3b73c Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 4 Apr 2018 00:46:31 +1000 Subject: py/vm: Don't do unnecessary updates of ip and sp variables. Neither the ip nor sp variables are used again after the execution of the RAISE_VARARGS opcode, so they don't need to be updated. --- py/vm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/vm.c b/py/vm.c index 2a8e3c990..7b3a0b324 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1118,7 +1118,7 @@ unwind_return: ENTRY(MP_BC_RAISE_VARARGS): { MARK_EXC_IP_SELECTIVE(); - mp_uint_t unum = *ip++; + mp_uint_t unum = *ip; mp_obj_t obj; if (unum == 2) { mp_warning("exception chaining not supported"); @@ -1139,7 +1139,7 @@ unwind_return: RAISE(obj); } } else { - obj = POP(); + obj = TOP(); } obj = mp_make_raise_obj(obj); RAISE(obj); -- cgit v1.2.3 From bc36521386a2c608be06fa8a5bf110b050dd1052 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 4 Apr 2018 00:51:10 +1000 Subject: py/vm: Optimise handling of stackless mode when pystack is enabled. When pystack is enabled mp_obj_fun_bc_prepare_codestate() will always return a valid pointer, and if there is no more pystack available then it will raise an exception (a RuntimeError). So having pystack enabled with stackless enabled automatically gives strict stackless mode. There is therefore no need to have code for strict stackless mode when pystack is enabled, and this patch optimises the VM for such a case. --- py/vm.c | 78 ++++++++++++++++++++++++++++++++++------------------------------- 1 file changed, 41 insertions(+), 37 deletions(-) (limited to 'py') diff --git a/py/vm.c b/py/vm.c index 7b3a0b324..8abe65e43 100644 --- a/py/vm.c +++ b/py/vm.c @@ -911,21 +911,22 @@ unwind_jump:; code_state->sp = sp; code_state->exc_sp = MP_TAGPTR_MAKE(exc_sp, currently_in_except_block); mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(*sp, unum & 0xff, (unum >> 8) & 0xff, sp + 1); - if (new_state) { + #if !MICROPY_ENABLE_PYSTACK + if (new_state == NULL) { + // Couldn't allocate codestate on heap: in the strict case raise + // an exception, otherwise just fall through to stack allocation. + #if MICROPY_STACKLESS_STRICT + deep_recursion_error: + mp_raise_recursion_depth(); + #endif + } else + #endif + { new_state->prev = code_state; code_state = new_state; nlr_pop(); goto run_code_state; } - #if MICROPY_STACKLESS_STRICT - else { - deep_recursion_error: - mp_raise_recursion_depth(); - } - #else - // If we couldn't allocate codestate on heap, in - // non non-strict case fall thru to stack allocation. - #endif } #endif SET_TOP(mp_call_function_n_kw(*sp, unum & 0xff, (unum >> 8) & 0xff, sp + 1)); @@ -956,20 +957,21 @@ unwind_jump:; // pystack is not enabled. For pystack, they are freed when code_state is. mp_nonlocal_free(out_args.args, out_args.n_alloc * sizeof(mp_obj_t)); #endif - if (new_state) { + #if !MICROPY_ENABLE_PYSTACK + if (new_state == NULL) { + // Couldn't allocate codestate on heap: in the strict case raise + // an exception, otherwise just fall through to stack allocation. + #if MICROPY_STACKLESS_STRICT + goto deep_recursion_error; + #endif + } else + #endif + { new_state->prev = code_state; code_state = new_state; nlr_pop(); goto run_code_state; } - #if MICROPY_STACKLESS_STRICT - else { - goto deep_recursion_error; - } - #else - // If we couldn't allocate codestate on heap, in - // non non-strict case fall thru to stack allocation. - #endif } #endif SET_TOP(mp_call_method_n_kw_var(false, unum, sp)); @@ -993,20 +995,21 @@ unwind_jump:; int adjust = (sp[1] == MP_OBJ_NULL) ? 0 : 1; mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(*sp, n_args + adjust, n_kw, sp + 2 - adjust); - if (new_state) { + #if !MICROPY_ENABLE_PYSTACK + if (new_state == NULL) { + // Couldn't allocate codestate on heap: in the strict case raise + // an exception, otherwise just fall through to stack allocation. + #if MICROPY_STACKLESS_STRICT + goto deep_recursion_error; + #endif + } else + #endif + { new_state->prev = code_state; code_state = new_state; nlr_pop(); goto run_code_state; } - #if MICROPY_STACKLESS_STRICT - else { - goto deep_recursion_error; - } - #else - // If we couldn't allocate codestate on heap, in - // non non-strict case fall thru to stack allocation. - #endif } #endif SET_TOP(mp_call_method_n_kw(unum & 0xff, (unum >> 8) & 0xff, sp)); @@ -1037,20 +1040,21 @@ unwind_jump:; // pystack is not enabled. For pystack, they are freed when code_state is. mp_nonlocal_free(out_args.args, out_args.n_alloc * sizeof(mp_obj_t)); #endif - if (new_state) { + #if !MICROPY_ENABLE_PYSTACK + if (new_state == NULL) { + // Couldn't allocate codestate on heap: in the strict case raise + // an exception, otherwise just fall through to stack allocation. + #if MICROPY_STACKLESS_STRICT + goto deep_recursion_error; + #endif + } else + #endif + { new_state->prev = code_state; code_state = new_state; nlr_pop(); goto run_code_state; } - #if MICROPY_STACKLESS_STRICT - else { - goto deep_recursion_error; - } - #else - // If we couldn't allocate codestate on heap, in - // non non-strict case fall thru to stack allocation. - #endif } #endif SET_TOP(mp_call_method_n_kw_var(true, unum, sp)); -- cgit v1.2.3 From 323b5f7270ee0e3c9000c0fb0d5f74ec049337c2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 4 Apr 2018 14:23:25 +1000 Subject: py/modsys: Don't compile getsizeof function if feature is disabled. --- py/modsys.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/modsys.c b/py/modsys.c index 84a4eb0f4..609f8b454 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -140,10 +140,12 @@ STATIC mp_obj_t mp_sys_exc_info(void) { MP_DEFINE_CONST_FUN_OBJ_0(mp_sys_exc_info_obj, mp_sys_exc_info); #endif +#if MICROPY_PY_SYS_GETSIZEOF STATIC mp_obj_t mp_sys_getsizeof(mp_obj_t obj) { return mp_unary_op(MP_UNARY_OP_SIZEOF, obj); } -MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_getsizeof_obj, mp_sys_getsizeof); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_getsizeof_obj, mp_sys_getsizeof); +#endif STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_sys) }, -- cgit v1.2.3 From 3f420c0c27bd6daa5af39517925be55b9b9a9ab3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 4 Apr 2018 14:24:03 +1000 Subject: py: Don't include mp_optimise_value or opt_level() if compiler disabled. Without the compiler enabled the mp_optimise_value is unused, and the micropython.opt_level() function is not useful, so exclude these from the build to save RAM and code size. --- py/modmicropython.c | 4 ++++ py/mpstate.h | 2 ++ py/runtime.c | 2 ++ 3 files changed, 8 insertions(+) (limited to 'py') diff --git a/py/modmicropython.c b/py/modmicropython.c index 5cc7bdbe2..864d1a5c5 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -35,6 +35,7 @@ // Various builtins specific to MicroPython runtime, // living in micropython module +#if MICROPY_ENABLE_COMPILER STATIC mp_obj_t mp_micropython_opt_level(size_t n_args, const mp_obj_t *args) { if (n_args == 0) { return MP_OBJ_NEW_SMALL_INT(MP_STATE_VM(mp_optimise_value)); @@ -44,6 +45,7 @@ STATIC mp_obj_t mp_micropython_opt_level(size_t n_args, const mp_obj_t *args) { } } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_micropython_opt_level_obj, 0, 1, mp_micropython_opt_level); +#endif #if MICROPY_PY_MICROPYTHON_MEM_INFO @@ -158,7 +160,9 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(mp_micropython_schedule_obj, mp_micropython_sch STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_micropython) }, { MP_ROM_QSTR(MP_QSTR_const), MP_ROM_PTR(&mp_identity_obj) }, + #if MICROPY_ENABLE_COMPILER { MP_ROM_QSTR(MP_QSTR_opt_level), MP_ROM_PTR(&mp_micropython_opt_level_obj) }, + #endif #if MICROPY_PY_MICROPYTHON_MEM_INFO #if MICROPY_MEM_STATS { MP_ROM_QSTR(MP_QSTR_mem_total), MP_ROM_PTR(&mp_micropython_mem_total_obj) }, diff --git a/py/mpstate.h b/py/mpstate.h index 816698f4e..16c4bf6c5 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -199,7 +199,9 @@ typedef struct _mp_state_vm_t { mp_thread_mutex_t qstr_mutex; #endif + #if MICROPY_ENABLE_COMPILER mp_uint_t mp_optimise_value; + #endif // size of the emergency exception buf, if it's dynamically allocated #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0 diff --git a/py/runtime.c b/py/runtime.c index 219ec22de..4efb29bec 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -83,8 +83,10 @@ void mp_init(void) { MICROPY_PORT_INIT_FUNC; #endif + #if MICROPY_ENABLE_COMPILER // optimization disabled by default MP_STATE_VM(mp_optimise_value) = 0; + #endif // init global module dict mp_obj_dict_init(&MP_STATE_VM(mp_loaded_modules_dict), 3); -- cgit v1.2.3 From f1df86a0177c77769d73bc477570580b4f705acf Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 5 Apr 2018 01:11:26 +1000 Subject: py/objint: Simplify LHS arg type checking in int binary op functions. The LHS passed to mp_obj_int_binary_op() will always be an integer, either a small int or a big int, so the test for this type doesn't need to include an "other, unsupported type" case. --- py/objint_longlong.c | 5 ++--- py/objint_mpz.c | 6 ++---- 2 files changed, 4 insertions(+), 7 deletions(-) (limited to 'py') diff --git a/py/objint_longlong.c b/py/objint_longlong.c index 3e5ebadaf..cb8d1672d 100644 --- a/py/objint_longlong.c +++ b/py/objint_longlong.c @@ -124,10 +124,9 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i if (MP_OBJ_IS_SMALL_INT(lhs_in)) { lhs_val = MP_OBJ_SMALL_INT_VALUE(lhs_in); - } else if (MP_OBJ_IS_TYPE(lhs_in, &mp_type_int)) { - lhs_val = ((mp_obj_int_t*)lhs_in)->val; } else { - return MP_OBJ_NULL; // op not supported + assert(MP_OBJ_IS_TYPE(lhs_in, &mp_type_int)); + lhs_val = ((mp_obj_int_t*)lhs_in)->val; } if (MP_OBJ_IS_SMALL_INT(rhs_in)) { diff --git a/py/objint_mpz.c b/py/objint_mpz.c index 17e3ee6d2..0f05c84f4 100644 --- a/py/objint_mpz.c +++ b/py/objint_mpz.c @@ -170,11 +170,9 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i if (MP_OBJ_IS_SMALL_INT(lhs_in)) { mpz_init_fixed_from_int(&z_int, z_int_dig, MPZ_NUM_DIG_FOR_INT, MP_OBJ_SMALL_INT_VALUE(lhs_in)); zlhs = &z_int; - } else if (MP_OBJ_IS_TYPE(lhs_in, &mp_type_int)) { - zlhs = &((mp_obj_int_t*)MP_OBJ_TO_PTR(lhs_in))->mpz; } else { - // unsupported type - return MP_OBJ_NULL; + assert(MP_OBJ_IS_TYPE(lhs_in, &mp_type_int)); + zlhs = &((mp_obj_int_t*)MP_OBJ_TO_PTR(lhs_in))->mpz; } // if rhs is small int, then lhs was not (otherwise mp_binary_op handles it) -- cgit v1.2.3 From d6cf5c674952d5ae3463d9b580dac6559576deac Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 31 Mar 2018 21:27:56 -0500 Subject: py/objstr: In find/rfind, don't crash when end < start. --- py/objstr.c | 5 +++++ tests/basics/string_find.py | 1 + tests/basics/string_rfind.py | 1 + 3 files changed, 7 insertions(+) (limited to 'py') diff --git a/py/objstr.c b/py/objstr.c index 0b11533f8..da925234e 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -699,8 +699,13 @@ STATIC mp_obj_t str_finder(size_t n_args, const mp_obj_t *args, int direction, b end = str_index_to_ptr(self_type, haystack, haystack_len, args[3], true); } + if (end < start) { + goto out_error; + } + const byte *p = find_subbytes(start, end - start, needle, needle_len, direction); if (p == NULL) { + out_error: // not found if (is_index) { mp_raise_ValueError("substring not found"); diff --git a/tests/basics/string_find.py b/tests/basics/string_find.py index 4a206eb0e..f9fcad3e5 100644 --- a/tests/basics/string_find.py +++ b/tests/basics/string_find.py @@ -21,6 +21,7 @@ print("0000".find('-1', 3)) print("0000".find('1', 3)) print("0000".find('1', 4)) print("0000".find('1', 5)) +print("aaaaaaaaaaa".find("bbb", 9, 2)) try: 'abc'.find(1) diff --git a/tests/basics/string_rfind.py b/tests/basics/string_rfind.py index 4d0e84018..54269d6f5 100644 --- a/tests/basics/string_rfind.py +++ b/tests/basics/string_rfind.py @@ -21,3 +21,4 @@ print("0000".rfind('-1', 3)) print("0000".rfind('1', 3)) print("0000".rfind('1', 4)) print("0000".rfind('1', 5)) +print("aaaaaaaaaaa".rfind("bbb", 9, 2)) -- cgit v1.2.3 From cf31d384f1d2ca09f817f154892eaa6860c10144 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 7 Mar 2018 17:48:53 +1100 Subject: py/stream: Switch stream close operation from method to ioctl. This patch moves the implementation of stream closure from a dedicated method to the ioctl of the stream protocol, for each type that implements closing. The benefits of this are: 1. Rounds out the stream ioctl function, which already includes flush, seek and poll (among other things). 2. Makes calling mp_stream_close() on an object slightly more efficient because it now no longer needs to lookup the close method and call it, rather it just delegates straight to the ioctl function (if it exists). 3. Reduces code size and allows future types that implement the stream protocol to be smaller because they don't need a dedicated close method. Code size reduction is around 200 bytes smaller for x86 archs and around 30 bytes smaller for the bare-metal archs. --- extmod/modlwip.c | 72 ++++++++++++++++++++---------------------- extmod/modussl_axtls.c | 35 ++++++++++++-------- extmod/modussl_mbedtls.c | 37 +++++++++++++--------- extmod/modwebrepl.c | 21 ++++++++---- extmod/modwebsocket.c | 15 ++++----- extmod/vfs_fat_file.c | 30 ++++++++---------- ports/cc3200/mods/modusocket.c | 15 +++------ ports/esp32/modsocket.c | 27 +++++++--------- ports/stm32/modusocket.c | 22 ++++++------- ports/unix/file.c | 20 +++++------- ports/unix/modusocket.c | 34 ++++++++++++-------- ports/zephyr/modusocket.c | 33 ++++++++++++------- py/objstringio.c | 30 ++++++++---------- py/stream.c | 18 +++++++---- py/stream.h | 3 +- 15 files changed, 216 insertions(+), 196 deletions(-) (limited to 'py') diff --git a/extmod/modlwip.c b/extmod/modlwip.c index 2c194e1bd..9810089da 100644 --- a/extmod/modlwip.c +++ b/extmod/modlwip.c @@ -637,42 +637,6 @@ STATIC mp_obj_t lwip_socket_make_new(const mp_obj_type_t *type, size_t n_args, s return socket; } -STATIC mp_obj_t lwip_socket_close(mp_obj_t self_in) { - lwip_socket_obj_t *socket = self_in; - bool socket_is_listener = false; - - if (socket->pcb.tcp == NULL) { - return mp_const_none; - } - switch (socket->type) { - case MOD_NETWORK_SOCK_STREAM: { - if (socket->pcb.tcp->state == LISTEN) { - socket_is_listener = true; - } - if (tcp_close(socket->pcb.tcp) != ERR_OK) { - DEBUG_printf("lwip_close: had to call tcp_abort()\n"); - tcp_abort(socket->pcb.tcp); - } - break; - } - case MOD_NETWORK_SOCK_DGRAM: udp_remove(socket->pcb.udp); break; - //case MOD_NETWORK_SOCK_RAW: raw_remove(socket->pcb.raw); break; - } - socket->pcb.tcp = NULL; - socket->state = _ERR_BADF; - if (socket->incoming.pbuf != NULL) { - if (!socket_is_listener) { - pbuf_free(socket->incoming.pbuf); - } else { - tcp_abort(socket->incoming.connection); - } - socket->incoming.pbuf = NULL; - } - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(lwip_socket_close_obj, lwip_socket_close); - STATIC mp_obj_t lwip_socket_bind(mp_obj_t self_in, mp_obj_t addr_in) { lwip_socket_obj_t *socket = self_in; @@ -1179,6 +1143,38 @@ STATIC mp_uint_t lwip_socket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_ ret |= flags & (MP_STREAM_POLL_RD | MP_STREAM_POLL_WR); } + } else if (request == MP_STREAM_CLOSE) { + bool socket_is_listener = false; + + if (socket->pcb.tcp == NULL) { + return 0; + } + switch (socket->type) { + case MOD_NETWORK_SOCK_STREAM: { + if (socket->pcb.tcp->state == LISTEN) { + socket_is_listener = true; + } + if (tcp_close(socket->pcb.tcp) != ERR_OK) { + DEBUG_printf("lwip_close: had to call tcp_abort()\n"); + tcp_abort(socket->pcb.tcp); + } + break; + } + case MOD_NETWORK_SOCK_DGRAM: udp_remove(socket->pcb.udp); break; + //case MOD_NETWORK_SOCK_RAW: raw_remove(socket->pcb.raw); break; + } + socket->pcb.tcp = NULL; + socket->state = _ERR_BADF; + if (socket->incoming.pbuf != NULL) { + if (!socket_is_listener) { + pbuf_free(socket->incoming.pbuf); + } else { + tcp_abort(socket->incoming.connection); + } + socket->incoming.pbuf = NULL; + } + ret = 0; + } else { *errcode = MP_EINVAL; ret = MP_STREAM_ERROR; @@ -1188,8 +1184,8 @@ STATIC mp_uint_t lwip_socket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_ } STATIC const mp_rom_map_elem_t lwip_socket_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&lwip_socket_close_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&lwip_socket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_bind), MP_ROM_PTR(&lwip_socket_bind_obj) }, { MP_ROM_QSTR(MP_QSTR_listen), MP_ROM_PTR(&lwip_socket_listen_obj) }, { MP_ROM_QSTR(MP_QSTR_accept), MP_ROM_PTR(&lwip_socket_accept_obj) }, diff --git a/extmod/modussl_axtls.c b/extmod/modussl_axtls.c index 35e3106cd..689d33305 100644 --- a/extmod/modussl_axtls.c +++ b/extmod/modussl_axtls.c @@ -175,6 +175,25 @@ STATIC mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, in return r; } +STATIC mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_ssl_socket_t *self = MP_OBJ_TO_PTR(o_in); + (void)arg; + switch (request) { + case MP_STREAM_CLOSE: + if (self->ssl_sock != NULL) { + ssl_free(self->ssl_sock); + ssl_ctx_free(self->ssl_ctx); + self->ssl_sock = NULL; + mp_stream_close(self->sock); + } + return 0; + + default: + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } +} + STATIC mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { // Currently supports only blocking mode (void)self_in; @@ -185,26 +204,13 @@ STATIC mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_2(socket_setblocking_obj, socket_setblocking); -STATIC mp_obj_t socket_close(mp_obj_t self_in) { - mp_obj_ssl_socket_t *self = MP_OBJ_TO_PTR(self_in); - if (self->ssl_sock != NULL) { - ssl_free(self->ssl_sock); - ssl_ctx_free(self->ssl_ctx); - self->ssl_sock = NULL; - return mp_stream_close(self->sock); - } - - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(socket_close_obj, socket_close); - STATIC const mp_rom_map_elem_t ussl_socket_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_setblocking), MP_ROM_PTR(&socket_setblocking_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&socket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, #if MICROPY_PY_USSL_FINALISER { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&socket_close_obj) }, #endif @@ -215,6 +221,7 @@ STATIC MP_DEFINE_CONST_DICT(ussl_socket_locals_dict, ussl_socket_locals_dict_tab STATIC const mp_stream_p_t ussl_socket_stream_p = { .read = socket_read, .write = socket_write, + .ioctl = socket_ioctl, }; STATIC const mp_obj_type_t ussl_socket_type = { diff --git a/extmod/modussl_mbedtls.c b/extmod/modussl_mbedtls.c index 197a0651c..bd4b0c725 100644 --- a/extmod/modussl_mbedtls.c +++ b/extmod/modussl_mbedtls.c @@ -274,20 +274,26 @@ STATIC mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_2(socket_setblocking_obj, socket_setblocking); -STATIC mp_obj_t socket_close(mp_obj_t self_in) { - mp_obj_ssl_socket_t *self = MP_OBJ_TO_PTR(self_in); - - mbedtls_pk_free(&self->pkey); - mbedtls_x509_crt_free(&self->cert); - mbedtls_x509_crt_free(&self->cacert); - mbedtls_ssl_free(&self->ssl); - mbedtls_ssl_config_free(&self->conf); - mbedtls_ctr_drbg_free(&self->ctr_drbg); - mbedtls_entropy_free(&self->entropy); - - return mp_stream_close(self->sock); +STATIC mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_ssl_socket_t *self = MP_OBJ_TO_PTR(o_in); + (void)arg; + switch (request) { + case MP_STREAM_CLOSE: + mbedtls_pk_free(&self->pkey); + mbedtls_x509_crt_free(&self->cert); + mbedtls_x509_crt_free(&self->cacert); + mbedtls_ssl_free(&self->ssl); + mbedtls_ssl_config_free(&self->conf); + mbedtls_ctr_drbg_free(&self->ctr_drbg); + mbedtls_entropy_free(&self->entropy); + mp_stream_close(self->sock); + return 0; + + default: + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(socket_close_obj, socket_close); STATIC const mp_rom_map_elem_t ussl_socket_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, @@ -295,9 +301,9 @@ STATIC const mp_rom_map_elem_t ussl_socket_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_setblocking), MP_ROM_PTR(&socket_setblocking_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&socket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, #if MICROPY_PY_USSL_FINALISER - { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&socket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, #endif { MP_ROM_QSTR(MP_QSTR_getpeercert), MP_ROM_PTR(&mod_ssl_getpeercert_obj) }, }; @@ -307,6 +313,7 @@ STATIC MP_DEFINE_CONST_DICT(ussl_socket_locals_dict, ussl_socket_locals_dict_tab STATIC const mp_stream_p_t ussl_socket_stream_p = { .read = socket_read, .write = socket_write, + .ioctl = socket_ioctl, }; STATIC const mp_obj_type_t ussl_socket_type = { diff --git a/extmod/modwebrepl.c b/extmod/modwebrepl.c index 42b30f5ea..983b5a10c 100644 --- a/extmod/modwebrepl.c +++ b/extmod/modwebrepl.c @@ -297,12 +297,20 @@ STATIC mp_uint_t webrepl_write(mp_obj_t self_in, const void *buf, mp_uint_t size return stream_p->write(self->sock, buf, size, errcode); } -STATIC mp_obj_t webrepl_close(mp_obj_t self_in) { - mp_obj_webrepl_t *self = MP_OBJ_TO_PTR(self_in); - // TODO: This is a place to do cleanup - return mp_stream_close(self->sock); +STATIC mp_uint_t webrepl_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_webrepl_t *self = MP_OBJ_TO_PTR(o_in); + (void)arg; + switch (request) { + case MP_STREAM_CLOSE: + // TODO: This is a place to do cleanup + mp_stream_close(self->sock); + return 0; + + default: + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(webrepl_close_obj, webrepl_close); STATIC mp_obj_t webrepl_set_password(mp_obj_t passwd_in) { size_t len; @@ -319,13 +327,14 @@ STATIC const mp_rom_map_elem_t webrepl_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&webrepl_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, }; STATIC MP_DEFINE_CONST_DICT(webrepl_locals_dict, webrepl_locals_dict_table); STATIC const mp_stream_p_t webrepl_stream_p = { .read = webrepl_read, .write = webrepl_write, + .ioctl = webrepl_ioctl, }; STATIC const mp_obj_type_t webrepl_type = { diff --git a/extmod/modwebsocket.c b/extmod/modwebsocket.c index a651164b2..5a826ec64 100644 --- a/extmod/modwebsocket.c +++ b/extmod/modwebsocket.c @@ -256,6 +256,11 @@ STATIC mp_uint_t websocket_write(mp_obj_t self_in, const void *buf, mp_uint_t si STATIC mp_uint_t websocket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) { mp_obj_websocket_t *self = MP_OBJ_TO_PTR(self_in); switch (request) { + case MP_STREAM_CLOSE: + // TODO: Send close signaling to the other side, otherwise it's + // abrupt close (connection abort). + mp_stream_close(self->sock); + return 0; case MP_STREAM_GET_DATA_OPTS: return self->ws_flags & FRAME_OPCODE_MASK; case MP_STREAM_SET_DATA_OPTS: { @@ -269,21 +274,13 @@ STATIC mp_uint_t websocket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t } } -STATIC mp_obj_t websocket_close(mp_obj_t self_in) { - mp_obj_websocket_t *self = MP_OBJ_TO_PTR(self_in); - // TODO: Send close signaling to the other side, otherwise it's - // abrupt close (connection abort). - return mp_stream_close(self->sock); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(websocket_close_obj, websocket_close); - STATIC const mp_rom_map_elem_t websocket_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_ioctl), MP_ROM_PTR(&mp_stream_ioctl_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&websocket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, }; STATIC MP_DEFINE_CONST_DICT(websocket_locals_dict, websocket_locals_dict_table); diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index 23e5aa10f..cbd013f16 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -103,22 +103,9 @@ STATIC mp_uint_t file_obj_write(mp_obj_t self_in, const void *buf, mp_uint_t siz } -STATIC mp_obj_t file_obj_close(mp_obj_t self_in) { - pyb_file_obj_t *self = MP_OBJ_TO_PTR(self_in); - // if fs==NULL then the file is closed and in that case this method is a no-op - if (self->fp.obj.fs != NULL) { - FRESULT res = f_close(&self->fp); - if (res != FR_OK) { - mp_raise_OSError(fresult_to_errno_table[res]); - } - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(file_obj_close_obj, file_obj_close); - STATIC mp_obj_t file_obj___exit__(size_t n_args, const mp_obj_t *args) { (void)n_args; - return file_obj_close(args[0]); + return mp_stream_close(args[0]); } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(file_obj___exit___obj, 4, 4, file_obj___exit__); @@ -153,6 +140,17 @@ STATIC mp_uint_t file_obj_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, } return 0; + } else if (request == MP_STREAM_CLOSE) { + // if fs==NULL then the file is closed and in that case this method is a no-op + if (self->fp.obj.fs != NULL) { + FRESULT res = f_close(&self->fp); + if (res != FR_OK) { + *errcode = fresult_to_errno_table[res]; + return MP_STREAM_ERROR; + } + } + return 0; + } else { *errcode = MP_EINVAL; return MP_STREAM_ERROR; @@ -234,10 +232,10 @@ STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) }, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&file_obj_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, - { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&file_obj_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&file_obj___exit___obj) }, }; diff --git a/ports/cc3200/mods/modusocket.c b/ports/cc3200/mods/modusocket.c index f587e765a..286b1fb02 100644 --- a/ports/cc3200/mods/modusocket.c +++ b/ports/cc3200/mods/modusocket.c @@ -336,6 +336,9 @@ STATIC int wlan_socket_ioctl (mod_network_socket_obj_t *s, mp_uint_t request, mp if (SL_FD_ISSET(sd, &xfds)) { ret |= MP_STREAM_POLL_HUP; } + } else if (request == MP_STREAM_CLOSE) { + wlan_socket_close(s); + ret = 0; } else { *_errno = MP_EINVAL; ret = MP_STREAM_ERROR; @@ -466,14 +469,6 @@ STATIC mp_obj_t socket_make_new(const mp_obj_type_t *type, size_t n_args, size_t return s; } -// method socket.close() -STATIC mp_obj_t socket_close(mp_obj_t self_in) { - mod_network_socket_obj_t *self = self_in; - wlan_socket_close(self); - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(socket_close_obj, socket_close); - // method socket.bind(address) STATIC mp_obj_t socket_bind(mp_obj_t self_in, mp_obj_t addr_in) { mod_network_socket_obj_t *self = self_in; @@ -704,8 +699,8 @@ STATIC mp_obj_t socket_makefile(size_t n_args, const mp_obj_t *args) { STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(socket_makefile_obj, 1, 6, socket_makefile); STATIC const mp_rom_map_elem_t socket_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&socket_close_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&socket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_bind), MP_ROM_PTR(&socket_bind_obj) }, { MP_ROM_QSTR(MP_QSTR_listen), MP_ROM_PTR(&socket_listen_obj) }, { MP_ROM_QSTR(MP_QSTR_accept), MP_ROM_PTR(&socket_accept_obj) }, diff --git a/ports/esp32/modsocket.c b/ports/esp32/modsocket.c index 31d153964..8b3c28063 100644 --- a/ports/esp32/modsocket.c +++ b/ports/esp32/modsocket.c @@ -83,19 +83,6 @@ void check_for_exceptions() { } } -STATIC mp_obj_t socket_close(const mp_obj_t arg0) { - socket_obj_t *self = MP_OBJ_TO_PTR(arg0); - if (self->fd >= 0) { - int ret = lwip_close_r(self->fd); - if (ret != 0) { - exception_from_errno(errno); - } - self->fd = -1; - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(socket_close_obj, socket_close); - static int _socket_getaddrinfo2(const mp_obj_t host, const mp_obj_t portx, struct addrinfo **resp) { const struct addrinfo hints = { .ai_family = AF_INET, @@ -450,6 +437,16 @@ STATIC mp_uint_t socket_stream_ioctl(mp_obj_t self_in, mp_uint_t request, uintpt if (FD_ISSET(socket->fd, &wfds)) ret |= MP_STREAM_POLL_WR; if (FD_ISSET(socket->fd, &efds)) ret |= MP_STREAM_POLL_HUP; return ret; + } else if (request == MP_STREAM_CLOSE) { + if (socket->fd >= 0) { + int ret = lwip_close_r(socket->fd); + if (ret != 0) { + *errcode = errno; + return MP_STREAM_ERROR; + } + socket->fd = -1; + } + return 0; } *errcode = MP_EINVAL; @@ -457,8 +454,8 @@ STATIC mp_uint_t socket_stream_ioctl(mp_obj_t self_in, mp_uint_t request, uintpt } STATIC const mp_map_elem_t socket_locals_dict_table[] = { - { MP_OBJ_NEW_QSTR(MP_QSTR___del__), (mp_obj_t)&socket_close_obj }, - { MP_OBJ_NEW_QSTR(MP_QSTR_close), (mp_obj_t)&socket_close_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR___del__), (mp_obj_t)&mp_stream_close_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_close), (mp_obj_t)&mp_stream_close_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_bind), (mp_obj_t)&socket_bind_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_listen), (mp_obj_t)&socket_listen_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_accept), (mp_obj_t)&socket_accept_obj }, diff --git a/ports/stm32/modusocket.c b/ports/stm32/modusocket.c index 71a237b0d..0c663437e 100644 --- a/ports/stm32/modusocket.c +++ b/ports/stm32/modusocket.c @@ -30,6 +30,7 @@ #include "py/objtuple.h" #include "py/objlist.h" #include "py/runtime.h" +#include "py/stream.h" #include "py/mperrno.h" #include "lib/netutils/netutils.h" #include "modnetwork.h" @@ -79,16 +80,6 @@ STATIC void socket_select_nic(mod_network_socket_obj_t *self, const byte *ip) { } } } -// method socket.close() -STATIC mp_obj_t socket_close(mp_obj_t self_in) { - mod_network_socket_obj_t *self = self_in; - if (self->nic != MP_OBJ_NULL) { - self->nic_type->close(self); - self->nic = MP_OBJ_NULL; - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(socket_close_obj, socket_close); // method socket.bind(address) STATIC mp_obj_t socket_bind(mp_obj_t self_in, mp_obj_t addr_in) { @@ -347,8 +338,8 @@ STATIC mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t blocking) { STATIC MP_DEFINE_CONST_FUN_OBJ_2(socket_setblocking_obj, socket_setblocking); STATIC const mp_rom_map_elem_t socket_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&socket_close_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&socket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_bind), MP_ROM_PTR(&socket_bind_obj) }, { MP_ROM_QSTR(MP_QSTR_listen), MP_ROM_PTR(&socket_listen_obj) }, { MP_ROM_QSTR(MP_QSTR_accept), MP_ROM_PTR(&socket_accept_obj) }, @@ -366,6 +357,13 @@ STATIC MP_DEFINE_CONST_DICT(socket_locals_dict, socket_locals_dict_table); mp_uint_t socket_ioctl(mp_obj_t self_in, mp_uint_t request, mp_uint_t arg, int *errcode) { mod_network_socket_obj_t *self = self_in; + if (request == MP_STREAM_CLOSE) { + if (self->nic != MP_OBJ_NULL) { + self->nic_type->close(self); + self->nic = MP_OBJ_NULL; + } + return 0; + } return self->nic_type->ioctl(self, request, arg, errcode); } diff --git a/ports/unix/file.c b/ports/unix/file.c index 84e918082..9bb44c6ab 100644 --- a/ports/unix/file.c +++ b/ports/unix/file.c @@ -118,25 +118,21 @@ STATIC mp_uint_t fdfile_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i return MP_STREAM_ERROR; } return 0; + case MP_STREAM_CLOSE: + close(o->fd); + #ifdef MICROPY_CPYTHON_COMPAT + o->fd = -1; + #endif + return 0; default: *errcode = EINVAL; return MP_STREAM_ERROR; } } -STATIC mp_obj_t fdfile_close(mp_obj_t self_in) { - mp_obj_fdfile_t *self = MP_OBJ_TO_PTR(self_in); - close(self->fd); -#ifdef MICROPY_CPYTHON_COMPAT - self->fd = -1; -#endif - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(fdfile_close_obj, fdfile_close); - STATIC mp_obj_t fdfile___exit__(size_t n_args, const mp_obj_t *args) { (void)n_args; - return fdfile_close(args[0]); + return mp_stream_close(args[0]); } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(fdfile___exit___obj, 4, 4, fdfile___exit__); @@ -224,7 +220,7 @@ STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&fdfile_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&fdfile___exit___obj) }, }; diff --git a/ports/unix/modusocket.c b/ports/unix/modusocket.c index cfb6a9f5e..ba50e6165 100644 --- a/ports/unix/modusocket.c +++ b/ports/unix/modusocket.c @@ -108,19 +108,26 @@ STATIC mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, in return r; } -STATIC mp_obj_t socket_close(mp_obj_t self_in) { - mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); - // There's a POSIX drama regarding return value of close in general, - // and EINTR error in particular. See e.g. - // http://lwn.net/Articles/576478/ - // http://austingroupbugs.net/view.php?id=529 - // The rationale MicroPython follows is that close() just releases - // file descriptor. If you're interested to catch I/O errors before - // closing fd, fsync() it. - close(self->fd); - return mp_const_none; +STATIC mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_socket_t *self = MP_OBJ_TO_PTR(o_in); + (void)arg; + switch (request) { + case MP_STREAM_CLOSE: + // There's a POSIX drama regarding return value of close in general, + // and EINTR error in particular. See e.g. + // http://lwn.net/Articles/576478/ + // http://austingroupbugs.net/view.php?id=529 + // The rationale MicroPython follows is that close() just releases + // file descriptor. If you're interested to catch I/O errors before + // closing fd, fsync() it. + close(self->fd); + return 0; + + default: + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(socket_close_obj, socket_close); STATIC mp_obj_t socket_fileno(mp_obj_t self_in) { mp_obj_socket_t *self = MP_OBJ_TO_PTR(self_in); @@ -359,7 +366,7 @@ STATIC const mp_rom_map_elem_t usocket_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_sendto), MP_ROM_PTR(&socket_sendto_obj) }, { MP_ROM_QSTR(MP_QSTR_setsockopt), MP_ROM_PTR(&socket_setsockopt_obj) }, { MP_ROM_QSTR(MP_QSTR_setblocking), MP_ROM_PTR(&socket_setblocking_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&socket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, }; STATIC MP_DEFINE_CONST_DICT(usocket_locals_dict, usocket_locals_dict_table); @@ -367,6 +374,7 @@ STATIC MP_DEFINE_CONST_DICT(usocket_locals_dict, usocket_locals_dict_table); STATIC const mp_stream_p_t usocket_stream_p = { .read = socket_read, .write = socket_write, + .ioctl = socket_ioctl, }; const mp_obj_type_t mp_type_socket = { diff --git a/ports/zephyr/modusocket.c b/ports/zephyr/modusocket.c index 95414a49b..84d6fa729 100644 --- a/ports/zephyr/modusocket.c +++ b/ports/zephyr/modusocket.c @@ -299,20 +299,31 @@ STATIC mp_obj_t socket_makefile(size_t n_args, const mp_obj_t *args) { } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(socket_makefile_obj, 1, 3, socket_makefile); -STATIC mp_obj_t socket_close(mp_obj_t self_in) { - socket_obj_t *socket = self_in; - if (socket->ctx != -1) { - int res = zsock_close(socket->ctx); - RAISE_SOCK_ERRNO(res); - socket->ctx = -1; +STATIC mp_uint_t sock_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + socket_obj_t *socket = o_in; + (void)arg; + switch (request) { + case MP_STREAM_CLOSE: + if (socket->ctx != -1) { + int res = zsock_close(socket->ctx); + RAISE_SOCK_ERRNO(res); + if (res == -1) { + *errcode = errno; + return MP_STREAM_ERROR; + } + socket->ctx = -1; + } + return 0; + + default: + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; } - return mp_const_none; } -STATIC MP_DEFINE_CONST_FUN_OBJ_1(socket_close_obj, socket_close); STATIC const mp_rom_map_elem_t socket_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&socket_close_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&socket_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_bind), MP_ROM_PTR(&socket_bind_obj) }, { MP_ROM_QSTR(MP_QSTR_connect), MP_ROM_PTR(&socket_connect_obj) }, { MP_ROM_QSTR(MP_QSTR_listen), MP_ROM_PTR(&socket_listen_obj) }, @@ -332,7 +343,7 @@ STATIC MP_DEFINE_CONST_DICT(socket_locals_dict, socket_locals_dict_table); STATIC const mp_stream_p_t socket_stream_p = { .read = sock_read, .write = sock_write, - //.ioctl = sock_ioctl, + .ioctl = sock_ioctl, }; STATIC const mp_obj_type_t socket_type = { diff --git a/py/objstringio.c b/py/objstringio.c index 5c50aa317..b405ee21e 100644 --- a/py/objstringio.c +++ b/py/objstringio.c @@ -143,6 +143,17 @@ STATIC mp_uint_t stringio_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, } case MP_STREAM_FLUSH: return 0; + case MP_STREAM_CLOSE: + #if MICROPY_CPYTHON_COMPAT + vstr_free(o->vstr); + o->vstr = NULL; + #else + vstr_clear(o->vstr); + o->vstr->alloc = 0; + o->vstr->len = 0; + o->pos = 0; + #endif + return 0; default: *errcode = MP_EINVAL; return MP_STREAM_ERROR; @@ -159,24 +170,9 @@ STATIC mp_obj_t stringio_getvalue(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(stringio_getvalue_obj, stringio_getvalue); -STATIC mp_obj_t stringio_close(mp_obj_t self_in) { - mp_obj_stringio_t *self = MP_OBJ_TO_PTR(self_in); -#if MICROPY_CPYTHON_COMPAT - vstr_free(self->vstr); - self->vstr = NULL; -#else - vstr_clear(self->vstr); - self->vstr->alloc = 0; - self->vstr->len = 0; - self->pos = 0; -#endif - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(stringio_close_obj, stringio_close); - STATIC mp_obj_t stringio___exit__(size_t n_args, const mp_obj_t *args) { (void)n_args; - return stringio_close(args[0]); + return mp_stream_close(args[0]); } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(stringio___exit___obj, 4, 4, stringio___exit__); @@ -233,7 +229,7 @@ STATIC const mp_rom_map_elem_t stringio_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&stringio_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_getvalue), MP_ROM_PTR(&stringio_getvalue_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&stringio___exit___obj) }, diff --git a/py/stream.c b/py/stream.c index 453dee769..f51f634b6 100644 --- a/py/stream.c +++ b/py/stream.c @@ -105,13 +105,6 @@ const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags) { return stream_p; } -mp_obj_t mp_stream_close(mp_obj_t stream) { - // TODO: Still consider using ioctl for close - mp_obj_t dest[2]; - mp_load_method(stream, MP_QSTR_close, dest); - return mp_call_method_n_kw(0, 0, dest); -} - STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte flags) { const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ); @@ -434,6 +427,17 @@ mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self) { return MP_OBJ_STOP_ITERATION; } +mp_obj_t mp_stream_close(mp_obj_t stream) { + const mp_stream_p_t *stream_p = mp_get_stream_raise(stream, MP_STREAM_OP_IOCTL); + int error; + mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_CLOSE, 0, &error); + if (res == MP_STREAM_ERROR) { + mp_raise_OSError(error); + } + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_close_obj, mp_stream_close); + STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) { const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL); diff --git a/py/stream.h b/py/stream.h index fbe3d7d85..a7d8d08f1 100644 --- a/py/stream.h +++ b/py/stream.h @@ -35,7 +35,7 @@ #define MP_STREAM_FLUSH (1) #define MP_STREAM_SEEK (2) #define MP_STREAM_POLL (3) -//#define MP_STREAM_CLOSE (4) // Not yet implemented +#define MP_STREAM_CLOSE (4) #define MP_STREAM_TIMEOUT (5) // Get/set timeout (single op) #define MP_STREAM_GET_OPTS (6) // Get stream options #define MP_STREAM_SET_OPTS (7) // Set stream options @@ -69,6 +69,7 @@ MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj); MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_write1_obj); +MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_close_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_tell_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_flush_obj); -- cgit v1.2.3 From cbf981f3307661c205d27f3a418be3989ab47c5e Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 1 Apr 2018 12:15:35 -0500 Subject: py/objgenerator: Check stack before resuming a generator. This turns a hard crash in a recursive generator into a 'maximum recursion depth exceeded' exception. --- py/objgenerator.c | 2 ++ tests/run-tests | 1 + tests/stress/recursive_gen.py | 9 +++++++++ 3 files changed, 12 insertions(+) create mode 100644 tests/stress/recursive_gen.py (limited to 'py') diff --git a/py/objgenerator.c b/py/objgenerator.c index 8c1260b60..5fd13f831 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -32,6 +32,7 @@ #include "py/bc.h" #include "py/objgenerator.h" #include "py/objfun.h" +#include "py/stackctrl.h" /******************************************************************************/ /* generator wrapper */ @@ -92,6 +93,7 @@ STATIC void gen_instance_print(const mp_print_t *print, mp_obj_t self_in, mp_pri } mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) { + MP_STACK_CHECK(); mp_check_self(MP_OBJ_IS_TYPE(self_in, &mp_type_gen_instance)); mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in); if (self->code_state.ip == 0) { diff --git a/tests/run-tests b/tests/run-tests index ef2f4bc6a..42037831d 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -367,6 +367,7 @@ def run_tests(pyb, tests, args, base_path="."): skip_tests.add('micropython/heapalloc_iter.py') # requires generators skip_tests.add('micropython/schedule.py') # native code doesn't check pending events skip_tests.add('stress/gc_trace.py') # requires yield + skip_tests.add('stress/recursive_gen.py') # requires yield for test_file in tests: test_file = test_file.replace('\\', '/') diff --git a/tests/stress/recursive_gen.py b/tests/stress/recursive_gen.py new file mode 100644 index 000000000..65f5d8d47 --- /dev/null +++ b/tests/stress/recursive_gen.py @@ -0,0 +1,9 @@ +# test deeply recursive generators + +def gen(): + yield from gen() + +try: + list(gen()) +except RuntimeError: + print('RuntimeError') -- cgit v1.2.3 From ef12a4bd05cdd531d9c130763ffb2470e9c0fb54 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 10 Apr 2018 15:06:47 +1000 Subject: py: Refactor how native emitter code is compiled with a file per arch. Instead of emitnative.c having configuration code for each supported architecture, and then compiling this file multiple times with different macros defined, this patch adds a file per architecture with the necessary code to configure the native emitter. These files then #include the emitnative.c file. This simplifies emitnative.c (which is already very large), and simplifies the build system because emitnative.c no longer needs special handling for compilation and qstr extraction. --- py/asmthumb.h | 1 + py/emitnarm.c | 15 +++++++++ py/emitnative.c | 97 +------------------------------------------------------- py/emitnthumb.c | 15 +++++++++ py/emitnx64.c | 15 +++++++++ py/emitnx86.c | 67 ++++++++++++++++++++++++++++++++++++++ py/emitnxtensa.c | 15 +++++++++ py/mkrules.mk | 5 +-- py/py.mk | 26 ++------------- 9 files changed, 132 insertions(+), 124 deletions(-) create mode 100644 py/emitnarm.c create mode 100644 py/emitnthumb.c create mode 100644 py/emitnx64.c create mode 100644 py/emitnx86.c create mode 100644 py/emitnxtensa.c (limited to 'py') diff --git a/py/asmthumb.h b/py/asmthumb.h index 552ad75fc..8a7df5d50 100644 --- a/py/asmthumb.h +++ b/py/asmthumb.h @@ -26,6 +26,7 @@ #ifndef MICROPY_INCLUDED_PY_ASMTHUMB_H #define MICROPY_INCLUDED_PY_ASMTHUMB_H +#include #include "py/misc.h" #include "py/asmbase.h" diff --git a/py/emitnarm.c b/py/emitnarm.c new file mode 100644 index 000000000..1b585f821 --- /dev/null +++ b/py/emitnarm.c @@ -0,0 +1,15 @@ +// ARM specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_ARM + +// This is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmarm.h" + +#define N_ARM (1) +#define EXPORT_FUN(name) emit_native_arm_##name +#include "py/emitnative.c" + +#endif diff --git a/py/emitnative.c b/py/emitnative.c index 964db9552..7e017ba39 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -57,14 +57,7 @@ #endif // wrapper around everything in this file -#if (MICROPY_EMIT_X64 && N_X64) \ - || (MICROPY_EMIT_X86 && N_X86) \ - || (MICROPY_EMIT_THUMB && N_THUMB) \ - || (MICROPY_EMIT_ARM && N_ARM) \ - || (MICROPY_EMIT_XTENSA && N_XTENSA) \ - -// this is defined so that the assembler exports generic assembler API macros -#define GENERIC_ASM_API (1) +#if N_X64 || N_X86 || N_THUMB || N_ARM || N_XTENSA // define additional generic helper macros #define ASM_MOV_LOCAL_IMM_VIA(as, local_num, imm, reg_temp) \ @@ -73,94 +66,6 @@ ASM_MOV_LOCAL_REG((as), (local_num), (reg_temp)); \ } while (false) -#if N_X64 - -// x64 specific stuff -#include "py/asmx64.h" -#define EXPORT_FUN(name) emit_native_x64_##name - -#elif N_X86 - -// x86 specific stuff - -STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { - [MP_F_CONVERT_OBJ_TO_NATIVE] = 2, - [MP_F_CONVERT_NATIVE_TO_OBJ] = 2, - [MP_F_LOAD_NAME] = 1, - [MP_F_LOAD_GLOBAL] = 1, - [MP_F_LOAD_BUILD_CLASS] = 0, - [MP_F_LOAD_ATTR] = 2, - [MP_F_LOAD_METHOD] = 3, - [MP_F_LOAD_SUPER_METHOD] = 2, - [MP_F_STORE_NAME] = 2, - [MP_F_STORE_GLOBAL] = 2, - [MP_F_STORE_ATTR] = 3, - [MP_F_OBJ_SUBSCR] = 3, - [MP_F_OBJ_IS_TRUE] = 1, - [MP_F_UNARY_OP] = 2, - [MP_F_BINARY_OP] = 3, - [MP_F_BUILD_TUPLE] = 2, - [MP_F_BUILD_LIST] = 2, - [MP_F_LIST_APPEND] = 2, - [MP_F_BUILD_MAP] = 1, - [MP_F_STORE_MAP] = 3, -#if MICROPY_PY_BUILTINS_SET - [MP_F_BUILD_SET] = 2, - [MP_F_STORE_SET] = 2, -#endif - [MP_F_MAKE_FUNCTION_FROM_RAW_CODE] = 3, - [MP_F_NATIVE_CALL_FUNCTION_N_KW] = 3, - [MP_F_CALL_METHOD_N_KW] = 3, - [MP_F_CALL_METHOD_N_KW_VAR] = 3, - [MP_F_NATIVE_GETITER] = 2, - [MP_F_NATIVE_ITERNEXT] = 1, - [MP_F_NLR_PUSH] = 1, - [MP_F_NLR_POP] = 0, - [MP_F_NATIVE_RAISE] = 1, - [MP_F_IMPORT_NAME] = 3, - [MP_F_IMPORT_FROM] = 2, - [MP_F_IMPORT_ALL] = 1, -#if MICROPY_PY_BUILTINS_SLICE - [MP_F_NEW_SLICE] = 3, -#endif - [MP_F_UNPACK_SEQUENCE] = 3, - [MP_F_UNPACK_EX] = 3, - [MP_F_DELETE_NAME] = 1, - [MP_F_DELETE_GLOBAL] = 1, - [MP_F_NEW_CELL] = 1, - [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3, - [MP_F_SETUP_CODE_STATE] = 5, - [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2, - [MP_F_SMALL_INT_MODULO] = 2, -}; - -#include "py/asmx86.h" -#define EXPORT_FUN(name) emit_native_x86_##name - -#elif N_THUMB - -// thumb specific stuff -#include "py/asmthumb.h" -#define EXPORT_FUN(name) emit_native_thumb_##name - -#elif N_ARM - -// ARM specific stuff -#include "py/asmarm.h" -#define EXPORT_FUN(name) emit_native_arm_##name - -#elif N_XTENSA - -// Xtensa specific stuff -#include "py/asmxtensa.h" -#define EXPORT_FUN(name) emit_native_xtensa_##name - -#else - -#error unknown native emitter - -#endif - #define EMIT_NATIVE_VIPER_TYPE_ERROR(emit, ...) do { \ *emit->error_slot = mp_obj_new_exception_msg_varg(&mp_type_ViperTypeError, __VA_ARGS__); \ } while (0) diff --git a/py/emitnthumb.c b/py/emitnthumb.c new file mode 100644 index 000000000..2b68ca3a1 --- /dev/null +++ b/py/emitnthumb.c @@ -0,0 +1,15 @@ +// thumb specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_THUMB + +// this is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmthumb.h" + +#define N_THUMB (1) +#define EXPORT_FUN(name) emit_native_thumb_##name +#include "py/emitnative.c" + +#endif diff --git a/py/emitnx64.c b/py/emitnx64.c new file mode 100644 index 000000000..b9800f636 --- /dev/null +++ b/py/emitnx64.c @@ -0,0 +1,15 @@ +// x64 specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_X64 + +// This is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmx64.h" + +#define N_X64 (1) +#define EXPORT_FUN(name) emit_native_x64_##name +#include "py/emitnative.c" + +#endif diff --git a/py/emitnx86.c b/py/emitnx86.c new file mode 100644 index 000000000..d4cd24d74 --- /dev/null +++ b/py/emitnx86.c @@ -0,0 +1,67 @@ +// x86 specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_X86 + +// This is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmx86.h" + +// x86 needs a table to know how many args a given function has +STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { + [MP_F_CONVERT_OBJ_TO_NATIVE] = 2, + [MP_F_CONVERT_NATIVE_TO_OBJ] = 2, + [MP_F_LOAD_NAME] = 1, + [MP_F_LOAD_GLOBAL] = 1, + [MP_F_LOAD_BUILD_CLASS] = 0, + [MP_F_LOAD_ATTR] = 2, + [MP_F_LOAD_METHOD] = 3, + [MP_F_LOAD_SUPER_METHOD] = 2, + [MP_F_STORE_NAME] = 2, + [MP_F_STORE_GLOBAL] = 2, + [MP_F_STORE_ATTR] = 3, + [MP_F_OBJ_SUBSCR] = 3, + [MP_F_OBJ_IS_TRUE] = 1, + [MP_F_UNARY_OP] = 2, + [MP_F_BINARY_OP] = 3, + [MP_F_BUILD_TUPLE] = 2, + [MP_F_BUILD_LIST] = 2, + [MP_F_LIST_APPEND] = 2, + [MP_F_BUILD_MAP] = 1, + [MP_F_STORE_MAP] = 3, + #if MICROPY_PY_BUILTINS_SET + [MP_F_BUILD_SET] = 2, + [MP_F_STORE_SET] = 2, + #endif + [MP_F_MAKE_FUNCTION_FROM_RAW_CODE] = 3, + [MP_F_NATIVE_CALL_FUNCTION_N_KW] = 3, + [MP_F_CALL_METHOD_N_KW] = 3, + [MP_F_CALL_METHOD_N_KW_VAR] = 3, + [MP_F_NATIVE_GETITER] = 2, + [MP_F_NATIVE_ITERNEXT] = 1, + [MP_F_NLR_PUSH] = 1, + [MP_F_NLR_POP] = 0, + [MP_F_NATIVE_RAISE] = 1, + [MP_F_IMPORT_NAME] = 3, + [MP_F_IMPORT_FROM] = 2, + [MP_F_IMPORT_ALL] = 1, + #if MICROPY_PY_BUILTINS_SLICE + [MP_F_NEW_SLICE] = 3, + #endif + [MP_F_UNPACK_SEQUENCE] = 3, + [MP_F_UNPACK_EX] = 3, + [MP_F_DELETE_NAME] = 1, + [MP_F_DELETE_GLOBAL] = 1, + [MP_F_NEW_CELL] = 1, + [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3, + [MP_F_SETUP_CODE_STATE] = 5, + [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2, + [MP_F_SMALL_INT_MODULO] = 2, +}; + +#define N_X86 (1) +#define EXPORT_FUN(name) emit_native_x86_##name +#include "py/emitnative.c" + +#endif diff --git a/py/emitnxtensa.c b/py/emitnxtensa.c new file mode 100644 index 000000000..1a423e21e --- /dev/null +++ b/py/emitnxtensa.c @@ -0,0 +1,15 @@ +// Xtensa specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_XTENSA + +// this is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmxtensa.h" + +#define N_XTENSA (1) +#define EXPORT_FUN(name) emit_native_xtensa_##name +#include "py/emitnative.c" + +#endif diff --git a/py/mkrules.mk b/py/mkrules.mk index fa7138695..850c2aa3a 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -46,10 +46,7 @@ vpath %.c . $(TOP) $(BUILD)/%.o: %.c $(call compile_c) -# List all native flags since the current build system doesn't have -# the MicroPython configuration available. However, these flags are -# needed to extract all qstrings -QSTR_GEN_EXTRA_CFLAGS += -DNO_QSTR -DN_X64 -DN_X86 -DN_THUMB -DN_ARM -DN_XTENSA +QSTR_GEN_EXTRA_CFLAGS += -DNO_QSTR QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp vpath %.c . $(TOP) diff --git a/py/py.mk b/py/py.mk index 7c4cf82d8..a91813526 100644 --- a/py/py.mk +++ b/py/py.mk @@ -267,8 +267,8 @@ PY_O += $(BUILD)/$(BUILD)/frozen_mpy.o endif # Sources that may contain qstrings -SRC_QSTR_IGNORE = py/nlr% py/emitnx86% py/emitnx64% py/emitnthumb% py/emitnarm% py/emitnxtensa% -SRC_QSTR = $(SRC_MOD) $(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) py/emitnative.c $(PY_EXTMOD_O_BASENAME:.o=.c) +SRC_QSTR_IGNORE = py/nlr% +SRC_QSTR = $(SRC_MOD) $(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) $(PY_EXTMOD_O_BASENAME:.o=.c) # Anything that depends on FORCE will be considered out-of-date FORCE: @@ -295,28 +295,6 @@ $(HEADER_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_C # that the function preludes are of a minimal and predictable form. $(PY_BUILD)/nlr%.o: CFLAGS += -Os -# emitters - -$(PY_BUILD)/emitnx64.o: CFLAGS += -DN_X64 -$(PY_BUILD)/emitnx64.o: py/emitnative.c - $(call compile_c) - -$(PY_BUILD)/emitnx86.o: CFLAGS += -DN_X86 -$(PY_BUILD)/emitnx86.o: py/emitnative.c - $(call compile_c) - -$(PY_BUILD)/emitnthumb.o: CFLAGS += -DN_THUMB -$(PY_BUILD)/emitnthumb.o: py/emitnative.c - $(call compile_c) - -$(PY_BUILD)/emitnarm.o: CFLAGS += -DN_ARM -$(PY_BUILD)/emitnarm.o: py/emitnative.c - $(call compile_c) - -$(PY_BUILD)/emitnxtensa.o: CFLAGS += -DN_XTENSA -$(PY_BUILD)/emitnxtensa.o: py/emitnative.c - $(call compile_c) - # optimising gc for speed; 5ms down to 4ms on pybv2 $(PY_BUILD)/gc.o: CFLAGS += $(CSUPEROPT) -- cgit v1.2.3 From deaa46aa66230071d355c0ffd004ccf0cbefd8b9 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Tue, 24 Apr 2018 17:47:45 +0200 Subject: py/nlrthumb: Fix Clang support wrt use of "return 0". Clang defines __GNUC__ so we have to check for it specifically. --- py/nlrthumb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/nlrthumb.c b/py/nlrthumb.c index 7dbeac1b6..c28302355 100644 --- a/py/nlrthumb.c +++ b/py/nlrthumb.c @@ -76,8 +76,9 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { #endif ); - #if defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)) + #if !defined(__clang__) && defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)) // Older versions of gcc give an error when naked functions don't return a value + // Additionally exclude Clang as it also defines __GNUC__ but doesn't need this statement return 0; #endif } -- cgit v1.2.3 From 96740be3574d91279ca883adbb19b976413ef52b Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 26 Apr 2018 15:02:59 +1000 Subject: py/mperrno: Define MP_EWOULDBLOCK as EWOULDBLOCK, not EAGAIN. Most modern systems have EWOULDBLOCK aliased to EAGAIN, ie they have the same value. But some systems use different values for these errnos and if a uPy port is using the system errno values (ie not the internal uPy values) then it's important to be able to distinguish EWOULDBLOCK from EAGAIN. Eg if a system call returned EWOULDBLOCK it must be possible to check for this return value, and this patch makes this now possible. --- py/mperrno.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/mperrno.h b/py/mperrno.h index f439f6555..0cad75a17 100644 --- a/py/mperrno.h +++ b/py/mperrno.h @@ -122,7 +122,7 @@ #define MP_EPIPE EPIPE #define MP_EDOM EDOM #define MP_ERANGE ERANGE -#define MP_EWOULDBLOCK EAGAIN +#define MP_EWOULDBLOCK EWOULDBLOCK #define MP_EOPNOTSUPP EOPNOTSUPP #define MP_EAFNOSUPPORT EAFNOSUPPORT #define MP_EADDRINUSE EADDRINUSE -- cgit v1.2.3 From d43c7377564b3e1be36f627a465df2c5779a1ae8 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Tue, 24 Apr 2018 17:43:14 +0200 Subject: py/stream: Use uPy errno instead of system's for non-blocking check. This is a more consistent use of errno codes. For example, it may be that a stream returns MP_EAGAIN but the mp_is_nonblocking_error() macro doesn't catch this value because it checks for EAGAIN instead (which may have a different value than MP_EAGAIN when MICROPY_USE_INTERNAL_ERRNO is enabled). --- py/stream.c | 7 ------- py/stream.h | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) (limited to 'py') diff --git a/py/stream.c b/py/stream.c index f51f634b6..393988d2c 100644 --- a/py/stream.c +++ b/py/stream.c @@ -32,13 +32,6 @@ #include "py/stream.h" #include "py/runtime.h" -#if MICROPY_STREAMS_NON_BLOCK -#include -#if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) -#define EWOULDBLOCK 140 -#endif -#endif - // This file defines generic Python stream read/write methods which // dispatch to the underlying stream interface of an object. diff --git a/py/stream.h b/py/stream.h index a7d8d08f1..a1d1c4f8a 100644 --- a/py/stream.h +++ b/py/stream.h @@ -107,7 +107,7 @@ int mp_stream_posix_fsync(mp_obj_t stream); #endif #if MICROPY_STREAMS_NON_BLOCK -#define mp_is_nonblocking_error(errno) ((errno) == EAGAIN || (errno) == EWOULDBLOCK) +#define mp_is_nonblocking_error(errno) ((errno) == MP_EAGAIN || (errno) == MP_EWOULDBLOCK) #else #define mp_is_nonblocking_error(errno) (0) #endif -- cgit v1.2.3 From 6b4b6d388be338d36de01b01e25f6e324321750a Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 1 May 2018 23:25:18 +1000 Subject: py/obj.h: Fix math.e constant for nan-boxing builds. Due to a typo, math.e was too small by around 6e-11. --- py/obj.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/obj.h b/py/obj.h index 80599966e..95a94836e 100644 --- a/py/obj.h +++ b/py/obj.h @@ -179,7 +179,7 @@ static inline bool MP_OBJ_IS_QSTR(mp_const_obj_t o) #define MP_OBJ_NEW_QSTR(qst) ((mp_obj_t)((((mp_uint_t)(qst)) << 1) | 0x0002000000000001)) #if MICROPY_PY_BUILTINS_FLOAT -#define mp_const_float_e {((mp_obj_t)((uint64_t)0x4005bf0a8b125769 + 0x8004000000000000))} +#define mp_const_float_e {((mp_obj_t)((uint64_t)0x4005bf0a8b145769 + 0x8004000000000000))} #define mp_const_float_pi {((mp_obj_t)((uint64_t)0x400921fb54442d18 + 0x8004000000000000))} static inline bool mp_obj_is_float(mp_const_obj_t o) { -- cgit v1.2.3 From 60db80920a1eac962e43991fcffafd16edd11885 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 2 May 2018 16:50:28 +1000 Subject: py/builtinhelp: Change occurrence of mp_uint_t to size_t. --- py/builtinhelp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/builtinhelp.c b/py/builtinhelp.c index 7106f3ced..6c2c3b92c 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -100,7 +100,7 @@ STATIC void mp_help_print_modules(void) { // print the list of modules in a column-first order #define NUM_COLUMNS (4) #define COLUMN_WIDTH (18) - mp_uint_t len; + size_t len; mp_obj_t *items; mp_obj_list_get(list, &len, &items); unsigned int num_rows = (len + NUM_COLUMNS - 1) / NUM_COLUMNS; -- cgit v1.2.3 From 3cf02be4e025ae74fa1a6924ff773e40fe5ef970 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 4 May 2018 20:39:16 +1000 Subject: py/emitnx86: Fix 32-bit x86 native emitter build by including header. --- py/emitnx86.c | 1 + 1 file changed, 1 insertion(+) (limited to 'py') diff --git a/py/emitnx86.c b/py/emitnx86.c index d4cd24d74..5d2bbb267 100644 --- a/py/emitnx86.c +++ b/py/emitnx86.c @@ -1,6 +1,7 @@ // x86 specific stuff #include "py/mpconfig.h" +#include "py/runtime0.h" #if MICROPY_EMIT_X86 -- cgit v1.2.3 From eb88803ac859db07726481501aea5c0e6bbf3705 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 9 May 2018 16:15:02 +1000 Subject: py/{modbuiltins,repl}: Start qstr probing from after empty qstr. The list of qstrs starts with MP_QSTR_NULL followed by MP_QSTR_, and these should never appear in dir() or REPL tab completion, so skip them. --- py/modbuiltins.c | 2 +- py/repl.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 5d4ec8ffe..e45c714e9 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -186,7 +186,7 @@ STATIC mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { // Make a list of names in the given object // Implemented by probing all possible qstrs with mp_load_method_maybe size_t nqstr = QSTR_TOTAL(); - for (size_t i = 1; i < nqstr; ++i) { + for (size_t i = MP_QSTR_ + 1; i < nqstr; ++i) { mp_obj_t dest[2]; mp_load_method_maybe(args[0], i, dest); if (dest[0] != MP_OBJ_NULL) { diff --git a/py/repl.c b/py/repl.c index 61ae2c1fe..56b1db011 100644 --- a/py/repl.c +++ b/py/repl.c @@ -176,7 +176,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print const char *match_str = NULL; size_t match_len = 0; qstr q_first = 0, q_last; - for (qstr q = 1; q < nqstr; ++q) { + for (qstr q = MP_QSTR_ + 1; q < nqstr; ++q) { size_t d_len; const char *d_str = (const char*)qstr_data(q, &d_len); if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { -- cgit v1.2.3 From bc87b862fd22afb38ed7392b69474286fa5fe19f Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 10 May 2018 23:00:04 +1000 Subject: py/runtime: Add mp_load_method_protected helper which catches exceptions This new helper function acts like mp_load_method_maybe but is wrapped in an NLR handler so it can catch exceptions. It prevents AttributeError from propagating out, and optionally all other exceptions. This helper can be used to fully implement hasattr (see follow-up commit), and also for cases where mp_load_method_maybe is used but it must now raise an exception. --- py/runtime.c | 16 ++++++++++++++++ py/runtime.h | 1 + 2 files changed, 17 insertions(+) (limited to 'py') diff --git a/py/runtime.c b/py/runtime.c index 4efb29bec..8f1063076 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1086,6 +1086,22 @@ void mp_load_method(mp_obj_t base, qstr attr, mp_obj_t *dest) { } } +// Acts like mp_load_method_maybe but catches AttributeError, and all other exceptions if requested +void mp_load_method_protected(mp_obj_t obj, qstr attr, mp_obj_t *dest, bool catch_all_exc) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_load_method_maybe(obj, attr, dest); + nlr_pop(); + } else { + if (!catch_all_exc + && !mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(((mp_obj_base_t*)nlr.ret_val)->type), + MP_OBJ_FROM_PTR(&mp_type_AttributeError))) { + // Re-raise the exception + nlr_raise(MP_OBJ_FROM_PTR(nlr.ret_val)); + } + } +} + void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t value) { DEBUG_OP_printf("store attr %p.%s <- %p\n", base, qstr_str(attr), value); mp_obj_type_t *type = mp_obj_get_type(base); diff --git a/py/runtime.h b/py/runtime.h index 6288e8836..ad65f3f46 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -132,6 +132,7 @@ mp_obj_t mp_load_attr(mp_obj_t base, qstr attr); void mp_convert_member_lookup(mp_obj_t obj, const mp_obj_type_t *type, mp_obj_t member, mp_obj_t *dest); void mp_load_method(mp_obj_t base, qstr attr, mp_obj_t *dest); void mp_load_method_maybe(mp_obj_t base, qstr attr, mp_obj_t *dest); +void mp_load_method_protected(mp_obj_t obj, qstr attr, mp_obj_t *dest, bool catch_all_exc); void mp_load_super_method(qstr attr, mp_obj_t *dest); void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t val); -- cgit v1.2.3 From 529860643bc321267376272356886ef39c4c6bd1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 10 May 2018 23:03:30 +1000 Subject: py/modbuiltins: Make built-in hasattr work properly for user types. It now allows __getattr__ in a user type to raise AttributeError when the attribute does not exist. --- py/modbuiltins.c | 8 +------- tests/basics/builtin_hasattr.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index e45c714e9..2f3526415 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -525,14 +525,8 @@ MP_DEFINE_CONST_FUN_OBJ_2(mp_builtin_delattr_obj, mp_builtin_delattr); STATIC mp_obj_t mp_builtin_hasattr(mp_obj_t object_in, mp_obj_t attr_in) { qstr attr = mp_obj_str_get_qstr(attr_in); - mp_obj_t dest[2]; - // TODO: https://docs.python.org/3/library/functions.html?highlight=hasattr#hasattr - // explicitly says "This is implemented by calling getattr(object, name) and seeing - // whether it raises an AttributeError or not.", so we should explicitly wrap this - // in nlr_push and handle exception. - mp_load_method_maybe(object_in, attr, dest); - + mp_load_method_protected(object_in, attr, dest, false); return mp_obj_new_bool(dest[0] != MP_OBJ_NULL); } MP_DEFINE_CONST_FUN_OBJ_2(mp_builtin_hasattr_obj, mp_builtin_hasattr); diff --git a/tests/basics/builtin_hasattr.py b/tests/basics/builtin_hasattr.py index 118a19e57..c60033b96 100644 --- a/tests/basics/builtin_hasattr.py +++ b/tests/basics/builtin_hasattr.py @@ -21,12 +21,19 @@ class C: def __getattr__(self, attr): if attr == "exists": return attr + elif attr == "raise": + raise Exception(123) raise AttributeError c = C() print(hasattr(c, "exists")) -# TODO -#print(hasattr(c, "doesnt_exist")) +print(hasattr(c, "doesnt_exist")) + +# ensure that non-AttributeError exceptions propagate out of hasattr +try: + hasattr(c, "raise") +except Exception as er: + print(er) try: hasattr(1, b'123') -- cgit v1.2.3 From 7241d90272f4dfe4100723393cc2f348f5d32c0a Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 10 May 2018 23:05:43 +1000 Subject: py/repl: Use mp_load_method_protected to prevent leaking of exceptions. This patch fixes the possibility of a crash of the REPL when tab-completing an object which raises an exception when its attributes are accessed. See issue #3729. --- py/repl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/repl.c b/py/repl.c index 56b1db011..a5a3ee007 100644 --- a/py/repl.c +++ b/py/repl.c @@ -158,7 +158,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print // lookup will fail return 0; } - mp_load_method_maybe(obj, q, dest); + mp_load_method_protected(obj, q, dest, true); obj = dest[0]; // attribute, method, or MP_OBJ_NULL if nothing found if (obj == MP_OBJ_NULL) { @@ -180,7 +180,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print size_t d_len; const char *d_str = (const char*)qstr_data(q, &d_len); if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { - mp_load_method_maybe(obj, q, dest); + mp_load_method_protected(obj, q, dest, true); if (dest[0] != MP_OBJ_NULL) { if (match_str == NULL) { match_str = d_str; @@ -234,7 +234,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print size_t d_len; const char *d_str = (const char*)qstr_data(q, &d_len); if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { - mp_load_method_maybe(obj, q, dest); + mp_load_method_protected(obj, q, dest, true); if (dest[0] != MP_OBJ_NULL) { int gap = (line_len + WORD_SLOT_LEN - 1) / WORD_SLOT_LEN * WORD_SLOT_LEN - line_len; if (gap < 2) { -- cgit v1.2.3 From 29d28c2574d6c0b93fd3fc869b389a61dee12102 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 10 May 2018 23:07:19 +1000 Subject: py/modbuiltins: In built-in dir make use of mp_load_method_protected. This gives dir() better behaviour when listing the attributes of a user type that defines __getattr__: it will now not list those attributes for which __getattr__ raises AttributeError (meaning the attribute is not supported by the object). --- py/modbuiltins.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 2f3526415..bc99f8261 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -188,7 +188,7 @@ STATIC mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { size_t nqstr = QSTR_TOTAL(); for (size_t i = MP_QSTR_ + 1; i < nqstr; ++i) { mp_obj_t dest[2]; - mp_load_method_maybe(args[0], i, dest); + mp_load_method_protected(args[0], i, dest, false); if (dest[0] != MP_OBJ_NULL) { mp_obj_list_append(dir, MP_OBJ_NEW_QSTR(i)); } -- cgit v1.2.3 From 3678a6bdc6a925f2bce6423c41f911229836f946 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 10 May 2018 23:10:46 +1000 Subject: py/modbuiltins: Make built-in dir support the __dir__ special method. If MICROPY_PY_ALL_SPECIAL_METHODS is enabled then dir() will now delegate to the special method __dir__ if the object it is listing has this method. --- py/makeqstrdata.py | 3 +++ py/modbuiltins.c | 7 +++++++ tests/basics/special_methods2.py | 9 +++++++++ 3 files changed, 19 insertions(+) (limited to 'py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 38fde1a9c..3c0a60909 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -114,6 +114,9 @@ def parse_input_headers(infiles): if ident == "": # Sort empty qstr above all still order = -200000 + elif ident == "__dir__": + # Put __dir__ after empty qstr for builtin dir() to work + order = -190000 elif ident.startswith("__"): order -= 100000 qstrs[ident] = (order, ident, qstr) diff --git a/py/modbuiltins.c b/py/modbuiltins.c index bc99f8261..0d511338b 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -190,6 +190,13 @@ STATIC mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { mp_obj_t dest[2]; mp_load_method_protected(args[0], i, dest, false); if (dest[0] != MP_OBJ_NULL) { + #if MICROPY_PY_ALL_SPECIAL_METHODS + // Support for __dir__: see if we can dispatch to this special method + // This relies on MP_QSTR__dir__ being first after MP_QSTR_ + if (i == MP_QSTR___dir__ && dest[1] != MP_OBJ_NULL) { + return mp_call_method_n_kw(0, 0, dest); + } + #endif mp_obj_list_append(dir, MP_OBJ_NEW_QSTR(i)); } } diff --git a/tests/basics/special_methods2.py b/tests/basics/special_methods2.py index ba7cf27cd..1a38a250c 100644 --- a/tests/basics/special_methods2.py +++ b/tests/basics/special_methods2.py @@ -94,6 +94,9 @@ class Cud(): print("__isub__ called") return self + def __dir__(self): + return ['a', 'b', 'c'] + cud1 = Cud() cud2 = Cud() @@ -113,6 +116,12 @@ cud2 // cud1 cud1 += cud2 cud1 -= cud2 +# test that dir() delegates to __dir__ special method +print(dir(cud1)) + +# test that dir() does not delegate to __dir__ for the type +print('a' in dir(Cud)) + # TODO: the following operations are not supported on every ports # # ne is not supported, !(eq) is called instead -- cgit v1.2.3 From 095d3970173f3d4ad6c43bd0c363b727d3a67241 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 11 May 2018 13:44:50 +1000 Subject: py/objdeque: Fix sign extension bug when computing len of deque object. For cases where size_t is smaller than mp_int_t (eg nan-boxing builds) the difference between two size_t's is not sign extended into mp_int_t and so the result is never negative. This patch fixes this bug by using ssize_t for the type of the result. --- py/objdeque.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/objdeque.c b/py/objdeque.c index bbb078103..1cff1f8d3 100644 --- a/py/objdeque.c +++ b/py/objdeque.c @@ -24,6 +24,7 @@ * THE SOFTWARE. */ +#include // for ssize_t #include #include "py/mpconfig.h" @@ -75,7 +76,7 @@ STATIC mp_obj_t deque_unary_op(mp_unary_op_t op, mp_obj_t self_in) { case MP_UNARY_OP_BOOL: return mp_obj_new_bool(self->i_get != self->i_put); case MP_UNARY_OP_LEN: { - mp_int_t len = self->i_put - self->i_get; + ssize_t len = self->i_put - self->i_get; if (len < 0) { len += self->alloc; } -- cgit v1.2.3 From 6046e68fe19ca3e88e6e55438be9b2dee98723af Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 11 May 2018 13:48:47 +1000 Subject: py/repl: Initialise q_last variable to prevent compiler warnings. Some older compilers cannot deduce that q_last is always written to before being read. --- py/repl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/repl.c b/py/repl.c index a5a3ee007..5dce8bbb7 100644 --- a/py/repl.c +++ b/py/repl.c @@ -175,7 +175,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print // look for matches const char *match_str = NULL; size_t match_len = 0; - qstr q_first = 0, q_last; + qstr q_first = 0, q_last = 0; for (qstr q = MP_QSTR_ + 1; q < nqstr; ++q) { size_t d_len; const char *d_str = (const char*)qstr_data(q, &d_len); -- cgit v1.2.3 From 9630376dbca0202321f2aae6ef88ef75cdb5374e Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 7 May 2018 13:36:52 +1000 Subject: py/mpconfig.h: Be stricter when autodetecting machine endianness. This patch changes 2 things in the endianness detection: 1. Don't assume that __BYTE_ORDER__ not being __ORDER_LITTLE_ENDIAN__ means that the machine is big endian, so add an explicit check that this macro is indeed __ORDER_BIG_ENDIAN__ (same with __BYTE_ORDER, __LITTLE_ENDIAN and __BIG_ENDIAN). A machine could have PDP endianness. 2. Remove the checks which base their autodetection decision on whether any little or big endian macros are defined (eg __LITTLE_ENDIAN__ or __BIG_ENDIAN__). Just because a system defines these does not mean it has that endianness. See issue #3760. --- py/mpconfig.h | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'py') diff --git a/py/mpconfig.h b/py/mpconfig.h index 532b54ab0..c42fe7853 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1256,29 +1256,26 @@ typedef double mp_float_t; #elif defined(MP_ENDIANNESS_BIG) #define MP_ENDIANNESS_LITTLE (!MP_ENDIANNESS_BIG) #else - // Endiannes not defined by port so try to autodetect it. + // Endianness not defined by port so try to autodetect it. #if defined(__BYTE_ORDER__) #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define MP_ENDIANNESS_LITTLE (1) - #else + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define MP_ENDIANNESS_LITTLE (0) #endif - #elif defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) - #define MP_ENDIANNESS_LITTLE (1) - #elif defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined (_BIG_ENDIAN) - #define MP_ENDIANNESS_LITTLE (0) #else #include #if defined(__BYTE_ORDER) #if __BYTE_ORDER == __LITTLE_ENDIAN #define MP_ENDIANNESS_LITTLE (1) - #else + #elif __BYTE_ORDER == __BIG_ENDIAN #define MP_ENDIANNESS_LITTLE (0) #endif - #else - #error endianness not defined and cannot detect it #endif #endif + #ifndef MP_ENDIANNESS_LITTLE + #error endianness not defined and cannot detect it + #endif #define MP_ENDIANNESS_BIG (!MP_ENDIANNESS_LITTLE) #endif -- cgit v1.2.3 From 749b16174b5292b2680ab027a6a1b3874e22ea43 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 12 May 2018 22:09:34 +1000 Subject: py/mpstate.h: Adjust start of root pointer section to exclude non-ptrs. This patch moves the start of the root pointer section in mp_state_ctx_t so that it skips entries that are not pointers and don't need scanning. Previously, the start of the root pointer section was at the very beginning of the mp_state_ctx_t struct (which is the beginning of mp_state_thread_t). This was the original assembler version of the NLR code was hard-coded to have the nlr_top pointer at the start of this state structure. But now that the NLR code is partially written in C there is no longer this restriction on the location of nlr_top (and a comment to this effect has been removed in this patch). So now the root pointer section starts part way through the mp_state_thread_t structure, after the entries which are not root pointers. This patch also moves the non-pointer entries for MICROPY_ENABLE_SCHEDULER outside the root pointer section. Moving non-pointer entries out of the root pointer section helps to make the GC more precise and should help to prevent some cases of collectable garbage being kept. This patch also has a measurable improvement in performance of the pystone.py benchmark: on unix x86-64 and stm32 there was an improvement of roughly 0.6% (tested with both gcc 7.3 and gcc 8.1). --- py/gc.c | 4 +++- py/mpstate.h | 34 +++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 14 deletions(-) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 84c9918fd..38de51399 100644 --- a/py/gc.c +++ b/py/gc.c @@ -328,7 +328,9 @@ void gc_collect_start(void) { // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, // dict_globals, then the root pointer section of mp_state_vm. void **ptrs = (void**)(void*)&mp_state_ctx; - gc_collect_root(ptrs, offsetof(mp_state_ctx_t, vm.qstr_last_chunk) / sizeof(void*)); + size_t root_start = offsetof(mp_state_ctx_t, thread.dict_locals); + size_t root_end = offsetof(mp_state_ctx_t, vm.qstr_last_chunk); + gc_collect_root(ptrs + root_start / sizeof(void*), (root_end - root_start) / sizeof(void*)); #if MICROPY_ENABLE_PYSTACK // Trace root pointers from the Python stack. diff --git a/py/mpstate.h b/py/mpstate.h index 16c4bf6c5..199fd2cb6 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -105,10 +105,11 @@ typedef struct _mp_state_mem_t { // This structure hold runtime and VM information. It includes a section // which contains root pointers that must be scanned by the GC. typedef struct _mp_state_vm_t { - //////////////////////////////////////////////////////////// - // START ROOT POINTER SECTION - // everything that needs GC scanning must go here - // this must start at the start of this structure + // + // CONTINUE ROOT POINTER SECTION + // This must start at the start of this structure and follows + // the state in the mp_state_thread_t structure, continuing + // the root pointer section from there. // qstr_pool_t *last_pool; @@ -139,8 +140,6 @@ typedef struct _mp_state_vm_t { volatile mp_obj_t mp_pending_exception; #if MICROPY_ENABLE_SCHEDULER - volatile int16_t sched_state; - uint16_t sched_sp; mp_sched_item_t sched_stack[MICROPY_SCHEDULER_DEPTH]; #endif @@ -208,6 +207,11 @@ typedef struct _mp_state_vm_t { mp_int_t mp_emergency_exception_buf_size; #endif + #if MICROPY_ENABLE_SCHEDULER + volatile int16_t sched_state; + uint16_t sched_sp; + #endif + #if MICROPY_PY_THREAD_GIL // This is a global mutex used to make the VM/runtime thread-safe. mp_thread_mutex_t gil_mutex; @@ -217,11 +221,6 @@ typedef struct _mp_state_vm_t { // This structure holds state that is specific to a given thread. // Everything in this structure is scanned for root pointers. typedef struct _mp_state_thread_t { - mp_obj_dict_t *dict_locals; - mp_obj_dict_t *dict_globals; - - nlr_buf_t *nlr_top; // ROOT POINTER - // Stack top at the start of program char *stack_top; @@ -234,12 +233,21 @@ typedef struct _mp_state_thread_t { uint8_t *pystack_end; uint8_t *pystack_cur; #endif + + //////////////////////////////////////////////////////////// + // START ROOT POINTER SECTION + // Everything that needs GC scanning must start here, and + // is followed by state in the mp_state_vm_t structure. + // + + mp_obj_dict_t *dict_locals; + mp_obj_dict_t *dict_globals; + + nlr_buf_t *nlr_top; } mp_state_thread_t; // This structure combines the above 3 structures. // The order of the entries are important for root pointer scanning in the GC to work. -// Note: if this structure changes then revisit all nlr asm code since they -// have the offset of nlr_top hard-coded. typedef struct _mp_state_ctx_t { mp_state_thread_t thread; mp_state_vm_t vm; -- cgit v1.2.3 From c97607db5ccc03afbccacf853f2cd06305c28251 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 15 May 2018 11:17:28 +1000 Subject: py/nlrx86: Use naked attribute on nlr_push for gcc 8.0 and higher. gcc 8.0 supports the naked attribute for x86 systems so it can now be used here. And in fact it is necessary to use this for nlr_push because gcc 8.0 no longer generates a prelude for this function (even without the naked attribute). --- py/nlrx86.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'py') diff --git a/py/nlrx86.c b/py/nlrx86.c index 23882cc30..59b97d8ee 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -39,15 +39,29 @@ unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); #endif +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 8 +// Since gcc 8.0 the naked attribute is supported +#define USE_NAKED (1) +#define UNDO_PRELUDE (0) +#elif defined(__ZEPHYR__) || defined(__ANDROID__) +// Zephyr and Android use a different calling convention by default +#define USE_NAKED (0) +#define UNDO_PRELUDE (0) +#else +#define USE_NAKED (0) +#define UNDO_PRELUDE (1) +#endif + +#if USE_NAKED +__attribute__((naked)) +#endif unsigned int nlr_push(nlr_buf_t *nlr) { + #if !USE_NAKED (void)nlr; + #endif __asm volatile ( - // Check for Zephyr, which uses a different calling convention - // by default. - // TODE: Better support for various x86 calling conventions - // (unfortunately, __attribute__((naked)) is not supported on x86). - #if !(defined(__ZEPHYR__) || defined(__ANDROID__)) + #if UNDO_PRELUDE "pop %ebp \n" // undo function's prelude #endif "mov 4(%esp), %edx \n" // load nlr_buf @@ -61,7 +75,9 @@ unsigned int nlr_push(nlr_buf_t *nlr) { "jmp nlr_push_tail \n" // do the rest in C ); + #if !USE_NAKED return 0; // needed to silence compiler warning + #endif } NORETURN void nlr_jump(void *val) { -- cgit v1.2.3 From 1b7487e519c2e2a66f2f5a45ebd238d0773ca3d3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 16 May 2018 12:33:39 +1000 Subject: py/vm: Adjust #if logic for gil_divisor so braces are balanced. Having balanced braces { and } makes it easier to navigate the function. --- py/vm.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'py') diff --git a/py/vm.c b/py/vm.c index 8abe65e43..b88cfd8d3 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1322,11 +1322,12 @@ pending_exception_check: #if MICROPY_PY_THREAD_GIL #if MICROPY_PY_THREAD_GIL_VM_DIVISOR - if (--gil_divisor == 0) { - gil_divisor = MICROPY_PY_THREAD_GIL_VM_DIVISOR; - #else - { + if (--gil_divisor == 0) #endif + { + #if MICROPY_PY_THREAD_GIL_VM_DIVISOR + gil_divisor = MICROPY_PY_THREAD_GIL_VM_DIVISOR; + #endif #if MICROPY_ENABLE_SCHEDULER // can only switch threads if the scheduler is unlocked if (MP_STATE_VM(sched_state) == MP_SCHED_IDLE) -- cgit v1.2.3 From a883fe12d95953e9aa705f0050da1b61ae54ec8d Mon Sep 17 00:00:00 2001 From: Tom Collins Date: Wed, 16 May 2018 17:08:34 -0700 Subject: py/objfun: Fix variable name in DECODE_CODESTATE_SIZE() macro. This patch fixes the macro so you can pass any name in, and the macro will make more sense if you're reading it on its own. It worked previously because n_state is always passed in as n_state_out_var. --- py/objfun.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/objfun.c b/py/objfun.c index e6d33d287..8c51d92e0 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -204,10 +204,11 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { n_state_out_var = mp_decode_uint_value(bytecode); \ size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(bytecode)); \ \ - n_state += VM_DETECT_STACK_OVERFLOW; \ + n_state_out_var += VM_DETECT_STACK_OVERFLOW; \ \ /* state size in bytes */ \ - state_size_out_var = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); \ + state_size_out_var = n_state_out_var * sizeof(mp_obj_t) \ + + n_exc_stack * sizeof(mp_exc_stack_t); \ } #define INIT_CODESTATE(code_state, _fun_bc, n_args, n_kw, args) \ -- cgit v1.2.3 From 46ce395130305ce3299ae0dfd42502d29837c39f Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 18 May 2018 11:44:26 +1000 Subject: py/vm: Use enum names instead of magic numbers in multi-opcode dispatch. --- py/vm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/vm.c b/py/vm.c index b88cfd8d3..52e15ae33 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1270,10 +1270,10 @@ yield: } else if (ip[-1] < MP_BC_STORE_FAST_MULTI + 16) { fastn[MP_BC_STORE_FAST_MULTI - (mp_int_t)ip[-1]] = POP(); DISPATCH(); - } else if (ip[-1] < MP_BC_UNARY_OP_MULTI + 7) { + } else if (ip[-1] < MP_BC_UNARY_OP_MULTI + MP_UNARY_OP_NUM_BYTECODE) { SET_TOP(mp_unary_op(ip[-1] - MP_BC_UNARY_OP_MULTI, TOP())); DISPATCH(); - } else if (ip[-1] < MP_BC_BINARY_OP_MULTI + 36) { + } else if (ip[-1] < MP_BC_BINARY_OP_MULTI + MP_BINARY_OP_NUM_BYTECODE) { mp_obj_t rhs = POP(); mp_obj_t lhs = TOP(); SET_TOP(mp_binary_op(ip[-1] - MP_BC_BINARY_OP_MULTI, lhs, rhs)); -- cgit v1.2.3 From 869024dd6e62905b7e1069b547856a769b3b24ba Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 18 May 2018 11:47:03 +1000 Subject: py/vm: Improve performance of opcode dispatch when using switch stmt. Before this patch, when using the switch statement for dispatch in the VM (not computed goto) a pending exception check was done after each opcode. This is not necessary and this patch makes the pending exception check only happen when explicitly requested by certain opcodes, like jump. This improves performance of the VM by about 2.5% when using the switch. --- py/vm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/vm.c b/py/vm.c index 52e15ae33..d24a024d5 100644 --- a/py/vm.c +++ b/py/vm.c @@ -136,7 +136,7 @@ mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp #define ENTRY(op) entry_##op #define ENTRY_DEFAULT entry_default #else - #define DISPATCH() break + #define DISPATCH() goto dispatch_loop #define DISPATCH_WITH_PEND_EXC_CHECK() goto pending_exception_check #define ENTRY(op) case op #define ENTRY_DEFAULT default -- cgit v1.2.3 From 3e6ab82179f7b9a1df915f4bd0d6e48a454d942c Mon Sep 17 00:00:00 2001 From: Li Weiwei Date: Fri, 18 May 2018 11:15:53 +0800 Subject: py/repl: Fix handling of unmatched brackets and unfinished quotes. Before this patch: >>> print(') ... ') Traceback (most recent call last): File "", line 1 SyntaxError: invalid syntax After this patch: >>> print(') Traceback (most recent call last): File "", line 1 SyntaxError: invalid syntax This matches CPython and prevents getting stuck in REPL continuation when a 1-quote is unmatched. --- py/repl.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/repl.c b/py/repl.c index 5dce8bbb7..da0fefb3a 100644 --- a/py/repl.c +++ b/py/repl.c @@ -106,8 +106,13 @@ bool mp_repl_continue_with_input(const char *input) { } } - // continue if unmatched brackets or quotes - if (n_paren > 0 || n_brack > 0 || n_brace > 0 || in_quote == Q_3_SINGLE || in_quote == Q_3_DOUBLE) { + // continue if unmatched 3-quotes + if (in_quote == Q_3_SINGLE || in_quote == Q_3_DOUBLE) { + return true; + } + + // continue if unmatched brackets, but only if not in a 1-quote + if ((n_paren > 0 || n_brack > 0 || n_brace > 0) && in_quote == Q_NONE) { return true; } -- cgit v1.2.3 From 43d08d6dd6fab6c88eeb20663ab97d622c2ea915 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 18 May 2018 13:10:31 +1000 Subject: py/misc.h: Add MP_STATIC_ASSERT macro to do static assertions. --- py/misc.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'py') diff --git a/py/misc.h b/py/misc.h index 72560da1e..e6d25b850 100644 --- a/py/misc.h +++ b/py/misc.h @@ -50,6 +50,9 @@ typedef unsigned int uint; #define _MP_STRINGIFY(x) #x #define MP_STRINGIFY(x) _MP_STRINGIFY(x) +// Static assertion macro +#define MP_STATIC_ASSERT(cond) ((void)sizeof(char[1 - 2 * !(cond)])) + /** memory allocation ******************************************/ // TODO make a lazy m_renew that can increase by a smaller amount than requested (but by at least 1 more element) -- cgit v1.2.3 From 828ce16dc8063f11a2743cfd8fc698e0afa23cac Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 18 May 2018 13:11:02 +1000 Subject: py/compile: Change comment about ITER_BUF_NSLOTS to a static assertion. --- py/compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 9200b346b..c2dd914f0 100644 --- a/py/compile.c +++ b/py/compile.c @@ -3052,7 +3052,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { // There are 4 slots on the stack for the iterator, and the first one is // NULL to indicate that the second one points to the iterator object. if (scope->kind == SCOPE_GEN_EXPR) { - // TODO static assert that MP_OBJ_ITER_BUF_NSLOTS == 4 + MP_STATIC_ASSERT(MP_OBJ_ITER_BUF_NSLOTS == 4); EMIT(load_null); compile_load_id(comp, qstr_arg); EMIT(load_null); -- cgit v1.2.3 From bc6c0b28bf830a75c817fb498b713779c92b731b Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 19 May 2018 10:52:43 -0500 Subject: py/emitbc: Avoid undefined behavior calling memset() with NULL 1st arg. Calling memset(NULL, value, 0) is not standards compliant so we must add an explicit check that emit->label_offsets is indeed not NULL before calling memset (this pointer will be NULL on the first pass of the parse tree and it's more logical / safer to check this pointer rather than check that the pass is not the first one). Code sanitizers will warn if NULL is passed as the first value to memset, and compilers may optimise the code based on the knowledge that any pointer passed to memset is guaranteed not to be NULL. --- py/emitbc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/emitbc.c b/py/emitbc.c index 32e833000..b1b61ba67 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -315,7 +315,7 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { emit->last_source_line = 1; #ifndef NDEBUG // With debugging enabled labels are checked for unique assignment - if (pass < MP_PASS_EMIT) { + if (pass < MP_PASS_EMIT && emit->label_offsets != NULL) { memset(emit->label_offsets, -1, emit->max_num_labels * sizeof(mp_uint_t)); } #endif -- cgit v1.2.3 From 5efc575067df17be785d2b60124706d789d6cfe0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 21 May 2018 12:27:38 +1000 Subject: py/parsenum: Use int instead of mp_int_t for parsing float exponent. There is no need to use the mp_int_t type which may be 64-bits wide, there is enough bit-width in a normal int to parse reasonable exponents. Using int helps to reduce code size for 64-bit ports, especially nan-boxing builds. (Similarly for the "dig" variable which is now an unsigned int.) --- py/parsenum.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/parsenum.c b/py/parsenum.c index 124489c66..8bd5232eb 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -227,10 +227,10 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool // string should be a decimal number parse_dec_in_t in = PARSE_DEC_IN_INTG; bool exp_neg = false; - mp_int_t exp_val = 0; - mp_int_t exp_extra = 0; + int exp_val = 0; + int exp_extra = 0; while (str < top) { - mp_uint_t dig = *str++; + unsigned int dig = *str++; if ('0' <= dig && dig <= '9') { dig -= '0'; if (in == PARSE_DEC_IN_EXP) { -- cgit v1.2.3 From 4f71a2a75a71b89dad2eed147d6e237b633b878e Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 19 May 2018 10:56:34 -0500 Subject: py/parsenum: Avoid undefined behavior parsing floats with large exponents. Fuzz testing combined with the undefined behavior sanitizer found that parsing unreasonable float literals like 1e+9999999999999 resulted in undefined behavior due to overflow in signed integer arithmetic, and a wrong result being returned. --- py/parsenum.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/parsenum.c b/py/parsenum.c index 8bd5232eb..ba7e40afd 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -234,7 +234,12 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool if ('0' <= dig && dig <= '9') { dig -= '0'; if (in == PARSE_DEC_IN_EXP) { - exp_val = 10 * exp_val + dig; + // don't overflow exp_val when adding next digit, instead just truncate + // it and the resulting float will still be correct, either inf or 0.0 + // (use INT_MAX/2 to allow adding exp_extra at the end without overflow) + if (exp_val < (INT_MAX / 2 - 9) / 10) { + exp_val = 10 * exp_val + dig; + } } else { if (dec_val < DEC_VAL_MAX) { // dec_val won't overflow so keep accumulating -- cgit v1.2.3 From 60eb5305f637e21f7ec4006924c06518ca6de476 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 19 May 2018 11:25:03 -0500 Subject: py/objfloat: Fix undefined shifting behavior in high-quality float hash. When computing e.g. hash(0.4e3) with ubsan enabled, a diagnostic like the following would occur: ../../py/objfloat.c:91:30: runtime error: shift exponent 44 is too large for 32-bit type 'int' By casting constant "1" to the right type the intended value is preserved. --- py/objfloat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/objfloat.c b/py/objfloat.c index e4d5a6570..31c624778 100644 --- a/py/objfloat.c +++ b/py/objfloat.c @@ -88,7 +88,7 @@ typedef uint32_t mp_float_uint_t; if (adj_exp <= MP_FLOAT_FRAC_BITS) { // number may have a fraction; xor the integer part with the fractional part val = (frc >> (MP_FLOAT_FRAC_BITS - adj_exp)) - ^ (frc & ((1 << (MP_FLOAT_FRAC_BITS - adj_exp)) - 1)); + ^ (frc & (((mp_float_uint_t)1 << (MP_FLOAT_FRAC_BITS - adj_exp)) - 1)); } else if ((unsigned int)adj_exp < BITS_PER_BYTE * sizeof(mp_int_t) - 1) { // the number is a (big) whole integer and will fit in val's signed-width val = (mp_int_t)frc << (adj_exp - MP_FLOAT_FRAC_BITS); -- cgit v1.2.3 From c4dafcef4fe9edaacaeeb16c412c298cbab3b414 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 19 May 2018 11:20:29 -0500 Subject: py/mpz: Avoid undefined behavior at integer overflow in mpz_hash. Before this, ubsan would detect a problem when executing hash(006699999999999999999999999999999999999999999999999999999999999999999999) ../../py/mpz.c:1539:20: runtime error: left shift of 1067371580458 by 32 places cannot be represented in type 'mp_int_t' (aka 'long') When the overflow does occur it now happens as defined by the rules of unsigned arithmetic. --- py/mpz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/mpz.c b/py/mpz.c index fa5086862..8687092d0 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -1532,7 +1532,7 @@ mpz_t *mpz_mod(const mpz_t *lhs, const mpz_t *rhs) { // must return actual int value if it fits in mp_int_t mp_int_t mpz_hash(const mpz_t *z) { - mp_int_t val = 0; + mp_uint_t val = 0; mpz_dig_t *d = z->dig + z->len; while (d-- > z->dig) { -- cgit v1.2.3 From 95e43efc994af7d7ff85ef1722eac163be9cc5fd Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 19 May 2018 13:13:02 -0500 Subject: py/objfloat: Fix undefined integer behavior hashing negative zero. Under ubsan, when evaluating hash(-0.) the following diagnostic occurs: ../../py/objfloat.c:102:15: runtime error: negation of -9223372036854775808 cannot be represented in type 'mp_int_t' (aka 'long'); cast to an unsigned type to negate this value to itself So do just that, to tell the compiler that we want to perform this operation using modulo arithmetic rules. --- py/objfloat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/objfloat.c b/py/objfloat.c index 31c624778..b62fe8e71 100644 --- a/py/objfloat.c +++ b/py/objfloat.c @@ -99,7 +99,7 @@ typedef uint32_t mp_float_uint_t; } if (u.p.sgn) { - val = -val; + val = -(mp_uint_t)val; } return val; -- cgit v1.2.3 From 6bd78741c13849da0570710ad6df80846df812c0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 21 May 2018 13:36:21 +1000 Subject: py/gc: When GC threshold is hit don't unnecessarily collect twice. Without this, if GC threshold is hit and there is not enough memory left to satisfy the request, gc_collect() will run a second time and the search for memory will happen again and will fail again. Thanks to @adritium for pointing out this issue, see #3786. --- py/gc.c | 1 + 1 file changed, 1 insertion(+) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index 38de51399..e92b81ece 100644 --- a/py/gc.c +++ b/py/gc.c @@ -453,6 +453,7 @@ void *gc_alloc(size_t n_bytes, bool has_finaliser) { if (!collected && MP_STATE_MEM(gc_alloc_amount) >= MP_STATE_MEM(gc_alloc_threshold)) { GC_EXIT(); gc_collect(); + collected = 1; GC_ENTER(); } #endif -- cgit v1.2.3 From f2ec7925542e5a75b2da7ef378f5df66fdb6fad9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 13:20:00 +1000 Subject: py/parsenum: Adjust braces so they are balanced. --- py/parsenum.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/parsenum.c b/py/parsenum.c index ba7e40afd..354d0f756 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -319,11 +319,13 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool return mp_obj_new_complex(0, dec_val); } else if (force_complex) { return mp_obj_new_complex(dec_val, 0); + } #else if (imag || force_complex) { raise_exc(mp_obj_new_exception_msg(&mp_type_ValueError, "complex values not supported"), lex); + } #endif - } else { + else { return mp_obj_new_float(dec_val); } -- cgit v1.2.3 From b318ebf1015ced6354f8bbaf035308214b3f5c5d Mon Sep 17 00:00:00 2001 From: Jan Klusacek Date: Tue, 9 Jan 2018 22:47:35 +0100 Subject: py/modbuiltins: Add support for rounding integers. As per CPython semantics. This feature is controlled by MICROPY_PY_BUILTINS_ROUND_INT which is disabled by default. --- py/modbuiltins.c | 31 ++++++++++++++++++++++++++++++- py/mpconfig.h | 5 +++++ tests/basics/builtin_round_int.py | 18 ++++++++++++++++++ tests/basics/builtin_round_intbig.py | 17 +++++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 tests/basics/builtin_round_int.py create mode 100644 tests/basics/builtin_round_intbig.py (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 0d511338b..b216e021f 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -445,7 +445,36 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_repr_obj, mp_builtin_repr); STATIC mp_obj_t mp_builtin_round(size_t n_args, const mp_obj_t *args) { mp_obj_t o_in = args[0]; if (MP_OBJ_IS_INT(o_in)) { - return o_in; + if (n_args <= 1) { + return o_in; + } + + #if !MICROPY_PY_BUILTINS_ROUND_INT + mp_raise_NotImplementedError(NULL); + #else + mp_int_t num_dig = mp_obj_get_int(args[1]); + if (num_dig >= 0) { + return o_in; + } + + mp_obj_t mult = mp_binary_op(MP_BINARY_OP_POWER, MP_OBJ_NEW_SMALL_INT(10), MP_OBJ_NEW_SMALL_INT(-num_dig)); + mp_obj_t half_mult = mp_binary_op(MP_BINARY_OP_FLOOR_DIVIDE, mult, MP_OBJ_NEW_SMALL_INT(2)); + mp_obj_t modulo = mp_binary_op(MP_BINARY_OP_MODULO, o_in, mult); + mp_obj_t rounded = mp_binary_op(MP_BINARY_OP_SUBTRACT, o_in, modulo); + if (mp_obj_is_true(mp_binary_op(MP_BINARY_OP_MORE, half_mult, modulo))) { + return rounded; + } else if (mp_obj_is_true(mp_binary_op(MP_BINARY_OP_MORE, modulo, half_mult))) { + return mp_binary_op(MP_BINARY_OP_ADD, rounded, mult); + } else { + // round to even number + mp_obj_t floor = mp_binary_op(MP_BINARY_OP_FLOOR_DIVIDE, o_in, mult); + if (mp_obj_is_true(mp_binary_op(MP_BINARY_OP_AND, floor, MP_OBJ_NEW_SMALL_INT(1)))) { + return mp_binary_op(MP_BINARY_OP_ADD, rounded, mult); + } else { + return rounded; + } + } + #endif } #if MICROPY_PY_BUILTINS_FLOAT mp_float_t val = mp_obj_get_float(o_in); diff --git a/py/mpconfig.h b/py/mpconfig.h index c42fe7853..100e2a981 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -802,6 +802,11 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_RANGE_BINOP (0) #endif +// Whether to support rounding of integers (incl bignum); eg round(123,-1)=120 +#ifndef MICROPY_PY_BUILTINS_ROUND_INT +#define MICROPY_PY_BUILTINS_ROUND_INT (0) +#endif + // Whether to support timeout exceptions (like socket.timeout) #ifndef MICROPY_PY_BUILTINS_TIMEOUTERROR #define MICROPY_PY_BUILTINS_TIMEOUTERROR (0) diff --git a/tests/basics/builtin_round_int.py b/tests/basics/builtin_round_int.py new file mode 100644 index 000000000..a2017622a --- /dev/null +++ b/tests/basics/builtin_round_int.py @@ -0,0 +1,18 @@ +# test round() with integer values and second arg + +# rounding integers is an optional feature so test for it +try: + round(1, -1) +except NotImplementedError: + print('SKIP') + raise SystemExit + +tests = [ + (1, False), (1, True), + (124, -1), (125, -1), (126, -1), + (5, -1), (15, -1), (25, -1), + (12345, 0), (12345, -1), (12345, 1), + (-1234, 0), (-1234, -1), (-1234, 1), +] +for t in tests: + print(round(*t)) diff --git a/tests/basics/builtin_round_intbig.py b/tests/basics/builtin_round_intbig.py new file mode 100644 index 000000000..adf9d29f2 --- /dev/null +++ b/tests/basics/builtin_round_intbig.py @@ -0,0 +1,17 @@ +# test round() with large integer values and second arg + +# rounding integers is an optional feature so test for it +try: + round(1, -1) +except NotImplementedError: + print('SKIP') + raise SystemExit + +i = 2**70 + +tests = [ + (i, 0), (i, -1), (i, -10), (i, 1), + (-i, 0), (-i, -1), (-i, -10), (-i, 1), +] +for t in tests: + print(round(*t)) -- cgit v1.2.3 From 771cb359af5242762baa29645c37cafa23c47b25 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 16:39:19 +1000 Subject: py/objgenerator: Save state in old_globals instead of local variable. The code_state.old_globals variable is there to save the globals state so should be used for this purpose, to avoid the need for additional local variables on the C stack. --- py/objgenerator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/objgenerator.c b/py/objgenerator.c index 5fd13f831..d500dbd9d 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -117,10 +117,10 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ *self->code_state.sp = send_value; } } - mp_obj_dict_t *old_globals = mp_globals_get(); + self->code_state.old_globals = mp_globals_get(); mp_globals_set(self->globals); mp_vm_return_kind_t ret_kind = mp_execute_bytecode(&self->code_state, throw_value); - mp_globals_set(old_globals); + mp_globals_set(self->code_state.old_globals); switch (ret_kind) { case MP_VM_RETURN_NORMAL: -- cgit v1.2.3 From 400273a799581e5eb86538d8c88fb872705475ab Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 16:54:03 +1000 Subject: py/objgenerator: Protect against reentering a generator. Generators that are already executing cannot be reexecuted. This patch puts in a check for such a case. Thanks to @jepler for finding the bug. --- py/objgenerator.c | 10 ++++++++++ tests/basics/gen_yield_from_executing.py | 15 +++++++++++++++ tests/run-tests | 2 +- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 tests/basics/gen_yield_from_executing.py (limited to 'py') diff --git a/py/objgenerator.c b/py/objgenerator.c index d500dbd9d..a2ad490d6 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -117,9 +117,19 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ *self->code_state.sp = send_value; } } + + // We set self->globals=NULL while executing, for a sentinel to ensure the generator + // cannot be reentered during execution + if (self->globals == NULL) { + mp_raise_ValueError("generator already executing"); + } + + // Set up the correct globals context for the generator and execute it self->code_state.old_globals = mp_globals_get(); mp_globals_set(self->globals); + self->globals = NULL; mp_vm_return_kind_t ret_kind = mp_execute_bytecode(&self->code_state, throw_value); + self->globals = mp_globals_get(); mp_globals_set(self->code_state.old_globals); switch (ret_kind) { diff --git a/tests/basics/gen_yield_from_executing.py b/tests/basics/gen_yield_from_executing.py new file mode 100644 index 000000000..cad0c7695 --- /dev/null +++ b/tests/basics/gen_yield_from_executing.py @@ -0,0 +1,15 @@ +# yielding from an already executing generator is not allowed + +def f(): + yield 1 + # g here is already executing so this will raise an exception + yield from g + +g = f() + +print(next(g)) + +try: + next(g) +except ValueError: + print('ValueError') diff --git a/tests/run-tests b/tests/run-tests index afefa264f..25fb33a3f 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -336,7 +336,7 @@ def run_tests(pyb, tests, args, base_path="."): # Some tests are known to fail with native emitter # Remove them from the below when they work if args.emit == 'native': - skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_pend_throw generator_return generator_send'.split()}) # require yield + skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_executing gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_pend_throw generator_return generator_send'.split()}) # require yield skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class globals_del string_join'.split()}) # require yield skip_tests.update({'basics/async_%s.py' % t for t in 'def await await2 for for2 with with2'.split()}) # require yield skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs -- cgit v1.2.3 From 0a25fff9562e3051d85db9ca773f203620004742 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 19 May 2018 00:11:04 +1000 Subject: py/emit: Combine fast and deref into one function for load/store/delete. Reduces code size by: bare-arm: -16 minimal x86: -208 unix x64: -408 unix nanbox: -248 stm32: -12 cc3200: -24 esp8266: -96 esp32: -44 --- py/compile.c | 4 ++-- py/emit.h | 16 ++++++++-------- py/emitbc.c | 57 ++++++++++++++++++++------------------------------------- py/emitcommon.c | 4 ++-- py/emitnative.c | 48 +++++++++++++++++++++++++++++------------------- 5 files changed, 61 insertions(+), 68 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index c2dd914f0..4d985a07f 100644 --- a/py/compile.c +++ b/py/compile.c @@ -65,7 +65,7 @@ typedef enum { // we need a method table to do the lookup for the emitter functions #define EMIT(fun) (comp->emit_method_table->fun(comp->emit)) #define EMIT_ARG(fun, ...) (comp->emit_method_table->fun(comp->emit, __VA_ARGS__)) -#define EMIT_LOAD_FAST(qst, local_num) (comp->emit_method_table->load_id.fast(comp->emit, qst, local_num)) +#define EMIT_LOAD_FAST(qst, local_num) (comp->emit_method_table->load_id.local(comp->emit, qst, local_num, MP_EMIT_IDOP_LOCAL_FAST)) #define EMIT_LOAD_GLOBAL(qst) (comp->emit_method_table->load_id.global(comp->emit, qst)) #else @@ -73,7 +73,7 @@ typedef enum { // if we only have the bytecode emitter enabled then we can do a direct call to the functions #define EMIT(fun) (mp_emit_bc_##fun(comp->emit)) #define EMIT_ARG(fun, ...) (mp_emit_bc_##fun(comp->emit, __VA_ARGS__)) -#define EMIT_LOAD_FAST(qst, local_num) (mp_emit_bc_load_fast(comp->emit, qst, local_num)) +#define EMIT_LOAD_FAST(qst, local_num) (mp_emit_bc_load_local(comp->emit, qst, local_num, MP_EMIT_IDOP_LOCAL_FAST)) #define EMIT_LOAD_GLOBAL(qst) (mp_emit_bc_load_global(comp->emit, qst)) #endif diff --git a/py/emit.h b/py/emit.h index 270a40633..5fe3c3d1f 100644 --- a/py/emit.h +++ b/py/emit.h @@ -55,11 +55,14 @@ typedef enum { #define MP_EMIT_NATIVE_TYPE_RETURN (1) #define MP_EMIT_NATIVE_TYPE_ARG (2) +// Kind for emit_id_ops->local() +#define MP_EMIT_IDOP_LOCAL_FAST (0) +#define MP_EMIT_IDOP_LOCAL_DEREF (1) + typedef struct _emit_t emit_t; typedef struct _mp_emit_method_table_id_ops_t { - void (*fast)(emit_t *emit, qstr qst, mp_uint_t local_num); - void (*deref)(emit_t *emit, qstr qst, mp_uint_t local_num); + void (*local)(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); void (*name)(emit_t *emit, qstr qst); void (*global)(emit_t *emit, qstr qst); } mp_emit_method_table_id_ops_t; @@ -180,16 +183,13 @@ bool mp_emit_bc_last_emit_was_return_value(emit_t *emit); void mp_emit_bc_adjust_stack_size(emit_t *emit, mp_int_t delta); void mp_emit_bc_set_source_line(emit_t *emit, mp_uint_t line); -void mp_emit_bc_load_fast(emit_t *emit, qstr qst, mp_uint_t local_num); -void mp_emit_bc_load_deref(emit_t *emit, qstr qst, mp_uint_t local_num); +void mp_emit_bc_load_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); void mp_emit_bc_load_name(emit_t *emit, qstr qst); void mp_emit_bc_load_global(emit_t *emit, qstr qst); -void mp_emit_bc_store_fast(emit_t *emit, qstr qst, mp_uint_t local_num); -void mp_emit_bc_store_deref(emit_t *emit, qstr qst, mp_uint_t local_num); +void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); void mp_emit_bc_store_name(emit_t *emit, qstr qst); void mp_emit_bc_store_global(emit_t *emit, qstr qst); -void mp_emit_bc_delete_fast(emit_t *emit, qstr qst, mp_uint_t local_num); -void mp_emit_bc_delete_deref(emit_t *emit, qstr qst, mp_uint_t local_num); +void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); void mp_emit_bc_delete_name(emit_t *emit, qstr qst); void mp_emit_bc_delete_global(emit_t *emit, qstr qst); diff --git a/py/emitbc.c b/py/emitbc.c index b1b61ba67..28d32d4ea 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -556,22 +556,18 @@ void mp_emit_bc_load_null(emit_t *emit) { emit_write_bytecode_byte(emit, MP_BC_LOAD_NULL); } -void mp_emit_bc_load_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { +void mp_emit_bc_load_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { + MP_STATIC_ASSERT(MP_BC_LOAD_FAST_N + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_LOAD_FAST_N); + MP_STATIC_ASSERT(MP_BC_LOAD_FAST_N + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_LOAD_DEREF); (void)qst; emit_bc_pre(emit, 1); - if (local_num <= 15) { + if (kind == MP_EMIT_IDOP_LOCAL_FAST && local_num <= 15) { emit_write_bytecode_byte(emit, MP_BC_LOAD_FAST_MULTI + local_num); } else { - emit_write_bytecode_byte_uint(emit, MP_BC_LOAD_FAST_N, local_num); + emit_write_bytecode_byte_uint(emit, MP_BC_LOAD_FAST_N + kind, local_num); } } -void mp_emit_bc_load_deref(emit_t *emit, qstr qst, mp_uint_t local_num) { - (void)qst; - emit_bc_pre(emit, 1); - emit_write_bytecode_byte_uint(emit, MP_BC_LOAD_DEREF, local_num); -} - void mp_emit_bc_load_name(emit_t *emit, qstr qst) { (void)qst; emit_bc_pre(emit, 1); @@ -613,22 +609,18 @@ void mp_emit_bc_load_subscr(emit_t *emit) { emit_write_bytecode_byte(emit, MP_BC_LOAD_SUBSCR); } -void mp_emit_bc_store_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { +void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { + MP_STATIC_ASSERT(MP_BC_STORE_FAST_N + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_STORE_FAST_N); + MP_STATIC_ASSERT(MP_BC_STORE_FAST_N + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_STORE_DEREF); (void)qst; emit_bc_pre(emit, -1); - if (local_num <= 15) { + if (kind == MP_EMIT_IDOP_LOCAL_FAST && local_num <= 15) { emit_write_bytecode_byte(emit, MP_BC_STORE_FAST_MULTI + local_num); } else { - emit_write_bytecode_byte_uint(emit, MP_BC_STORE_FAST_N, local_num); + emit_write_bytecode_byte_uint(emit, MP_BC_STORE_FAST_N + kind, local_num); } } -void mp_emit_bc_store_deref(emit_t *emit, qstr qst, mp_uint_t local_num) { - (void)qst; - emit_bc_pre(emit, -1); - emit_write_bytecode_byte_uint(emit, MP_BC_STORE_DEREF, local_num); -} - void mp_emit_bc_store_name(emit_t *emit, qstr qst) { emit_bc_pre(emit, -1); emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_NAME, qst); @@ -652,14 +644,11 @@ void mp_emit_bc_store_subscr(emit_t *emit) { emit_write_bytecode_byte(emit, MP_BC_STORE_SUBSCR); } -void mp_emit_bc_delete_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { - (void)qst; - emit_write_bytecode_byte_uint(emit, MP_BC_DELETE_FAST, local_num); -} - -void mp_emit_bc_delete_deref(emit_t *emit, qstr qst, mp_uint_t local_num) { +void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { + MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_DELETE_FAST); + MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_DELETE_DEREF); (void)qst; - emit_write_bytecode_byte_uint(emit, MP_BC_DELETE_DEREF, local_num); + emit_write_bytecode_byte_uint(emit, MP_BC_DELETE_FAST + kind, local_num); } void mp_emit_bc_delete_name(emit_t *emit, qstr qst) { @@ -972,20 +961,17 @@ const emit_method_table_t emit_bc_method_table = { mp_emit_bc_set_source_line, { - mp_emit_bc_load_fast, - mp_emit_bc_load_deref, + mp_emit_bc_load_local, mp_emit_bc_load_name, mp_emit_bc_load_global, }, { - mp_emit_bc_store_fast, - mp_emit_bc_store_deref, + mp_emit_bc_store_local, mp_emit_bc_store_name, mp_emit_bc_store_global, }, { - mp_emit_bc_delete_fast, - mp_emit_bc_delete_deref, + mp_emit_bc_delete_local, mp_emit_bc_delete_name, mp_emit_bc_delete_global, }, @@ -1056,22 +1042,19 @@ const emit_method_table_t emit_bc_method_table = { }; #else const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_load_id_ops = { - mp_emit_bc_load_fast, - mp_emit_bc_load_deref, + mp_emit_bc_load_local, mp_emit_bc_load_name, mp_emit_bc_load_global, }; const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_store_id_ops = { - mp_emit_bc_store_fast, - mp_emit_bc_store_deref, + mp_emit_bc_store_local, mp_emit_bc_store_name, mp_emit_bc_store_global, }; const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_delete_id_ops = { - mp_emit_bc_delete_fast, - mp_emit_bc_delete_deref, + mp_emit_bc_delete_local, mp_emit_bc_delete_name, mp_emit_bc_delete_global, }; diff --git a/py/emitcommon.c b/py/emitcommon.c index 07b1dbb4c..47fd41ef2 100644 --- a/py/emitcommon.c +++ b/py/emitcommon.c @@ -67,10 +67,10 @@ void mp_emit_common_id_op(emit_t *emit, const mp_emit_method_table_id_ops_t *emi } else if (id->kind == ID_INFO_KIND_GLOBAL_EXPLICIT) { emit_method_table->global(emit, qst); } else if (id->kind == ID_INFO_KIND_LOCAL) { - emit_method_table->fast(emit, qst, id->local_num); + emit_method_table->local(emit, qst, id->local_num, MP_EMIT_IDOP_LOCAL_FAST); } else { assert(id->kind == ID_INFO_KIND_CELL || id->kind == ID_INFO_KIND_FREE); - emit_method_table->deref(emit, qst, id->local_num); + emit_method_table->local(emit, qst, id->local_num, MP_EMIT_IDOP_LOCAL_DEREF); } } diff --git a/py/emitnative.c b/py/emitnative.c index 7e017ba39..9e16ef4bd 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -938,6 +938,14 @@ STATIC void emit_native_load_deref(emit_t *emit, qstr qst, mp_uint_t local_num) emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } +STATIC void emit_native_load_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { + if (kind == MP_EMIT_IDOP_LOCAL_FAST) { + emit_native_load_fast(emit, qst, local_num); + } else { + emit_native_load_deref(emit, qst, local_num); + } +} + STATIC void emit_native_load_name(emit_t *emit, qstr qst) { DEBUG_printf("load_name(%s)\n", qstr_str(qst)); emit_native_pre(emit); @@ -1178,6 +1186,14 @@ STATIC void emit_native_store_deref(emit_t *emit, qstr qst, mp_uint_t local_num) emit_post(emit); } +STATIC void emit_native_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { + if (kind == MP_EMIT_IDOP_LOCAL_FAST) { + emit_native_store_fast(emit, qst, local_num); + } else { + emit_native_store_deref(emit, qst, local_num); + } +} + STATIC void emit_native_store_name(emit_t *emit, qstr qst) { // mp_store_name, but needs conversion of object (maybe have mp_viper_store_name(obj, type)) vtype_kind_t vtype; @@ -1389,19 +1405,16 @@ STATIC void emit_native_store_subscr(emit_t *emit) { } } -STATIC void emit_native_delete_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { - // TODO: This is not compliant implementation. We could use MP_OBJ_SENTINEL - // to mark deleted vars but then every var would need to be checked on - // each access. Very inefficient, so just set value to None to enable GC. - emit_native_load_const_tok(emit, MP_TOKEN_KW_NONE); - emit_native_store_fast(emit, qst, local_num); -} - -STATIC void emit_native_delete_deref(emit_t *emit, qstr qst, mp_uint_t local_num) { - // TODO implement me! - (void)emit; - (void)qst; - (void)local_num; +STATIC void emit_native_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { + if (kind == MP_EMIT_IDOP_LOCAL_FAST) { + // TODO: This is not compliant implementation. We could use MP_OBJ_SENTINEL + // to mark deleted vars but then every var would need to be checked on + // each access. Very inefficient, so just set value to None to enable GC. + emit_native_load_const_tok(emit, MP_TOKEN_KW_NONE); + emit_native_store_fast(emit, qst, local_num); + } else { + // TODO implement me! + } } STATIC void emit_native_delete_name(emit_t *emit, qstr qst) { @@ -2192,20 +2205,17 @@ const emit_method_table_t EXPORT_FUN(method_table) = { emit_native_set_source_line, { - emit_native_load_fast, - emit_native_load_deref, + emit_native_load_local, emit_native_load_name, emit_native_load_global, }, { - emit_native_store_fast, - emit_native_store_deref, + emit_native_store_local, emit_native_store_name, emit_native_store_global, }, { - emit_native_delete_fast, - emit_native_delete_deref, + emit_native_delete_local, emit_native_delete_name, emit_native_delete_global, }, -- cgit v1.2.3 From e686c940525c5b87d02e3dd7bfcdec23cfa996dd Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 19 May 2018 00:30:42 +1000 Subject: py/emit: Combine yield value and yield-from emit funcs into one. Reduces code size by: bare-arm: -24 minimal x86: -72 unix x64: -200 unix nanbox: -72 stm32: -52 cc3200: -32 esp8266: -84 esp32: -24 --- py/compile.c | 8 ++++---- py/emit.h | 10 ++++++---- py/emitbc.c | 16 +++++----------- py/emitnative.c | 11 +++-------- tests/micropython/viper_error.py.exp | 2 +- 5 files changed, 19 insertions(+), 28 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 4d985a07f..743034e57 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1699,7 +1699,7 @@ STATIC void compile_with_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { STATIC void compile_yield_from(compiler_t *comp) { EMIT_ARG(get_iter, false); EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); - EMIT(yield_from); + EMIT_ARG(yield, MP_EMIT_YIELD_FROM); } #if MICROPY_PY_ASYNC_AWAIT @@ -2621,14 +2621,14 @@ STATIC void compile_yield_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { } if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) { EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); - EMIT(yield_value); + EMIT_ARG(yield, MP_EMIT_YIELD_VALUE); } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_yield_arg_from)) { pns = (mp_parse_node_struct_t*)pns->nodes[0]; compile_node(comp, pns->nodes[0]); compile_yield_from(comp); } else { compile_node(comp, pns->nodes[0]); - EMIT(yield_value); + EMIT_ARG(yield, MP_EMIT_YIELD_VALUE); } } @@ -2862,7 +2862,7 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, mp_parse_node_struct_t *pn // no more nested if/for; compile inner expression compile_node(comp, pn_inner_expr); if (comp->scope_cur->kind == SCOPE_GEN_EXPR) { - EMIT(yield_value); + EMIT_ARG(yield, MP_EMIT_YIELD_VALUE); EMIT(pop_top); } else { EMIT_ARG(store_comp, comp->scope_cur->kind, 4 * for_depth + 5); diff --git a/py/emit.h b/py/emit.h index 5fe3c3d1f..c85df0a67 100644 --- a/py/emit.h +++ b/py/emit.h @@ -59,6 +59,10 @@ typedef enum { #define MP_EMIT_IDOP_LOCAL_FAST (0) #define MP_EMIT_IDOP_LOCAL_DEREF (1) +// Kind for emit->yield() +#define MP_EMIT_YIELD_VALUE (0) +#define MP_EMIT_YIELD_FROM (1) + typedef struct _emit_t emit_t; typedef struct _mp_emit_method_table_id_ops_t { @@ -137,8 +141,7 @@ typedef struct _emit_method_table_t { void (*call_method)(emit_t *emit, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags); void (*return_value)(emit_t *emit); void (*raise_varargs)(emit_t *emit, mp_uint_t n_args); - void (*yield_value)(emit_t *emit); - void (*yield_from)(emit_t *emit); + void (*yield)(emit_t *emit, int kind); // these methods are used to control entry to/exit from an exception handler // they may or may not emit code @@ -252,8 +255,7 @@ void mp_emit_bc_call_function(emit_t *emit, mp_uint_t n_positional, mp_uint_t n_ void mp_emit_bc_call_method(emit_t *emit, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags); void mp_emit_bc_return_value(emit_t *emit); void mp_emit_bc_raise_varargs(emit_t *emit, mp_uint_t n_args); -void mp_emit_bc_yield_value(emit_t *emit); -void mp_emit_bc_yield_from(emit_t *emit); +void mp_emit_bc_yield(emit_t *emit, int kind); void mp_emit_bc_start_except_handler(emit_t *emit); void mp_emit_bc_end_except_handler(emit_t *emit); diff --git a/py/emitbc.c b/py/emitbc.c index 28d32d4ea..bff2c5a4c 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -931,16 +931,11 @@ void mp_emit_bc_raise_varargs(emit_t *emit, mp_uint_t n_args) { emit_write_bytecode_byte_byte(emit, MP_BC_RAISE_VARARGS, n_args); } -void mp_emit_bc_yield_value(emit_t *emit) { - emit_bc_pre(emit, 0); - emit->scope->scope_flags |= MP_SCOPE_FLAG_GENERATOR; - emit_write_bytecode_byte(emit, MP_BC_YIELD_VALUE); -} - -void mp_emit_bc_yield_from(emit_t *emit) { - emit_bc_pre(emit, -1); +void mp_emit_bc_yield(emit_t *emit, int kind) { + MP_STATIC_ASSERT(MP_BC_YIELD_VALUE + 1 == MP_BC_YIELD_FROM); + emit_bc_pre(emit, -kind); emit->scope->scope_flags |= MP_SCOPE_FLAG_GENERATOR; - emit_write_bytecode_byte(emit, MP_BC_YIELD_FROM); + emit_write_bytecode_byte(emit, MP_BC_YIELD_VALUE + kind); } void mp_emit_bc_start_except_handler(emit_t *emit) { @@ -1034,8 +1029,7 @@ const emit_method_table_t emit_bc_method_table = { mp_emit_bc_call_method, mp_emit_bc_return_value, mp_emit_bc_raise_varargs, - mp_emit_bc_yield_value, - mp_emit_bc_yield_from, + mp_emit_bc_yield, mp_emit_bc_start_except_handler, mp_emit_bc_end_except_handler, diff --git a/py/emitnative.c b/py/emitnative.c index 9e16ef4bd..26bcebfcb 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -2170,16 +2170,12 @@ STATIC void emit_native_raise_varargs(emit_t *emit, mp_uint_t n_args) { emit_call(emit, MP_F_NATIVE_RAISE); } -STATIC void emit_native_yield_value(emit_t *emit) { +STATIC void emit_native_yield(emit_t *emit, int kind) { // not supported (for now) (void)emit; + (void)kind; mp_raise_NotImplementedError("native yield"); } -STATIC void emit_native_yield_from(emit_t *emit) { - // not supported (for now) - (void)emit; - mp_raise_NotImplementedError("native yield from"); -} STATIC void emit_native_start_except_handler(emit_t *emit) { // This instruction follows an nlr_pop, so the stack counter is back to zero, when really @@ -2278,8 +2274,7 @@ const emit_method_table_t EXPORT_FUN(method_table) = { emit_native_call_method, emit_native_return_value, emit_native_raise_varargs, - emit_native_yield_value, - emit_native_yield_from, + emit_native_yield, emit_native_start_except_handler, emit_native_end_except_handler, diff --git a/tests/micropython/viper_error.py.exp b/tests/micropython/viper_error.py.exp index a44fb3ff0..3a8cb0299 100644 --- a/tests/micropython/viper_error.py.exp +++ b/tests/micropython/viper_error.py.exp @@ -20,6 +20,6 @@ ViperTypeError('unary op __neg__ not implemented',) ViperTypeError('unary op __invert__ not implemented',) ViperTypeError('binary op not implemented',) NotImplementedError('native yield',) -NotImplementedError('native yield from',) +NotImplementedError('native yield',) NotImplementedError('conversion to object',) NotImplementedError('casting',) -- cgit v1.2.3 From 26b5754092134b53e03eed8c5e6c580d28a2a829 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 19 May 2018 00:41:40 +1000 Subject: py/emit: Combine build tuple/list/map emit funcs into one. Reduces code size by: bare-arm: -24 minimal x86: -192 unix x64: -288 unix nanbox: -184 stm32: -72 cc3200: -16 esp8266: -148 esp32: -32 --- py/compile.c | 32 ++++++++++++++++---------------- py/emit.h | 13 +++++++------ py/emitbc.c | 27 +++++++++++---------------- py/emitnative.c | 30 ++++++++++-------------------- 4 files changed, 44 insertions(+), 58 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 743034e57..6f0be8fa8 100644 --- a/py/compile.c +++ b/py/compile.c @@ -273,7 +273,7 @@ STATIC void c_tuple(compiler_t *comp, mp_parse_node_t pn, mp_parse_node_struct_t } total += n; } - EMIT_ARG(build_tuple, total); + EMIT_ARG(build, total, MP_EMIT_BUILD_TUPLE); } STATIC void compile_generic_tuple(compiler_t *comp, mp_parse_node_struct_t *pns) { @@ -652,12 +652,12 @@ STATIC void compile_funcdef_lambdef_param(compiler_t *comp, mp_parse_node_t pn) // in MicroPython we put the default positional parameters into a tuple using the bytecode // we need to do this here before we start building the map for the default keywords if (comp->num_default_params > 0) { - EMIT_ARG(build_tuple, comp->num_default_params); + EMIT_ARG(build, comp->num_default_params, MP_EMIT_BUILD_TUPLE); } else { EMIT(load_null); // sentinel indicating empty default positional args } // first default dict param, so make the map - EMIT_ARG(build_map, 0); + EMIT_ARG(build, 0, MP_EMIT_BUILD_MAP); } // compile value then key, then store it to the dict @@ -693,7 +693,7 @@ STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, mp_parse_n // in MicroPython we put the default positional parameters into a tuple using the bytecode // the default keywords args may have already made the tuple; if not, do it now if (comp->num_default_params > 0 && comp->num_dict_params == 0) { - EMIT_ARG(build_tuple, comp->num_default_params); + EMIT_ARG(build, comp->num_default_params, MP_EMIT_BUILD_TUPLE); EMIT(load_null); // sentinel indicating empty default keyword args } @@ -1122,7 +1122,7 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) { // build the "fromlist" tuple EMIT_ARG(load_const_str, MP_QSTR__star_); - EMIT_ARG(build_tuple, 1); + EMIT_ARG(build, 1, MP_EMIT_BUILD_TUPLE); // do the import qstr dummy_q; @@ -1141,7 +1141,7 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) { qstr id2 = MP_PARSE_NODE_LEAF_ARG(pns3->nodes[0]); // should be id EMIT_ARG(load_const_str, id2); } - EMIT_ARG(build_tuple, n); + EMIT_ARG(build, n, MP_EMIT_BUILD_TUPLE); // do the import qstr dummy_q; @@ -2397,7 +2397,7 @@ STATIC void compile_atom_paren(compiler_t *comp, mp_parse_node_struct_t *pns) { STATIC void compile_atom_bracket(compiler_t *comp, mp_parse_node_struct_t *pns) { if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) { // empty list - EMIT_ARG(build_list, 0); + EMIT_ARG(build, 0, MP_EMIT_BUILD_LIST); } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_comp)) { mp_parse_node_struct_t *pns2 = (mp_parse_node_struct_t*)pns->nodes[0]; if (MP_PARSE_NODE_IS_STRUCT(pns2->nodes[1])) { @@ -2406,12 +2406,12 @@ STATIC void compile_atom_bracket(compiler_t *comp, mp_parse_node_struct_t *pns) // list of one item, with trailing comma assert(MP_PARSE_NODE_IS_NULL(pns3->nodes[0])); compile_node(comp, pns2->nodes[0]); - EMIT_ARG(build_list, 1); + EMIT_ARG(build, 1, MP_EMIT_BUILD_LIST); } else if (MP_PARSE_NODE_STRUCT_KIND(pns3) == PN_testlist_comp_3c) { // list of many items compile_node(comp, pns2->nodes[0]); compile_generic_all_nodes(comp, pns3); - EMIT_ARG(build_list, 1 + MP_PARSE_NODE_STRUCT_NUM_NODES(pns3)); + EMIT_ARG(build, 1 + MP_PARSE_NODE_STRUCT_NUM_NODES(pns3), MP_EMIT_BUILD_LIST); } else if (MP_PARSE_NODE_STRUCT_KIND(pns3) == PN_comp_for) { // list comprehension compile_comprehension(comp, pns2, SCOPE_LIST_COMP); @@ -2424,12 +2424,12 @@ STATIC void compile_atom_bracket(compiler_t *comp, mp_parse_node_struct_t *pns) list_with_2_items: compile_node(comp, pns2->nodes[0]); compile_node(comp, pns2->nodes[1]); - EMIT_ARG(build_list, 2); + EMIT_ARG(build, 2, MP_EMIT_BUILD_LIST); } } else { // list with 1 item compile_node(comp, pns->nodes[0]); - EMIT_ARG(build_list, 1); + EMIT_ARG(build, 1, MP_EMIT_BUILD_LIST); } } @@ -2437,12 +2437,12 @@ STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) { mp_parse_node_t pn = pns->nodes[0]; if (MP_PARSE_NODE_IS_NULL(pn)) { // empty dict - EMIT_ARG(build_map, 0); + EMIT_ARG(build, 0, MP_EMIT_BUILD_MAP); } else if (MP_PARSE_NODE_IS_STRUCT(pn)) { pns = (mp_parse_node_struct_t*)pn; if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_dictorsetmaker_item) { // dict with one element - EMIT_ARG(build_map, 1); + EMIT_ARG(build, 1, MP_EMIT_BUILD_MAP); compile_node(comp, pn); EMIT(store_map); } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_dictorsetmaker) { @@ -2459,7 +2459,7 @@ STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) { bool is_dict; if (!MICROPY_PY_BUILTINS_SET || MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_dictorsetmaker_item)) { // a dictionary - EMIT_ARG(build_map, 1 + n); + EMIT_ARG(build, 1 + n, MP_EMIT_BUILD_MAP); compile_node(comp, pns->nodes[0]); EMIT(store_map); is_dict = true; @@ -3040,9 +3040,9 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { } if (scope->kind == SCOPE_LIST_COMP) { - EMIT_ARG(build_list, 0); + EMIT_ARG(build, 0, MP_EMIT_BUILD_LIST); } else if (scope->kind == SCOPE_DICT_COMP) { - EMIT_ARG(build_map, 0); + EMIT_ARG(build, 0, MP_EMIT_BUILD_MAP); #if MICROPY_PY_BUILTINS_SET } else if (scope->kind == SCOPE_SET_COMP) { EMIT_ARG(build_set, 0); diff --git a/py/emit.h b/py/emit.h index c85df0a67..8caf92882 100644 --- a/py/emit.h +++ b/py/emit.h @@ -59,6 +59,11 @@ typedef enum { #define MP_EMIT_IDOP_LOCAL_FAST (0) #define MP_EMIT_IDOP_LOCAL_DEREF (1) +// Kind for emit->build() +#define MP_EMIT_BUILD_TUPLE (0) +#define MP_EMIT_BUILD_LIST (1) +#define MP_EMIT_BUILD_MAP (3) + // Kind for emit->yield() #define MP_EMIT_YIELD_VALUE (0) #define MP_EMIT_YIELD_FROM (1) @@ -122,9 +127,7 @@ typedef struct _emit_method_table_t { void (*pop_except)(emit_t *emit); void (*unary_op)(emit_t *emit, mp_unary_op_t op); void (*binary_op)(emit_t *emit, mp_binary_op_t op); - void (*build_tuple)(emit_t *emit, mp_uint_t n_args); - void (*build_list)(emit_t *emit, mp_uint_t n_args); - void (*build_map)(emit_t *emit, mp_uint_t n_args); + void (*build)(emit_t *emit, mp_uint_t n_args, int kind); void (*store_map)(emit_t *emit); #if MICROPY_PY_BUILTINS_SET void (*build_set)(emit_t *emit, mp_uint_t n_args); @@ -236,9 +239,7 @@ void mp_emit_bc_pop_block(emit_t *emit); void mp_emit_bc_pop_except(emit_t *emit); void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op); void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op); -void mp_emit_bc_build_tuple(emit_t *emit, mp_uint_t n_args); -void mp_emit_bc_build_list(emit_t *emit, mp_uint_t n_args); -void mp_emit_bc_build_map(emit_t *emit, mp_uint_t n_args); +void mp_emit_bc_build(emit_t *emit, mp_uint_t n_args, int kind); void mp_emit_bc_store_map(emit_t *emit); #if MICROPY_PY_BUILTINS_SET void mp_emit_bc_build_set(emit_t *emit, mp_uint_t n_args); diff --git a/py/emitbc.c b/py/emitbc.c index bff2c5a4c..3cfc05935 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -816,19 +816,16 @@ void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op) { } } -void mp_emit_bc_build_tuple(emit_t *emit, mp_uint_t n_args) { - emit_bc_pre(emit, 1 - n_args); - emit_write_bytecode_byte_uint(emit, MP_BC_BUILD_TUPLE, n_args); -} - -void mp_emit_bc_build_list(emit_t *emit, mp_uint_t n_args) { - emit_bc_pre(emit, 1 - n_args); - emit_write_bytecode_byte_uint(emit, MP_BC_BUILD_LIST, n_args); -} - -void mp_emit_bc_build_map(emit_t *emit, mp_uint_t n_args) { - emit_bc_pre(emit, 1); - emit_write_bytecode_byte_uint(emit, MP_BC_BUILD_MAP, n_args); +void mp_emit_bc_build(emit_t *emit, mp_uint_t n_args, int kind) { + MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_TUPLE == MP_BC_BUILD_TUPLE); + MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_LIST == MP_BC_BUILD_LIST); + MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_MAP == MP_BC_BUILD_MAP); + if (kind == MP_EMIT_BUILD_MAP) { + emit_bc_pre(emit, 1); + } else { + emit_bc_pre(emit, 1 - n_args); + } + emit_write_bytecode_byte_uint(emit, MP_BC_BUILD_TUPLE + kind, n_args); } void mp_emit_bc_store_map(emit_t *emit) { @@ -1010,9 +1007,7 @@ const emit_method_table_t emit_bc_method_table = { mp_emit_bc_pop_except, mp_emit_bc_unary_op, mp_emit_bc_binary_op, - mp_emit_bc_build_tuple, - mp_emit_bc_build_list, - mp_emit_bc_build_map, + mp_emit_bc_build, mp_emit_bc_store_map, #if MICROPY_PY_BUILTINS_SET mp_emit_bc_build_set, diff --git a/py/emitnative.c b/py/emitnative.c index 26bcebfcb..3b4e9edb2 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1923,26 +1923,18 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { } } -STATIC void emit_native_build_tuple(emit_t *emit, mp_uint_t n_args) { +STATIC void emit_native_build(emit_t *emit, mp_uint_t n_args, int kind) { // for viper: call runtime, with types of args // if wrapped in byte_array, or something, allocates memory and fills it + MP_STATIC_ASSERT(MP_F_BUILD_TUPLE + MP_EMIT_BUILD_TUPLE == MP_F_BUILD_TUPLE); + MP_STATIC_ASSERT(MP_F_BUILD_TUPLE + MP_EMIT_BUILD_LIST == MP_F_BUILD_LIST); + MP_STATIC_ASSERT(MP_F_BUILD_TUPLE + MP_EMIT_BUILD_MAP == MP_F_BUILD_MAP); emit_native_pre(emit); - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_2, n_args); // pointer to items - emit_call_with_imm_arg(emit, MP_F_BUILD_TUPLE, n_args, REG_ARG_1); - emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); // new tuple -} - -STATIC void emit_native_build_list(emit_t *emit, mp_uint_t n_args) { - emit_native_pre(emit); - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_2, n_args); // pointer to items - emit_call_with_imm_arg(emit, MP_F_BUILD_LIST, n_args, REG_ARG_1); - emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); // new list -} - -STATIC void emit_native_build_map(emit_t *emit, mp_uint_t n_args) { - emit_native_pre(emit); - emit_call_with_imm_arg(emit, MP_F_BUILD_MAP, n_args, REG_ARG_1); - emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); // new map + if (kind == MP_EMIT_BUILD_TUPLE || kind == MP_EMIT_BUILD_LIST) { + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_2, n_args); // pointer to items + } + emit_call_with_imm_arg(emit, MP_F_BUILD_TUPLE + kind, n_args, REG_ARG_1); + emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); // new tuple/list/map } STATIC void emit_native_store_map(emit_t *emit) { @@ -2255,9 +2247,7 @@ const emit_method_table_t EXPORT_FUN(method_table) = { emit_native_pop_except, emit_native_unary_op, emit_native_binary_op, - emit_native_build_tuple, - emit_native_build_list, - emit_native_build_map, + emit_native_build, emit_native_store_map, #if MICROPY_PY_BUILTINS_SET emit_native_build_set, -- cgit v1.2.3 From d298013939b38fb05961cf05c03ac3aef6a4f00c Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 21:16:30 +1000 Subject: py/emit: Combine name and global into one func for load/store/delete. Reduces code size by: bare-arm: -56 minimal x86: -300 unix x64: -576 unix nanbox: -300 stm32: -164 cc3200: -56 esp8266: -236 esp32: -76 --- py/compile.c | 4 +-- py/emit.h | 16 ++++----- py/emitbc.c | 43 +++++++----------------- py/emitcommon.c | 4 +-- py/emitnative.c | 101 ++++++++++++++++++++++++++++---------------------------- 5 files changed, 74 insertions(+), 94 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 6f0be8fa8..3d443c6d8 100644 --- a/py/compile.c +++ b/py/compile.c @@ -66,7 +66,7 @@ typedef enum { #define EMIT(fun) (comp->emit_method_table->fun(comp->emit)) #define EMIT_ARG(fun, ...) (comp->emit_method_table->fun(comp->emit, __VA_ARGS__)) #define EMIT_LOAD_FAST(qst, local_num) (comp->emit_method_table->load_id.local(comp->emit, qst, local_num, MP_EMIT_IDOP_LOCAL_FAST)) -#define EMIT_LOAD_GLOBAL(qst) (comp->emit_method_table->load_id.global(comp->emit, qst)) +#define EMIT_LOAD_GLOBAL(qst) (comp->emit_method_table->load_id.global(comp->emit, qst, MP_EMIT_IDOP_GLOBAL_GLOBAL)) #else @@ -74,7 +74,7 @@ typedef enum { #define EMIT(fun) (mp_emit_bc_##fun(comp->emit)) #define EMIT_ARG(fun, ...) (mp_emit_bc_##fun(comp->emit, __VA_ARGS__)) #define EMIT_LOAD_FAST(qst, local_num) (mp_emit_bc_load_local(comp->emit, qst, local_num, MP_EMIT_IDOP_LOCAL_FAST)) -#define EMIT_LOAD_GLOBAL(qst) (mp_emit_bc_load_global(comp->emit, qst)) +#define EMIT_LOAD_GLOBAL(qst) (mp_emit_bc_load_global(comp->emit, qst, MP_EMIT_IDOP_GLOBAL_GLOBAL)) #endif diff --git a/py/emit.h b/py/emit.h index 8caf92882..3574d00e3 100644 --- a/py/emit.h +++ b/py/emit.h @@ -59,6 +59,10 @@ typedef enum { #define MP_EMIT_IDOP_LOCAL_FAST (0) #define MP_EMIT_IDOP_LOCAL_DEREF (1) +// Kind for emit_id_ops->global() +#define MP_EMIT_IDOP_GLOBAL_NAME (0) +#define MP_EMIT_IDOP_GLOBAL_GLOBAL (1) + // Kind for emit->build() #define MP_EMIT_BUILD_TUPLE (0) #define MP_EMIT_BUILD_LIST (1) @@ -72,8 +76,7 @@ typedef struct _emit_t emit_t; typedef struct _mp_emit_method_table_id_ops_t { void (*local)(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); - void (*name)(emit_t *emit, qstr qst); - void (*global)(emit_t *emit, qstr qst); + void (*global)(emit_t *emit, qstr qst, int kind); } mp_emit_method_table_id_ops_t; typedef struct _emit_method_table_t { @@ -190,14 +193,11 @@ void mp_emit_bc_adjust_stack_size(emit_t *emit, mp_int_t delta); void mp_emit_bc_set_source_line(emit_t *emit, mp_uint_t line); void mp_emit_bc_load_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); -void mp_emit_bc_load_name(emit_t *emit, qstr qst); -void mp_emit_bc_load_global(emit_t *emit, qstr qst); +void mp_emit_bc_load_global(emit_t *emit, qstr qst, int kind); void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); -void mp_emit_bc_store_name(emit_t *emit, qstr qst); -void mp_emit_bc_store_global(emit_t *emit, qstr qst); +void mp_emit_bc_store_global(emit_t *emit, qstr qst, int kind); void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); -void mp_emit_bc_delete_name(emit_t *emit, qstr qst); -void mp_emit_bc_delete_global(emit_t *emit, qstr qst); +void mp_emit_bc_delete_global(emit_t *emit, qstr qst, int kind); void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l); void mp_emit_bc_import_name(emit_t *emit, qstr qst); diff --git a/py/emitbc.c b/py/emitbc.c index 3cfc05935..ed043de3a 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -568,19 +568,12 @@ void mp_emit_bc_load_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind } } -void mp_emit_bc_load_name(emit_t *emit, qstr qst) { +void mp_emit_bc_load_global(emit_t *emit, qstr qst, int kind) { + MP_STATIC_ASSERT(MP_BC_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_LOAD_NAME); + MP_STATIC_ASSERT(MP_BC_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_LOAD_GLOBAL); (void)qst; emit_bc_pre(emit, 1); - emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_NAME, qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { - emit_write_bytecode_byte(emit, 0); - } -} - -void mp_emit_bc_load_global(emit_t *emit, qstr qst) { - (void)qst; - emit_bc_pre(emit, 1); - emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_GLOBAL, qst); + emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_NAME + kind, qst); if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { emit_write_bytecode_byte(emit, 0); } @@ -621,14 +614,11 @@ void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kin } } -void mp_emit_bc_store_name(emit_t *emit, qstr qst) { +void mp_emit_bc_store_global(emit_t *emit, qstr qst, int kind) { + MP_STATIC_ASSERT(MP_BC_STORE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_STORE_NAME); + MP_STATIC_ASSERT(MP_BC_STORE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_STORE_GLOBAL); emit_bc_pre(emit, -1); - emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_NAME, qst); -} - -void mp_emit_bc_store_global(emit_t *emit, qstr qst) { - emit_bc_pre(emit, -1); - emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_GLOBAL, qst); + emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_NAME + kind, qst); } void mp_emit_bc_store_attr(emit_t *emit, qstr qst) { @@ -651,14 +641,11 @@ void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int ki emit_write_bytecode_byte_uint(emit, MP_BC_DELETE_FAST + kind, local_num); } -void mp_emit_bc_delete_name(emit_t *emit, qstr qst) { - emit_bc_pre(emit, 0); - emit_write_bytecode_byte_qstr(emit, MP_BC_DELETE_NAME, qst); -} - -void mp_emit_bc_delete_global(emit_t *emit, qstr qst) { +void mp_emit_bc_delete_global(emit_t *emit, qstr qst, int kind) { + MP_STATIC_ASSERT(MP_BC_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_DELETE_NAME); + MP_STATIC_ASSERT(MP_BC_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_DELETE_GLOBAL); emit_bc_pre(emit, 0); - emit_write_bytecode_byte_qstr(emit, MP_BC_DELETE_GLOBAL, qst); + emit_write_bytecode_byte_qstr(emit, MP_BC_DELETE_NAME + kind, qst); } void mp_emit_bc_delete_attr(emit_t *emit, qstr qst) { @@ -954,17 +941,14 @@ const emit_method_table_t emit_bc_method_table = { { mp_emit_bc_load_local, - mp_emit_bc_load_name, mp_emit_bc_load_global, }, { mp_emit_bc_store_local, - mp_emit_bc_store_name, mp_emit_bc_store_global, }, { mp_emit_bc_delete_local, - mp_emit_bc_delete_name, mp_emit_bc_delete_global, }, @@ -1032,19 +1016,16 @@ const emit_method_table_t emit_bc_method_table = { #else const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_load_id_ops = { mp_emit_bc_load_local, - mp_emit_bc_load_name, mp_emit_bc_load_global, }; const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_store_id_ops = { mp_emit_bc_store_local, - mp_emit_bc_store_name, mp_emit_bc_store_global, }; const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_delete_id_ops = { mp_emit_bc_delete_local, - mp_emit_bc_delete_name, mp_emit_bc_delete_global, }; #endif diff --git a/py/emitcommon.c b/py/emitcommon.c index 47fd41ef2..89cc2c959 100644 --- a/py/emitcommon.c +++ b/py/emitcommon.c @@ -63,9 +63,9 @@ void mp_emit_common_id_op(emit_t *emit, const mp_emit_method_table_id_ops_t *emi // call the emit backend with the correct code if (id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { - emit_method_table->name(emit, qst); + emit_method_table->global(emit, qst, MP_EMIT_IDOP_GLOBAL_NAME); } else if (id->kind == ID_INFO_KIND_GLOBAL_EXPLICIT) { - emit_method_table->global(emit, qst); + emit_method_table->global(emit, qst, MP_EMIT_IDOP_GLOBAL_GLOBAL); } else if (id->kind == ID_INFO_KIND_LOCAL) { emit_method_table->local(emit, qst, id->local_num, MP_EMIT_IDOP_LOCAL_FAST); } else { diff --git a/py/emitnative.c b/py/emitnative.c index 3b4e9edb2..203cacff4 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -946,33 +946,39 @@ STATIC void emit_native_load_local(emit_t *emit, qstr qst, mp_uint_t local_num, } } -STATIC void emit_native_load_name(emit_t *emit, qstr qst) { - DEBUG_printf("load_name(%s)\n", qstr_str(qst)); +STATIC void emit_native_load_global(emit_t *emit, qstr qst, int kind) { + MP_STATIC_ASSERT(MP_F_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_F_LOAD_NAME); + MP_STATIC_ASSERT(MP_F_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_F_LOAD_GLOBAL); emit_native_pre(emit); - emit_call_with_imm_arg(emit, MP_F_LOAD_NAME, qst, REG_ARG_1); - emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); -} - -STATIC void emit_native_load_global(emit_t *emit, qstr qst) { - DEBUG_printf("load_global(%s)\n", qstr_str(qst)); - emit_native_pre(emit); - // check for builtin casting operators - if (emit->do_viper_types && qst == MP_QSTR_int) { - emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_INT); - } else if (emit->do_viper_types && qst == MP_QSTR_uint) { - emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_UINT); - } else if (emit->do_viper_types && qst == MP_QSTR_ptr) { - emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_PTR); - } else if (emit->do_viper_types && qst == MP_QSTR_ptr8) { - emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_PTR8); - } else if (emit->do_viper_types && qst == MP_QSTR_ptr16) { - emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_PTR16); - } else if (emit->do_viper_types && qst == MP_QSTR_ptr32) { - emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_PTR32); + if (kind == MP_EMIT_IDOP_GLOBAL_NAME) { + DEBUG_printf("load_name(%s)\n", qstr_str(qst)); } else { - emit_call_with_imm_arg(emit, MP_F_LOAD_GLOBAL, qst, REG_ARG_1); - emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); + DEBUG_printf("load_global(%s)\n", qstr_str(qst)); + if (emit->do_viper_types) { + // check for builtin casting operators + if (qst == MP_QSTR_int) { + emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_INT); + return; + } else if (qst == MP_QSTR_uint) { + emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_UINT); + return; + } else if (qst == MP_QSTR_ptr) { + emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_PTR); + return; + } else if (qst == MP_QSTR_ptr8) { + emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_PTR8); + return; + } else if (qst == MP_QSTR_ptr16) { + emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_PTR16); + return; + } else if (qst == MP_QSTR_ptr32) { + emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, VTYPE_PTR32); + return; + } + } } + emit_call_with_imm_arg(emit, MP_F_LOAD_NAME + kind, qst, REG_ARG_1); + emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } STATIC void emit_native_load_attr(emit_t *emit, qstr qst) { @@ -1194,25 +1200,25 @@ STATIC void emit_native_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, } } -STATIC void emit_native_store_name(emit_t *emit, qstr qst) { - // mp_store_name, but needs conversion of object (maybe have mp_viper_store_name(obj, type)) - vtype_kind_t vtype; - emit_pre_pop_reg(emit, &vtype, REG_ARG_2); - assert(vtype == VTYPE_PYOBJ); - emit_call_with_imm_arg(emit, MP_F_STORE_NAME, qst, REG_ARG_1); // arg1 = name - emit_post(emit); -} - -STATIC void emit_native_store_global(emit_t *emit, qstr qst) { - vtype_kind_t vtype = peek_vtype(emit, 0); - if (vtype == VTYPE_PYOBJ) { +STATIC void emit_native_store_global(emit_t *emit, qstr qst, int kind) { + MP_STATIC_ASSERT(MP_F_STORE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_F_STORE_NAME); + MP_STATIC_ASSERT(MP_F_STORE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_F_STORE_GLOBAL); + if (kind == MP_EMIT_IDOP_GLOBAL_NAME) { + // mp_store_name, but needs conversion of object (maybe have mp_viper_store_name(obj, type)) + vtype_kind_t vtype; emit_pre_pop_reg(emit, &vtype, REG_ARG_2); + assert(vtype == VTYPE_PYOBJ); } else { - emit_pre_pop_reg(emit, &vtype, REG_ARG_1); - emit_call_with_imm_arg(emit, MP_F_CONVERT_NATIVE_TO_OBJ, vtype, REG_ARG_2); // arg2 = type - ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_RET); + vtype_kind_t vtype = peek_vtype(emit, 0); + if (vtype == VTYPE_PYOBJ) { + emit_pre_pop_reg(emit, &vtype, REG_ARG_2); + } else { + emit_pre_pop_reg(emit, &vtype, REG_ARG_1); + emit_call_with_imm_arg(emit, MP_F_CONVERT_NATIVE_TO_OBJ, vtype, REG_ARG_2); // arg2 = type + ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_RET); + } } - emit_call_with_imm_arg(emit, MP_F_STORE_GLOBAL, qst, REG_ARG_1); // arg1 = name + emit_call_with_imm_arg(emit, MP_F_STORE_NAME + kind, qst, REG_ARG_1); // arg1 = name emit_post(emit); } @@ -1417,15 +1423,11 @@ STATIC void emit_native_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num } } -STATIC void emit_native_delete_name(emit_t *emit, qstr qst) { - emit_native_pre(emit); - emit_call_with_imm_arg(emit, MP_F_DELETE_NAME, qst, REG_ARG_1); - emit_post(emit); -} - -STATIC void emit_native_delete_global(emit_t *emit, qstr qst) { +STATIC void emit_native_delete_global(emit_t *emit, qstr qst, int kind) { + MP_STATIC_ASSERT(MP_F_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_F_DELETE_NAME); + MP_STATIC_ASSERT(MP_F_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_F_DELETE_GLOBAL); emit_native_pre(emit); - emit_call_with_imm_arg(emit, MP_F_DELETE_GLOBAL, qst, REG_ARG_1); + emit_call_with_imm_arg(emit, MP_F_DELETE_NAME + kind, qst, REG_ARG_1); emit_post(emit); } @@ -2194,17 +2196,14 @@ const emit_method_table_t EXPORT_FUN(method_table) = { { emit_native_load_local, - emit_native_load_name, emit_native_load_global, }, { emit_native_store_local, - emit_native_store_name, emit_native_store_global, }, { emit_native_delete_local, - emit_native_delete_name, emit_native_delete_global, }, -- cgit v1.2.3 From a4941a8ba49e3503f1a87f318b79b137a70b803b Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 21:31:56 +1000 Subject: py/emit: Combine load/store/delete subscr into one emit function. Reduces code size by: bare-arm: -8 minimal x86: -104 unix x64: -312 unix nanbox: -120 stm32: -60 cc3200: -16 esp8266: -92 esp32: -24 --- py/compile.c | 10 +++++----- py/emit.h | 13 +++++++------ py/emitbc.c | 30 +++++++++++++----------------- py/emitnative.c | 14 +++++++++++--- 4 files changed, 36 insertions(+), 31 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 3d443c6d8..6aba2b896 100644 --- a/py/compile.c +++ b/py/compile.c @@ -366,14 +366,14 @@ STATIC void c_assign_atom_expr(compiler_t *comp, mp_parse_node_struct_t *pns, as if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_bracket) { if (assign_kind == ASSIGN_AUG_STORE) { EMIT(rot_three); - EMIT(store_subscr); + EMIT_ARG(subscr, MP_EMIT_SUBSCR_STORE); } else { compile_node(comp, pns1->nodes[0]); if (assign_kind == ASSIGN_AUG_LOAD) { EMIT(dup_top_two); - EMIT(load_subscr); + EMIT_ARG(subscr, MP_EMIT_SUBSCR_LOAD); } else { - EMIT(store_subscr); + EMIT_ARG(subscr, MP_EMIT_SUBSCR_STORE); } } return; @@ -884,7 +884,7 @@ STATIC void c_del_stmt(compiler_t *comp, mp_parse_node_t pn) { } if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_bracket) { compile_node(comp, pns1->nodes[0]); - EMIT(delete_subscr); + EMIT_ARG(subscr, MP_EMIT_SUBSCR_DELETE); } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_period) { assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0])); EMIT_ARG(delete_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0])); @@ -2536,7 +2536,7 @@ STATIC void compile_trailer_paren(compiler_t *comp, mp_parse_node_struct_t *pns) STATIC void compile_trailer_bracket(compiler_t *comp, mp_parse_node_struct_t *pns) { // object who's index we want is on top of stack compile_node(comp, pns->nodes[0]); // the index - EMIT(load_subscr); + EMIT_ARG(subscr, MP_EMIT_SUBSCR_LOAD); } STATIC void compile_trailer_period(compiler_t *comp, mp_parse_node_struct_t *pns) { diff --git a/py/emit.h b/py/emit.h index 3574d00e3..6467f84e8 100644 --- a/py/emit.h +++ b/py/emit.h @@ -63,6 +63,11 @@ typedef enum { #define MP_EMIT_IDOP_GLOBAL_NAME (0) #define MP_EMIT_IDOP_GLOBAL_GLOBAL (1) +// Kind for emit->subscr() +#define MP_EMIT_SUBSCR_LOAD (0) +#define MP_EMIT_SUBSCR_STORE (1) +#define MP_EMIT_SUBSCR_DELETE (2) + // Kind for emit->build() #define MP_EMIT_BUILD_TUPLE (0) #define MP_EMIT_BUILD_LIST (1) @@ -103,11 +108,9 @@ typedef struct _emit_method_table_t { void (*load_attr)(emit_t *emit, qstr qst); void (*load_method)(emit_t *emit, qstr qst, bool is_super); void (*load_build_class)(emit_t *emit); - void (*load_subscr)(emit_t *emit); + void (*subscr)(emit_t *emit, int kind); void (*store_attr)(emit_t *emit, qstr qst); - void (*store_subscr)(emit_t *emit); void (*delete_attr)(emit_t *emit, qstr qst); - void (*delete_subscr)(emit_t *emit); void (*dup_top)(emit_t *emit); void (*dup_top_two)(emit_t *emit); void (*pop_top)(emit_t *emit); @@ -211,11 +214,9 @@ void mp_emit_bc_load_null(emit_t *emit); void mp_emit_bc_load_attr(emit_t *emit, qstr qst); void mp_emit_bc_load_method(emit_t *emit, qstr qst, bool is_super); void mp_emit_bc_load_build_class(emit_t *emit); -void mp_emit_bc_load_subscr(emit_t *emit); +void mp_emit_bc_subscr(emit_t *emit, int kind); void mp_emit_bc_store_attr(emit_t *emit, qstr qst); -void mp_emit_bc_store_subscr(emit_t *emit); void mp_emit_bc_delete_attr(emit_t *emit, qstr qst); -void mp_emit_bc_delete_subscr(emit_t *emit); void mp_emit_bc_dup_top(emit_t *emit); void mp_emit_bc_dup_top_two(emit_t *emit); void mp_emit_bc_pop_top(emit_t *emit); diff --git a/py/emitbc.c b/py/emitbc.c index ed043de3a..b342c21a4 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -597,9 +597,18 @@ void mp_emit_bc_load_build_class(emit_t *emit) { emit_write_bytecode_byte(emit, MP_BC_LOAD_BUILD_CLASS); } -void mp_emit_bc_load_subscr(emit_t *emit) { - emit_bc_pre(emit, -1); - emit_write_bytecode_byte(emit, MP_BC_LOAD_SUBSCR); +void mp_emit_bc_subscr(emit_t *emit, int kind) { + if (kind == MP_EMIT_SUBSCR_LOAD) { + emit_bc_pre(emit, -1); + emit_write_bytecode_byte(emit, MP_BC_LOAD_SUBSCR); + } else { + if (kind == MP_EMIT_SUBSCR_DELETE) { + mp_emit_bc_load_null(emit); + mp_emit_bc_rot_three(emit); + } + emit_bc_pre(emit, -3); + emit_write_bytecode_byte(emit, MP_BC_STORE_SUBSCR); + } } void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { @@ -629,11 +638,6 @@ void mp_emit_bc_store_attr(emit_t *emit, qstr qst) { } } -void mp_emit_bc_store_subscr(emit_t *emit) { - emit_bc_pre(emit, -3); - emit_write_bytecode_byte(emit, MP_BC_STORE_SUBSCR); -} - void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_DELETE_FAST); MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_DELETE_DEREF); @@ -654,12 +658,6 @@ void mp_emit_bc_delete_attr(emit_t *emit, qstr qst) { mp_emit_bc_store_attr(emit, qst); } -void mp_emit_bc_delete_subscr(emit_t *emit) { - mp_emit_bc_load_null(emit); - mp_emit_bc_rot_three(emit); - mp_emit_bc_store_subscr(emit); -} - void mp_emit_bc_dup_top(emit_t *emit) { emit_bc_pre(emit, 1); emit_write_bytecode_byte(emit, MP_BC_DUP_TOP); @@ -964,11 +962,9 @@ const emit_method_table_t emit_bc_method_table = { mp_emit_bc_load_attr, mp_emit_bc_load_method, mp_emit_bc_load_build_class, - mp_emit_bc_load_subscr, + mp_emit_bc_subscr, mp_emit_bc_store_attr, - mp_emit_bc_store_subscr, mp_emit_bc_delete_attr, - mp_emit_bc_delete_subscr, mp_emit_bc_dup_top, mp_emit_bc_dup_top_two, mp_emit_bc_pop_top, diff --git a/py/emitnative.c b/py/emitnative.c index 203cacff4..725ee21ed 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1447,6 +1447,16 @@ STATIC void emit_native_delete_subscr(emit_t *emit) { emit_call_with_imm_arg(emit, MP_F_OBJ_SUBSCR, (mp_uint_t)MP_OBJ_NULL, REG_ARG_3); } +STATIC void emit_native_subscr(emit_t *emit, int kind) { + if (kind == MP_EMIT_SUBSCR_LOAD) { + emit_native_load_subscr(emit); + } else if (kind == MP_EMIT_SUBSCR_STORE) { + emit_native_store_subscr(emit); + } else { + emit_native_delete_subscr(emit); + } +} + STATIC void emit_native_dup_top(emit_t *emit) { DEBUG_printf("dup_top\n"); vtype_kind_t vtype; @@ -2219,11 +2229,9 @@ const emit_method_table_t EXPORT_FUN(method_table) = { emit_native_load_attr, emit_native_load_method, emit_native_load_build_class, - emit_native_load_subscr, + emit_native_subscr, emit_native_store_attr, - emit_native_store_subscr, emit_native_delete_attr, - emit_native_delete_subscr, emit_native_dup_top, emit_native_dup_top_two, emit_native_pop_top, -- cgit v1.2.3 From 6211d979eed0a809cef03230e14b6d264f6f92ee Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 21:43:41 +1000 Subject: py/emit: Combine load/store/delete attr into one emit function. Reduces code size by: bare-arm: -20 minimal x86: -140 unix x64: -408 unix nanbox: -140 stm32: -68 cc3200: -16 esp8266: -80 esp32: -32 --- py/compile.c | 14 +++++++------- py/emit.h | 13 +++++++------ py/emitbc.c | 43 ++++++++++++++++++------------------------- py/emitnative.c | 14 +++++++++++--- 4 files changed, 43 insertions(+), 41 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 6aba2b896..e3735bf3d 100644 --- a/py/compile.c +++ b/py/compile.c @@ -381,12 +381,12 @@ STATIC void c_assign_atom_expr(compiler_t *comp, mp_parse_node_struct_t *pns, as assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0])); if (assign_kind == ASSIGN_AUG_LOAD) { EMIT(dup_top); - EMIT_ARG(load_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0])); + EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]), MP_EMIT_ATTR_LOAD); } else { if (assign_kind == ASSIGN_AUG_STORE) { EMIT(rot_two); } - EMIT_ARG(store_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0])); + EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]), MP_EMIT_ATTR_STORE); } return; } @@ -820,7 +820,7 @@ STATIC void compile_decorated(compiler_t *comp, mp_parse_node_struct_t *pns) { compile_node(comp, name_nodes[0]); for (int j = 1; j < name_len; j++) { assert(MP_PARSE_NODE_IS_ID(name_nodes[j])); // should be - EMIT_ARG(load_attr, MP_PARSE_NODE_LEAF_ARG(name_nodes[j])); + EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(name_nodes[j]), MP_EMIT_ATTR_LOAD); } // nodes[1] contains arguments to the decorator function, if any @@ -887,7 +887,7 @@ STATIC void c_del_stmt(compiler_t *comp, mp_parse_node_t pn) { EMIT_ARG(subscr, MP_EMIT_SUBSCR_DELETE); } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_period) { assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0])); - EMIT_ARG(delete_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0])); + EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]), MP_EMIT_ATTR_DELETE); } else { goto cannot_delete; } @@ -1061,7 +1061,7 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { EMIT_ARG(import_name, q_full); if (is_as) { for (int i = 1; i < n; i++) { - EMIT_ARG(load_attr, MP_PARSE_NODE_LEAF_ARG(pns->nodes[i])); + EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]), MP_EMIT_ATTR_LOAD); } } } @@ -1811,7 +1811,7 @@ STATIC void compile_async_with_stmt_helper(compiler_t *comp, int n, mp_parse_nod // at this point the stack contains: ..., __aexit__, self, exc EMIT(dup_top); #if MICROPY_CPYTHON_COMPAT - EMIT_ARG(load_attr, MP_QSTR___class__); // get type(exc) + EMIT_ARG(attr, MP_QSTR___class__, MP_EMIT_ATTR_LOAD); // get type(exc) #else compile_load_id(comp, MP_QSTR_type); EMIT(rot_two); @@ -2541,7 +2541,7 @@ STATIC void compile_trailer_bracket(compiler_t *comp, mp_parse_node_struct_t *pn STATIC void compile_trailer_period(compiler_t *comp, mp_parse_node_struct_t *pns) { // object who's attribute we want is on top of stack - EMIT_ARG(load_attr, MP_PARSE_NODE_LEAF_ARG(pns->nodes[0])); // attribute to get + EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]), MP_EMIT_ATTR_LOAD); // attribute to get } #if MICROPY_PY_BUILTINS_SLICE diff --git a/py/emit.h b/py/emit.h index 6467f84e8..c9e4f0c71 100644 --- a/py/emit.h +++ b/py/emit.h @@ -68,6 +68,11 @@ typedef enum { #define MP_EMIT_SUBSCR_STORE (1) #define MP_EMIT_SUBSCR_DELETE (2) +// Kind for emit->attr() +#define MP_EMIT_ATTR_LOAD (0) +#define MP_EMIT_ATTR_STORE (1) +#define MP_EMIT_ATTR_DELETE (2) + // Kind for emit->build() #define MP_EMIT_BUILD_TUPLE (0) #define MP_EMIT_BUILD_LIST (1) @@ -105,12 +110,10 @@ typedef struct _emit_method_table_t { void (*load_const_str)(emit_t *emit, qstr qst); void (*load_const_obj)(emit_t *emit, mp_obj_t obj); void (*load_null)(emit_t *emit); - void (*load_attr)(emit_t *emit, qstr qst); void (*load_method)(emit_t *emit, qstr qst, bool is_super); void (*load_build_class)(emit_t *emit); void (*subscr)(emit_t *emit, int kind); - void (*store_attr)(emit_t *emit, qstr qst); - void (*delete_attr)(emit_t *emit, qstr qst); + void (*attr)(emit_t *emit, qstr qst, int kind); void (*dup_top)(emit_t *emit); void (*dup_top_two)(emit_t *emit); void (*pop_top)(emit_t *emit); @@ -211,12 +214,10 @@ void mp_emit_bc_load_const_small_int(emit_t *emit, mp_int_t arg); void mp_emit_bc_load_const_str(emit_t *emit, qstr qst); void mp_emit_bc_load_const_obj(emit_t *emit, mp_obj_t obj); void mp_emit_bc_load_null(emit_t *emit); -void mp_emit_bc_load_attr(emit_t *emit, qstr qst); void mp_emit_bc_load_method(emit_t *emit, qstr qst, bool is_super); void mp_emit_bc_load_build_class(emit_t *emit); void mp_emit_bc_subscr(emit_t *emit, int kind); -void mp_emit_bc_store_attr(emit_t *emit, qstr qst); -void mp_emit_bc_delete_attr(emit_t *emit, qstr qst); +void mp_emit_bc_attr(emit_t *emit, qstr qst, int kind); void mp_emit_bc_dup_top(emit_t *emit); void mp_emit_bc_dup_top_two(emit_t *emit); void mp_emit_bc_pop_top(emit_t *emit); diff --git a/py/emitbc.c b/py/emitbc.c index b342c21a4..774343f2b 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -579,14 +579,6 @@ void mp_emit_bc_load_global(emit_t *emit, qstr qst, int kind) { } } -void mp_emit_bc_load_attr(emit_t *emit, qstr qst) { - emit_bc_pre(emit, 0); - emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_ATTR, qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { - emit_write_bytecode_byte(emit, 0); - } -} - void mp_emit_bc_load_method(emit_t *emit, qstr qst, bool is_super) { emit_bc_pre(emit, 1 - 2 * is_super); emit_write_bytecode_byte_qstr(emit, is_super ? MP_BC_LOAD_SUPER_METHOD : MP_BC_LOAD_METHOD, qst); @@ -611,6 +603,23 @@ void mp_emit_bc_subscr(emit_t *emit, int kind) { } } +void mp_emit_bc_attr(emit_t *emit, qstr qst, int kind) { + if (kind == MP_EMIT_ATTR_LOAD) { + emit_bc_pre(emit, 0); + emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_ATTR, qst); + } else { + if (kind == MP_EMIT_ATTR_DELETE) { + mp_emit_bc_load_null(emit); + mp_emit_bc_rot_two(emit); + } + emit_bc_pre(emit, -2); + emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_ATTR, qst); + } + if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { + emit_write_bytecode_byte(emit, 0); + } +} + void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { MP_STATIC_ASSERT(MP_BC_STORE_FAST_N + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_STORE_FAST_N); MP_STATIC_ASSERT(MP_BC_STORE_FAST_N + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_STORE_DEREF); @@ -630,14 +639,6 @@ void mp_emit_bc_store_global(emit_t *emit, qstr qst, int kind) { emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_NAME + kind, qst); } -void mp_emit_bc_store_attr(emit_t *emit, qstr qst) { - emit_bc_pre(emit, -2); - emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_ATTR, qst); - if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) { - emit_write_bytecode_byte(emit, 0); - } -} - void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) { MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_DELETE_FAST); MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_DELETE_DEREF); @@ -652,12 +653,6 @@ void mp_emit_bc_delete_global(emit_t *emit, qstr qst, int kind) { emit_write_bytecode_byte_qstr(emit, MP_BC_DELETE_NAME + kind, qst); } -void mp_emit_bc_delete_attr(emit_t *emit, qstr qst) { - mp_emit_bc_load_null(emit); - mp_emit_bc_rot_two(emit); - mp_emit_bc_store_attr(emit, qst); -} - void mp_emit_bc_dup_top(emit_t *emit) { emit_bc_pre(emit, 1); emit_write_bytecode_byte(emit, MP_BC_DUP_TOP); @@ -959,12 +954,10 @@ const emit_method_table_t emit_bc_method_table = { mp_emit_bc_load_const_str, mp_emit_bc_load_const_obj, mp_emit_bc_load_null, - mp_emit_bc_load_attr, mp_emit_bc_load_method, mp_emit_bc_load_build_class, mp_emit_bc_subscr, - mp_emit_bc_store_attr, - mp_emit_bc_delete_attr, + mp_emit_bc_attr, mp_emit_bc_dup_top, mp_emit_bc_dup_top_two, mp_emit_bc_pop_top, diff --git a/py/emitnative.c b/py/emitnative.c index 725ee21ed..8624680a6 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1457,6 +1457,16 @@ STATIC void emit_native_subscr(emit_t *emit, int kind) { } } +STATIC void emit_native_attr(emit_t *emit, qstr qst, int kind) { + if (kind == MP_EMIT_ATTR_LOAD) { + emit_native_load_attr(emit, qst); + } else if (kind == MP_EMIT_ATTR_STORE) { + emit_native_store_attr(emit, qst); + } else { + emit_native_delete_attr(emit, qst); + } +} + STATIC void emit_native_dup_top(emit_t *emit) { DEBUG_printf("dup_top\n"); vtype_kind_t vtype; @@ -2226,12 +2236,10 @@ const emit_method_table_t EXPORT_FUN(method_table) = { emit_native_load_const_str, emit_native_load_const_obj, emit_native_load_null, - emit_native_load_attr, emit_native_load_method, emit_native_load_build_class, emit_native_subscr, - emit_native_store_attr, - emit_native_delete_attr, + emit_native_attr, emit_native_dup_top, emit_native_dup_top_two, emit_native_pop_top, -- cgit v1.2.3 From 8a513da5a560e9c6afa5f0a0f8d44c5fb1ed552d Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 21:50:22 +1000 Subject: py/emit: Combine break_loop and continue_loop into one emit function. Reduces code size by: bare-arm: +0 minimal x86: +0 unix x64: -80 unix nanbox: +0 stm32: -12 cc3200: +0 esp8266: -28 esp32: +0 --- py/compile.c | 4 ++-- py/emit.h | 5 +---- py/emitbc.c | 1 - py/emitnative.c | 10 ++-------- 4 files changed, 5 insertions(+), 15 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index e3735bf3d..411201931 100644 --- a/py/compile.c +++ b/py/compile.c @@ -950,7 +950,7 @@ STATIC void compile_break_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { compile_syntax_error(comp, (mp_parse_node_t)pns, "'break' outside loop"); } assert(comp->cur_except_level >= comp->break_continue_except_level); - EMIT_ARG(break_loop, comp->break_label, comp->cur_except_level - comp->break_continue_except_level); + EMIT_ARG(unwind_jump, comp->break_label, comp->cur_except_level - comp->break_continue_except_level); } STATIC void compile_continue_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { @@ -958,7 +958,7 @@ STATIC void compile_continue_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) compile_syntax_error(comp, (mp_parse_node_t)pns, "'continue' outside loop"); } assert(comp->cur_except_level >= comp->break_continue_except_level); - EMIT_ARG(continue_loop, comp->continue_label, comp->cur_except_level - comp->break_continue_except_level); + EMIT_ARG(unwind_jump, comp->continue_label, comp->cur_except_level - comp->break_continue_except_level); } STATIC void compile_return_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { diff --git a/py/emit.h b/py/emit.h index c9e4f0c71..4fd115e23 100644 --- a/py/emit.h +++ b/py/emit.h @@ -122,8 +122,7 @@ typedef struct _emit_method_table_t { void (*jump)(emit_t *emit, mp_uint_t label); void (*pop_jump_if)(emit_t *emit, bool cond, mp_uint_t label); void (*jump_if_or_pop)(emit_t *emit, bool cond, mp_uint_t label); - void (*break_loop)(emit_t *emit, mp_uint_t label, mp_uint_t except_depth); - void (*continue_loop)(emit_t *emit, mp_uint_t label, mp_uint_t except_depth); + void (*unwind_jump)(emit_t *emit, mp_uint_t label, mp_uint_t except_depth); void (*setup_with)(emit_t *emit, mp_uint_t label); void (*with_cleanup)(emit_t *emit, mp_uint_t label); void (*setup_except)(emit_t *emit, mp_uint_t label); @@ -227,8 +226,6 @@ void mp_emit_bc_jump(emit_t *emit, mp_uint_t label); void mp_emit_bc_pop_jump_if(emit_t *emit, bool cond, mp_uint_t label); void mp_emit_bc_jump_if_or_pop(emit_t *emit, bool cond, mp_uint_t label); void mp_emit_bc_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_depth); -#define mp_emit_bc_break_loop mp_emit_bc_unwind_jump -#define mp_emit_bc_continue_loop mp_emit_bc_unwind_jump void mp_emit_bc_setup_with(emit_t *emit, mp_uint_t label); void mp_emit_bc_with_cleanup(emit_t *emit, mp_uint_t label); void mp_emit_bc_setup_except(emit_t *emit, mp_uint_t label); diff --git a/py/emitbc.c b/py/emitbc.c index 774343f2b..6f5530366 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -967,7 +967,6 @@ const emit_method_table_t emit_bc_method_table = { mp_emit_bc_pop_jump_if, mp_emit_bc_jump_if_or_pop, mp_emit_bc_unwind_jump, - mp_emit_bc_unwind_jump, mp_emit_bc_setup_with, mp_emit_bc_with_cleanup, mp_emit_bc_setup_except, diff --git a/py/emitnative.c b/py/emitnative.c index 8624680a6..b4a3f987c 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1560,16 +1560,11 @@ STATIC void emit_native_jump_if_or_pop(emit_t *emit, bool cond, mp_uint_t label) emit_post(emit); } -STATIC void emit_native_break_loop(emit_t *emit, mp_uint_t label, mp_uint_t except_depth) { +STATIC void emit_native_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_depth) { (void)except_depth; emit_native_jump(emit, label & ~MP_EMIT_BREAK_FROM_FOR); // TODO properly } -STATIC void emit_native_continue_loop(emit_t *emit, mp_uint_t label, mp_uint_t except_depth) { - (void)except_depth; - emit_native_jump(emit, label); // TODO properly -} - STATIC void emit_native_setup_with(emit_t *emit, mp_uint_t label) { // the context manager is on the top of the stack // stack: (..., ctx_mgr) @@ -2248,8 +2243,7 @@ const emit_method_table_t EXPORT_FUN(method_table) = { emit_native_jump, emit_native_pop_jump_if, emit_native_jump_if_or_pop, - emit_native_break_loop, - emit_native_continue_loop, + emit_native_unwind_jump, emit_native_setup_with, emit_native_with_cleanup, emit_native_setup_except, -- cgit v1.2.3 From d97906ca9a0f1e0728cefb930d458bb8fba3bd17 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 21:58:25 +1000 Subject: py/emit: Combine import from/name/star into one emit function. Change in code size is: bare-arm: +4 minimal x86: -88 unix x64: -456 unix nanbox: -88 stm32: -44 cc3200: +0 esp8266: -104 esp32: +8 --- py/compile.c | 10 +++++----- py/emit.h | 13 +++++++------ py/emitbc.c | 30 ++++++++++++++---------------- py/emitnative.c | 14 +++++++++++--- 4 files changed, 37 insertions(+), 30 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 411201931..1789530f6 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1024,14 +1024,14 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { if (MP_PARSE_NODE_IS_NULL(pn)) { // empty name (eg, from . import x) *q_base = MP_QSTR_; - EMIT_ARG(import_name, MP_QSTR_); // import the empty string + EMIT_ARG(import, MP_QSTR_, MP_EMIT_IMPORT_NAME); // import the empty string } else if (MP_PARSE_NODE_IS_ID(pn)) { // just a simple name qstr q_full = MP_PARSE_NODE_LEAF_ARG(pn); if (!is_as) { *q_base = q_full; } - EMIT_ARG(import_name, q_full); + EMIT_ARG(import, q_full, MP_EMIT_IMPORT_NAME); } else { assert(MP_PARSE_NODE_IS_STRUCT_KIND(pn, PN_dotted_name)); // should be mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; @@ -1058,7 +1058,7 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { } qstr q_full = qstr_from_strn(q_ptr, len); mp_local_free(q_ptr); - EMIT_ARG(import_name, q_full); + EMIT_ARG(import, q_full, MP_EMIT_IMPORT_NAME); if (is_as) { for (int i = 1; i < n; i++) { EMIT_ARG(attr, MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]), MP_EMIT_ATTR_LOAD); @@ -1127,7 +1127,7 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) { // do the import qstr dummy_q; do_import_name(comp, pn_import_source, &dummy_q); - EMIT(import_star); + EMIT_ARG(import, MP_QSTR_NULL, MP_EMIT_IMPORT_STAR); } else { EMIT_ARG(load_const_small_int, import_level); @@ -1150,7 +1150,7 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) { assert(MP_PARSE_NODE_IS_STRUCT_KIND(pn_nodes[i], PN_import_as_name)); mp_parse_node_struct_t *pns3 = (mp_parse_node_struct_t*)pn_nodes[i]; qstr id2 = MP_PARSE_NODE_LEAF_ARG(pns3->nodes[0]); // should be id - EMIT_ARG(import_from, id2); + EMIT_ARG(import, id2, MP_EMIT_IMPORT_FROM); if (MP_PARSE_NODE_IS_NULL(pns3->nodes[1])) { compile_store_id(comp, id2); } else { diff --git a/py/emit.h b/py/emit.h index 4fd115e23..d6ebcf791 100644 --- a/py/emit.h +++ b/py/emit.h @@ -63,6 +63,11 @@ typedef enum { #define MP_EMIT_IDOP_GLOBAL_NAME (0) #define MP_EMIT_IDOP_GLOBAL_GLOBAL (1) +// Kind for emit->import() +#define MP_EMIT_IMPORT_NAME (0) +#define MP_EMIT_IMPORT_FROM (1) +#define MP_EMIT_IMPORT_STAR (2) + // Kind for emit->subscr() #define MP_EMIT_SUBSCR_LOAD (0) #define MP_EMIT_SUBSCR_STORE (1) @@ -102,9 +107,7 @@ typedef struct _emit_method_table_t { mp_emit_method_table_id_ops_t delete_id; void (*label_assign)(emit_t *emit, mp_uint_t l); - void (*import_name)(emit_t *emit, qstr qst); - void (*import_from)(emit_t *emit, qstr qst); - void (*import_star)(emit_t *emit); + void (*import)(emit_t *emit, qstr qst, int kind); void (*load_const_tok)(emit_t *emit, mp_token_kind_t tok); void (*load_const_small_int)(emit_t *emit, mp_int_t arg); void (*load_const_str)(emit_t *emit, qstr qst); @@ -205,9 +208,7 @@ void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int ki void mp_emit_bc_delete_global(emit_t *emit, qstr qst, int kind); void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l); -void mp_emit_bc_import_name(emit_t *emit, qstr qst); -void mp_emit_bc_import_from(emit_t *emit, qstr qst); -void mp_emit_bc_import_star(emit_t *emit); +void mp_emit_bc_import(emit_t *emit, qstr qst, int kind); void mp_emit_bc_load_const_tok(emit_t *emit, mp_token_kind_t tok); void mp_emit_bc_load_const_small_int(emit_t *emit, mp_int_t arg); void mp_emit_bc_load_const_str(emit_t *emit, qstr qst); diff --git a/py/emitbc.c b/py/emitbc.c index 6f5530366..6be9f900c 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -504,19 +504,19 @@ void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l) { } } -void mp_emit_bc_import_name(emit_t *emit, qstr qst) { - emit_bc_pre(emit, -1); - emit_write_bytecode_byte_qstr(emit, MP_BC_IMPORT_NAME, qst); -} - -void mp_emit_bc_import_from(emit_t *emit, qstr qst) { - emit_bc_pre(emit, 1); - emit_write_bytecode_byte_qstr(emit, MP_BC_IMPORT_FROM, qst); -} - -void mp_emit_bc_import_star(emit_t *emit) { - emit_bc_pre(emit, -1); - emit_write_bytecode_byte(emit, MP_BC_IMPORT_STAR); +void mp_emit_bc_import(emit_t *emit, qstr qst, int kind) { + MP_STATIC_ASSERT(MP_BC_IMPORT_NAME + MP_EMIT_IMPORT_NAME == MP_BC_IMPORT_NAME); + MP_STATIC_ASSERT(MP_BC_IMPORT_NAME + MP_EMIT_IMPORT_FROM == MP_BC_IMPORT_FROM); + if (kind == MP_EMIT_IMPORT_FROM) { + emit_bc_pre(emit, 1); + } else { + emit_bc_pre(emit, -1); + } + if (kind == MP_EMIT_IMPORT_STAR) { + emit_write_bytecode_byte(emit, MP_BC_IMPORT_STAR); + } else { + emit_write_bytecode_byte_qstr(emit, MP_BC_IMPORT_NAME + kind, qst); + } } void mp_emit_bc_load_const_tok(emit_t *emit, mp_token_kind_t tok) { @@ -946,9 +946,7 @@ const emit_method_table_t emit_bc_method_table = { }, mp_emit_bc_label_assign, - mp_emit_bc_import_name, - mp_emit_bc_import_from, - mp_emit_bc_import_star, + mp_emit_bc_import, mp_emit_bc_load_const_tok, mp_emit_bc_load_const_small_int, mp_emit_bc_load_const_str, diff --git a/py/emitnative.c b/py/emitnative.c index b4a3f987c..e06198bc9 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -837,6 +837,16 @@ STATIC void emit_native_import_star(emit_t *emit) { emit_post(emit); } +STATIC void emit_native_import(emit_t *emit, qstr qst, int kind) { + if (kind == MP_EMIT_IMPORT_NAME) { + emit_native_import_name(emit, qst); + } else if (kind == MP_EMIT_IMPORT_FROM) { + emit_native_import_from(emit, qst); + } else { + emit_native_import_star(emit); + } +} + STATIC void emit_native_load_const_tok(emit_t *emit, mp_token_kind_t tok) { DEBUG_printf("load_const_tok(tok=%u)\n", tok); emit_native_pre(emit); @@ -2223,9 +2233,7 @@ const emit_method_table_t EXPORT_FUN(method_table) = { }, emit_native_label_assign, - emit_native_import_name, - emit_native_import_from, - emit_native_import_star, + emit_native_import, emit_native_load_const_tok, emit_native_load_const_small_int, emit_native_load_const_str, -- cgit v1.2.3 From 436e0d4c54ab22050072d392f0822e555bcc70f1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 22:18:42 +1000 Subject: py/emit: Merge build set/slice into existing build emit function. Reduces code size by: bare-arm: +0 minimal x86: +0 unix x64: -368 unix nanbox: -248 stm32: -128 cc3200: -48 esp8266: -184 esp32: -40 --- py/compile.c | 20 ++++++++++---------- py/emit.h | 14 ++------------ py/emitbc.c | 22 ++-------------------- py/emitnative.c | 30 +++++++++++++----------------- py/nativeglue.c | 2 +- py/runtime0.h | 2 +- 6 files changed, 29 insertions(+), 61 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 1789530f6..be1fd8b1f 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2499,7 +2499,7 @@ STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) { #if MICROPY_PY_BUILTINS_SET // if it's a set, build it if (!is_dict) { - EMIT_ARG(build_set, 1 + n); + EMIT_ARG(build, 1 + n, MP_EMIT_BUILD_SET); } #endif } else { @@ -2522,7 +2522,7 @@ STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) { set_with_one_element: #if MICROPY_PY_BUILTINS_SET compile_node(comp, pn); - EMIT_ARG(build_set, 1); + EMIT_ARG(build, 1, MP_EMIT_BUILD_SET); #else assert(0); #endif @@ -2551,7 +2551,7 @@ STATIC void compile_subscript_3_helper(compiler_t *comp, mp_parse_node_struct_t if (MP_PARSE_NODE_IS_NULL(pn)) { // [?:] EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); - EMIT_ARG(build_slice, 2); + EMIT_ARG(build, 2, MP_EMIT_BUILD_SLICE); } else if (MP_PARSE_NODE_IS_STRUCT(pn)) { pns = (mp_parse_node_struct_t*)pn; if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_subscript_3c) { @@ -2559,11 +2559,11 @@ STATIC void compile_subscript_3_helper(compiler_t *comp, mp_parse_node_struct_t pn = pns->nodes[0]; if (MP_PARSE_NODE_IS_NULL(pn)) { // [?::] - EMIT_ARG(build_slice, 2); + EMIT_ARG(build, 2, MP_EMIT_BUILD_SLICE); } else { // [?::x] compile_node(comp, pn); - EMIT_ARG(build_slice, 3); + EMIT_ARG(build, 3, MP_EMIT_BUILD_SLICE); } } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_subscript_3d) { compile_node(comp, pns->nodes[0]); @@ -2572,21 +2572,21 @@ STATIC void compile_subscript_3_helper(compiler_t *comp, mp_parse_node_struct_t assert(MP_PARSE_NODE_STRUCT_KIND(pns) == PN_sliceop); // should always be if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) { // [?:x:] - EMIT_ARG(build_slice, 2); + EMIT_ARG(build, 2, MP_EMIT_BUILD_SLICE); } else { // [?:x:x] compile_node(comp, pns->nodes[0]); - EMIT_ARG(build_slice, 3); + EMIT_ARG(build, 3, MP_EMIT_BUILD_SLICE); } } else { // [?:x] compile_node(comp, pn); - EMIT_ARG(build_slice, 2); + EMIT_ARG(build, 2, MP_EMIT_BUILD_SLICE); } } else { // [?:x] compile_node(comp, pn); - EMIT_ARG(build_slice, 2); + EMIT_ARG(build, 2, MP_EMIT_BUILD_SLICE); } } @@ -3045,7 +3045,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { EMIT_ARG(build, 0, MP_EMIT_BUILD_MAP); #if MICROPY_PY_BUILTINS_SET } else if (scope->kind == SCOPE_SET_COMP) { - EMIT_ARG(build_set, 0); + EMIT_ARG(build, 0, MP_EMIT_BUILD_SET); #endif } diff --git a/py/emit.h b/py/emit.h index d6ebcf791..7f0d8c0f7 100644 --- a/py/emit.h +++ b/py/emit.h @@ -82,6 +82,8 @@ typedef enum { #define MP_EMIT_BUILD_TUPLE (0) #define MP_EMIT_BUILD_LIST (1) #define MP_EMIT_BUILD_MAP (3) +#define MP_EMIT_BUILD_SET (6) +#define MP_EMIT_BUILD_SLICE (8) // Kind for emit->yield() #define MP_EMIT_YIELD_VALUE (0) @@ -140,12 +142,6 @@ typedef struct _emit_method_table_t { void (*binary_op)(emit_t *emit, mp_binary_op_t op); void (*build)(emit_t *emit, mp_uint_t n_args, int kind); void (*store_map)(emit_t *emit); - #if MICROPY_PY_BUILTINS_SET - void (*build_set)(emit_t *emit, mp_uint_t n_args); - #endif - #if MICROPY_PY_BUILTINS_SLICE - void (*build_slice)(emit_t *emit, mp_uint_t n_args); - #endif void (*store_comp)(emit_t *emit, scope_kind_t kind, mp_uint_t set_stack_index); void (*unpack_sequence)(emit_t *emit, mp_uint_t n_args); void (*unpack_ex)(emit_t *emit, mp_uint_t n_left, mp_uint_t n_right); @@ -241,12 +237,6 @@ void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op); void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op); void mp_emit_bc_build(emit_t *emit, mp_uint_t n_args, int kind); void mp_emit_bc_store_map(emit_t *emit); -#if MICROPY_PY_BUILTINS_SET -void mp_emit_bc_build_set(emit_t *emit, mp_uint_t n_args); -#endif -#if MICROPY_PY_BUILTINS_SLICE -void mp_emit_bc_build_slice(emit_t *emit, mp_uint_t n_args); -#endif void mp_emit_bc_store_comp(emit_t *emit, scope_kind_t kind, mp_uint_t list_stack_index); void mp_emit_bc_unpack_sequence(emit_t *emit, mp_uint_t n_args); void mp_emit_bc_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_right); diff --git a/py/emitbc.c b/py/emitbc.c index 6be9f900c..4c587c72e 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -800,6 +800,8 @@ void mp_emit_bc_build(emit_t *emit, mp_uint_t n_args, int kind) { MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_TUPLE == MP_BC_BUILD_TUPLE); MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_LIST == MP_BC_BUILD_LIST); MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_MAP == MP_BC_BUILD_MAP); + MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_SET == MP_BC_BUILD_SET); + MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_SLICE == MP_BC_BUILD_SLICE); if (kind == MP_EMIT_BUILD_MAP) { emit_bc_pre(emit, 1); } else { @@ -813,20 +815,6 @@ void mp_emit_bc_store_map(emit_t *emit) { emit_write_bytecode_byte(emit, MP_BC_STORE_MAP); } -#if MICROPY_PY_BUILTINS_SET -void mp_emit_bc_build_set(emit_t *emit, mp_uint_t n_args) { - emit_bc_pre(emit, 1 - n_args); - emit_write_bytecode_byte_uint(emit, MP_BC_BUILD_SET, n_args); -} -#endif - -#if MICROPY_PY_BUILTINS_SLICE -void mp_emit_bc_build_slice(emit_t *emit, mp_uint_t n_args) { - emit_bc_pre(emit, 1 - n_args); - emit_write_bytecode_byte_uint(emit, MP_BC_BUILD_SLICE, n_args); -} -#endif - void mp_emit_bc_store_comp(emit_t *emit, scope_kind_t kind, mp_uint_t collection_stack_index) { int t; int n; @@ -979,12 +967,6 @@ const emit_method_table_t emit_bc_method_table = { mp_emit_bc_binary_op, mp_emit_bc_build, mp_emit_bc_store_map, - #if MICROPY_PY_BUILTINS_SET - mp_emit_bc_build_set, - #endif - #if MICROPY_PY_BUILTINS_SLICE - mp_emit_bc_build_slice, - #endif mp_emit_bc_store_comp, mp_emit_bc_unpack_sequence, mp_emit_bc_unpack_ex, diff --git a/py/emitnative.c b/py/emitnative.c index e06198bc9..331e4cf18 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1950,18 +1950,29 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { } } +#if MICROPY_PY_BUILTINS_SLICE +STATIC void emit_native_build_slice(emit_t *emit, mp_uint_t n_args); +#endif + STATIC void emit_native_build(emit_t *emit, mp_uint_t n_args, int kind) { // for viper: call runtime, with types of args // if wrapped in byte_array, or something, allocates memory and fills it MP_STATIC_ASSERT(MP_F_BUILD_TUPLE + MP_EMIT_BUILD_TUPLE == MP_F_BUILD_TUPLE); MP_STATIC_ASSERT(MP_F_BUILD_TUPLE + MP_EMIT_BUILD_LIST == MP_F_BUILD_LIST); MP_STATIC_ASSERT(MP_F_BUILD_TUPLE + MP_EMIT_BUILD_MAP == MP_F_BUILD_MAP); + MP_STATIC_ASSERT(MP_F_BUILD_TUPLE + MP_EMIT_BUILD_SET == MP_F_BUILD_SET); + #if MICROPY_PY_BUILTINS_SLICE + if (kind == MP_EMIT_BUILD_SLICE) { + emit_native_build_slice(emit, n_args); + return; + } + #endif emit_native_pre(emit); - if (kind == MP_EMIT_BUILD_TUPLE || kind == MP_EMIT_BUILD_LIST) { + if (kind == MP_EMIT_BUILD_TUPLE || kind == MP_EMIT_BUILD_LIST || kind == MP_EMIT_BUILD_SET) { emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_2, n_args); // pointer to items } emit_call_with_imm_arg(emit, MP_F_BUILD_TUPLE + kind, n_args, REG_ARG_1); - emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); // new tuple/list/map + emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); // new tuple/list/map/set } STATIC void emit_native_store_map(emit_t *emit) { @@ -1974,15 +1985,6 @@ STATIC void emit_native_store_map(emit_t *emit) { emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); // map } -#if MICROPY_PY_BUILTINS_SET -STATIC void emit_native_build_set(emit_t *emit, mp_uint_t n_args) { - emit_native_pre(emit); - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_2, n_args); // pointer to items - emit_call_with_imm_arg(emit, MP_F_BUILD_SET, n_args, REG_ARG_1); - emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); // new set -} -#endif - #if MICROPY_PY_BUILTINS_SLICE STATIC void emit_native_build_slice(emit_t *emit, mp_uint_t n_args) { DEBUG_printf("build_slice %d\n", n_args); @@ -2266,12 +2268,6 @@ const emit_method_table_t EXPORT_FUN(method_table) = { emit_native_binary_op, emit_native_build, emit_native_store_map, - #if MICROPY_PY_BUILTINS_SET - emit_native_build_set, - #endif - #if MICROPY_PY_BUILTINS_SLICE - emit_native_build_slice, - #endif emit_native_store_comp, emit_native_unpack_sequence, emit_native_unpack_ex, diff --git a/py/nativeglue.c b/py/nativeglue.c index e63c2fcda..b87da6931 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -146,8 +146,8 @@ void *const mp_fun_table[MP_F_NUMBER_OF] = { mp_obj_new_dict, mp_obj_dict_store, #if MICROPY_PY_BUILTINS_SET - mp_obj_new_set, mp_obj_set_store, + mp_obj_new_set, #endif mp_make_function_from_raw_code, mp_native_call_function_n_kw, diff --git a/py/runtime0.h b/py/runtime0.h index 960532d17..2e89de9f4 100644 --- a/py/runtime0.h +++ b/py/runtime0.h @@ -164,8 +164,8 @@ typedef enum { MP_F_BUILD_MAP, MP_F_STORE_MAP, #if MICROPY_PY_BUILTINS_SET - MP_F_BUILD_SET, MP_F_STORE_SET, + MP_F_BUILD_SET, #endif MP_F_MAKE_FUNCTION_FROM_RAW_CODE, MP_F_NATIVE_CALL_FUNCTION_N_KW, -- cgit v1.2.3 From 18e6358480e640fd94a9383d5fa7d9b8cc2b9f73 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 22 May 2018 22:33:26 +1000 Subject: py/emit: Combine setup with/except/finally into one emit function. This patch reduces code size by: bare-arm: -16 minimal x86: -156 unix x64: -288 unix nanbox: -184 stm32: -48 cc3200: -16 esp8266: -96 esp32: -16 The last 10 patches combined reduce code size by: bare-arm: -164 minimal x86: -1260 unix x64: -3416 unix nanbox: -1616 stm32: -676 cc3200: -232 esp8266: -1144 esp32: -268 --- py/compile.c | 14 +++++++------- py/emit.h | 13 +++++++------ py/emitbc.c | 29 ++++++++++++----------------- py/emitnative.c | 33 ++++++++++++++++----------------- 4 files changed, 42 insertions(+), 47 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index be1fd8b1f..c62ed057d 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1516,7 +1516,7 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_ uint l1 = comp_next_label(comp); uint success_label = comp_next_label(comp); - EMIT_ARG(setup_except, l1); + EMIT_ARG(setup_block, l1, MP_EMIT_SETUP_BLOCK_EXCEPT); compile_increase_except_level(comp); compile_node(comp, pn_body); // body @@ -1571,7 +1571,7 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_ uint l3 = 0; if (qstr_exception_local != 0) { l3 = comp_next_label(comp); - EMIT_ARG(setup_finally, l3); + EMIT_ARG(setup_block, l3, MP_EMIT_SETUP_BLOCK_FINALLY); compile_increase_except_level(comp); } compile_node(comp, pns_except->nodes[1]); @@ -1606,7 +1606,7 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_ STATIC void compile_try_finally(compiler_t *comp, mp_parse_node_t pn_body, int n_except, mp_parse_node_t *pn_except, mp_parse_node_t pn_else, mp_parse_node_t pn_finally) { uint l_finally_block = comp_next_label(comp); - EMIT_ARG(setup_finally, l_finally_block); + EMIT_ARG(setup_block, l_finally_block, MP_EMIT_SETUP_BLOCK_FINALLY); compile_increase_except_level(comp); if (n_except == 0) { @@ -1668,12 +1668,12 @@ STATIC void compile_with_stmt_helper(compiler_t *comp, int n, mp_parse_node_t *n // this pre-bit is of the form "a as b" mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)nodes[0]; compile_node(comp, pns->nodes[0]); - EMIT_ARG(setup_with, l_end); + EMIT_ARG(setup_block, l_end, MP_EMIT_SETUP_BLOCK_WITH); c_assign(comp, pns->nodes[1], ASSIGN_STORE); } else { // this pre-bit is just an expression compile_node(comp, nodes[0]); - EMIT_ARG(setup_with, l_end); + EMIT_ARG(setup_block, l_end, MP_EMIT_SETUP_BLOCK_WITH); EMIT(pop_top); } compile_increase_except_level(comp); @@ -1726,7 +1726,7 @@ STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns EMIT_ARG(label_assign, continue_label); - EMIT_ARG(setup_except, try_exception_label); + EMIT_ARG(setup_block, try_exception_label, MP_EMIT_SETUP_BLOCK_EXCEPT); compile_increase_except_level(comp); compile_load_id(comp, context); @@ -1797,7 +1797,7 @@ STATIC void compile_async_with_stmt_helper(compiler_t *comp, int n, mp_parse_nod compile_load_id(comp, context); EMIT_ARG(load_method, MP_QSTR___aexit__, false); - EMIT_ARG(setup_except, try_exception_label); + EMIT_ARG(setup_block, try_exception_label, MP_EMIT_SETUP_BLOCK_EXCEPT); compile_increase_except_level(comp); // compile additional pre-bits and the body compile_async_with_stmt_helper(comp, n - 1, nodes + 1, body); diff --git a/py/emit.h b/py/emit.h index 7f0d8c0f7..aa98efa77 100644 --- a/py/emit.h +++ b/py/emit.h @@ -78,6 +78,11 @@ typedef enum { #define MP_EMIT_ATTR_STORE (1) #define MP_EMIT_ATTR_DELETE (2) +// Kind for emit->setup_block() +#define MP_EMIT_SETUP_BLOCK_WITH (0) +#define MP_EMIT_SETUP_BLOCK_EXCEPT (2) +#define MP_EMIT_SETUP_BLOCK_FINALLY (3) + // Kind for emit->build() #define MP_EMIT_BUILD_TUPLE (0) #define MP_EMIT_BUILD_LIST (1) @@ -128,10 +133,8 @@ typedef struct _emit_method_table_t { void (*pop_jump_if)(emit_t *emit, bool cond, mp_uint_t label); void (*jump_if_or_pop)(emit_t *emit, bool cond, mp_uint_t label); void (*unwind_jump)(emit_t *emit, mp_uint_t label, mp_uint_t except_depth); - void (*setup_with)(emit_t *emit, mp_uint_t label); + void (*setup_block)(emit_t *emit, mp_uint_t label, int kind); void (*with_cleanup)(emit_t *emit, mp_uint_t label); - void (*setup_except)(emit_t *emit, mp_uint_t label); - void (*setup_finally)(emit_t *emit, mp_uint_t label); void (*end_finally)(emit_t *emit); void (*get_iter)(emit_t *emit, bool use_stack); void (*for_iter)(emit_t *emit, mp_uint_t label); @@ -223,10 +226,8 @@ void mp_emit_bc_jump(emit_t *emit, mp_uint_t label); void mp_emit_bc_pop_jump_if(emit_t *emit, bool cond, mp_uint_t label); void mp_emit_bc_jump_if_or_pop(emit_t *emit, bool cond, mp_uint_t label); void mp_emit_bc_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_depth); -void mp_emit_bc_setup_with(emit_t *emit, mp_uint_t label); +void mp_emit_bc_setup_block(emit_t *emit, mp_uint_t label, int kind); void mp_emit_bc_with_cleanup(emit_t *emit, mp_uint_t label); -void mp_emit_bc_setup_except(emit_t *emit, mp_uint_t label); -void mp_emit_bc_setup_finally(emit_t *emit, mp_uint_t label); void mp_emit_bc_end_finally(emit_t *emit); void mp_emit_bc_get_iter(emit_t *emit, bool use_stack); void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label); diff --git a/py/emitbc.c b/py/emitbc.c index 4c587c72e..f3951e9cb 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -719,11 +719,18 @@ void mp_emit_bc_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_dept } } -void mp_emit_bc_setup_with(emit_t *emit, mp_uint_t label) { +void mp_emit_bc_setup_block(emit_t *emit, mp_uint_t label, int kind) { + MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_WITH == MP_BC_SETUP_WITH); + MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_EXCEPT == MP_BC_SETUP_EXCEPT); + MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_FINALLY == MP_BC_SETUP_FINALLY); + if (kind == MP_EMIT_SETUP_BLOCK_WITH) { // The SETUP_WITH opcode pops ctx_mgr from the top of the stack // and then pushes 3 entries: __exit__, ctx_mgr, as_value. - emit_bc_pre(emit, 2); - emit_write_bytecode_byte_unsigned_label(emit, MP_BC_SETUP_WITH, label); + emit_bc_pre(emit, 2); + } else { + emit_bc_pre(emit, 0); + } + emit_write_bytecode_byte_unsigned_label(emit, MP_BC_SETUP_WITH + kind, label); } void mp_emit_bc_with_cleanup(emit_t *emit, mp_uint_t label) { @@ -732,17 +739,7 @@ void mp_emit_bc_with_cleanup(emit_t *emit, mp_uint_t label) { mp_emit_bc_label_assign(emit, label); emit_bc_pre(emit, 2); // ensure we have enough stack space to call the __exit__ method emit_write_bytecode_byte(emit, MP_BC_WITH_CLEANUP); - emit_bc_pre(emit, -4); // cancel the 2 above, plus the 2 from mp_emit_bc_setup_with -} - -void mp_emit_bc_setup_except(emit_t *emit, mp_uint_t label) { - emit_bc_pre(emit, 0); - emit_write_bytecode_byte_unsigned_label(emit, MP_BC_SETUP_EXCEPT, label); -} - -void mp_emit_bc_setup_finally(emit_t *emit, mp_uint_t label) { - emit_bc_pre(emit, 0); - emit_write_bytecode_byte_unsigned_label(emit, MP_BC_SETUP_FINALLY, label); + emit_bc_pre(emit, -4); // cancel the 2 above, plus the 2 from mp_emit_bc_setup_block(MP_EMIT_SETUP_BLOCK_WITH) } void mp_emit_bc_end_finally(emit_t *emit) { @@ -953,10 +950,8 @@ const emit_method_table_t emit_bc_method_table = { mp_emit_bc_pop_jump_if, mp_emit_bc_jump_if_or_pop, mp_emit_bc_unwind_jump, - mp_emit_bc_setup_with, + mp_emit_bc_setup_block, mp_emit_bc_with_cleanup, - mp_emit_bc_setup_except, - mp_emit_bc_setup_finally, mp_emit_bc_end_finally, mp_emit_bc_get_iter, mp_emit_bc_for_iter, diff --git a/py/emitnative.c b/py/emitnative.c index 331e4cf18..ad8f04aac 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1617,6 +1617,21 @@ STATIC void emit_native_setup_with(emit_t *emit, mp_uint_t label) { // stack: (..., __exit__, self, as_value, nlr_buf, as_value) } +STATIC void emit_native_setup_block(emit_t *emit, mp_uint_t label, int kind) { + if (kind == MP_EMIT_SETUP_BLOCK_WITH) { + emit_native_setup_with(emit, label); + } else { + // Set up except and finally + emit_native_pre(emit); + // need to commit stack because we may jump elsewhere + need_stack_settled(emit); + emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_1, sizeof(nlr_buf_t) / sizeof(mp_uint_t)); // arg1 = pointer to nlr buf + emit_call(emit, MP_F_NLR_PUSH); + ASM_JUMP_IF_REG_NONZERO(emit->as, REG_RET, label); + emit_post(emit); + } +} + STATIC void emit_native_with_cleanup(emit_t *emit, mp_uint_t label) { // note: label+1 is available as an auxiliary label @@ -1686,20 +1701,6 @@ STATIC void emit_native_with_cleanup(emit_t *emit, mp_uint_t label) { emit_native_label_assign(emit, label + 1); } -STATIC void emit_native_setup_except(emit_t *emit, mp_uint_t label) { - emit_native_pre(emit); - // need to commit stack because we may jump elsewhere - need_stack_settled(emit); - emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_1, sizeof(nlr_buf_t) / sizeof(mp_uint_t)); // arg1 = pointer to nlr buf - emit_call(emit, MP_F_NLR_PUSH); - ASM_JUMP_IF_REG_NONZERO(emit->as, REG_RET, label); - emit_post(emit); -} - -STATIC void emit_native_setup_finally(emit_t *emit, mp_uint_t label) { - emit_native_setup_except(emit, label); -} - STATIC void emit_native_end_finally(emit_t *emit) { // logic: // exc = pop_stack @@ -2254,10 +2255,8 @@ const emit_method_table_t EXPORT_FUN(method_table) = { emit_native_pop_jump_if, emit_native_jump_if_or_pop, emit_native_unwind_jump, - emit_native_setup_with, + emit_native_setup_block, emit_native_with_cleanup, - emit_native_setup_except, - emit_native_setup_finally, emit_native_end_finally, emit_native_get_iter, emit_native_for_iter, -- cgit v1.2.3 From dfeaea144168c3ec3b4ec513cfb201ce93f99f5e Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 25 May 2018 10:59:40 +1000 Subject: py/objtype: Remove TODO comment about needing to check for property. Instance members are always treated as values, even if they are properties. A test is added to show this is the case. --- py/objtype.c | 1 - tests/basics/builtin_property.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'py') diff --git a/py/objtype.c b/py/objtype.c index 7df349ce9..2ec27c762 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -562,7 +562,6 @@ STATIC void mp_obj_instance_load_attr(mp_obj_t self_in, qstr attr, mp_obj_t *des mp_map_elem_t *elem = mp_map_lookup(&self->members, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP); if (elem != NULL) { // object member, always treated as a value - // TODO should we check for properties? dest[0] = elem->value; return; } diff --git a/tests/basics/builtin_property.py b/tests/basics/builtin_property.py index 89c3d4936..4b08ee9d3 100644 --- a/tests/basics/builtin_property.py +++ b/tests/basics/builtin_property.py @@ -105,3 +105,9 @@ class E: # not tested for because the other keyword arguments are not accepted # q = property(fget=lambda self: 21, doc="Half the truth.") print(E().p) + +# a property as an instance member should not be delegated to +class F: + def __init__(self): + self.prop_member = property() +print(type(F().prop_member)) -- cgit v1.2.3 From 05b13fd292b42f20affc7cae218d92447efbf6d6 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 25 Mar 2018 16:05:32 -0500 Subject: py/objtype: Fix assertion failures in mp_obj_new_type by checking types. Fixes assertion failures when the arguments to type() were not of valid types, e.g., when making calls like: type("", (), 3) type("", 3, {}) --- py/objtype.c | 13 ++++++++++--- tests/basics/builtin_type.py | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) (limited to 'py') diff --git a/py/objtype.c b/py/objtype.c index 2ec27c762..9b57a5051 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -1005,8 +1005,13 @@ const mp_obj_type_t mp_type_type = { }; mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) { - assert(MP_OBJ_IS_TYPE(bases_tuple, &mp_type_tuple)); // MicroPython restriction, for now - assert(MP_OBJ_IS_TYPE(locals_dict, &mp_type_dict)); // MicroPython restriction, for now + // Verify input objects have expected type + if (!MP_OBJ_IS_TYPE(bases_tuple, &mp_type_tuple)) { + mp_raise_TypeError(NULL); + } + if (!MP_OBJ_IS_TYPE(locals_dict, &mp_type_dict)) { + mp_raise_TypeError(NULL); + } // TODO might need to make a copy of locals_dict; at least that's how CPython does it @@ -1015,7 +1020,9 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) mp_obj_t *bases_items; mp_obj_tuple_get(bases_tuple, &bases_len, &bases_items); for (size_t i = 0; i < bases_len; i++) { - assert(MP_OBJ_IS_TYPE(bases_items[i], &mp_type_type)); + if (!MP_OBJ_IS_TYPE(bases_items[i], &mp_type_type)) { + mp_raise_TypeError(NULL); + } mp_obj_type_t *t = MP_OBJ_TO_PTR(bases_items[i]); // TODO: Verify with CPy, tested on function type if (t->make_new == NULL) { diff --git a/tests/basics/builtin_type.py b/tests/basics/builtin_type.py index 83c45c64b..c5fb36626 100644 --- a/tests/basics/builtin_type.py +++ b/tests/basics/builtin_type.py @@ -11,3 +11,21 @@ try: type(1, 2) except TypeError: print('TypeError') + +# second arg should be a tuple +try: + type('abc', None, None) +except TypeError: + print('TypeError') + +# third arg should be a dict +try: + type('abc', (), None) +except TypeError: + print('TypeError') + +# elements of second arg (the bases) should be types +try: + type('abc', (1,), {}) +except TypeError: + print('TypeError') -- cgit v1.2.3 From c60589c02b998794837489a6c6e51c4723af097f Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 25 Mar 2018 16:13:49 -0500 Subject: py/objtype: Fix assertion failures in super_attr by checking type. Fixes assertion failures and segmentation faults when making calls like: super(1, 1).x --- py/objtype.c | 3 +++ tests/basics/class_super.py | 6 ++++++ 2 files changed, 9 insertions(+) (limited to 'py') diff --git a/py/objtype.c b/py/objtype.c index 9b57a5051..77810ce70 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -1112,6 +1112,9 @@ STATIC mp_obj_t super_make_new(const mp_obj_type_t *type_in, size_t n_args, size // 0 arguments are turned into 2 in the compiler // 1 argument is not yet implemented mp_arg_check_num(n_args, n_kw, 2, 2, false); + if (!MP_OBJ_IS_TYPE(args[0], &mp_type_type)) { + mp_raise_TypeError(NULL); + } mp_obj_super_t *o = m_new_obj(mp_obj_super_t); *o = (mp_obj_super_t){{type_in}, args[0], args[1]}; return MP_OBJ_FROM_PTR(o); diff --git a/tests/basics/class_super.py b/tests/basics/class_super.py index 698fe5dff..b6ee68308 100644 --- a/tests/basics/class_super.py +++ b/tests/basics/class_super.py @@ -35,6 +35,12 @@ class B(A): return super().foo().count(2) # calling a subsequent method print(B().foo()) +# first arg to super must be a type +try: + super(1, 1) +except TypeError: + print('TypeError') + # store/delete of super attribute not allowed assert hasattr(super(B, B()), 'foo') try: -- cgit v1.2.3 From 1427f8f593062a9f428d40e8d3536c6a97adc479 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 4 Jun 2018 16:53:17 +1000 Subject: py/stream: Move definition of mp_stream_p_t from obj.h to stream.h. Since a long time now, mp_obj_type_t no longer refers explicitly to mp_stream_p_t but rather to an abstract "const void *protocol". So there's no longer any need to define mp_stream_p_t in obj.h and it can go with all its associated definitions in stream.h. Pretty much all users of this type will already include the stream header. --- py/obj.h | 10 ---------- py/stream.h | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'py') diff --git a/py/obj.h b/py/obj.h index 95a94836e..8e4083920 100644 --- a/py/obj.h +++ b/py/obj.h @@ -451,16 +451,6 @@ typedef struct _mp_buffer_p_t { bool mp_get_buffer(mp_obj_t obj, mp_buffer_info_t *bufinfo, mp_uint_t flags); void mp_get_buffer_raise(mp_obj_t obj, mp_buffer_info_t *bufinfo, mp_uint_t flags); -// Stream protocol -typedef struct _mp_stream_p_t { - // On error, functions should return MP_STREAM_ERROR and fill in *errcode (values - // are implementation-dependent, but will be exposed to user, e.g. via exception). - mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode); - mp_uint_t (*write)(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode); - mp_uint_t (*ioctl)(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode); - mp_uint_t is_text : 1; // default is bytes, set this for text stream -} mp_stream_p_t; - struct _mp_obj_type_t { // A type is an object so must start with this entry, which points to mp_type_type. mp_obj_base_t base; diff --git a/py/stream.h b/py/stream.h index a1d1c4f8a..3dec49a2e 100644 --- a/py/stream.h +++ b/py/stream.h @@ -62,6 +62,16 @@ struct mp_stream_seek_t { #define MP_SEEK_CUR (1) #define MP_SEEK_END (2) +// Stream protocol +typedef struct _mp_stream_p_t { + // On error, functions should return MP_STREAM_ERROR and fill in *errcode (values + // are implementation-dependent, but will be exposed to user, e.g. via exception). + mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode); + mp_uint_t (*write)(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode); + mp_uint_t (*ioctl)(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode); + mp_uint_t is_text : 1; // default is bytes, set this for text stream +} mp_stream_p_t; + MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read1_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj); -- cgit v1.2.3 From 8d82b0edbd38cd6019ec1c55dbc886c9058be7d4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 6 Jun 2018 13:11:33 +1000 Subject: extmod: Add VfsPosix filesystem component. This VFS component allows to mount a host POSIX filesystem within the uPy VFS sub-system. All traditional POSIX file access then goes through the VFS, allowing to sandbox a uPy process to a certain sub-dir of the host system, as well as mount other filesystem types alongside the host filesystem. --- extmod/vfs_posix.c | 357 ++++++++++++++++++++++++++++++++++++++++++++++++ extmod/vfs_posix.h | 41 ++++++ extmod/vfs_posix_file.c | 261 +++++++++++++++++++++++++++++++++++ py/mpconfig.h | 5 + py/py.mk | 2 + 5 files changed, 666 insertions(+) create mode 100644 extmod/vfs_posix.c create mode 100644 extmod/vfs_posix.h create mode 100644 extmod/vfs_posix_file.c (limited to 'py') diff --git a/extmod/vfs_posix.c b/extmod/vfs_posix.c new file mode 100644 index 000000000..0db45a0c1 --- /dev/null +++ b/extmod/vfs_posix.c @@ -0,0 +1,357 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017-2018 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/runtime.h" +#include "py/mperrno.h" +#include "extmod/vfs.h" +#include "extmod/vfs_posix.h" + +#if MICROPY_VFS_POSIX + +#include +#include +#include + +typedef struct _mp_obj_vfs_posix_t { + mp_obj_base_t base; + vstr_t root; + size_t root_len; + bool readonly; +} mp_obj_vfs_posix_t; + +STATIC const char *vfs_posix_get_path_str(mp_obj_vfs_posix_t *self, mp_obj_t path) { + if (self->root_len == 0) { + return mp_obj_str_get_str(path); + } else { + self->root.len = self->root_len; + vstr_add_str(&self->root, mp_obj_str_get_str(path)); + return vstr_null_terminated_str(&self->root); + } +} + +STATIC mp_obj_t vfs_posix_get_path_obj(mp_obj_vfs_posix_t *self, mp_obj_t path) { + if (self->root_len == 0) { + return path; + } else { + self->root.len = self->root_len; + vstr_add_str(&self->root, mp_obj_str_get_str(path)); + return mp_obj_new_str(self->root.buf, self->root.len); + } +} + +STATIC mp_obj_t vfs_posix_fun1_helper(mp_obj_t self_in, mp_obj_t path_in, int (*f)(const char*)) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + int ret = f(vfs_posix_get_path_str(self, path_in)); + if (ret != 0) { + mp_raise_OSError(errno); + } + return mp_const_none; +} + +mp_import_stat_t mp_vfs_posix_import_stat(mp_obj_vfs_posix_t *self, const char *path) { + if (self->root_len != 0) { + self->root.len = self->root_len; + vstr_add_str(&self->root, path); + path = vstr_null_terminated_str(&self->root); + } + struct stat st; + if (stat(path, &st) == 0) { + if (S_ISDIR(st.st_mode)) { + return MP_IMPORT_STAT_DIR; + } else if (S_ISREG(st.st_mode)) { + return MP_IMPORT_STAT_FILE; + } + } + return MP_IMPORT_STAT_NO_EXIST; +} + +STATIC mp_obj_t vfs_posix_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 0, 1, false); + + mp_obj_vfs_posix_t *vfs = m_new_obj(mp_obj_vfs_posix_t); + vfs->base.type = type; + vstr_init(&vfs->root, 0); + if (n_args == 1) { + vstr_add_str(&vfs->root, mp_obj_str_get_str(args[0])); + vstr_add_char(&vfs->root, '/'); + } + vfs->root_len = vfs->root.len; + vfs->readonly = false; + + return MP_OBJ_FROM_PTR(vfs); +} + +STATIC mp_obj_t vfs_posix_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + if (mp_obj_is_true(readonly)) { + self->readonly = true; + } + if (mp_obj_is_true(mkfs)) { + mp_raise_OSError(MP_EPERM); + } + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_3(vfs_posix_mount_obj, vfs_posix_mount); + +STATIC mp_obj_t vfs_posix_umount(mp_obj_t self_in) { + (void)self_in; + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(vfs_posix_umount_obj, vfs_posix_umount); + +STATIC mp_obj_t vfs_posix_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + const char *mode = mp_obj_str_get_str(mode_in); + if (self->readonly + && (strchr(mode, 'w') != NULL || strchr(mode, 'a') != NULL || strchr(mode, '+') != NULL)) { + mp_raise_OSError(MP_EROFS); + } + if (!MP_OBJ_IS_SMALL_INT(path_in)) { + path_in = vfs_posix_get_path_obj(self, path_in); + } + return mp_vfs_posix_file_open(&mp_type_textio, path_in, mode_in); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_3(vfs_posix_open_obj, vfs_posix_open); + +STATIC mp_obj_t vfs_posix_chdir(mp_obj_t self_in, mp_obj_t path_in) { + return vfs_posix_fun1_helper(self_in, path_in, chdir); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_chdir_obj, vfs_posix_chdir); + +STATIC mp_obj_t vfs_posix_getcwd(mp_obj_t self_in) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + char buf[MICROPY_ALLOC_PATH_MAX + 1]; + const char *ret = getcwd(buf, sizeof(buf)); + if (ret == NULL) { + mp_raise_OSError(errno); + } + ret += self->root_len; + return mp_obj_new_str(ret, strlen(ret)); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(vfs_posix_getcwd_obj, vfs_posix_getcwd); + +typedef struct _vfs_posix_ilistdir_it_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + bool is_str; + DIR *dir; +} vfs_posix_ilistdir_it_t; + +STATIC mp_obj_t vfs_posix_ilistdir_it_iternext(mp_obj_t self_in) { + vfs_posix_ilistdir_it_t *self = MP_OBJ_TO_PTR(self_in); + + if (self->dir == NULL) { + return MP_OBJ_STOP_ITERATION; + } + + for (;;) { + struct dirent *dirent = readdir(self->dir); + if (dirent == NULL) { + closedir(self->dir); + self->dir = NULL; + return MP_OBJ_STOP_ITERATION; + } + const char *fn = dirent->d_name; + + if (fn[0] == '.' && (fn[1] == 0 || fn[1] == '.')) { + // skip . and .. + continue; + } + + // make 3-tuple with info about this entry + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(3, NULL)); + + if (self->is_str) { + t->items[0] = mp_obj_new_str(fn, strlen(fn)); + } else { + t->items[0] = mp_obj_new_bytes((const byte*)fn, strlen(fn)); + } + + #ifdef _DIRENT_HAVE_D_TYPE + if (dirent->d_type == DT_DIR) { + t->items[1] = MP_OBJ_NEW_SMALL_INT(MP_S_IFDIR); + } else if (dirent->d_type == DT_REG) { + t->items[1] = MP_OBJ_NEW_SMALL_INT(MP_S_IFREG); + } else { + t->items[1] = MP_OBJ_NEW_SMALL_INT(dirent->d_type); + } + #else + // DT_UNKNOWN should have 0 value on any reasonable system + t->items[1] = MP_OBJ_NEW_SMALL_INT(0); + #endif + #ifdef _DIRENT_HAVE_D_INO + t->items[2] = MP_OBJ_NEW_SMALL_INT(dirent->d_ino); + #else + t->items[2] = MP_OBJ_NEW_SMALL_INT(0); + #endif + + return MP_OBJ_FROM_PTR(t); + } +} + +STATIC mp_obj_t vfs_posix_ilistdir(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + vfs_posix_ilistdir_it_t *iter = m_new_obj(vfs_posix_ilistdir_it_t); + iter->base.type = &mp_type_polymorph_iter; + iter->iternext = vfs_posix_ilistdir_it_iternext; + iter->is_str = mp_obj_get_type(path_in) == &mp_type_str; + const char *path = vfs_posix_get_path_str(self, path_in); + iter->dir = opendir(path); + if (iter->dir == NULL) { + mp_raise_OSError(errno); + } + return MP_OBJ_FROM_PTR(iter); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_ilistdir_obj, vfs_posix_ilistdir); + +typedef struct _mp_obj_listdir_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + DIR *dir; +} mp_obj_listdir_t; + +STATIC mp_obj_t vfs_posix_mkdir(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + int ret = mkdir(vfs_posix_get_path_str(self, path_in), 0777); + if (ret != 0) { + mp_raise_OSError(errno); + } + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_mkdir_obj, vfs_posix_mkdir); + +STATIC mp_obj_t vfs_posix_remove(mp_obj_t self_in, mp_obj_t path_in) { + return vfs_posix_fun1_helper(self_in, path_in, unlink); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_remove_obj, vfs_posix_remove); + +STATIC mp_obj_t vfs_posix_rename(mp_obj_t self_in, mp_obj_t old_path_in, mp_obj_t new_path_in) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + const char *old_path = vfs_posix_get_path_str(self, old_path_in); + const char *new_path = vfs_posix_get_path_str(self, new_path_in); + int ret = rename(old_path, new_path); + if (ret != 0) { + mp_raise_OSError(errno); + } + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_3(vfs_posix_rename_obj, vfs_posix_rename); + +STATIC mp_obj_t vfs_posix_rmdir(mp_obj_t self_in, mp_obj_t path_in) { + return vfs_posix_fun1_helper(self_in, path_in, rmdir); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_rmdir_obj, vfs_posix_rmdir); + +STATIC mp_obj_t vfs_posix_stat(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + struct stat sb; + int ret = stat(vfs_posix_get_path_str(self, path_in), &sb); + if (ret != 0) { + mp_raise_OSError(errno); + } + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(sb.st_mode); + t->items[1] = MP_OBJ_NEW_SMALL_INT(sb.st_ino); + t->items[2] = MP_OBJ_NEW_SMALL_INT(sb.st_dev); + t->items[3] = MP_OBJ_NEW_SMALL_INT(sb.st_nlink); + t->items[4] = MP_OBJ_NEW_SMALL_INT(sb.st_uid); + t->items[5] = MP_OBJ_NEW_SMALL_INT(sb.st_gid); + t->items[6] = MP_OBJ_NEW_SMALL_INT(sb.st_size); + t->items[7] = MP_OBJ_NEW_SMALL_INT(sb.st_atime); + t->items[8] = MP_OBJ_NEW_SMALL_INT(sb.st_mtime); + t->items[9] = MP_OBJ_NEW_SMALL_INT(sb.st_ctime); + return MP_OBJ_FROM_PTR(t); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_stat_obj, vfs_posix_stat); + +#ifdef __ANDROID__ +#define USE_STATFS 1 +#endif + +#if USE_STATFS +#include +#define STRUCT_STATVFS struct statfs +#define STATVFS statfs +#define F_FAVAIL sb.f_ffree +#define F_NAMEMAX sb.f_namelen +#define F_FLAG sb.f_flags +#else +#include +#define STRUCT_STATVFS struct statvfs +#define STATVFS statvfs +#define F_FAVAIL sb.f_favail +#define F_NAMEMAX sb.f_namemax +#define F_FLAG sb.f_flag +#endif + +STATIC mp_obj_t vfs_posix_statvfs(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); + STRUCT_STATVFS sb; + const char *path = vfs_posix_get_path_str(self, path_in); + int ret = STATVFS(path, &sb); + if (ret != 0) { + mp_raise_OSError(errno); + } + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(sb.f_bsize); + t->items[1] = MP_OBJ_NEW_SMALL_INT(sb.f_frsize); + t->items[2] = MP_OBJ_NEW_SMALL_INT(sb.f_blocks); + t->items[3] = MP_OBJ_NEW_SMALL_INT(sb.f_bfree); + t->items[4] = MP_OBJ_NEW_SMALL_INT(sb.f_bavail); + t->items[5] = MP_OBJ_NEW_SMALL_INT(sb.f_files); + t->items[6] = MP_OBJ_NEW_SMALL_INT(sb.f_ffree); + t->items[7] = MP_OBJ_NEW_SMALL_INT(F_FAVAIL); + t->items[8] = MP_OBJ_NEW_SMALL_INT(F_FLAG); + t->items[9] = MP_OBJ_NEW_SMALL_INT(F_NAMEMAX); + return MP_OBJ_FROM_PTR(t); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(vfs_posix_statvfs_obj, vfs_posix_statvfs); + +STATIC const mp_rom_map_elem_t vfs_posix_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_posix_mount_obj) }, + { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&vfs_posix_umount_obj) }, + { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&vfs_posix_open_obj) }, + + { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&vfs_posix_chdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_getcwd), MP_ROM_PTR(&vfs_posix_getcwd_obj) }, + { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&vfs_posix_ilistdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_mkdir), MP_ROM_PTR(&vfs_posix_mkdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&vfs_posix_remove_obj) }, + { MP_ROM_QSTR(MP_QSTR_rename), MP_ROM_PTR(&vfs_posix_rename_obj) }, + { MP_ROM_QSTR(MP_QSTR_rmdir), MP_ROM_PTR(&vfs_posix_rmdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_posix_stat_obj) }, + { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_posix_statvfs_obj) }, +}; +STATIC MP_DEFINE_CONST_DICT(vfs_posix_locals_dict, vfs_posix_locals_dict_table); + +const mp_obj_type_t mp_type_vfs_posix = { + { &mp_type_type }, + .name = MP_QSTR_VfsPosix, + .make_new = vfs_posix_make_new, + .locals_dict = (mp_obj_dict_t*)&vfs_posix_locals_dict, +}; + +#endif // MICROPY_VFS_POSIX diff --git a/extmod/vfs_posix.h b/extmod/vfs_posix.h new file mode 100644 index 000000000..35a255ff6 --- /dev/null +++ b/extmod/vfs_posix.h @@ -0,0 +1,41 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2018 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_EXTMOD_VFS_POSIX_H +#define MICROPY_INCLUDED_EXTMOD_VFS_POSIX_H + +#include "py/lexer.h" +#include "py/obj.h" + +struct _mp_obj_vfs_posix_t; + +extern const mp_obj_type_t mp_type_vfs_posix; +extern const mp_obj_type_t mp_type_vfs_posix_fileio; +extern const mp_obj_type_t mp_type_vfs_posix_textio; + +mp_import_stat_t mp_vfs_posix_import_stat(struct _mp_obj_vfs_posix_t *self, const char *path_in); +mp_obj_t mp_vfs_posix_file_open(const mp_obj_type_t *type, mp_obj_t file_in, mp_obj_t mode_in); + +#endif // MICROPY_INCLUDED_EXTMOD_VFS_POSIX_H diff --git a/extmod/vfs_posix_file.c b/extmod/vfs_posix_file.c new file mode 100644 index 000000000..435ac65cd --- /dev/null +++ b/extmod/vfs_posix_file.c @@ -0,0 +1,261 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/runtime.h" +#include "py/stream.h" +#include "extmod/vfs_posix.h" + +#if MICROPY_VFS_POSIX + +#include + +#ifdef _WIN32 +#define fsync _commit +#endif + +typedef struct _mp_obj_vfs_posix_file_t { + mp_obj_base_t base; + int fd; +} mp_obj_vfs_posix_file_t; + +#ifdef MICROPY_CPYTHON_COMPAT +STATIC void check_fd_is_open(const mp_obj_vfs_posix_file_t *o) { + if (o->fd < 0) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "I/O operation on closed file")); + } +} +#else +#define check_fd_is_open(o) +#endif + +STATIC void vfs_posix_file_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { + (void)kind; + mp_obj_vfs_posix_file_t *self = MP_OBJ_TO_PTR(self_in); + mp_printf(print, "", mp_obj_get_type_str(self_in), self->fd); +} + +mp_obj_t mp_vfs_posix_file_open(const mp_obj_type_t *type, mp_obj_t file_in, mp_obj_t mode_in) { + mp_obj_vfs_posix_file_t *o = m_new_obj(mp_obj_vfs_posix_file_t); + const char *mode_s = mp_obj_str_get_str(mode_in); + + int mode_rw = 0, mode_x = 0; + while (*mode_s) { + switch (*mode_s++) { + case 'r': + mode_rw = O_RDONLY; + break; + case 'w': + mode_rw = O_WRONLY; + mode_x = O_CREAT | O_TRUNC; + break; + case 'a': + mode_rw = O_WRONLY; + mode_x = O_CREAT | O_APPEND; + break; + case '+': + mode_rw = O_RDWR; + break; + #if MICROPY_PY_IO_FILEIO + // If we don't have io.FileIO, then files are in text mode implicitly + case 'b': + type = &mp_type_vfs_posix_fileio; + break; + case 't': + type = &mp_type_vfs_posix_textio; + break; + #endif + } + } + + o->base.type = type; + + mp_obj_t fid = file_in; + + if (MP_OBJ_IS_SMALL_INT(fid)) { + o->fd = MP_OBJ_SMALL_INT_VALUE(fid); + return MP_OBJ_FROM_PTR(o); + } + + const char *fname = mp_obj_str_get_str(fid); + int fd = open(fname, mode_x | mode_rw, 0644); + if (fd == -1) { + mp_raise_OSError(errno); + } + o->fd = fd; + return MP_OBJ_FROM_PTR(o); +} + +STATIC mp_obj_t vfs_posix_file_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + static const mp_arg_t allowed_args[] = { + { MP_QSTR_file, MP_ARG_OBJ | MP_ARG_REQUIRED, {.u_rom_obj = MP_ROM_PTR(&mp_const_none_obj)} }, + { MP_QSTR_mode, MP_ARG_OBJ, {.u_rom_obj = MP_ROM_QSTR(MP_QSTR_r)} }, + }; + + mp_arg_val_t arg_vals[MP_ARRAY_SIZE(allowed_args)]; + mp_arg_parse_all_kw_array(n_args, n_kw, args, MP_ARRAY_SIZE(allowed_args), allowed_args, arg_vals); + return mp_vfs_posix_file_open(type, arg_vals[0].u_obj, arg_vals[1].u_obj); +} + +STATIC mp_obj_t vfs_posix_file_fileno(mp_obj_t self_in) { + mp_obj_vfs_posix_file_t *self = MP_OBJ_TO_PTR(self_in); + check_fd_is_open(self); + return MP_OBJ_NEW_SMALL_INT(self->fd); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(vfs_posix_file_fileno_obj, vfs_posix_file_fileno); + +STATIC mp_obj_t vfs_posix_file___exit__(size_t n_args, const mp_obj_t *args) { + (void)n_args; + return mp_stream_close(args[0]); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(vfs_posix_file___exit___obj, 4, 4, vfs_posix_file___exit__); + +STATIC mp_uint_t vfs_posix_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { + mp_obj_vfs_posix_file_t *o = MP_OBJ_TO_PTR(o_in); + check_fd_is_open(o); + mp_int_t r = read(o->fd, buf, size); + if (r == -1) { + *errcode = errno; + return MP_STREAM_ERROR; + } + return r; +} + +STATIC mp_uint_t vfs_posix_file_write(mp_obj_t o_in, const void *buf, mp_uint_t size, int *errcode) { + mp_obj_vfs_posix_file_t *o = MP_OBJ_TO_PTR(o_in); + check_fd_is_open(o); + #if MICROPY_PY_OS_DUPTERM + if (o->fd <= STDERR_FILENO) { + mp_hal_stdout_tx_strn(buf, size); + return size; + } + #endif + mp_int_t r = write(o->fd, buf, size); + while (r == -1 && errno == EINTR) { + if (MP_STATE_VM(mp_pending_exception) != MP_OBJ_NULL) { + mp_obj_t obj = MP_STATE_VM(mp_pending_exception); + MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL; + nlr_raise(obj); + } + r = write(o->fd, buf, size); + } + if (r == -1) { + *errcode = errno; + return MP_STREAM_ERROR; + } + return r; +} + +STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_vfs_posix_file_t *o = MP_OBJ_TO_PTR(o_in); + check_fd_is_open(o); + switch (request) { + case MP_STREAM_FLUSH: + if (fsync(o->fd) < 0) { + *errcode = errno; + return MP_STREAM_ERROR; + } + return 0; + case MP_STREAM_SEEK: { + struct mp_stream_seek_t *s = (struct mp_stream_seek_t*)arg; + off_t off = lseek(o->fd, s->offset, s->whence); + if (off == (off_t)-1) { + *errcode = errno; + return MP_STREAM_ERROR; + } + s->offset = off; + return 0; + } + case MP_STREAM_CLOSE: + close(o->fd); + #ifdef MICROPY_CPYTHON_COMPAT + o->fd = -1; + #endif + return 0; + default: + *errcode = EINVAL; + return MP_STREAM_ERROR; + } +} + +STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_fileno), MP_ROM_PTR(&vfs_posix_file_fileno_obj) }, + { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, + { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, + { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) }, + { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, + { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, + { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, + { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&vfs_posix_file___exit___obj) }, +}; + +STATIC MP_DEFINE_CONST_DICT(rawfile_locals_dict, rawfile_locals_dict_table); + +#if MICROPY_PY_IO_FILEIO +STATIC const mp_stream_p_t fileio_stream_p = { + .read = vfs_posix_file_read, + .write = vfs_posix_file_write, + .ioctl = vfs_posix_file_ioctl, +}; + +const mp_obj_type_t mp_type_vfs_posix_fileio = { + { &mp_type_type }, + .name = MP_QSTR_FileIO, + .print = vfs_posix_file_print, + .make_new = vfs_posix_file_make_new, + .getiter = mp_identity_getiter, + .iternext = mp_stream_unbuffered_iter, + .protocol = &fileio_stream_p, + .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict, +}; +#endif + +STATIC const mp_stream_p_t textio_stream_p = { + .read = vfs_posix_file_read, + .write = vfs_posix_file_write, + .ioctl = vfs_posix_file_ioctl, + .is_text = true, +}; + +const mp_obj_type_t mp_type_vfs_posix_textio = { + { &mp_type_type }, + .name = MP_QSTR_TextIOWrapper, + .print = vfs_posix_file_print, + .make_new = vfs_posix_file_make_new, + .getiter = mp_identity_getiter, + .iternext = mp_stream_unbuffered_iter, + .protocol = &textio_stream_p, + .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict, +}; + +const mp_obj_vfs_posix_file_t mp_sys_stdin_obj = {{&mp_type_textio}, STDIN_FILENO}; +const mp_obj_vfs_posix_file_t mp_sys_stdout_obj = {{&mp_type_textio}, STDOUT_FILENO}; +const mp_obj_vfs_posix_file_t mp_sys_stderr_obj = {{&mp_type_textio}, STDERR_FILENO}; + +#endif // MICROPY_VFS_POSIX diff --git a/py/mpconfig.h b/py/mpconfig.h index 100e2a981..dc0fc5d40 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -681,6 +681,11 @@ typedef double mp_float_t; #define MICROPY_VFS (0) #endif +// Support for VFS POSIX component, to mount a POSIX filesystem within VFS +#ifndef MICROPY_VFS +#define MICROPY_VFS_POSIX (0) +#endif + /*****************************************************************************/ /* Fine control over Python builtins, classes, modules, etc */ diff --git a/py/py.mk b/py/py.mk index a91813526..f4b62a88a 100644 --- a/py/py.mk +++ b/py/py.mk @@ -241,6 +241,8 @@ PY_EXTMOD_O_BASENAME = \ extmod/modframebuf.o \ extmod/vfs.o \ extmod/vfs_reader.o \ + extmod/vfs_posix.o \ + extmod/vfs_posix_file.o \ extmod/vfs_fat.o \ extmod/vfs_fat_diskio.o \ extmod/vfs_fat_file.o \ -- cgit v1.2.3 From a93144cb65c4e68cba137aa82413c56e0a409d81 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 6 Jun 2018 13:12:05 +1000 Subject: py/reader: Allow MICROPY_VFS_POSIX to work with MICROPY_READER_POSIX. --- py/reader.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'py') diff --git a/py/reader.c b/py/reader.c index c4d18d53d..80364104b 100644 --- a/py/reader.c +++ b/py/reader.c @@ -124,6 +124,8 @@ void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) { reader->close = mp_reader_posix_close; } +#if !MICROPY_VFS_POSIX +// If MICROPY_VFS_POSIX is defined then this function is provided by the VFS layer void mp_reader_new_file(mp_reader_t *reader, const char *filename) { int fd = open(filename, O_RDONLY, 0644); if (fd < 0) { @@ -131,5 +133,6 @@ void mp_reader_new_file(mp_reader_t *reader, const char *filename) { } mp_reader_new_file_from_fd(reader, fd, true); } +#endif #endif -- cgit v1.2.3 From a8b9e71ac13c1cfad7bda7449683ff4b69886df0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 6 Jun 2018 14:31:29 +1000 Subject: py/mpconfig.h: Add default MICROPY_VFS_FAT config value. At least to document it's existence. --- py/mpconfig.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'py') diff --git a/py/mpconfig.h b/py/mpconfig.h index dc0fc5d40..0ea087fb3 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -686,6 +686,11 @@ typedef double mp_float_t; #define MICROPY_VFS_POSIX (0) #endif +// Support for VFS FAT component, to mount a FAT filesystem within VFS +#ifndef MICROPY_VFS +#define MICROPY_VFS_FAT (0) +#endif + /*****************************************************************************/ /* Fine control over Python builtins, classes, modules, etc */ -- cgit v1.2.3 From db5d8c97f1ee74bbd0d5b86991c6de33d0f985b6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 25 May 2018 16:48:19 +1000 Subject: py/obj.h: Introduce a "flags" entry in mp_obj_type_t. --- py/obj.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/obj.h b/py/obj.h index 8e4083920..a64cd0f69 100644 --- a/py/obj.h +++ b/py/obj.h @@ -455,8 +455,11 @@ struct _mp_obj_type_t { // A type is an object so must start with this entry, which points to mp_type_type. mp_obj_base_t base; - // The name of this type. - qstr name; + // Flags associated with this type. + uint16_t flags; + + // The name of this type, a qstr. + uint16_t name; // Corresponds to __repr__ and __str__ special methods. mp_print_fun_t print; -- cgit v1.2.3 From bace1a16d056cc755f12f525b9f2bfb3cb4b4b50 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 25 May 2018 17:08:09 +1000 Subject: py/objtype: Don't expose mp_obj_instance_attr(). mp_obj_is_instance_type() can be used instead to check for instance types. --- py/objtype.c | 2 +- py/objtype.h | 3 --- py/vm.c | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) (limited to 'py') diff --git a/py/objtype.c b/py/objtype.c index 77810ce70..d7d0ed7ff 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -764,7 +764,7 @@ STATIC bool mp_obj_instance_store_attr(mp_obj_t self_in, qstr attr, mp_obj_t val } } -void mp_obj_instance_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { +STATIC void mp_obj_instance_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (dest[0] == MP_OBJ_NULL) { mp_obj_instance_load_attr(self_in, attr, dest); } else { diff --git a/py/objtype.h b/py/objtype.h index 1f4313084..3fc8c6e1b 100644 --- a/py/objtype.h +++ b/py/objtype.h @@ -42,9 +42,6 @@ typedef struct _mp_obj_instance_t { mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *cls, const mp_obj_type_t **native_base); #endif -// this needs to be exposed for MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE to work -void mp_obj_instance_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); - // these need to be exposed so mp_obj_is_callable can work correctly bool mp_obj_instance_is_callable(mp_obj_t self_in); mp_obj_t mp_obj_instance_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args); diff --git a/py/vm.c b/py/vm.c index d24a024d5..498ecb491 100644 --- a/py/vm.c +++ b/py/vm.c @@ -336,7 +336,7 @@ dispatch_loop: MARK_EXC_IP_SELECTIVE(); DECODE_QSTR; mp_obj_t top = TOP(); - if (mp_obj_get_type(top)->attr == mp_obj_instance_attr) { + if (mp_obj_is_instance_type(mp_obj_get_type(top))) { mp_obj_instance_t *self = MP_OBJ_TO_PTR(top); mp_uint_t x = *ip; mp_obj_t key = MP_OBJ_NEW_QSTR(qst); @@ -434,7 +434,7 @@ dispatch_loop: MARK_EXC_IP_SELECTIVE(); DECODE_QSTR; mp_obj_t top = TOP(); - if (mp_obj_get_type(top)->attr == mp_obj_instance_attr && sp[-1] != MP_OBJ_NULL) { + if (mp_obj_is_instance_type(mp_obj_get_type(top)) && sp[-1] != MP_OBJ_NULL) { mp_obj_instance_t *self = MP_OBJ_TO_PTR(top); mp_uint_t x = *ip; mp_obj_t key = MP_OBJ_NEW_QSTR(qst); -- cgit v1.2.3 From 36c105218321167d47b0fb67575a7cddc36d17e7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 25 May 2018 17:09:54 +1000 Subject: py/objtype: Optimise instance get/set/del by skipping special accessors. This patch is a code optimisation, trading text bytes for speed. On pyboard it's an increase of 0.06% in code size for a gain (in pystone performance) of roughly 6.5%. The patch optimises load/store/delete of attributes in user defined classes by not looking up special accessors (@property, __get__, __delete__, __set__, __setattr__ and __getattr_) if they are guaranteed not to exist in the class. Currently, if you do my_obj.foo() then the runtime has to do a few checks to see if foo is a property or has __get__, and if so delegate the call. And for stores things like my_obj.foo = 1 has to first check if foo is a property or has __set__ defined on it. Doing all those checks each and every time the attribute is accessed has a performance penalty. This patch eliminates all those checks for cases when it's guaranteed that the checks will always fail, ie no attributes are properties nor have any special accessor methods defined on them. To make this guarantee it checks all attributes of a user-defined class when it is first created. If any of the attributes of the user class are properties or have special accessors, or any of the base classes of the user class have them, then it sets a flag in the class to indicate that special accessors must be checked for. Then in the load/store/delete code it checks this flag to see if it can take the shortcut and optimise the lookup. It's an optimisation that's pretty widely applicable because it improves lookup performance for all methods of user defined classes, and stores of attributes, at least for those that don't have special accessors. And, it allows to enable descriptors with minimal additional runtime overhead if they are not used for a particular user class. There is one restriction on dynamic class creation that has been introduced by this patch: a user-defined class cannot go from zero special accessors to one special accessor (or more) after that class has been subclassed. If the script attempts this an AttributeError is raised (see addition to tests/misc/non_compliant.py for an example of this case). The cost in code space bytes for the optimisation in this patch is: unix x64: +528 unix nanbox: +508 stm32: +192 cc3200: +200 esp8266: +332 esp32: +244 Performance tests that were done: - on unix x86-64, pystone improved by about 5% - on pyboard, pystone improved by about 6.5%, from 1683 up to 1794 - on pyboard, bm_chaos (from CPython benchmark suite) improved by about 5% - on esp32, pystone improved by about 30% (but there are caching effects) - on esp32, bm_chaos improved by about 11% --- py/mpconfig.h | 8 ++- py/objtype.c | 101 ++++++++++++++++++++++++++++--- tests/basics/builtin_property_inherit.py | 53 ++++++++++++++++ tests/misc/non_compliant.py | 10 +++ tests/misc/non_compliant.py.exp | 1 + 5 files changed, 161 insertions(+), 12 deletions(-) create mode 100644 tests/basics/builtin_property_inherit.py (limited to 'py') diff --git a/py/mpconfig.h b/py/mpconfig.h index 0ea087fb3..26c9cd92b 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -706,14 +706,16 @@ typedef double mp_float_t; #define MICROPY_PY_FUNCTION_ATTRS (0) #endif -// Whether to support descriptors (__get__ and __set__) -// This costs some code size and makes all load attrs and store attrs slow +// Whether to support the descriptors __get__, __set__, __delete__ +// This costs some code size and makes load/store/delete of instance +// attributes slower for the classes that use this feature #ifndef MICROPY_PY_DESCRIPTORS #define MICROPY_PY_DESCRIPTORS (0) #endif // Whether to support class __delattr__ and __setattr__ methods -// This costs some code size and makes all del attrs and store attrs slow +// This costs some code size and makes store/delete of instance +// attributes slower for the classes that use this feature #ifndef MICROPY_PY_DELATTR_SETATTR #define MICROPY_PY_DELATTR_SETATTR (0) #endif diff --git a/py/objtype.c b/py/objtype.c index d7d0ed7ff..ef70dfce0 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013-2018 Damien P. George * Copyright (c) 2014-2016 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -41,6 +41,12 @@ #define DEBUG_printf(...) (void)0 #endif +#define ENABLE_SPECIAL_ACCESSORS \ + (MICROPY_PY_DESCRIPTORS || MICROPY_PY_DELATTR_SETATTR || MICROPY_PY_BUILTINS_PROPERTY) + +#define TYPE_FLAG_IS_SUBCLASSED (0x0001) +#define TYPE_FLAG_HAS_SPECIAL_ACCESSORS (0x0002) + STATIC mp_obj_t static_class_method_make_new(const mp_obj_type_t *self_in, size_t n_args, size_t n_kw, const mp_obj_t *args); /******************************************************************************/ @@ -591,6 +597,11 @@ STATIC void mp_obj_instance_load_attr(mp_obj_t self_in, qstr attr, mp_obj_t *des mp_obj_class_lookup(&lookup, self->base.type); mp_obj_t member = dest[0]; if (member != MP_OBJ_NULL) { + if (!(self->base.type->flags & TYPE_FLAG_HAS_SPECIAL_ACCESSORS)) { + // Class doesn't have any special accessors to check so return straightaway + return; + } + #if MICROPY_PY_BUILTINS_PROPERTY if (MP_OBJ_IS_TYPE(member, &mp_type_property)) { // object member is a property; delegate the load to the property @@ -652,11 +663,15 @@ STATIC void mp_obj_instance_load_attr(mp_obj_t self_in, qstr attr, mp_obj_t *des STATIC bool mp_obj_instance_store_attr(mp_obj_t self_in, qstr attr, mp_obj_t value) { mp_obj_instance_t *self = MP_OBJ_TO_PTR(self_in); + if (!(self->base.type->flags & TYPE_FLAG_HAS_SPECIAL_ACCESSORS)) { + // Class doesn't have any special accessors so skip their checks + goto skip_special_accessors; + } + #if MICROPY_PY_BUILTINS_PROPERTY || MICROPY_PY_DESCRIPTORS // With property and/or descriptors enabled we need to do a lookup // first in the class dict for the attribute to see if the store should // be delegated. - // Note: this makes all stores slow... how to fix? mp_obj_t member[2] = {MP_OBJ_NULL}; struct class_lookup_data lookup = { .obj = self, @@ -728,9 +743,9 @@ STATIC bool mp_obj_instance_store_attr(mp_obj_t self_in, qstr attr, mp_obj_t val } #endif + #if MICROPY_PY_DELATTR_SETATTR if (value == MP_OBJ_NULL) { // delete attribute - #if MICROPY_PY_DELATTR_SETATTR // try __delattr__ first mp_obj_t attr_delattr_method[3]; mp_load_method_maybe(self_in, MP_QSTR___delattr__, attr_delattr_method); @@ -740,13 +755,8 @@ STATIC bool mp_obj_instance_store_attr(mp_obj_t self_in, qstr attr, mp_obj_t val mp_call_method_n_kw(1, 0, attr_delattr_method); return true; } - #endif - - mp_map_elem_t *elem = mp_map_lookup(&self->members, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_REMOVE_IF_FOUND); - return elem != NULL; } else { // store attribute - #if MICROPY_PY_DELATTR_SETATTR // try __setattr__ first mp_obj_t attr_setattr_method[4]; mp_load_method_maybe(self_in, MP_QSTR___setattr__, attr_setattr_method); @@ -757,8 +767,17 @@ STATIC bool mp_obj_instance_store_attr(mp_obj_t self_in, qstr attr, mp_obj_t val mp_call_method_n_kw(2, 0, attr_setattr_method); return true; } - #endif + } + #endif + +skip_special_accessors: + if (value == MP_OBJ_NULL) { + // delete attribute + mp_map_elem_t *elem = mp_map_lookup(&self->members, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_REMOVE_IF_FOUND); + return elem != NULL; + } else { + // store attribute mp_map_lookup(&self->members, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = value; return true; } @@ -899,6 +918,34 @@ STATIC mp_int_t instance_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, // - there is a constant mp_obj_type_t (called mp_type_type) for the 'type' object // - creating a new class (a new type) creates a new mp_obj_type_t +#if ENABLE_SPECIAL_ACCESSORS +STATIC bool check_for_special_accessors(mp_obj_t key, mp_obj_t value) { + #if MICROPY_PY_DELATTR_SETATTR + if (key == MP_OBJ_NEW_QSTR(MP_QSTR___setattr__) || key == MP_OBJ_NEW_QSTR(MP_QSTR___delattr__)) { + return true; + } + #endif + #if MICROPY_PY_BUILTINS_PROPERTY + if (MP_OBJ_IS_TYPE(value, &mp_type_property)) { + return true; + } + #endif + #if MICROPY_PY_DESCRIPTORS + static const uint8_t to_check[] = { + MP_QSTR___get__, MP_QSTR___set__, MP_QSTR___delete__, + }; + for (size_t i = 0; i < MP_ARRAY_SIZE(to_check); ++i) { + mp_obj_t dest_temp[2]; + mp_load_method_protected(value, to_check[i], dest_temp, true); + if (dest_temp[0] != MP_OBJ_NULL) { + return true; + } + } + #endif + return false; +} +#endif + STATIC void type_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_type_t *self = MP_OBJ_TO_PTR(self_in); @@ -985,6 +1032,19 @@ STATIC void type_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { dest[0] = MP_OBJ_NULL; // indicate success } } else { + #if ENABLE_SPECIAL_ACCESSORS + // Check if we add any special accessor methods with this store + if (!(self->flags & TYPE_FLAG_HAS_SPECIAL_ACCESSORS)) { + if (check_for_special_accessors(MP_OBJ_NEW_QSTR(attr), dest[1])) { + if (self->flags & TYPE_FLAG_IS_SUBCLASSED) { + // This class is already subclassed so can't have special accessors added + mp_raise_msg(&mp_type_AttributeError, "can't add special method to already-subclassed class"); + } + self->flags |= TYPE_FLAG_HAS_SPECIAL_ACCESSORS; + } + } + #endif + // store attribute mp_map_elem_t *elem = mp_map_lookup(locals_map, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); elem->value = dest[1]; @@ -1016,6 +1076,7 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) // TODO might need to make a copy of locals_dict; at least that's how CPython does it // Basic validation of base classes + uint16_t base_flags = 0; size_t bases_len; mp_obj_t *bases_items; mp_obj_tuple_get(bases_tuple, &bases_len, &bases_items); @@ -1033,10 +1094,17 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) "type '%q' is not an acceptable base type", t->name)); } } + #if ENABLE_SPECIAL_ACCESSORS + if (mp_obj_is_instance_type(t)) { + t->flags |= TYPE_FLAG_IS_SUBCLASSED; + base_flags |= t->flags & TYPE_FLAG_HAS_SPECIAL_ACCESSORS; + } + #endif } mp_obj_type_t *o = m_new0(mp_obj_type_t, 1); o->base.type = &mp_type_type; + o->flags = base_flags; o->name = name; o->print = instance_print; o->make_new = mp_obj_instance_make_new; @@ -1069,6 +1137,21 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) o->locals_dict = MP_OBJ_TO_PTR(locals_dict); + #if ENABLE_SPECIAL_ACCESSORS + // Check if the class has any special accessor methods + if (!(o->flags & TYPE_FLAG_HAS_SPECIAL_ACCESSORS)) { + for (size_t i = 0; i < o->locals_dict->map.alloc; i++) { + if (MP_MAP_SLOT_IS_FILLED(&o->locals_dict->map, i)) { + const mp_map_elem_t *elem = &o->locals_dict->map.table[i]; + if (check_for_special_accessors(elem->key, elem->value)) { + o->flags |= TYPE_FLAG_HAS_SPECIAL_ACCESSORS; + break; + } + } + } + } + #endif + const mp_obj_type_t *native_base; size_t num_native_bases = instance_count_native_bases(o, &native_base); if (num_native_bases > 1) { diff --git a/tests/basics/builtin_property_inherit.py b/tests/basics/builtin_property_inherit.py new file mode 100644 index 000000000..27a091393 --- /dev/null +++ b/tests/basics/builtin_property_inherit.py @@ -0,0 +1,53 @@ +# test builtin property combined with inheritance +try: + property +except: + print("SKIP") + raise SystemExit + +# test property in a base class works for derived classes +class A: + @property + def x(self): + print('A x') + return 123 +class B(A): + pass +class C(B): + pass +class D: + pass +class E(C, D): + pass +print(A().x) +print(B().x) +print(C().x) +print(E().x) + +# test that we can add a property to base class after creation +class F: + pass +F.foo = property(lambda self: print('foo get')) +class G(F): + pass +F().foo +G().foo + +# should be able to add a property to already-subclassed class because it already has one +F.bar = property(lambda self: print('bar get')) +F().bar +G().bar + +# test case where class (H here) is already subclassed before adding attributes +class H: + pass +class I(H): + pass + +# should be able to add a normal member to already-subclassed class +H.val = 2 +print(I().val) + +# should be able to add a property to the derived class +I.baz = property(lambda self: print('baz get')) +I().baz diff --git a/tests/misc/non_compliant.py b/tests/misc/non_compliant.py index 31129f075..580583bf3 100644 --- a/tests/misc/non_compliant.py +++ b/tests/misc/non_compliant.py @@ -139,3 +139,13 @@ class A: class B(object, A): pass B().foo() + +# can't assign property (or other special accessors) to already-subclassed class +class A: + pass +class B(A): + pass +try: + A.bar = property() +except AttributeError: + print('AttributeError') diff --git a/tests/misc/non_compliant.py.exp b/tests/misc/non_compliant.py.exp index 061e3fcc8..3f15a1440 100644 --- a/tests/misc/non_compliant.py.exp +++ b/tests/misc/non_compliant.py.exp @@ -20,3 +20,4 @@ NotImplementedError AttributeError TypeError A.foo +AttributeError -- cgit v1.2.3 From 522ea80f06a80fd799bd57de351373ceaeeae325 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 31 May 2018 23:10:48 +1000 Subject: py/gc: Add gc_sweep_all() function to run all remaining finalisers. This patch adds the gc_sweep_all() function which does a garbage collection without tracing any root pointers, so frees all the memory, and most importantly runs any remaining finalisers. This helps primarily for soft reset: it will close any open files, any open sockets, and help to get the system back to a clean state upon soft reset. --- py/gc.c | 7 +++++++ py/gc.h | 3 +++ 2 files changed, 10 insertions(+) (limited to 'py') diff --git a/py/gc.c b/py/gc.c index e92b81ece..0fc43ef49 100644 --- a/py/gc.c +++ b/py/gc.c @@ -362,6 +362,13 @@ void gc_collect_end(void) { GC_EXIT(); } +void gc_sweep_all(void) { + GC_ENTER(); + MP_STATE_MEM(gc_lock_depth)++; + MP_STATE_MEM(gc_stack_overflow) = 0; + gc_collect_end(); +} + void gc_info(gc_info_t *info) { GC_ENTER(); info->total = MP_STATE_MEM(gc_pool_end) - MP_STATE_MEM(gc_pool_start); diff --git a/py/gc.h b/py/gc.h index 739349c1f..73d86e6c3 100644 --- a/py/gc.h +++ b/py/gc.h @@ -45,6 +45,9 @@ void gc_collect_start(void); void gc_collect_root(void **ptrs, size_t len); void gc_collect_end(void); +// Use this function to sweep the whole heap and run all finalisers +void gc_sweep_all(void); + void *gc_alloc(size_t n_bytes, bool has_finaliser); void gc_free(void *ptr); // does not call finaliser size_t gc_nbytes(const void *ptr); -- cgit v1.2.3 From 6a445b60fad87c94a1d00ce3a043b881a1f7a5a4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 2 Jun 2018 00:21:46 +1000 Subject: py/lexer: Add support for underscores in numeric literals. This is a very convenient feature introduced in Python 3.6 by PEP 515. --- py/lexer.c | 2 ++ py/parsenum.c | 4 ++++ tests/basics/python36.py | 10 ++++++++++ tests/basics/python36.py.exp | 5 +++++ tests/float/python36.py | 10 ++++++++++ tests/float/python36.py.exp | 5 +++++ 6 files changed, 36 insertions(+) create mode 100644 tests/basics/python36.py create mode 100644 tests/basics/python36.py.exp create mode 100644 tests/float/python36.py create mode 100644 tests/float/python36.py.exp (limited to 'py') diff --git a/py/lexer.c b/py/lexer.c index 6017d69d6..e161700b1 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -590,6 +590,8 @@ void mp_lexer_to_next(mp_lexer_t *lex) { } vstr_add_char(&lex->vstr, CUR_CHAR(lex)); next_char(lex); + } else if (is_char(lex, '_')) { + next_char(lex); } else { break; } diff --git a/py/parsenum.c b/py/parsenum.c index 354d0f756..b7e5a3c83 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -83,6 +83,8 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m mp_uint_t dig = *str; if ('0' <= dig && dig <= '9') { dig -= '0'; + } else if (dig == '_') { + continue; } else { dig |= 0x20; // make digit lower-case if ('a' <= dig && dig <= 'z') { @@ -273,6 +275,8 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool } else if (allow_imag && (dig | 0x20) == 'j') { imag = true; break; + } else if (dig == '_') { + continue; } else { // unknown character str--; diff --git a/tests/basics/python36.py b/tests/basics/python36.py new file mode 100644 index 000000000..20ecd9227 --- /dev/null +++ b/tests/basics/python36.py @@ -0,0 +1,10 @@ +# tests for things that only Python 3.6 supports + +# underscores in numeric literals +print(100_000) +print(0b1010_0101) +print(0xff_ff) + +# underscore supported by int constructor +print(int('1_2_3')) +print(int('0o1_2_3', 8)) diff --git a/tests/basics/python36.py.exp b/tests/basics/python36.py.exp new file mode 100644 index 000000000..4b65daafa --- /dev/null +++ b/tests/basics/python36.py.exp @@ -0,0 +1,5 @@ +100000 +165 +65535 +123 +83 diff --git a/tests/float/python36.py b/tests/float/python36.py new file mode 100644 index 000000000..6e8fb1f21 --- /dev/null +++ b/tests/float/python36.py @@ -0,0 +1,10 @@ +# tests for things that only Python 3.6 supports, needing floats + +# underscores in numeric literals +print(1_000.1_8) +print('%.2g' % 1e1_2) + +# underscore supported by int/float constructors +print(float('1_2_3')) +print(float('1_2_3.4')) +print('%.2g' % float('1e1_3')) diff --git a/tests/float/python36.py.exp b/tests/float/python36.py.exp new file mode 100644 index 000000000..3febfed9a --- /dev/null +++ b/tests/float/python36.py.exp @@ -0,0 +1,5 @@ +1000.18 +1e+12 +123.0 +123.4 +1e+13 -- cgit v1.2.3 From af0932a7793478f0b90b754d38955d69700b0bee Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 4 Jun 2018 15:54:26 +1000 Subject: py/modio: Add uio.IOBase class to allow to define user streams. A user class derived from IOBase and implementing readinto/write/ioctl can now be used anywhere a native stream object is accepted. The mapping from C to Python is: stream_p->read --> readinto(buf) stream_p->write --> write(buf) stream_p->ioctl --> ioctl(request, arg) Among other things it allows the user to: - create an object which can be passed as the file argument to print: print(..., file=myobj), and then print will pass all the data to the object via the objects write method (same as CPython) - pass a user object to uio.BufferedWriter to buffer the writes (same as CPython) - use select.select on a user object - register user objects with select.poll, in particular so user objects can be used with uasyncio - create user files that can be returned from user filesystems, and import can import scripts from these user files For example: class MyOut(io.IOBase): def write(self, buf): print('write', repr(buf)) return len(buf) print('hello', file=MyOut()) The feature is enabled via MICROPY_PY_IO_IOBASE which is disabled by default. --- py/modio.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ py/mpconfig.h | 5 +++++ 2 files changed, 74 insertions(+) (limited to 'py') diff --git a/py/modio.c b/py/modio.c index 3a5c69c4c..e75432b28 100644 --- a/py/modio.c +++ b/py/modio.c @@ -30,6 +30,8 @@ #include "py/runtime.h" #include "py/builtin.h" #include "py/stream.h" +#include "py/binary.h" +#include "py/objarray.h" #include "py/objstringio.h" #include "py/frozenmod.h" @@ -38,6 +40,70 @@ extern const mp_obj_type_t mp_type_fileio; extern const mp_obj_type_t mp_type_textio; +#if MICROPY_PY_IO_IOBASE + +STATIC const mp_obj_type_t mp_type_iobase; + +STATIC mp_obj_base_t iobase_singleton = {&mp_type_iobase}; + +STATIC mp_obj_t iobase_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + (void)type; + (void)n_args; + (void)n_kw; + (void)args; + return MP_OBJ_FROM_PTR(&iobase_singleton); +} + +STATIC mp_uint_t iobase_read_write(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode, qstr qst) { + mp_obj_t dest[3]; + mp_load_method(obj, qst, dest); + mp_obj_array_t ar = {{&mp_type_bytearray}, BYTEARRAY_TYPECODE, 0, size, buf}; + dest[2] = MP_OBJ_FROM_PTR(&ar); + mp_obj_t ret = mp_call_method_n_kw(1, 0, dest); + if (ret == mp_const_none) { + *errcode = MP_EAGAIN; + return MP_STREAM_ERROR; + } else { + return mp_obj_get_int(ret); + } +} +STATIC mp_uint_t iobase_read(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode) { + return iobase_read_write(obj, buf, size, errcode, MP_QSTR_readinto); +} + +STATIC mp_uint_t iobase_write(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode) { + return iobase_read_write(obj, (void*)buf, size, errcode, MP_QSTR_write); +} + +STATIC mp_uint_t iobase_ioctl(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_t dest[4]; + mp_load_method(obj, MP_QSTR_ioctl, dest); + dest[2] = mp_obj_new_int_from_uint(request); + dest[3] = mp_obj_new_int_from_uint(arg); + mp_int_t ret = mp_obj_get_int(mp_call_method_n_kw(2, 0, dest)); + if (ret >= 0) { + return ret; + } else { + *errcode = -ret; + return MP_STREAM_ERROR; + } +} + +STATIC const mp_stream_p_t iobase_p = { + .read = iobase_read, + .write = iobase_write, + .ioctl = iobase_ioctl, +}; + +STATIC const mp_obj_type_t mp_type_iobase = { + { &mp_type_type }, + .name = MP_QSTR_IOBase, + .make_new = iobase_make_new, + .protocol = &iobase_p, +}; + +#endif // MICROPY_PY_IO_IOBASE + #if MICROPY_PY_IO_BUFFEREDWRITER typedef struct _mp_obj_bufwriter_t { mp_obj_base_t base; @@ -187,6 +253,9 @@ STATIC const mp_rom_map_elem_t mp_module_io_globals_table[] = { // Note: mp_builtin_open_obj should be defined by port, it's not // part of the core. { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&mp_builtin_open_obj) }, + #if MICROPY_PY_IO_IOBASE + { MP_ROM_QSTR(MP_QSTR_IOBase), MP_ROM_PTR(&mp_type_iobase) }, + #endif #if MICROPY_PY_IO_RESOURCE_STREAM { MP_ROM_QSTR(MP_QSTR_resource_stream), MP_ROM_PTR(&resource_stream_obj) }, #endif diff --git a/py/mpconfig.h b/py/mpconfig.h index 26c9cd92b..a85b22128 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -998,6 +998,11 @@ typedef double mp_float_t; #define MICROPY_PY_IO (1) #endif +// Whether to provide "io.IOBase" class to support user streams +#ifndef MICROPY_PY_IO_IOBASE +#define MICROPY_PY_IO_IOBASE (0) +#endif + // Whether to provide "uio.resource_stream()" function with // the semantics of CPython's pkg_resources.resource_stream() // (allows to access binary resources in frozen source packages). -- cgit v1.2.3 From 6630354ffe9f33d2af8154fbac003cb4db0cc348 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 9 Jun 2018 02:48:29 +0300 Subject: extmod/moduhashlib: Allow to disable the sha256 class. Via the config value MICROPY_PY_UHASHLIB_SHA256. Default to enabled to keep backwards compatibility. Also add default value for the sha1 class, to at least document its existence. --- extmod/moduhashlib.c | 8 ++++++++ py/mpconfig.h | 8 ++++++++ 2 files changed, 16 insertions(+) (limited to 'py') diff --git a/extmod/moduhashlib.c b/extmod/moduhashlib.c index 2eb90ed73..e47bae0a0 100644 --- a/extmod/moduhashlib.c +++ b/extmod/moduhashlib.c @@ -31,7 +31,9 @@ #if MICROPY_PY_UHASHLIB +#if MICROPY_PY_UHASHLIB_SHA256 #include "crypto-algorithms/sha256.h" +#endif #if MICROPY_PY_UHASHLIB_SHA1 @@ -50,6 +52,7 @@ typedef struct _mp_obj_hash_t { char state[0]; } mp_obj_hash_t; +#if MICROPY_PY_UHASHLIB_SHA256 STATIC mp_obj_t uhashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg); STATIC mp_obj_t uhashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { @@ -94,6 +97,7 @@ STATIC const mp_obj_type_t uhashlib_sha256_type = { .make_new = uhashlib_sha256_make_new, .locals_dict = (void*)&uhashlib_sha256_locals_dict, }; +#endif #if MICROPY_PY_UHASHLIB_SHA1 STATIC mp_obj_t uhashlib_sha1_update(mp_obj_t self_in, mp_obj_t arg); @@ -177,7 +181,9 @@ STATIC const mp_obj_type_t uhashlib_sha1_type = { STATIC const mp_rom_map_elem_t mp_module_uhashlib_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uhashlib) }, + #if MICROPY_PY_UHASHLIB_SHA256 { MP_ROM_QSTR(MP_QSTR_sha256), MP_ROM_PTR(&uhashlib_sha256_type) }, + #endif #if MICROPY_PY_UHASHLIB_SHA1 { MP_ROM_QSTR(MP_QSTR_sha1), MP_ROM_PTR(&uhashlib_sha1_type) }, #endif @@ -190,6 +196,8 @@ const mp_obj_module_t mp_module_uhashlib = { .globals = (mp_obj_dict_t*)&mp_module_uhashlib_globals, }; +#if MICROPY_PY_UHASHLIB_SHA256 #include "crypto-algorithms/sha256.c" +#endif #endif //MICROPY_PY_UHASHLIB diff --git a/py/mpconfig.h b/py/mpconfig.h index a85b22128..08d100549 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1155,6 +1155,14 @@ typedef double mp_float_t; #define MICROPY_PY_UHASHLIB (0) #endif +#ifndef MICROPY_PY_UHASHLIB_SHA1 +#define MICROPY_PY_UHASHLIB_SHA1 (0) +#endif + +#ifndef MICROPY_PY_UHASHLIB_SHA256 +#define MICROPY_PY_UHASHLIB_SHA256 (1) +#endif + #ifndef MICROPY_PY_UBINASCII #define MICROPY_PY_UBINASCII (0) #endif -- cgit v1.2.3 From 7ad04d17dabd503b0a79426166bcdc9e8ea4bf91 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 9 Jun 2018 15:23:13 +1000 Subject: py/mkrules.mk: Regenerate all qstrs when config files change. A port can define QSTR_GLOBAL_DEPENDENCIES to add extra files. --- py/mkrules.mk | 8 ++++++-- py/py.mk | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/mkrules.mk b/py/mkrules.mk index 850c2aa3a..30ac520aa 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -66,9 +66,13 @@ $(BUILD)/%.pp: %.c # to get built before we try to compile any of them. $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h -$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) | $(HEADER_BUILD)/mpversion.h +# The logic for qstr regeneration is: +# - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^) +# - else, if list of newer prerequisites ($?) is not empty, then process just these ($?) +# - else, process all source files ($^) [this covers "make -B" which can set $? to empty] +$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h $(ECHO) "GEN $@" - $(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(if $?,$?,$^) >$(HEADER_BUILD)/qstr.i.last; + $(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) >$(HEADER_BUILD)/qstr.i.last; $(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last $(ECHO) "GEN $@" diff --git a/py/py.mk b/py/py.mk index f4b62a88a..0027fbb88 100644 --- a/py/py.mk +++ b/py/py.mk @@ -13,6 +13,9 @@ ifneq ($(QSTR_AUTOGEN_DISABLE),1) QSTR_DEFS_COLLECTED = $(HEADER_BUILD)/qstrdefs.collected.h endif +# Any files listed by this variable will cause a full regeneration of qstrs +QSTR_GLOBAL_DEPENDENCIES += $(PY_SRC)/mpconfig.h mpconfigport.h + # some code is performance bottleneck and compiled with other optimization options CSUPEROPT = -O3 -- cgit v1.2.3 From 035906419d925bf723dba21f9fbb22ca1c8021f1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 12 Jun 2018 15:06:11 +1000 Subject: extmod/uos_dupterm: Use native C stream methods on dupterm object. This patch changes dupterm to call the native C stream methods on the connected stream objects, instead of calling the Python readinto/write methods. This is much more efficient for native stream objects like UART and webrepl and doesn't require allocating a special dupterm array. This change is a minor breaking change from the user's perspective because dupterm no longer accepts pure user stream objects to duplicate on. But with the recent addition of uio.IOBase it is possible to still create such classes just by inheriting from uio.IOBase, for example: import uio, uos class MyStream(uio.IOBase): def write(self, buf): # existing write implementation def readinto(self, buf): # existing readinto implementation uos.dupterm(MyStream()) --- extmod/uos_dupterm.c | 42 ++++++++++++++++-------------------------- py/mpstate.h | 1 - py/runtime.c | 1 - 3 files changed, 16 insertions(+), 28 deletions(-) (limited to 'py') diff --git a/extmod/uos_dupterm.c b/extmod/uos_dupterm.c index f77cff577..822d64037 100644 --- a/extmod/uos_dupterm.c +++ b/extmod/uos_dupterm.c @@ -60,25 +60,29 @@ int mp_uos_dupterm_rx_chr(void) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_obj_t readinto_m[3]; - mp_load_method(MP_STATE_VM(dupterm_objs[idx]), MP_QSTR_readinto, readinto_m); - readinto_m[2] = MP_STATE_VM(dupterm_arr_obj); - mp_obj_t res = mp_call_method_n_kw(1, 0, readinto_m); - if (res == mp_const_none) { - nlr_pop(); - } else if (res == MP_OBJ_NEW_SMALL_INT(0)) { + byte buf[1]; + int errcode; + const mp_stream_p_t *stream_p = mp_get_stream_raise(MP_STATE_VM(dupterm_objs[idx]), MP_STREAM_OP_READ); + mp_uint_t out_sz = stream_p->read(MP_STATE_VM(dupterm_objs[idx]), buf, 1, &errcode); + if (out_sz == 0) { nlr_pop(); mp_uos_deactivate(idx, "dupterm: EOF received, deactivating\n", MP_OBJ_NULL); + } else if (out_sz == MP_STREAM_ERROR) { + // errcode is valid + if (mp_is_nonblocking_error(errcode)) { + nlr_pop(); + } else { + mp_raise_OSError(errcode); + } } else { - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(MP_STATE_VM(dupterm_arr_obj), &bufinfo, MP_BUFFER_READ); + // read 1 byte nlr_pop(); - if (*(byte*)bufinfo.buf == mp_interrupt_char) { + if (buf[0] == mp_interrupt_char) { // Signal keyboard interrupt to be raised as soon as the VM resumes mp_keyboard_interrupt(); return -2; } - return *(byte*)bufinfo.buf; + return buf[0]; } } else { mp_uos_deactivate(idx, "dupterm: Exception in read() method, deactivating: ", nlr.ret_val); @@ -96,18 +100,7 @@ void mp_uos_dupterm_tx_strn(const char *str, size_t len) { } nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_obj_t write_m[3]; - mp_load_method(MP_STATE_VM(dupterm_objs[idx]), MP_QSTR_write, write_m); - - mp_obj_array_t *arr = MP_OBJ_TO_PTR(MP_STATE_VM(dupterm_arr_obj)); - void *org_items = arr->items; - arr->items = (void*)str; - arr->len = len; - write_m[2] = MP_STATE_VM(dupterm_arr_obj); - mp_call_method_n_kw(1, 0, write_m); - arr = MP_OBJ_TO_PTR(MP_STATE_VM(dupterm_arr_obj)); - arr->items = org_items; - arr->len = 1; + mp_stream_write(MP_STATE_VM(dupterm_objs[idx]), str, len, MP_STREAM_RW_WRITE); nlr_pop(); } else { mp_uos_deactivate(idx, "dupterm: Exception in write() method, deactivating: ", nlr.ret_val); @@ -133,9 +126,6 @@ STATIC mp_obj_t mp_uos_dupterm(size_t n_args, const mp_obj_t *args) { MP_STATE_VM(dupterm_objs[idx]) = MP_OBJ_NULL; } else { MP_STATE_VM(dupterm_objs[idx]) = args[0]; - if (MP_STATE_VM(dupterm_arr_obj) == MP_OBJ_NULL) { - MP_STATE_VM(dupterm_arr_obj) = mp_obj_new_bytearray(1, ""); - } } return previous_obj; diff --git a/py/mpstate.h b/py/mpstate.h index 199fd2cb6..8c3b710cb 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -171,7 +171,6 @@ typedef struct _mp_state_vm_t { #if MICROPY_PY_OS_DUPTERM mp_obj_t dupterm_objs[MICROPY_PY_OS_DUPTERM]; - mp_obj_t dupterm_arr_obj; #endif #if MICROPY_PY_LWIP_SLIP diff --git a/py/runtime.c b/py/runtime.c index 8f1063076..a2dac46a4 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -108,7 +108,6 @@ void mp_init(void) { for (size_t i = 0; i < MICROPY_PY_OS_DUPTERM; ++i) { MP_STATE_VM(dupterm_objs[i]) = MP_OBJ_NULL; } - MP_STATE_VM(dupterm_arr_obj) = MP_OBJ_NULL; #endif #if MICROPY_FSUSERMOUNT -- cgit v1.2.3 From 6abede2ca9e221b6aefcaccbda0c89e367507df1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 13 Jun 2018 11:54:44 +1000 Subject: py/stream: Introduce and use efficient mp_get_stream to access stream_p. The existing mp_get_stream_raise() helper does explicit checks that the input object is a real pointer object, has a non-NULL stream protocol, and has the desired stream C method (read/write/ioctl). In most cases it is not necessary to do these checks because it is guaranteed that the input object has the stream protocol and desired C methods. For example, native objects that use the stream wrappers (eg mp_stream_readinto_obj) in their locals dict always have the stream protocol (or else they shouldn't have these wrappers in their locals dict). This patch introduces an efficient mp_get_stream() which doesn't do any checks and just extracts the stream protocol struct. This should be used in all cases where the argument object is known to be a stream. The existing mp_get_stream_raise() should be used primarily to verify that an object does have the correct stream protocol methods. All uses of mp_get_stream_raise() in py/stream.c have been converted to use mp_get_stream() because the argument is guaranteed to be a proper stream object. This patch improves efficiency of stream operations and reduces code size. --- py/stream.c | 25 ++++++++++--------------- py/stream.h | 5 +++++ 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'py') diff --git a/py/stream.c b/py/stream.c index 393988d2c..a65ba4c91 100644 --- a/py/stream.c +++ b/py/stream.c @@ -1,3 +1,4 @@ + /* * This file is part of the MicroPython project, http://micropython.org/ * @@ -47,10 +48,9 @@ STATIC mp_obj_t stream_readall(mp_obj_t self_in); // be equal to input size). mp_uint_t mp_stream_rw(mp_obj_t stream, void *buf_, mp_uint_t size, int *errcode, byte flags) { byte *buf = buf_; - mp_obj_base_t* s = (mp_obj_base_t*)MP_OBJ_TO_PTR(stream); typedef mp_uint_t (*io_func_t)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode); io_func_t io_func; - const mp_stream_p_t *stream_p = s->type->protocol; + const mp_stream_p_t *stream_p = mp_get_stream(stream); if (flags & MP_STREAM_RW_WRITE) { io_func = (io_func_t)stream_p->write; } else { @@ -99,8 +99,6 @@ const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags) { } STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte flags) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ); - // What to do if sz < -1? Python docs don't specify this case. // CPython does a readall, but here we silently let negatives through, // and they will cause a MemoryError. @@ -109,6 +107,8 @@ STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte fl return stream_readall(args[0]); } + const mp_stream_p_t *stream_p = mp_get_stream(args[0]); + #if MICROPY_PY_BUILTINS_STR_UNICODE if (stream_p->is_text) { // We need to read sz number of unicode characters. Because we don't have any @@ -227,8 +227,6 @@ STATIC mp_obj_t stream_read1(size_t n_args, const mp_obj_t *args) { MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read1_obj, 1, 2, stream_read1); mp_obj_t mp_stream_write(mp_obj_t self_in, const void *buf, size_t len, byte flags) { - mp_get_stream_raise(self_in, MP_STREAM_OP_WRITE); - int error; mp_uint_t out_sz = mp_stream_rw(self_in, (void*)buf, len, &error, flags); if (error != 0) { @@ -276,7 +274,6 @@ STATIC mp_obj_t stream_write1_method(mp_obj_t self_in, mp_obj_t arg) { MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write1_obj, stream_write1_method); STATIC mp_obj_t stream_readinto(size_t n_args, const mp_obj_t *args) { - mp_get_stream_raise(args[0], MP_STREAM_OP_READ); mp_buffer_info_t bufinfo; mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_WRITE); @@ -305,7 +302,7 @@ STATIC mp_obj_t stream_readinto(size_t n_args, const mp_obj_t *args) { MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj, 2, 3, stream_readinto); STATIC mp_obj_t stream_readall(mp_obj_t self_in) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(self_in, MP_STREAM_OP_READ); + const mp_stream_p_t *stream_p = mp_get_stream(self_in); mp_uint_t total_size = 0; vstr_t vstr; @@ -346,7 +343,7 @@ STATIC mp_obj_t stream_readall(mp_obj_t self_in) { // Unbuffered, inefficient implementation of readline() for raw I/O files. STATIC mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ); + const mp_stream_p_t *stream_p = mp_get_stream(args[0]); mp_int_t max_size = -1; if (n_args > 1) { @@ -421,7 +418,7 @@ mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self) { } mp_obj_t mp_stream_close(mp_obj_t stream) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(stream, MP_STREAM_OP_IOCTL); + const mp_stream_p_t *stream_p = mp_get_stream(stream); int error; mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_CLOSE, 0, &error); if (res == MP_STREAM_ERROR) { @@ -432,8 +429,6 @@ mp_obj_t mp_stream_close(mp_obj_t stream) { MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_close_obj, mp_stream_close); STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL); - struct mp_stream_seek_t seek_s; // TODO: Could be uint64 seek_s.offset = mp_obj_get_int(args[1]); @@ -447,6 +442,7 @@ STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) { mp_raise_OSError(MP_EINVAL); } + const mp_stream_p_t *stream_p = mp_get_stream(args[0]); int error; mp_uint_t res = stream_p->ioctl(args[0], MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &error); if (res == MP_STREAM_ERROR) { @@ -467,7 +463,7 @@ STATIC mp_obj_t stream_tell(mp_obj_t self) { MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_tell_obj, stream_tell); STATIC mp_obj_t stream_flush(mp_obj_t self) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(self, MP_STREAM_OP_IOCTL); + const mp_stream_p_t *stream_p = mp_get_stream(self); int error; mp_uint_t res = stream_p->ioctl(self, MP_STREAM_FLUSH, 0, &error); if (res == MP_STREAM_ERROR) { @@ -478,8 +474,6 @@ STATIC mp_obj_t stream_flush(mp_obj_t self) { MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_flush_obj, stream_flush); STATIC mp_obj_t stream_ioctl(size_t n_args, const mp_obj_t *args) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL); - mp_buffer_info_t bufinfo; uintptr_t val = 0; if (n_args > 2) { @@ -490,6 +484,7 @@ STATIC mp_obj_t stream_ioctl(size_t n_args, const mp_obj_t *args) { } } + const mp_stream_p_t *stream_p = mp_get_stream(args[0]); int error; mp_uint_t res = stream_p->ioctl(args[0], mp_obj_get_int(args[1]), val, &error); if (res == MP_STREAM_ERROR) { diff --git a/py/stream.h b/py/stream.h index 3dec49a2e..7b953138c 100644 --- a/py/stream.h +++ b/py/stream.h @@ -90,6 +90,11 @@ MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_ioctl_obj); #define MP_STREAM_OP_WRITE (2) #define MP_STREAM_OP_IOCTL (4) +// Object is assumed to have a non-NULL stream protocol with valid r/w/ioctl methods +static inline const mp_stream_p_t *mp_get_stream(mp_const_obj_t self) { + return (const mp_stream_p_t*)((const mp_obj_base_t*)MP_OBJ_TO_PTR(self))->type->protocol; +} + const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags); mp_obj_t mp_stream_close(mp_obj_t stream); -- cgit v1.2.3 From c00ee200accb0f79fc2a7829ebe4567258ed40c0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 18 Jun 2018 13:40:53 +1000 Subject: py/objarray: Replace 0x80 with new MP_OBJ_ARRAY_TYPECODE_FLAG_RW macro. --- py/objarray.c | 8 ++++---- py/objarray.h | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'py') diff --git a/py/objarray.c b/py/objarray.c index a35539484..aa9fa3b73 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -222,7 +222,7 @@ STATIC mp_obj_t memoryview_make_new(const mp_obj_type_t *type_in, size_t n_args, // test if the object can be written to if (mp_get_buffer(args[0], &bufinfo, MP_BUFFER_RW)) { - self->typecode |= 0x80; // used to indicate writable buffer + self->typecode |= MP_OBJ_ARRAY_TYPECODE_FLAG_RW; // indicate writable buffer } return MP_OBJ_FROM_PTR(self); @@ -414,7 +414,7 @@ STATIC mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value uint8_t* dest_items = o->items; #if MICROPY_PY_BUILTINS_MEMORYVIEW if (o->base.type == &mp_type_memoryview) { - if ((o->typecode & 0x80) == 0) { + if (!(o->typecode & MP_OBJ_ARRAY_TYPECODE_FLAG_RW)) { // store to read-only memoryview not allowed return MP_OBJ_NULL; } @@ -471,7 +471,7 @@ STATIC mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value #if MICROPY_PY_BUILTINS_MEMORYVIEW if (o->base.type == &mp_type_memoryview) { index += o->free; - if (value != MP_OBJ_SENTINEL && (o->typecode & 0x80) == 0) { + if (value != MP_OBJ_SENTINEL && !(o->typecode & MP_OBJ_ARRAY_TYPECODE_FLAG_RW)) { // store to read-only memoryview return MP_OBJ_NULL; } @@ -497,7 +497,7 @@ STATIC mp_int_t array_get_buffer(mp_obj_t o_in, mp_buffer_info_t *bufinfo, mp_ui bufinfo->typecode = o->typecode & TYPECODE_MASK; #if MICROPY_PY_BUILTINS_MEMORYVIEW if (o->base.type == &mp_type_memoryview) { - if ((o->typecode & 0x80) == 0 && (flags & MP_BUFFER_WRITE)) { + if (!(o->typecode & MP_OBJ_ARRAY_TYPECODE_FLAG_RW) && (flags & MP_BUFFER_WRITE)) { // read-only memoryview return 1; } diff --git a/py/objarray.h b/py/objarray.h index 038966845..0dad70571 100644 --- a/py/objarray.h +++ b/py/objarray.h @@ -29,6 +29,9 @@ #include "py/obj.h" +// Used only for memoryview types, set in "typecode" to indicate a writable memoryview +#define MP_OBJ_ARRAY_TYPECODE_FLAG_RW (0x80) + typedef struct _mp_obj_array_t { mp_obj_base_t base; size_t typecode : 8; -- cgit v1.2.3 From 2c8d130f702d07041e30d808b974b0ba2a1e51e0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 20 Jun 2018 15:56:32 +1000 Subject: py/stream: Update comment for mp_stream_write_adaptor. --- py/stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'py') diff --git a/py/stream.c b/py/stream.c index a65ba4c91..b9932b35e 100644 --- a/py/stream.c +++ b/py/stream.c @@ -242,7 +242,7 @@ mp_obj_t mp_stream_write(mp_obj_t self_in, const void *buf, size_t len, byte fla } } -// XXX hack +// This is used to adapt a stream object to an mp_print_t interface void mp_stream_write_adaptor(void *self, const char *buf, size_t len) { mp_stream_write(MP_OBJ_FROM_PTR(self), buf, len, MP_STREAM_RW_WRITE); } -- cgit v1.2.3 From 582b190764641e5049768da1ba8962c841ec014b Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 20 Jun 2018 15:57:10 +1000 Subject: py: Add checks for stream objects in print() and sys.print_exception(). --- py/modbuiltins.c | 2 +- py/modsys.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'py') diff --git a/py/modbuiltins.c b/py/modbuiltins.c index b216e021f..c169b2ee4 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -386,7 +386,7 @@ STATIC mp_obj_t mp_builtin_print(size_t n_args, const mp_obj_t *pos_args, mp_map mp_arg_parse_all(0, NULL, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, u.args); #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - // TODO file may not be a concrete object (eg it could be a small-int) + mp_get_stream_raise(u.args[ARG_file].u_obj, MP_STREAM_OP_WRITE); mp_print_t print = {MP_OBJ_TO_PTR(u.args[ARG_file].u_obj), mp_stream_write_adaptor}; #endif diff --git a/py/modsys.c b/py/modsys.c index 609f8b454..98addfcfc 100644 --- a/py/modsys.c +++ b/py/modsys.c @@ -106,7 +106,8 @@ STATIC mp_obj_t mp_sys_print_exception(size_t n_args, const mp_obj_t *args) { #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES void *stream_obj = &mp_sys_stdout_obj; if (n_args > 1) { - stream_obj = MP_OBJ_TO_PTR(args[1]); // XXX may fail + mp_get_stream_raise(args[1], MP_STREAM_OP_WRITE); + stream_obj = MP_OBJ_TO_PTR(args[1]); } mp_print_t print = {stream_obj, mp_stream_write_adaptor}; -- cgit v1.2.3 From 34344a413fb1939d476f43429a8a4e142f6fe3c0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 20 Jun 2018 16:26:12 +1000 Subject: py/stream: Remove stray empty line at start of file. This was accidentally added in 6abede2ca9e221b6aefcaccbda0c89e367507df1 --- py/stream.c | 1 - 1 file changed, 1 deletion(-) (limited to 'py') diff --git a/py/stream.c b/py/stream.c index b9932b35e..448de41bb 100644 --- a/py/stream.c +++ b/py/stream.c @@ -1,4 +1,3 @@ - /* * This file is part of the MicroPython project, http://micropython.org/ * -- cgit v1.2.3 From c14919792868d0e42bb84d61e584797c6c977114 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Jun 2018 13:54:03 +1000 Subject: py/compile: Combine break and continue compile functions. --- py/compile.c | 23 ++++++++++++----------- py/grammar.h | 4 ++-- 2 files changed, 14 insertions(+), 13 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index c62ed057d..f242e7458 100644 --- a/py/compile.c +++ b/py/compile.c @@ -945,20 +945,21 @@ STATIC void compile_del_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { apply_to_single_or_list(comp, pns->nodes[0], PN_exprlist, c_del_stmt); } -STATIC void compile_break_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { - if (comp->break_label == INVALID_LABEL) { - compile_syntax_error(comp, (mp_parse_node_t)pns, "'break' outside loop"); +STATIC void compile_break_cont_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { + uint16_t label; + const char *error_msg; + if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_break_stmt) { + label = comp->break_label; + error_msg = "'break' outside loop"; + } else { + label = comp->continue_label; + error_msg = "'continue' outside loop"; } - assert(comp->cur_except_level >= comp->break_continue_except_level); - EMIT_ARG(unwind_jump, comp->break_label, comp->cur_except_level - comp->break_continue_except_level); -} - -STATIC void compile_continue_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { - if (comp->continue_label == INVALID_LABEL) { - compile_syntax_error(comp, (mp_parse_node_t)pns, "'continue' outside loop"); + if (label == INVALID_LABEL) { + compile_syntax_error(comp, (mp_parse_node_t)pns, error_msg); } assert(comp->cur_except_level >= comp->break_continue_except_level); - EMIT_ARG(unwind_jump, comp->continue_label, comp->cur_except_level - comp->break_continue_except_level); + EMIT_ARG(unwind_jump, label, comp->cur_except_level - comp->break_continue_except_level); } STATIC void compile_return_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { diff --git a/py/grammar.h b/py/grammar.h index 6abb1de8c..ffedd2f8a 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -122,8 +122,8 @@ DEF_RULE_NC(augassign, or(12), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DE DEF_RULE(del_stmt, c(del_stmt), and(2), tok(KW_DEL), rule(exprlist)) DEF_RULE(pass_stmt, c(generic_all_nodes), and(1), tok(KW_PASS)) DEF_RULE_NC(flow_stmt, or(5), rule(break_stmt), rule(continue_stmt), rule(return_stmt), rule(raise_stmt), rule(yield_stmt)) -DEF_RULE(break_stmt, c(break_stmt), and(1), tok(KW_BREAK)) -DEF_RULE(continue_stmt, c(continue_stmt), and(1), tok(KW_CONTINUE)) +DEF_RULE(break_stmt, c(break_cont_stmt), and(1), tok(KW_BREAK)) +DEF_RULE(continue_stmt, c(break_cont_stmt), and(1), tok(KW_CONTINUE)) DEF_RULE(return_stmt, c(return_stmt), and(2), tok(KW_RETURN), opt_rule(testlist)) DEF_RULE(yield_stmt, c(yield_stmt), and(1), rule(yield_expr)) DEF_RULE(raise_stmt, c(raise_stmt), and(2), tok(KW_RAISE), opt_rule(raise_stmt_arg)) -- cgit v1.2.3 From d23bec3fc89d1c1f7a2ac8023161e3f2620ffa13 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Jun 2018 13:57:55 +1000 Subject: py/compile: Combine subscript_2 and subscript_3 into one function. --- py/compile.c | 22 ++++++++++------------ py/grammar.h | 4 ++-- 2 files changed, 12 insertions(+), 14 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index f242e7458..a857598c5 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2546,7 +2546,16 @@ STATIC void compile_trailer_period(compiler_t *comp, mp_parse_node_struct_t *pns } #if MICROPY_PY_BUILTINS_SLICE -STATIC void compile_subscript_3_helper(compiler_t *comp, mp_parse_node_struct_t *pns) { +STATIC void compile_subscript(compiler_t *comp, mp_parse_node_struct_t *pns) { + if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_subscript_2) { + compile_node(comp, pns->nodes[0]); // start of slice + assert(MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])); // should always be + pns = (mp_parse_node_struct_t*)pns->nodes[1]; + } else { + // pns is a PN_subscript_3, load None for start of slice + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + } + assert(MP_PARSE_NODE_STRUCT_KIND(pns) == PN_subscript_3); // should always be mp_parse_node_t pn = pns->nodes[0]; if (MP_PARSE_NODE_IS_NULL(pn)) { @@ -2590,17 +2599,6 @@ STATIC void compile_subscript_3_helper(compiler_t *comp, mp_parse_node_struct_t EMIT_ARG(build, 2, MP_EMIT_BUILD_SLICE); } } - -STATIC void compile_subscript_2(compiler_t *comp, mp_parse_node_struct_t *pns) { - compile_node(comp, pns->nodes[0]); // start of slice - assert(MP_PARSE_NODE_IS_STRUCT(pns->nodes[1])); // should always be - compile_subscript_3_helper(comp, (mp_parse_node_struct_t*)pns->nodes[1]); -} - -STATIC void compile_subscript_3(compiler_t *comp, mp_parse_node_struct_t *pns) { - EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); - compile_subscript_3_helper(comp, pns); -} #endif // MICROPY_PY_BUILTINS_SLICE STATIC void compile_dictorsetmaker_item(compiler_t *comp, mp_parse_node_struct_t *pns) { diff --git a/py/grammar.h b/py/grammar.h index ffedd2f8a..bd0a20876 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -290,8 +290,8 @@ DEF_RULE(trailer_period, c(trailer_period), and(2), tok(DEL_PERIOD), tok(NAME)) #if MICROPY_PY_BUILTINS_SLICE DEF_RULE(subscriptlist, c(generic_tuple), list_with_end, rule(subscript), tok(DEL_COMMA)) DEF_RULE_NC(subscript, or(2), rule(subscript_3), rule(subscript_2)) -DEF_RULE(subscript_2, c(subscript_2), and_ident(2), rule(test), opt_rule(subscript_3)) -DEF_RULE(subscript_3, c(subscript_3), and(2), tok(DEL_COLON), opt_rule(subscript_3b)) +DEF_RULE(subscript_2, c(subscript), and_ident(2), rule(test), opt_rule(subscript_3)) +DEF_RULE(subscript_3, c(subscript), and(2), tok(DEL_COLON), opt_rule(subscript_3b)) DEF_RULE_NC(subscript_3b, or(2), rule(subscript_3c), rule(subscript_3d)) DEF_RULE_NC(subscript_3c, and(2), tok(DEL_COLON), opt_rule(test)) DEF_RULE_NC(subscript_3d, and_ident(2), rule(test), opt_rule(sliceop)) -- cgit v1.2.3 From 1a7109d65aa87596c7ce1824037fe298109962e9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Jun 2018 14:04:05 +1000 Subject: py/compile: Combine global and nonlocal statement compile functions. --- py/compile.c | 34 +++++++++++++++------------------- py/grammar.h | 4 ++-- 2 files changed, 17 insertions(+), 21 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index a857598c5..7e8ebfbab 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1162,9 +1162,7 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) { } } -STATIC void compile_declare_global(compiler_t *comp, mp_parse_node_t pn, qstr qst) { - bool added; - id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, qst, &added); +STATIC void compile_declare_global(compiler_t *comp, mp_parse_node_t pn, qstr qst, bool added, id_info_t *id_info) { if (!added && id_info->kind != ID_INFO_KIND_GLOBAL_EXPLICIT) { compile_syntax_error(comp, pn, "identifier redefined as global"); return; @@ -1178,19 +1176,7 @@ STATIC void compile_declare_global(compiler_t *comp, mp_parse_node_t pn, qstr qs } } -STATIC void compile_global_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { - if (comp->pass == MP_PASS_SCOPE) { - mp_parse_node_t *nodes; - int n = mp_parse_node_extract_list(&pns->nodes[0], PN_name_list, &nodes); - for (int i = 0; i < n; i++) { - compile_declare_global(comp, (mp_parse_node_t)pns, MP_PARSE_NODE_LEAF_ARG(nodes[i])); - } - } -} - -STATIC void compile_declare_nonlocal(compiler_t *comp, mp_parse_node_t pn, qstr qst) { - bool added; - id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, qst, &added); +STATIC void compile_declare_nonlocal(compiler_t *comp, mp_parse_node_t pn, qstr qst, bool added, id_info_t *id_info) { if (added) { scope_find_local_and_close_over(comp->scope_cur, id_info, qst); if (id_info->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) { @@ -1201,16 +1187,26 @@ STATIC void compile_declare_nonlocal(compiler_t *comp, mp_parse_node_t pn, qstr } } -STATIC void compile_nonlocal_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { +STATIC void compile_global_nonlocal_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { if (comp->pass == MP_PASS_SCOPE) { - if (comp->scope_cur->kind == SCOPE_MODULE) { + bool is_global = MP_PARSE_NODE_STRUCT_KIND(pns) == PN_global_stmt; + + if (!is_global && comp->scope_cur->kind == SCOPE_MODULE) { compile_syntax_error(comp, (mp_parse_node_t)pns, "can't declare nonlocal in outer code"); return; } + mp_parse_node_t *nodes; int n = mp_parse_node_extract_list(&pns->nodes[0], PN_name_list, &nodes); for (int i = 0; i < n; i++) { - compile_declare_nonlocal(comp, (mp_parse_node_t)pns, MP_PARSE_NODE_LEAF_ARG(nodes[i])); + qstr qst = MP_PARSE_NODE_LEAF_ARG(nodes[i]); + bool added; + id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, qst, &added); + if (is_global) { + compile_declare_global(comp, (mp_parse_node_t)pns, qst, added, id_info); + } else { + compile_declare_nonlocal(comp, (mp_parse_node_t)pns, qst, added, id_info); + } } } } diff --git a/py/grammar.h b/py/grammar.h index bd0a20876..50611bb9c 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -157,8 +157,8 @@ DEF_RULE_NC(as_name, and_ident(2), tok(KW_AS), tok(NAME)) DEF_RULE_NC(import_as_names, list_with_end, rule(import_as_name), tok(DEL_COMMA)) DEF_RULE_NC(dotted_as_names, list, rule(dotted_as_name), tok(DEL_COMMA)) DEF_RULE_NC(dotted_name, list, tok(NAME), tok(DEL_PERIOD)) -DEF_RULE(global_stmt, c(global_stmt), and(2), tok(KW_GLOBAL), rule(name_list)) -DEF_RULE(nonlocal_stmt, c(nonlocal_stmt), and(2), tok(KW_NONLOCAL), rule(name_list)) +DEF_RULE(global_stmt, c(global_nonlocal_stmt), and(2), tok(KW_GLOBAL), rule(name_list)) +DEF_RULE(nonlocal_stmt, c(global_nonlocal_stmt), and(2), tok(KW_NONLOCAL), rule(name_list)) DEF_RULE_NC(name_list, list, tok(NAME), tok(DEL_COMMA)) DEF_RULE(assert_stmt, c(assert_stmt), and(3), tok(KW_ASSERT), rule(test), opt_rule(assert_stmt_extra)) DEF_RULE_NC(assert_stmt_extra, and_ident(2), tok(DEL_COMMA), rule(test)) -- cgit v1.2.3 From 36e474e83fa26cb78a9312dce5dc53a467c5d8b7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Jun 2018 14:10:29 +1000 Subject: py/compile: Combine or_test and and_test compile functions. --- py/compile.c | 11 ++--------- py/grammar.h | 4 ++-- 2 files changed, 4 insertions(+), 11 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 7e8ebfbab..032e0a6ae 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2024,7 +2024,8 @@ STATIC void compile_lambdef(compiler_t *comp, mp_parse_node_struct_t *pns) { compile_funcdef_lambdef(comp, this_scope, pns->nodes[0], PN_varargslist); } -STATIC void compile_or_and_test(compiler_t *comp, mp_parse_node_struct_t *pns, bool cond) { +STATIC void compile_or_and_test(compiler_t *comp, mp_parse_node_struct_t *pns) { + bool cond = MP_PARSE_NODE_STRUCT_KIND(pns) == PN_or_test; uint l_end = comp_next_label(comp); int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); for (int i = 0; i < n; i += 1) { @@ -2036,14 +2037,6 @@ STATIC void compile_or_and_test(compiler_t *comp, mp_parse_node_struct_t *pns, b EMIT_ARG(label_assign, l_end); } -STATIC void compile_or_test(compiler_t *comp, mp_parse_node_struct_t *pns) { - compile_or_and_test(comp, pns, true); -} - -STATIC void compile_and_test(compiler_t *comp, mp_parse_node_struct_t *pns) { - compile_or_and_test(comp, pns, false); -} - STATIC void compile_not_test_2(compiler_t *comp, mp_parse_node_struct_t *pns) { compile_node(comp, pns->nodes[0]); EMIT_ARG(unary_op, MP_UNARY_OP_NOT); diff --git a/py/grammar.h b/py/grammar.h index 50611bb9c..37b97a191 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -231,8 +231,8 @@ DEF_RULE(lambdef_nocond, c(lambdef), and_blank(4), tok(KW_LAMBDA), opt_rule(vara // power: atom_expr ['**' factor] // atom_expr: 'await' atom trailer* | atom trailer* -DEF_RULE(or_test, c(or_test), list, rule(and_test), tok(KW_OR)) -DEF_RULE(and_test, c(and_test), list, rule(not_test), tok(KW_AND)) +DEF_RULE(or_test, c(or_and_test), list, rule(and_test), tok(KW_OR)) +DEF_RULE(and_test, c(or_and_test), list, rule(not_test), tok(KW_AND)) DEF_RULE_NC(not_test, or(2), rule(not_test_2), rule(comparison)) DEF_RULE(not_test_2, c(not_test_2), and(2), tok(KW_NOT), rule(not_test)) DEF_RULE(comparison, c(comparison), list, rule(expr), rule(comp_op)) -- cgit v1.2.3 From 25ae98f07cb3c4488cb955403dfe56b8bb8db6f0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 19 Jun 2018 14:20:42 +1000 Subject: py/compile: Combine expr, xor_expr and and_expr into one function. This and the previous 4 commits combined have change in code size of: bare-arm: -92 minimal x86: -544 unix x64: -544 unix nanbox: -712 stm32: -116 cc3200: -128 esp8266: -348 esp32: -232 --- py/compile.c | 29 ++++++++++------------------- py/grammar.h | 6 +++--- 2 files changed, 13 insertions(+), 22 deletions(-) (limited to 'py') diff --git a/py/compile.c b/py/compile.c index 032e0a6ae..7daf91103 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1985,15 +1985,6 @@ STATIC void compile_expr_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { } } -STATIC void c_binary_op(compiler_t *comp, mp_parse_node_struct_t *pns, mp_binary_op_t binary_op) { - int num_nodes = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); - compile_node(comp, pns->nodes[0]); - for (int i = 1; i < num_nodes; i += 1) { - compile_node(comp, pns->nodes[i]); - EMIT_ARG(binary_op, binary_op); - } -} - STATIC void compile_test_if_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { assert(MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_test_if_else)); mp_parse_node_struct_t *pns_test_if_else = (mp_parse_node_struct_t*)pns->nodes[1]; @@ -2102,16 +2093,16 @@ STATIC void compile_star_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { compile_syntax_error(comp, (mp_parse_node_t)pns, "*x must be assignment target"); } -STATIC void compile_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { - c_binary_op(comp, pns, MP_BINARY_OP_OR); -} - -STATIC void compile_xor_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { - c_binary_op(comp, pns, MP_BINARY_OP_XOR); -} - -STATIC void compile_and_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { - c_binary_op(comp, pns, MP_BINARY_OP_AND); +STATIC void compile_binary_op(compiler_t *comp, mp_parse_node_struct_t *pns) { + MP_STATIC_ASSERT(MP_BINARY_OP_OR + PN_xor_expr - PN_expr == MP_BINARY_OP_XOR); + MP_STATIC_ASSERT(MP_BINARY_OP_OR + PN_and_expr - PN_expr == MP_BINARY_OP_AND); + mp_binary_op_t binary_op = MP_BINARY_OP_OR + MP_PARSE_NODE_STRUCT_KIND(pns) - PN_expr; + int num_nodes = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); + compile_node(comp, pns->nodes[0]); + for (int i = 1; i < num_nodes; ++i) { + compile_node(comp, pns->nodes[i]); + EMIT_ARG(binary_op, binary_op); + } } STATIC void compile_term(compiler_t *comp, mp_parse_node_struct_t *pns) { diff --git a/py/grammar.h b/py/grammar.h index 37b97a191..5a5b682ac 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -241,9 +241,9 @@ DEF_RULE_NC(comp_op_not_in, and(2), tok(KW_NOT), tok(KW_IN)) DEF_RULE_NC(comp_op_is, and(2), tok(KW_IS), opt_rule(comp_op_is_not)) DEF_RULE_NC(comp_op_is_not, and(1), tok(KW_NOT)) DEF_RULE(star_expr, c(star_expr), and(2), tok(OP_STAR), rule(expr)) -DEF_RULE(expr, c(expr), list, rule(xor_expr), tok(OP_PIPE)) -DEF_RULE(xor_expr, c(xor_expr), list, rule(and_expr), tok(OP_CARET)) -DEF_RULE(and_expr, c(and_expr), list, rule(shift_expr), tok(OP_AMPERSAND)) +DEF_RULE(expr, c(binary_op), list, rule(xor_expr), tok(OP_PIPE)) +DEF_RULE(xor_expr, c(binary_op), list, rule(and_expr), tok(OP_CARET)) +DEF_RULE(and_expr, c(binary_op), list, rule(shift_expr), tok(OP_AMPERSAND)) DEF_RULE(shift_expr, c(term), list, rule(arith_expr), rule(shift_op)) DEF_RULE_NC(shift_op, or(2), tok(OP_DBL_LESS), tok(OP_DBL_MORE)) DEF_RULE(arith_expr, c(term), list, rule(term), rule(arith_op)) -- cgit v1.2.3