diff options
Diffstat (limited to 'py')
| -rw-r--r-- | py/binary.c | 9 | ||||
| -rw-r--r-- | py/circuitpy_defns.mk | 24 | ||||
| -rw-r--r-- | py/circuitpy_mpconfig.h | 30 | ||||
| -rw-r--r-- | py/circuitpy_mpconfig.mk | 19 | ||||
| -rw-r--r-- | py/compile.c | 17 | ||||
| -rw-r--r-- | py/enum.c | 52 | ||||
| -rw-r--r-- | py/enum.h | 65 | ||||
| -rw-r--r-- | py/genlast.py | 71 | ||||
| -rw-r--r-- | py/makeqstrdata.py | 264 | ||||
| -rw-r--r-- | py/makeqstrdefs.py | 4 | ||||
| -rw-r--r-- | py/mkenv.mk | 2 | ||||
| -rw-r--r-- | py/mkrules.mk | 10 | ||||
| -rw-r--r-- | py/modstruct.c | 17 | ||||
| -rw-r--r-- | py/objarray.c | 7 | ||||
| -rw-r--r-- | py/objgenerator.c | 18 | ||||
| -rw-r--r-- | py/objmodule.c | 7 | ||||
| -rw-r--r-- | py/objnamedtuple.c | 2 | ||||
| -rw-r--r-- | py/objset.c | 1 | ||||
| -rw-r--r-- | py/py.mk | 1 |
19 files changed, 490 insertions, 130 deletions
diff --git a/py/binary.c b/py/binary.c index cd0f1aa4d..b85edba62 100644 --- a/py/binary.c +++ b/py/binary.c @@ -126,7 +126,6 @@ mp_obj_t mp_binary_get_val_array(char typecode, void *p, mp_uint_t index) { break; case BYTEARRAY_TYPECODE: case 'B': - case 'x': // value will be discarded val = ((unsigned char*)p)[index]; break; case 'h': @@ -330,7 +329,11 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte ** } } - mp_binary_set_int(MIN((size_t)size, sizeof(val)), struct_type == '>', p, val); + if (val_type == 'x') { + memset(p, 0, 1); + } else { + mp_binary_set_int(MIN((size_t)size, sizeof(val)), struct_type == '>', p, val); + } } void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t val_in) { @@ -379,8 +382,6 @@ void mp_binary_set_val_array_from_int(char typecode, void *p, mp_uint_t index, m case 'B': ((unsigned char*)p)[index] = val; break; - case 'x': - ((unsigned char*)p)[index] = 0; case 'h': ((short*)p)[index] = val; break; diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk index 04e38d81e..ccdf973e9 100644 --- a/py/circuitpy_defns.mk +++ b/py/circuitpy_defns.mk @@ -31,6 +31,7 @@ BASE_CFLAGS = \ -fsingle-precision-constant \ -fno-strict-aliasing \ -Wdouble-promotion \ + -Wimplicit-fallthrough=2 \ -Wno-endif-labels \ -Wstrict-prototypes \ -Werror-implicit-function-declaration \ @@ -44,7 +45,6 @@ BASE_CFLAGS = \ -Wnested-externs \ -Wunreachable-code \ -Wcast-align \ - -Wno-error=lto-type-mismatch \ -D__$(CHIP_VARIANT)__ \ -ffunction-sections \ -fdata-sections \ @@ -65,7 +65,6 @@ ifneq ($(FROZEN_DIR),) # To use frozen source modules, put your .py files in a subdirectory (eg scripts/) # and then invoke make with FROZEN_DIR=scripts (be sure to build from scratch). CFLAGS += -DMICROPY_MODULE_FROZEN_STR -CFLAGS += -Wno-error=lto-type-mismatch endif # To use frozen bytecode, put your .py files in a subdirectory (eg frozen/) and @@ -75,7 +74,6 @@ endif ifneq ($(FROZEN_MPY_DIRS),) CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool CFLAGS += -DMICROPY_MODULE_FROZEN_MPY -CFLAGS += -Wno-error=lto-type-mismatch endif @@ -138,6 +136,12 @@ endif ifeq ($(CIRCUITPY_BUSIO),1) SRC_PATTERNS += busio/% bitbangio/OneWire.% endif +ifeq ($(CIRCUITPY_CAMERA),1) +SRC_PATTERNS += camera/% +endif +ifeq ($(CIRCUITPY_CANIO),1) +SRC_PATTERNS += canio/% +endif ifeq ($(CIRCUITPY_COUNTIO),1) SRC_PATTERNS += countio/% endif @@ -309,6 +313,11 @@ SRC_COMMON_HAL_ALL = \ busio/SPI.c \ busio/UART.c \ busio/__init__.c \ + camera/__init__.c \ + camera/Camera.c \ + canio/CAN.c \ + canio/Listener.c \ + canio/__init__.c \ countio/Counter.c \ countio/__init__.c \ digitalio/DigitalInOut.c \ @@ -378,7 +387,9 @@ $(filter $(SRC_PATTERNS), \ _bleio/Address.c \ _bleio/Attribute.c \ _bleio/ScanEntry.c \ + canio/Match.c \ _eve/__init__.c \ + camera/ImageFormat.c \ digitalio/Direction.c \ digitalio/DriveMode.c \ digitalio/Pull.c \ @@ -395,6 +406,9 @@ SRC_SHARED_MODULE_ALL = \ _bleio/Attribute.c \ _bleio/ScanEntry.c \ _bleio/ScanResults.c \ + canio/Match.c \ + canio/Message.c \ + canio/RemoteTransmissionRequest.c \ _eve/__init__.c \ _pixelbuf/PixelBuf.c \ _pixelbuf/__init__.c \ @@ -511,10 +525,10 @@ SRC_MOD += $(addprefix lib/mp3/src/, \ $(BUILD)/lib/mp3/src/buffers.o: CFLAGS += -include "py/misc.h" -D'MPDEC_ALLOCATOR(x)=m_malloc(x,0)' -D'MPDEC_FREE(x)=m_free(x)' endif ifeq ($(CIRCUITPY_RGBMATRIX),1) -SRC_MOD += $(addprefix lib/protomatter/, \ +SRC_MOD += $(addprefix lib/protomatter/src/, \ core.c \ ) -$(BUILD)/lib/protomatter/core.o: CFLAGS += -include "shared-module/rgbmatrix/allocator.h" -DCIRCUITPY -Wno-missing-braces +$(BUILD)/lib/protomatter/src/core.o: CFLAGS += -include "shared-module/rgbmatrix/allocator.h" -DCIRCUITPY -Wno-missing-braces endif # All possible sources are listed here, and are filtered by SRC_PATTERNS. diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index 0fafbe876..08efabddb 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -79,7 +79,6 @@ #define MICROPY_PY_ARRAY (1) #define MICROPY_PY_ARRAY_SLICE_ASSIGN (1) -#define MICROPY_PY_ASYNC_AWAIT (0) #define MICROPY_PY_ATTRTUPLE (1) #define MICROPY_PY_BUILTINS_BYTEARRAY (1) @@ -195,6 +194,12 @@ typedef long mp_off_t; #define MICROPY_PY_BUILTINS_STR_PARTITION (CIRCUITPY_FULL_BUILD) #define MICROPY_PY_BUILTINS_STR_SPLITLINES (CIRCUITPY_FULL_BUILD) #define MICROPY_PY_UERRNO (CIRCUITPY_FULL_BUILD) +#ifndef MICROPY_PY_COLLECTIONS_ORDEREDDICT +#define MICROPY_PY_COLLECTIONS_ORDEREDDICT (CIRCUITPY_FULL_BUILD) +#endif +#ifndef MICROPY_PY_UBINASCII +#define MICROPY_PY_UBINASCII (CIRCUITPY_FULL_BUILD) +#endif // Opposite setting is deliberate. #define MICROPY_PY_UERRNO_ERRORCODE (!CIRCUITPY_FULL_BUILD) #ifndef MICROPY_PY_URE @@ -329,6 +334,20 @@ extern const struct _mp_obj_module_t busio_module; #define BUSIO_MODULE #endif +#if CIRCUITPY_CAMERA +extern const struct _mp_obj_module_t camera_module; +#define CAMERA_MODULE { MP_OBJ_NEW_QSTR(MP_QSTR_camera), (mp_obj_t)&camera_module }, +#else +#define CAMERA_MODULE +#endif + +#if CIRCUITPY_CANIO +extern const struct _mp_obj_module_t canio_module; +#define CANIO_MODULE { MP_OBJ_NEW_QSTR(MP_QSTR_canio), (mp_obj_t)&canio_module }, +#else +#define CANIO_MODULE +#endif + #if CIRCUITPY_COUNTIO extern const struct _mp_obj_module_t countio_module; #define COUNTIO_MODULE { MP_OBJ_NEW_QSTR(MP_QSTR_countio), (mp_obj_t)&countio_module }, @@ -683,6 +702,12 @@ extern const struct _mp_obj_module_t ustack_module; #endif // These modules are not yet in shared-bindings, but we prefer the non-uxxx names. +#if MICROPY_PY_UBINASCII +#define BINASCII_MODULE { MP_ROM_QSTR(MP_QSTR_binascii), MP_ROM_PTR(&mp_module_ubinascii) }, +#else +#define BINASCII_MODULE +#endif + #if MICROPY_PY_UERRNO #define ERRNO_MODULE { MP_ROM_QSTR(MP_QSTR_errno), MP_ROM_PTR(&mp_module_uerrno) }, #else @@ -754,10 +779,13 @@ extern const struct _mp_obj_module_t wifi_module; AUDIOMIXER_MODULE \ AUDIOMP3_MODULE \ AUDIOPWMIO_MODULE \ + BINASCII_MODULE \ BITBANGIO_MODULE \ BLEIO_MODULE \ BOARD_MODULE \ BUSIO_MODULE \ + CAMERA_MODULE \ + CANIO_MODULE \ COUNTIO_MODULE \ DIGITALIO_MODULE \ DISPLAYIO_MODULE \ diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index a2da9c42f..a6aabec33 100644 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -32,6 +32,9 @@ CIRCUITPY_FULL_BUILD ?= 1 CFLAGS += -DCIRCUITPY_FULL_BUILD=$(CIRCUITPY_FULL_BUILD) +# async/await language keyword support +MICROPY_PY_ASYNC_AWAIT ?= $(CIRCUITPY_FULL_BUILD) +CFLAGS += -DMICROPY_PY_ASYNC_AWAIT=$(MICROPY_PY_ASYNC_AWAIT) CIRCUITPY_AESIO ?= 0 CFLAGS += -DCIRCUITPY_AESIO=$(CIRCUITPY_AESIO) @@ -76,20 +79,26 @@ CFLAGS += -DCIRCUITPY_AUDIOMP3=$(CIRCUITPY_AUDIOMP3) CIRCUITPY_BITBANGIO ?= $(CIRCUITPY_FULL_BUILD) CFLAGS += -DCIRCUITPY_BITBANGIO=$(CIRCUITPY_BITBANGIO) -# Explicitly enabled for boards that support _bleio. -CIRCUITPY_BLEIO ?= 0 -CFLAGS += -DCIRCUITPY_BLEIO=$(CIRCUITPY_BLEIO) - # _bleio can be supported on most any board via HCI -CIRCUITPY_BLEIO_HCI ?= 0 +CIRCUITPY_BLEIO_HCI ?= $(CIRCUITPY_FULL_BUILD) CFLAGS += -DCIRCUITPY_BLEIO_HCI=$(CIRCUITPY_BLEIO_HCI) +# Explicitly enabled for boards that support _bleio. +CIRCUITPY_BLEIO ?= $(CIRCUITPY_BLEIO_HCI) +CFLAGS += -DCIRCUITPY_BLEIO=$(CIRCUITPY_BLEIO) + CIRCUITPY_BOARD ?= 1 CFLAGS += -DCIRCUITPY_BOARD=$(CIRCUITPY_BOARD) CIRCUITPY_BUSIO ?= 1 CFLAGS += -DCIRCUITPY_BUSIO=$(CIRCUITPY_BUSIO) +CIRCUITPY_CAMERA ?= 0 +CFLAGS += -DCIRCUITPY_CAMERA=$(CIRCUITPY_CAMERA) + +CIRCUITPY_CANIO ?= 0 +CFLAGS += -DCIRCUITPY_CANIO=$(CIRCUITPY_CANIO) + CIRCUITPY_DIGITALIO ?= 1 CFLAGS += -DCIRCUITPY_DIGITALIO=$(CIRCUITPY_DIGITALIO) diff --git a/py/compile.c b/py/compile.c index 653aae004..04bcf5bc1 100644 --- a/py/compile.c +++ b/py/compile.c @@ -853,7 +853,7 @@ STATIC void compile_decorated(compiler_t *comp, mp_parse_node_struct_t *pns) { mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns_body->nodes[0]; body_name = compile_funcdef_helper(comp, pns0, emit_options); scope_t *fscope = (scope_t*)pns0->nodes[4]; - fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR; + fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR | MP_SCOPE_FLAG_ASYNC; #endif } else { assert(MP_PARSE_NODE_STRUCT_KIND(pns_body) == PN_classdef); // should be @@ -2632,6 +2632,12 @@ STATIC void compile_yield_expr(compiler_t *comp, mp_parse_node_struct_t *pns) { EMIT_ARG(yield, MP_EMIT_YIELD_VALUE); } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_yield_arg_from)) { pns = (mp_parse_node_struct_t*)pns->nodes[0]; +#if MICROPY_PY_ASYNC_AWAIT + if(comp->scope_cur->scope_flags & MP_SCOPE_FLAG_ASYNC) { + compile_syntax_error(comp, (mp_parse_node_t)pns, translate("'yield from' inside async function")); + return; + } +#endif compile_node(comp, pns->nodes[0]); compile_yield_from(comp); } else { @@ -2648,7 +2654,14 @@ STATIC void compile_atom_expr_await(compiler_t *comp, mp_parse_node_struct_t *pn } compile_require_async_context(comp, pns); compile_atom_expr_normal(comp, pns); - compile_yield_from(comp); + + // If it's an awaitable thing, need to reach for the __await__ method for the coroutine. + // async def functions' __await__ return themselves, which are able to receive a send(), + // while other types with custom __await__ implementations return async generators. + EMIT_ARG(load_method, MP_QSTR___await__, false); + EMIT_ARG(call_method, 0, 0, 0); + EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); + EMIT_ARG(yield, MP_EMIT_YIELD_FROM); } #endif diff --git a/py/enum.c b/py/enum.c new file mode 100644 index 000000000..02a85a168 --- /dev/null +++ b/py/enum.c @@ -0,0 +1,52 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * SPDX-FileCopyrightText: Copyright (c) 2020 Jeff Epler for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/enum.h" +#include "py/runtime.h" + +mp_obj_t cp_enum_find(const mp_obj_type_t *type, int value) { + const mp_obj_dict_t *dict = type->locals_dict; + for (size_t i=0; i<dict->map.used; i++) { + const cp_enum_obj_t *v = dict->map.table[i].value; + if (v->value == value) { + return (mp_obj_t)v; + } + } + return mp_const_none; +} + +int cp_enum_value(const mp_obj_type_t *type, mp_obj_t *obj) { + if (!MP_OBJ_IS_TYPE(obj, type)) { + mp_raise_TypeError_varg(translate("Expected a %q"), type->name); + } + return ((cp_enum_obj_t*)MP_OBJ_TO_PTR(obj))->value; +} + +void cp_enum_obj_print_helper(uint16_t module, const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { + (void) kind; + cp_enum_obj_t *self = self_in; + mp_printf(print, "%q.%q.%q", module, self->base.type->name, self->name); +} diff --git a/py/enum.h b/py/enum.h new file mode 100644 index 000000000..1c38ae5ae --- /dev/null +++ b/py/enum.h @@ -0,0 +1,65 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * SPDX-FileCopyrightText: Copyright (c) 2020 Jeff Epler for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include "py/obj.h" + +typedef struct { + mp_obj_base_t base; + int16_t value; + int16_t name; +} cp_enum_obj_t; + +#define MAKE_ENUM_VALUE(type, prefix, name, value) \ + STATIC const cp_enum_obj_t prefix ## _ ## name ## _obj = { \ + { &type }, value, MP_QSTR_ ## name, \ + } + +#define MAKE_ENUM_MAP(name) \ + STATIC const mp_rom_map_elem_t name ## _locals_table[] = + +#define MAKE_ENUM_MAP_ENTRY(prefix, name) \ + { MP_ROM_QSTR(MP_QSTR_ ## name), MP_ROM_PTR(&prefix ## _ ## name ## _obj) } + +#define MAKE_PRINTER(module, typename) \ + STATIC void typename ## _ ## print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { \ + cp_enum_obj_print_helper(MP_QSTR_ ## module, print, self_in, kind); \ + } + +#define MAKE_ENUM_TYPE(module, type, typename) \ + const mp_obj_type_t typename ## _type = { \ + { &mp_type_type }, \ + .name = MP_QSTR_ ## type, \ + .print = typename ## _print, \ + .locals_dict = (mp_obj_t)&typename ## _locals_dict, \ + } + + + +mp_obj_t cp_enum_find(const mp_obj_type_t *type, int value); +int cp_enum_value(const mp_obj_type_t *type, mp_obj_t *obj); +void cp_enum_obj_print_helper(uint16_t module, const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind); diff --git a/py/genlast.py b/py/genlast.py new file mode 100644 index 000000000..1df2a2482 --- /dev/null +++ b/py/genlast.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import sys +import re +import os +import itertools +from concurrent.futures import ProcessPoolExecutor +import multiprocessing +import threading +import subprocess + +from makeqstrdefs import qstr_unescape, QSTRING_BLACK_LIST + +re_line = re.compile(r"#[line]*\s(\d+)\s\"([^\"]+)\"", re.DOTALL) +re_qstr = re.compile(r'MP_QSTR_[_a-zA-Z0-9]+', re.DOTALL) +re_translate = re.compile(r'translate\(\"((?:(?=(\\?))\2.)*?)\"\)', re.DOTALL) + +def write_out(fname, output_dir, output): + if output: + for m, r in [("/", "__"), ("\\", "__"), (":", "@"), ("..", "@@")]: + fname = fname.replace(m, r) + with open(output_dir + "/" + fname + ".qstr", "w") as f: + f.write("\n".join(output) + "\n") + +def process_file(fname, output_dir, content): + content = content.decode('utf-8', errors='ignore') + output = [] + for match in re_qstr.findall(content): + name = match.replace('MP_QSTR_', '') + if name not in QSTRING_BLACK_LIST: + output.append('Q(' + qstr_unescape(name) + ')') + for match in re_translate.findall(content): + output.append('TRANSLATE("' + match[0] + '")') + + write_out(fname, output_dir, output) + + +def checkoutput1(args): + info = subprocess.run(args, check=True, stdout=subprocess.PIPE, input='') + return info.stdout + +def preprocess(command, output_dir, fn): + try: + output = checkoutput1(command + [fn]) + process_file(fn, output_dir, output) + except Exception as e: + print(e, file=sys.stderr) + +def maybe_preprocess(command, output_dir, fn): + if subprocess.call(["grep", "-lqE", "(MP_QSTR|translate)", fn]) == 0: + preprocess(command, output_dir, fn) + +if __name__ == '__main__': + + + idx1 = sys.argv.index('--') + idx2 = sys.argv.index('--', idx1+1) + output_dir = sys.argv[1] + check = sys.argv[2:idx1] + always = sys.argv[idx1+1:idx2] + command = sys.argv[idx2+1:] + + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + + # Mac and Windows use 'spawn'. Uncomment this during testing to catch spawn-specific problems on Linux. + #multiprocessing.set_start_method("spawn") + executor = ProcessPoolExecutor(max_workers=multiprocessing.cpu_count() + 1) + executor.map(maybe_preprocess, itertools.repeat(command), itertools.repeat(output_dir), check) + executor.map(preprocess, itertools.repeat(command), itertools.repeat(output_dir), always) + executor.shutdown() diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 721fa8320..b4f4f1b03 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -12,10 +12,15 @@ from __future__ import print_function import re import sys +from math import log import collections import gettext import os.path +if hasattr(sys.stdout, 'reconfigure'): + sys.stdout.reconfigure(encoding='utf-8') + sys.stderr.reconfigure(errors='backslashreplace') + py = os.path.dirname(sys.argv[0]) top = os.path.dirname(py) @@ -100,77 +105,177 @@ def translate(translation_file, i18ns): translations.append((original, translation)) return translations -def frequent_ngrams(corpus, sz, n): - return collections.Counter(corpus[i:i+sz] for i in range(len(corpus)-sz)).most_common(n) +class TextSplitter: + def __init__(self, words): + words.sort(key=lambda x: len(x), reverse=True) + self.words = set(words) + if words: + pat = "|".join(re.escape(w) for w in words) + "|." + else: + pat = "." + self.pat = re.compile(pat, flags=re.DOTALL) + + def iter_words(self, text): + s = [] + words = self.words + for m in self.pat.finditer(text): + t = m.group(0) + if t in words: + if s: + yield (False, "".join(s)) + s = [] + yield (True, t) + else: + s.append(t) + if s: + yield (False, "".join(s)) + + def iter(self, text): + for m in self.pat.finditer(text): + yield m.group(0) + +def iter_substrings(s, minlen, maxlen): + len_s = len(s) + maxlen = min(len_s, maxlen) + for n in range(minlen, maxlen + 1): + for begin in range(0, len_s - n + 1): + yield s[begin : begin + n] + +def compute_huffman_coding(translations, compression_filename): + texts = [t[1] for t in translations] + words = [] + + start_unused = 0x80 + end_unused = 0xff + max_ord = 0 + for text in texts: + for c in text: + ord_c = ord(c) + max_ord = max(ord_c, max_ord) + if 0x80 <= ord_c < 0xff: + end_unused = min(ord_c, end_unused) + max_words = end_unused - 0x80 + + values_type = "uint16_t" if max_ord > 255 else "uint8_t" + max_words_len = 160 if max_ord > 255 else 255 + + sum_len = 0 + while True: + # Until the dictionary is filled to capacity, use a heuristic to find + # the best "word" (2- to 9-gram) to add to it. + # + # The TextSplitter allows us to avoid considering parts of the text + # that are already covered by a previously chosen word, for example + # if "the" is in words then not only will "the" not be considered + # again, neither will "there" or "wither", since they have "the" + # as substrings. + extractor = TextSplitter(words) + counter = collections.Counter() + for t in texts: + for (found, word) in extractor.iter_words(t): + if not found: + for substr in iter_substrings(word, minlen=2, maxlen=9): + counter[substr] += 1 + + # Score the candidates we found. This is an empirical formula only, + # chosen for its effectiveness. + scores = sorted( + ( + (s, (len(s) - 1) ** log(max(occ - 2, 1)), occ) + for (s, occ) in counter.items() + ), + key=lambda x: x[1], + reverse=True, + ) + + # Do we have a "word" that occurred 5 times and got a score of at least + # 5? Horray. Pick the one with the highest score. + word = None + for (s, score, occ) in scores: + if occ < 5: + continue + if score < 5: + break + word = s + break + + # If we can successfully add it to the dictionary, do so. Otherwise, + # we've filled the dictionary to capacity and are done. + if not word: + break + if sum_len + len(word) - 2 > max_words_len: + break + if len(words) == max_words: + break + words.append(word) + sum_len += len(word) - 2 + + extractor = TextSplitter(words) + counter = collections.Counter() + for t in texts: + for atom in extractor.iter(t): + counter[atom] += 1 + cb = huffman.codebook(counter.items()) + + word_start = start_unused + word_end = word_start + len(words) - 1 + print("// # words", len(words)) + print("// words", words) -def encode_ngrams(translation, ngrams): - if len(ngrams) > 32: - start = 0xe000 - else: - start = 0x80 - for i, g in enumerate(ngrams): - translation = translation.replace(g, chr(start + i)) - return translation - -def decode_ngrams(compressed, ngrams): - if len(ngrams) > 32: - start, end = 0xe000, 0xf8ff - else: - start, end = 0x80, 0x9f - return "".join(ngrams[ord(c) - start] if (start <= ord(c) <= end) else c for c in compressed) - -def compute_huffman_coding(translations, qstrs, compression_filename): - all_strings = [x[1] for x in translations] - all_strings_concat = "".join(all_strings) - ngrams = [i[0] for i in frequent_ngrams(all_strings_concat, 2, 32)] - all_strings_concat = encode_ngrams(all_strings_concat, ngrams) - counts = collections.Counter(all_strings_concat) - cb = huffman.codebook(counts.items()) values = [] length_count = {} renumbered = 0 - last_l = None + last_length = None canonical = {} - for ch, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])): - values.append(ch) - l = len(code) - if l not in length_count: - length_count[l] = 0 - length_count[l] += 1 - if last_l: - renumbered <<= (l - last_l) - canonical[ch] = '{0:0{width}b}'.format(renumbered, width=l) - s = C_ESCAPES.get(ch, ch) - print("//", ord(ch), s, counts[ch], canonical[ch], renumbered) + for atom, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])): + values.append(atom) + length = len(code) + if length not in length_count: + length_count[length] = 0 + length_count[length] += 1 + if last_length: + renumbered <<= (length - last_length) + canonical[atom] = '{0:0{width}b}'.format(renumbered, width=length) + # print(f"atom={repr(atom)} code={code}", file=sys.stderr) + if len(atom) > 1: + o = words.index(atom) + 0x80 + s = "".join(C_ESCAPES.get(ch1, ch1) for ch1 in atom) + else: + s = C_ESCAPES.get(atom, atom) + o = ord(atom) + print("//", o, s, counter[atom], canonical[atom], renumbered) renumbered += 1 - last_l = l + last_length = length lengths = bytearray() print("// length count", length_count) - print("// bigrams", ngrams) + for i in range(1, max(length_count) + 2): lengths.append(length_count.get(i, 0)) print("// values", values, "lengths", len(lengths), lengths) - ngramdata = [ord(ni) for i in ngrams for ni in i] - print("// estimated total memory size", len(lengths) + 2*len(values) + 2 * len(ngramdata) + sum((len(cb[u]) + 7)//8 for u in all_strings_concat)) + + print("//", values, lengths) + values = [(atom if len(atom) == 1 else chr(0x80 + words.index(atom))) for atom in values] print("//", values, lengths) - values_type = "uint16_t" if max(ord(u) for u in values) > 255 else "uint8_t" - max_translation_encoded_length = max(len(translation.encode("utf-8")) for original,translation in translations) + max_translation_encoded_length = max( + len(translation.encode("utf-8")) for (original, translation) in translations) + + wends = list(len(w) - 2 for w in words) + for i in range(1, len(wends)): + wends[i] += wends[i - 1] + with open(compression_filename, "w") as f: f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths)))) f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values))) f.write("#define compress_max_length_bits ({})\n".format(max_translation_encoded_length.bit_length())) - f.write("const {} bigrams[] = {{ {} }};\n".format(values_type, ", ".join(str(u) for u in ngramdata))) - if len(ngrams) > 32: - bigram_start = 0xe000 - else: - bigram_start = 0x80 - bigram_end = bigram_start + len(ngrams) - 1 # End is inclusive - f.write("#define bigram_start {}\n".format(bigram_start)) - f.write("#define bigram_end {}\n".format(bigram_end)) - return values, lengths, ngrams + f.write("const {} words[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(c)) for w in words for c in w))) + f.write("const uint8_t wends[] = {{ {} }};\n".format(", ".join(str(p) for p in wends))) + f.write("#define word_start {}\n".format(word_start)) + f.write("#define word_end {}\n".format(word_end)) + + return (values, lengths, words, canonical, extractor) def decompress(encoding_table, encoded, encoded_length_bits): - values, lengths, ngrams = encoding_table + (values, lengths, words, _, _) = encoding_table dec = [] this_byte = 0 this_bit = 7 @@ -218,7 +323,8 @@ def decompress(encoding_table, encoded, encoded_length_bits): searched_length += lengths[bit_length] v = values[searched_length + bits - max_code] - v = decode_ngrams(v, ngrams) + if v >= chr(0x80) and v < chr(0x80 + len(words)): + v = words[ord(v) - 0x80] i += len(v.encode('utf-8')) dec.append(v) return ''.join(dec) @@ -226,66 +332,32 @@ def decompress(encoding_table, encoded, encoded_length_bits): def compress(encoding_table, decompressed, encoded_length_bits, len_translation_encoded): if not isinstance(decompressed, str): raise TypeError() - values, lengths, ngrams = encoding_table - decompressed = encode_ngrams(decompressed, ngrams) + (_, _, _, canonical, extractor) = encoding_table + enc = bytearray(len(decompressed) * 3) - #print(decompressed) - #print(lengths) current_bit = 7 current_byte = 0 - code = len_translation_encoded - bits = encoded_length_bits+1 + bits = encoded_length_bits + 1 for i in range(bits - 1, 0, -1): if len_translation_encoded & (1 << (i - 1)): enc[current_byte] |= 1 << current_bit if current_bit == 0: current_bit = 7 - #print("packed {0:0{width}b}".format(enc[current_byte], width=8)) current_byte += 1 else: current_bit -= 1 - for c in decompressed: - #print() - #print("char", c, values.index(c)) - start = 0 - end = lengths[0] - bits = 1 - compressed = None - code = 0 - while compressed is None: - s = start - e = end - #print("{0:0{width}b}".format(code, width=bits)) - # Binary search! - while e > s: - midpoint = (s + e) // 2 - #print(s, e, midpoint) - if values[midpoint] == c: - compressed = code + (midpoint - start) - #print("found {0:0{width}b}".format(compressed, width=bits)) - break - elif c < values[midpoint]: - e = midpoint - else: - s = midpoint + 1 - code += end - start - code <<= 1 - start = end - end += lengths[bits] - bits += 1 - #print("next bit", bits) - - for i in range(bits - 1, 0, -1): - if compressed & (1 << (i - 1)): + for atom in extractor.iter(decompressed): + for b in canonical[atom]: + if b == "1": enc[current_byte] |= 1 << current_bit if current_bit == 0: current_bit = 7 - #print("packed {0:0{width}b}".format(enc[current_byte], width=8)) current_byte += 1 else: current_bit -= 1 + if current_bit != 7: current_byte += 1 return enc[:current_byte] @@ -452,7 +524,7 @@ if __name__ == "__main__": if args.translation: i18ns = sorted(i18ns) translations = translate(args.translation, i18ns) - encoding_table = compute_huffman_coding(translations, qstrs, args.compression_filename) + encoding_table = compute_huffman_coding(translations, args.compression_filename) print_qstr_data(encoding_table, qcfgs, qstrs, translations) else: print_qstr_enums(qstrs) diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index cb27ced35..d3e90ba48 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -57,6 +57,10 @@ name2codepoint['caret'] = ord('^') name2codepoint['pipe'] = ord('|') name2codepoint['tilde'] = ord('~') +# These are just vexing! +del name2codepoint['and'] +del name2codepoint['or'] + def write_out(fname, output): if output: for m, r in [("/", "__"), ("\\", "__"), (":", "@"), ("..", "@@")]: diff --git a/py/mkenv.mk b/py/mkenv.mk index b76dd60f8..cb678e8e7 100644 --- a/py/mkenv.mk +++ b/py/mkenv.mk @@ -55,6 +55,8 @@ PYTHON3 ?= python3 RM = rm RSYNC = rsync SED = sed +# Linux has 'nproc', macOS has 'sysctl -n hw.logicalcpu', this is cross-platform +NPROC = $(PYTHON) -c 'import multiprocessing as mp; print(mp.cpu_count())' AS = $(CROSS_COMPILE)as CC = $(CROSS_COMPILE)gcc diff --git a/py/mkrules.mk b/py/mkrules.mk index 0b3ecc0e1..549b0e5e4 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -76,18 +76,14 @@ $(OBJ): | $(HEADER_BUILD)/qstrdefs.enum.h $(HEADER_BUILD)/mpversion.h # - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^) # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?) # - else, process all source files ($^) [this covers "make -B" which can set $? to empty] -$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(SRC_QSTR_PREPROCESSOR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h +$(HEADER_BUILD)/qstr.split: $(SRC_QSTR) $(SRC_QSTR_PREPROCESSOR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h $(PY_SRC)/genlast.py $(STEPECHO) "GEN $@" - $(Q)grep -lE "(MP_QSTR|translate)" $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) | xargs $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(SRC_QSTR_PREPROCESSOR) >$(HEADER_BUILD)/qstr.i.last; - -$(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last $(PY_SRC)/makeqstrdefs.py - $(STEPECHO) "GEN $@" - $(Q)$(PYTHON3) $(PY_SRC)/makeqstrdefs.py split $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED) + $(Q)$(PYTHON3) $(PY_SRC)/genlast.py $(HEADER_BUILD)/qstr $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) -- $(SRC_QSTR_PREPROCESSOR) -- $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(Q)touch $@ $(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split $(PY_SRC)/makeqstrdefs.py $(STEPECHO) "GEN $@" - $(Q)$(PYTHON3) $(PY_SRC)/makeqstrdefs.py cat $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED) + $(Q)$(PYTHON3) $(PY_SRC)/makeqstrdefs.py cat - $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED) # $(sort $(var)) removes duplicates # diff --git a/py/modstruct.c b/py/modstruct.c index fe766a4de..7675de275 100644 --- a/py/modstruct.c +++ b/py/modstruct.c @@ -183,16 +183,21 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_unpack_from_obj, 2, 3, struct_unpack_ // This function assumes there is enough room in p to store all the values STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, const mp_obj_t *args) { + size_t size; + size_t count = calc_size_items(mp_obj_str_get_str(fmt_in), &size); + if (count != n_args) { +#if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE + mp_raise_ValueError(NULL); +#else + mp_raise_ValueError_varg(translate("pack expected %d items for packing (got %d)"), count, n_args); +#endif + } const char *fmt = mp_obj_str_get_str(fmt_in); char fmt_type = get_fmt_type(&fmt); size_t i; for (i = 0; i < n_args;) { mp_uint_t cnt = 1; - if (*fmt == '\0') { - // more arguments given than used by format string; CPython raises struct.error here - break; - } if (unichar_isdigit(*fmt)) { cnt = get_fmt_num(&fmt); } @@ -208,8 +213,7 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c memset(p + to_copy, 0, cnt - to_copy); p += cnt; } else { - // If we run out of args then we just finish; CPython would raise struct.error - while (cnt-- && i < n_args) { + while (cnt--) { mp_binary_set_val(fmt_type, *fmt, args[i], &p); // Pad bytes don't have a corresponding argument. if (*fmt != 'x') { @@ -222,7 +226,6 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c } STATIC mp_obj_t struct_pack(size_t n_args, const mp_obj_t *args) { - // TODO: "The arguments must match the values required by the format exactly." mp_int_t size = MP_OBJ_SMALL_INT_VALUE(struct_calcsize(args[0])); vstr_t vstr; vstr_init_len(&vstr, size); diff --git a/py/objarray.c b/py/objarray.c index 5d83f0697..e0b4cbd55 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -97,6 +97,9 @@ STATIC void array_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t #if MICROPY_PY_BUILTINS_BYTEARRAY || MICROPY_PY_ARRAY STATIC mp_obj_array_t *array_new(char typecode, size_t n) { + if (typecode == 'x') { + mp_raise_ValueError(translate("bad typecode")); + } int typecode_size = mp_binary_get_size('@', typecode, NULL); mp_obj_array_t *o = m_new_obj(mp_obj_array_t); #if MICROPY_PY_BUILTINS_BYTEARRAY && MICROPY_PY_ARRAY @@ -126,8 +129,10 @@ STATIC mp_obj_t array_construct(char typecode, mp_obj_t initializer) { || (MICROPY_PY_BUILTINS_BYTEARRAY && MP_OBJ_IS_TYPE(initializer, &mp_type_bytearray))))) && mp_get_buffer(initializer, &bufinfo, MP_BUFFER_READ)) { // construct array from raw bytes - // we round-down the len to make it a multiple of sz (CPython raises error) size_t sz = mp_binary_get_size('@', typecode, NULL); + if (bufinfo.len % sz) { + mp_raise_ValueError(translate("bytes length not a multiple of item size")); + } size_t len = bufinfo.len / sz; mp_obj_array_t *o = array_new(typecode, len); memcpy(o->items, bufinfo.buf, len * sz); diff --git a/py/objgenerator.c b/py/objgenerator.c index df421b60c..5e651ac26 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -225,6 +225,21 @@ STATIC mp_obj_t gen_instance_send(mp_obj_t self_in, mp_obj_t send_value) { STATIC MP_DEFINE_CONST_FUN_OBJ_2(gen_instance_send_obj, gen_instance_send); +#if MICROPY_PY_ASYNC_AWAIT +STATIC mp_obj_t gen_instance_await(mp_obj_t self_in) { + mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in); + if ( !self->coroutine_generator ) { + // Pretend like a generator does not have this coroutine behavior. + // Pay no attention to the dir() behind the curtain + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_AttributeError, + translate("type object 'generator' has no attribute '__await__'"))); + } + // You can directly call send on a coroutine generator or you can __await__ then send on the return of that. + return self; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(gen_instance_await_obj, gen_instance_await); +#endif + STATIC mp_obj_t gen_instance_close(mp_obj_t self_in); STATIC mp_obj_t gen_instance_throw(size_t n_args, const mp_obj_t *args) { mp_obj_t exc = (n_args == 2) ? args[1] : args[2]; @@ -280,6 +295,9 @@ STATIC const mp_rom_map_elem_t gen_instance_locals_dict_table[] = { #if MICROPY_PY_GENERATOR_PEND_THROW { MP_ROM_QSTR(MP_QSTR_pend_throw), MP_ROM_PTR(&gen_instance_pend_throw_obj) }, #endif + #if MICROPY_PY_ASYNC_AWAIT + { MP_ROM_QSTR(MP_QSTR___await__), MP_ROM_PTR(&gen_instance_await_obj) }, + #endif }; STATIC MP_DEFINE_CONST_DICT(gen_instance_locals_dict, gen_instance_locals_dict_table); diff --git a/py/objmodule.c b/py/objmodule.c index 757aece04..90796c535 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -239,7 +239,12 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = { { MP_ROM_QSTR(MP_QSTR_hashlib), MP_ROM_PTR(&mp_module_uhashlib) }, #endif #if MICROPY_PY_UBINASCII - { MP_ROM_QSTR(MP_QSTR_binascii), MP_ROM_PTR(&mp_module_ubinascii) }, +#if CIRCUITPY +// CircuitPython: Defined in MICROPY_PORT_BUILTIN_MODULES, so not defined here. +// TODO: move to shared-bindings/ +#else + { MP_ROM_QSTR(MP_QSTR_ubinascii), MP_ROM_PTR(&mp_module_ubinascii) }, +#endif #endif #if MICROPY_PY_URANDOM { MP_ROM_QSTR(MP_QSTR_urandom), MP_ROM_PTR(&mp_module_urandom) }, diff --git a/py/objnamedtuple.c b/py/objnamedtuple.c index 6c36909e7..ab2f2f3c0 100644 --- a/py/objnamedtuple.c +++ b/py/objnamedtuple.c @@ -46,7 +46,7 @@ size_t mp_obj_namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr n return (size_t)-1; } -#if MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT +#if MICROPY_PY_COLLECTIONS_ORDEREDDICT STATIC mp_obj_t namedtuple_asdict(mp_obj_t self_in) { mp_obj_namedtuple_t *self = MP_OBJ_TO_PTR(self_in); const qstr *fields = ((mp_obj_namedtuple_type_t*)self->tuple.base.type)->fields; diff --git a/py/objset.c b/py/objset.c index b8f0f07ed..45b5c1260 100644 --- a/py/objset.c +++ b/py/objset.c @@ -450,6 +450,7 @@ STATIC mp_obj_t set_unary_op(mp_unary_op_t op, mp_obj_t self_in) { return MP_OBJ_NEW_SMALL_INT(hash); } #endif + /* FALLTHROUGH */ default: return MP_OBJ_NULL; // op not supported } } @@ -181,6 +181,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ argcheck.o \ warning.o \ map.o \ + enum.o \ obj.o \ objarray.o \ objattrtuple.o \ |
