diff options
| author | hathach <thach@tinyusb.org> | 2018-08-22 15:24:12 +0700 |
|---|---|---|
| committer | hathach <thach@tinyusb.org> | 2018-08-22 15:24:12 +0700 |
| commit | 02b1b531534400f6028c75a34bc29bc66964697e (patch) | |
| tree | 6b566401ecf3e5ce62b7ad6983bcdb556e11a488 /py | |
| parent | fc955b1591798ec3a6b1cb3b5d28290e478a19b1 (diff) | |
| parent | 580a191cc83a4722f6208343163238d41c54981c (diff) | |
Merge branch 'master' into nrf52_dfu_touch1200
Diffstat (limited to 'py')
| -rw-r--r-- | py/compile.c | 4 | ||||
| -rw-r--r-- | py/emitinlinextensa.c | 22 | ||||
| -rw-r--r-- | py/emitnative.c | 36 | ||||
| -rw-r--r-- | py/makeqstrdata.py | 171 | ||||
| -rw-r--r-- | py/modbuiltins.c | 2 | ||||
| -rw-r--r-- | py/moduerrno.c | 32 | ||||
| -rw-r--r-- | py/mperrno.h | 2 | ||||
| -rw-r--r-- | py/obj.c | 24 | ||||
| -rw-r--r-- | py/obj.h | 15 | ||||
| -rw-r--r-- | py/objexcept.c | 29 | ||||
| -rw-r--r-- | py/parsenum.c | 10 | ||||
| -rw-r--r-- | py/py.mk | 2 | ||||
| -rwxr-xr-x | py/qstr.c | 2 | ||||
| -rw-r--r-- | py/runtime.c | 22 | ||||
| -rw-r--r-- | py/runtime.h | 22 | ||||
| -rw-r--r-- | py/vm.c | 6 |
16 files changed, 297 insertions, 104 deletions
diff --git a/py/compile.c b/py/compile.c index c4f447c56..1218aa07e 100644 --- a/py/compile.c +++ b/py/compile.c @@ -156,7 +156,7 @@ STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) { } } -STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const char *msg) { +STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const compressed_string_t *msg) { // only register the error if there has been no other error if (comp->compile_error == MP_OBJ_NULL) { comp->compile_error = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); @@ -949,7 +949,7 @@ STATIC void compile_del_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { STATIC void compile_break_cont_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { uint16_t label; - const char *error_msg; + const compressed_string_t *error_msg; if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_break_stmt) { label = comp->break_label; error_msg = translate("'break' outside loop"); diff --git a/py/emitinlinextensa.c b/py/emitinlinextensa.c index 3d3217f5b..9cd65824b 100644 --- a/py/emitinlinextensa.c +++ b/py/emitinlinextensa.c @@ -43,7 +43,7 @@ struct _emit_inline_asm_t { qstr *label_lookup; }; -STATIC void emit_inline_xtensa_error_msg(emit_inline_asm_t *emit, const char *msg) { +STATIC void emit_inline_xtensa_error_msg(emit_inline_asm_t *emit, const compressed_string_t *msg) { *emit->error_slot = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg); } @@ -83,17 +83,17 @@ STATIC void emit_inline_xtensa_end_pass(emit_inline_asm_t *emit, mp_uint_t type_ STATIC mp_uint_t emit_inline_xtensa_count_params(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params) { if (n_params > 4) { - emit_inline_xtensa_error_msg(emit, "can only have up to 4 parameters to Xtensa assembly"); + emit_inline_xtensa_error_msg(emit, translate("can only have up to 4 parameters to Xtensa assembly")); return 0; } for (mp_uint_t i = 0; i < n_params; i++) { if (!MP_PARSE_NODE_IS_ID(pn_params[i])) { - emit_inline_xtensa_error_msg(emit, "parameters must be registers in sequence a2 to a5"); + emit_inline_xtensa_error_msg(emit, translate("parameters must be registers in sequence a2 to a5")); return 0; } const char *p = qstr_str(MP_PARSE_NODE_LEAF_ARG(pn_params[i])); if (!(strlen(p) == 2 && p[0] == 'a' && p[1] == '2' + i)) { - emit_inline_xtensa_error_msg(emit, "parameters must be registers in sequence a2 to a5"); + emit_inline_xtensa_error_msg(emit, translate("parameters must be registers in sequence a2 to a5")); return 0; } } @@ -159,19 +159,19 @@ STATIC mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, mp_parse_n } emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - "'%s' expects a register", op)); + translate("'%s' expects a register"), op)); return 0; } STATIC uint32_t get_arg_i(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn, int min, int max) { mp_obj_t o; if (!mp_parse_node_get_int_maybe(pn, &o)) { - emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' expects an integer", op)); + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, translate("'%s' expects an integer"), op)); return 0; } uint32_t i = mp_obj_get_int_truncated(o); if (min != max && ((int)i < min || (int)i > max)) { - emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' integer %d is not within range %d..%d", op, i, min, max)); + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, translate("'%s' integer %d is not within range %d..%d"), op, i, min, max)); return 0; } return i; @@ -179,7 +179,7 @@ STATIC uint32_t get_arg_i(emit_inline_asm_t *emit, const char *op, mp_parse_node STATIC int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { if (!MP_PARSE_NODE_IS_ID(pn)) { - emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "'%s' expects a label", op)); + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, translate("'%s' expects a label"), op)); return 0; } qstr label_qstr = MP_PARSE_NODE_LEAF_ARG(pn); @@ -190,7 +190,7 @@ STATIC int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_ } // only need to have the labels on the last pass if (emit->pass == MP_PASS_EMIT) { - emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "label '%q' not defined", label_qstr)); + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, translate("label '%q' not defined"), label_qstr)); } return 0; } @@ -324,12 +324,12 @@ STATIC void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_ return; unknown_op: - emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, "unsupported Xtensa instruction '%s' with %d arguments", op_str, n_args)); + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, translate("unsupported Xtensa instruction '%s' with %d arguments"), op_str, n_args)); return; /* branch_not_in_range: - emit_inline_xtensa_error_msg(emit, "branch not in range"); + emit_inline_xtensa_error_msg(emit, translate("branch not in range")); return; */ } diff --git a/py/emitnative.c b/py/emitnative.c index 47309040e..67b0025c6 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -180,7 +180,7 @@ STATIC void emit_native_set_native_type(emit_t *emit, mp_uint_t op, mp_uint_t ar case MP_QSTR_ptr8: type = VTYPE_PTR8; break; case MP_QSTR_ptr16: type = VTYPE_PTR16; break; case MP_QSTR_ptr32: type = VTYPE_PTR32; break; - default: EMIT_NATIVE_VIPER_TYPE_ERROR(emit, "unknown type '%q'", arg2); return; + default: EMIT_NATIVE_VIPER_TYPE_ERROR(emit, translate("unknown type '%q'"), arg2); return; } if (op == MP_EMIT_NATIVE_TYPE_RETURN) { emit->return_vtype = type; @@ -257,7 +257,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // right now we have a restriction of maximum of 4 arguments if (scope->num_pos_args >= 5) { - EMIT_NATIVE_VIPER_TYPE_ERROR(emit, "Viper functions don't currently support more than 4 arguments"); + EMIT_NATIVE_VIPER_TYPE_ERROR(emit, translate("Viper functions don't currently support more than 4 arguments")); return; } @@ -918,7 +918,7 @@ STATIC void emit_native_load_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { DEBUG_printf("load_fast(%s, " UINT_FMT ")\n", qstr_str(qst), local_num); vtype_kind_t vtype = emit->local_vtype[local_num]; if (vtype == VTYPE_UNBOUND) { - EMIT_NATIVE_VIPER_TYPE_ERROR(emit, "local '%q' used before type known", qst); + EMIT_NATIVE_VIPER_TYPE_ERROR(emit, translate("local '%q' used before type known"), qst); } emit_native_pre(emit); if (local_num == 0) { @@ -1115,7 +1115,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { } default: EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't load from '%q'", vtype_to_qstr(vtype_base)); + translate("can't load from '%q'"), vtype_to_qstr(vtype_base)); } } else { // index is not an immediate @@ -1125,7 +1125,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); if (vtype_index != VTYPE_INT && vtype_index != VTYPE_UINT) { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't load with '%q' index", vtype_to_qstr(vtype_index)); + translate("can't load with '%q' index"), vtype_to_qstr(vtype_index)); } switch (vtype_base) { case VTYPE_PTR8: { @@ -1153,7 +1153,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { } default: EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't load from '%q'", vtype_to_qstr(vtype_base)); + translate("can't load from '%q'"), vtype_to_qstr(vtype_base)); } } emit_post_push_reg(emit, VTYPE_INT, REG_RET); @@ -1185,7 +1185,7 @@ STATIC void emit_native_store_fast(emit_t *emit, qstr qst, mp_uint_t local_num) } else if (emit->local_vtype[local_num] != vtype) { // type of local is not the same as object stored in it EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "local '%q' has type '%q' but source is '%q'", + translate("local '%q' has type '%q' but source is '%q'"), qst, vtype_to_qstr(emit->local_vtype[local_num]), vtype_to_qstr(vtype)); } } @@ -1286,7 +1286,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { #endif if (vtype_value != VTYPE_BOOL && vtype_value != VTYPE_INT && vtype_value != VTYPE_UINT) { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't store '%q'", vtype_to_qstr(vtype_value)); + translate("can't store '%q'"), vtype_to_qstr(vtype_value)); } switch (vtype_base) { case VTYPE_PTR8: { @@ -1355,7 +1355,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { } default: EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't store to '%q'", vtype_to_qstr(vtype_base)); + translate("can't store to '%q'"), vtype_to_qstr(vtype_base)); } } else { // index is not an immediate @@ -1366,7 +1366,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); if (vtype_index != VTYPE_INT && vtype_index != VTYPE_UINT) { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't store with '%q' index", vtype_to_qstr(vtype_index)); + translate("can't store with '%q' index"), vtype_to_qstr(vtype_index)); } #if N_X86 // special case: x86 needs byte stores to be from lower 4 regs (REG_ARG_3 is EDX) @@ -1376,7 +1376,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { #endif if (vtype_value != VTYPE_BOOL && vtype_value != VTYPE_INT && vtype_value != VTYPE_UINT) { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't store '%q'", vtype_to_qstr(vtype_value)); + translate("can't store '%q'"), vtype_to_qstr(vtype_value)); } switch (vtype_base) { case VTYPE_PTR8: { @@ -1416,7 +1416,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { } default: EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't store to '%q'", vtype_to_qstr(vtype_base)); + translate("can't store to '%q'"), vtype_to_qstr(vtype_base)); } } @@ -1537,7 +1537,7 @@ STATIC void emit_native_jump_helper(emit_t *emit, bool pop) { } if (!(vtype == VTYPE_BOOL || vtype == VTYPE_INT || vtype == VTYPE_UINT)) { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't implicitly convert '%q' to 'bool'", vtype_to_qstr(vtype)); + translate("can't implicitly convert '%q' to 'bool'"), vtype_to_qstr(vtype)); } } // For non-pop need to save the vtype so that emit_native_adjust_stack_size @@ -1771,7 +1771,7 @@ STATIC void emit_native_unary_op(emit_t *emit, mp_unary_op_t op) { } else { adjust_stack(emit, 1); EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "unary op %q not implemented", mp_unary_op_method_name[op]); + translate("unary op %q not implemented"), mp_unary_op_method_name[op]); } } @@ -1927,7 +1927,7 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { // TODO other ops not yet implemented adjust_stack(emit, 1); EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "binary op %q not implemented", mp_binary_op_method_name[op]); + translate("binary op %q not implemented"), mp_binary_op_method_name[op]); } } else if (vtype_lhs == VTYPE_PYOBJ && vtype_rhs == VTYPE_PYOBJ) { emit_pre_pop_reg_reg(emit, &vtype_rhs, REG_ARG_3, &vtype_lhs, REG_ARG_2); @@ -1948,7 +1948,7 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { } else { adjust_stack(emit, -1); EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "can't do binary op between '%q' and '%q'", + translate("can't do binary op between '%q' and '%q'"), vtype_to_qstr(vtype_lhs), vtype_to_qstr(vtype_rhs)); } } @@ -2170,7 +2170,7 @@ STATIC void emit_native_return_value(emit_t *emit) { emit_pre_pop_reg(emit, &vtype, REG_RET); if (vtype != emit->return_vtype) { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, - "return expected '%q' but got '%q'", + translate("return expected '%q' but got '%q'"), vtype_to_qstr(emit->return_vtype), vtype_to_qstr(vtype)); } } @@ -2188,7 +2188,7 @@ STATIC void emit_native_raise_varargs(emit_t *emit, mp_uint_t n_args) { vtype_kind_t vtype_exc; emit_pre_pop_reg(emit, &vtype_exc, REG_ARG_1); // arg1 = object to raise if (vtype_exc != VTYPE_PYOBJ) { - EMIT_NATIVE_VIPER_TYPE_ERROR(emit, "must raise an object"); + EMIT_NATIVE_VIPER_TYPE_ERROR(emit, translate("must raise an object")); } // TODO probably make this 1 call to the runtime (which could even call convert, native_raise(obj, type)) emit_call(emit, MP_F_NATIVE_RAISE); diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index adf065c72..b0a043ee3 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -11,6 +11,14 @@ import sys import collections import gettext +import os.path + +py = os.path.dirname(sys.argv[0]) +top = os.path.dirname(py) + +sys.path.append(os.path.join(top, "tools/huffman")) + +import huffman # Python 2/3 compatibility: # - iterating through bytes is different @@ -83,9 +91,144 @@ def translate(translation_file, i18ns): unescaped = original for s in C_ESCAPES: unescaped = unescaped.replace(C_ESCAPES[s], s) - translations.append((original, table.gettext(unescaped))) + translation = table.gettext(unescaped) + # Add in carriage returns to work in terminals + translation = translation.replace("\n", "\r\n") + translations.append((original, translation)) return translations +def compute_huffman_coding(translations, qstrs, compression_filename): + all_strings = [x[1] for x in translations] + + # go through each qstr and print it out + for _, _, qstr in qstrs.values(): + all_strings.append(qstr) + all_strings_concat = "".join(all_strings).encode("utf-8") + counts = collections.Counter(all_strings_concat) + # add other values + for i in range(256): + if i not in counts: + counts[i] = 0 + cb = huffman.codebook(counts.items()) + values = bytearray() + length_count = {} + renumbered = 0 + last_l = None + canonical = {} + for ch, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])): + values.append(ch) + l = len(code) + if l not in length_count: + length_count[l] = 0 + length_count[l] += 1 + if last_l: + renumbered <<= (l - last_l) + canonical[ch] = '{0:0{width}b}'.format(renumbered, width=l) + if chr(ch) in C_ESCAPES: + s = C_ESCAPES[chr(ch)] + else: + s = chr(ch) + print("//", ch, s, counts[ch], canonical[ch], renumbered) + renumbered += 1 + last_l = l + lengths = bytearray() + for i in range(1, max(length_count) + 1): + lengths.append(length_count.get(i, 0)) + print("//", values, lengths) + with open(compression_filename, "w") as f: + f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths)))) + f.write("const uint8_t values[256] = {{ {} }};\n".format(", ".join(map(str, values)))) + return values, lengths + +def decompress(encoding_table, length, encoded): + values, lengths = encoding_table + #print(l, encoded) + dec = bytearray(length) + this_byte = 0 + this_bit = 7 + b = encoded[this_byte] + for i in range(length): + bits = 0 + bit_length = 0 + max_code = lengths[0] + searched_length = lengths[0] + while True: + bits <<= 1 + if 0x80 & b: + bits |= 1 + + b <<= 1 + bit_length += 1 + if this_bit == 0: + this_bit = 7 + this_byte += 1 + if this_byte < len(encoded): + b = encoded[this_byte] + else: + this_bit -= 1 + if max_code > 0 and bits < max_code: + #print('{0:0{width}b}'.format(bits, width=bit_length)) + break + max_code = (max_code << 1) + lengths[bit_length] + searched_length += lengths[bit_length] + + v = values[searched_length + bits - max_code] + dec[i] = v + return dec + +def compress(encoding_table, decompressed): + if not isinstance(decompressed, bytes): + raise TypeError() + values, lengths = encoding_table + enc = bytearray(len(decompressed)) + #print(decompressed) + #print(lengths) + current_bit = 7 + current_byte = 0 + for c in decompressed: + #print() + #print("char", c, values.index(c)) + start = 0 + end = lengths[0] + bits = 1 + compressed = None + code = 0 + while compressed is None: + s = start + e = end + #print("{0:0{width}b}".format(code, width=bits)) + # Binary search! + while e > s: + midpoint = (s + e) // 2 + #print(s, e, midpoint) + if values[midpoint] == c: + compressed = code + (midpoint - start) + #print("found {0:0{width}b}".format(compressed, width=bits)) + break + elif c < values[midpoint]: + e = midpoint + else: + s = midpoint + 1 + code += end - start + code <<= 1 + start = end + end += lengths[bits] + bits += 1 + #print("next bit", bits) + + for i in range(bits - 1, 0, -1): + if compressed & (1 << (i - 1)): + enc[current_byte] |= 1 << current_bit + if current_bit == 0: + current_bit = 7 + #print("packed {0:0{width}b}".format(enc[current_byte], width=8)) + current_byte += 1 + else: + current_bit -= 1 + if current_bit != 7: + current_byte += 1 + return enc[:current_byte] + def qstr_escape(qst): def esc_char(m): c = ord(m.group(0)) @@ -178,7 +321,7 @@ def make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr): qhash_str = ('\\x%02x' * cfg_bytes_hash) % tuple(((qhash >> (8 * i)) & 0xff) for i in range(cfg_bytes_hash)) return '(const byte*)"%s%s" "%s"' % (qhash_str, qlen_str, qdata) -def print_qstr_data(qcfgs, qstrs, i18ns): +def print_qstr_data(encoding_table, qcfgs, qstrs, i18ns): # get config variables cfg_bytes_len = int(qcfgs['BYTES_IN_LEN']) cfg_bytes_hash = int(qcfgs['BYTES_IN_HASH']) @@ -191,6 +334,7 @@ def print_qstr_data(qcfgs, qstrs, i18ns): print('QDEF(MP_QSTR_NULL, (const byte*)"%s%s" "")' % ('\\x00' * cfg_bytes_hash, '\\x00' * cfg_bytes_len)) total_qstr_size = 0 + total_qstr_compressed_size = 0 # go through each qstr and print it out for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]): qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr) @@ -198,17 +342,23 @@ def print_qstr_data(qcfgs, qstrs, i18ns): total_qstr_size += len(qstr) total_text_size = 0 + total_text_compressed_size = 0 for original, translation in i18ns: - # Add in carriage returns to work in terminals - translation = translation.replace("\n", "\r\n") - for s in C_ESCAPES: - translation = translation.replace(s, C_ESCAPES[s]) - print("TRANSLATION(\"{}\", \"{}\")".format(original, translation)) - total_text_size += len(translation) + translation_encoded = translation.encode("utf-8") + compressed = compress(encoding_table, translation_encoded) + total_text_compressed_size += len(compressed) + decompressed = decompress(encoding_table, len(translation_encoded), compressed).decode("utf-8") + for c in C_ESCAPES: + decompressed.replace(c, C_ESCAPES[c]) + #print("// \"{}\"".format(translation)) + print("TRANSLATION(\"{}\", {}, {{ {} }}) // {}".format(original, len(translation_encoded)+1, ", ".join(["0x{:02x}".format(x) for x in compressed]), decompressed)) + total_text_size += len(translation.encode("utf-8")) print() print("// {} bytes worth of qstr".format(total_qstr_size)) print("// {} bytes worth of translations".format(total_text_size)) + print("// {} bytes worth of translations compressed".format(total_text_compressed_size)) + print("// {} bytes saved".format(total_text_size - total_text_compressed_size)) def print_qstr_enums(qstrs): # print out the starter of the generated C header file @@ -230,12 +380,15 @@ if __name__ == "__main__": help='an integer for the accumulator') parser.add_argument('--translation', default=None, type=str, help='translations for i18n() items') + parser.add_argument('--compression_filename', default=None, type=str, + help='header for compression info') args = parser.parse_args() qcfgs, qstrs, i18ns = parse_input_headers(args.infiles) if args.translation: translations = translate(args.translation, i18ns) - print_qstr_data(qcfgs, qstrs, translations) + encoding_table = compute_huffman_coding(translations, qstrs, args.compression_filename) + print_qstr_data(encoding_table, qcfgs, qstrs, translations) else: print_qstr_enums(qstrs) diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 4b24e8811..fc7ec24c7 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -316,7 +316,7 @@ MP_DEFINE_CONST_FUN_OBJ_KW(mp_builtin_min_obj, 1, mp_builtin_min); STATIC mp_obj_t mp_builtin_next(mp_obj_t o) { mp_obj_t ret = mp_iternext_allow_raise(o); if (ret == MP_OBJ_STOP_ITERATION) { - mp_raise_msg(&mp_type_StopIteration, ""); + mp_raise_msg(&mp_type_StopIteration, NULL); } else { return ret; } diff --git a/py/moduerrno.c b/py/moduerrno.c index 1b925dfc9..255c1a73e 100644 --- a/py/moduerrno.c +++ b/py/moduerrno.c @@ -102,20 +102,6 @@ const mp_obj_module_t mp_module_uerrno = { }; const char* mp_errno_to_str(mp_obj_t errno_val) { - // For commonly encountered errors, return human readable strings - if (MP_OBJ_IS_SMALL_INT(errno_val)) { - switch (MP_OBJ_SMALL_INT_VALUE(errno_val)) { - case EPERM: return translate("Permission denied"); - case ENOENT: return translate("No such file/directory"); - case EIO: return translate("Input/output error"); - case EACCES: return translate("Permission denied"); - case EEXIST: return translate("File exists"); - case ENODEV: return translate("Unsupported operation"); - case EINVAL: return translate("Invalid argument"); - case EROFS: return translate("Read-only filesystem"); - } - } - // Otherwise, return the Exxxx string for that error code #if MICROPY_PY_UERRNO_ERRORCODE // We have the errorcode dict so can do a lookup using the hash map @@ -148,3 +134,21 @@ const char* mp_errno_to_str(mp_obj_t errno_val) { } #endif //MICROPY_PY_UERRNO + + +// For commonly encountered errors, return human readable strings +const compressed_string_t* mp_common_errno_to_str(mp_obj_t errno_val) { + if (MP_OBJ_IS_SMALL_INT(errno_val)) { + switch (MP_OBJ_SMALL_INT_VALUE(errno_val)) { + case EPERM: return translate("Permission denied"); + case ENOENT: return translate("No such file/directory"); + case EIO: return translate("Input/output error"); + case EACCES: return translate("Permission denied"); + case EEXIST: return translate("File exists"); + case ENODEV: return translate("Unsupported operation"); + case EINVAL: return translate("Invalid argument"); + case EROFS: return translate("Read-only filesystem"); + } + } + return NULL; +} diff --git a/py/mperrno.h b/py/mperrno.h index c530ab183..876b090b5 100644 --- a/py/mperrno.h +++ b/py/mperrno.h @@ -141,5 +141,7 @@ #endif const char* mp_errno_to_str(mp_obj_t errno_val); +// For commonly encountered errors, return compressed human readable strings +const compressed_string_t* mp_common_errno_to_str(mp_obj_t errno_val); #endif // MICROPY_INCLUDED_PY_MPERRNO_H @@ -86,19 +86,35 @@ void mp_obj_print_exception(const mp_print_t *print, mp_obj_t exc) { mp_obj_exception_get_traceback(exc, &n, &values); if (n > 0) { assert(n % 3 == 0); - mp_print_str(print, translate("Traceback (most recent call last):\n")); + // Decompress the format strings + const compressed_string_t* traceback = translate("Traceback (most recent call last):\n"); + char decompressed[traceback->length]; + decompress(traceback, decompressed); +#if MICROPY_ENABLE_SOURCE_LINE + const compressed_string_t* frame = translate(" File \"%q\", line %d"); +#else + const compressed_string_t* frame = translate(" File \"%q\""); +#endif + char decompressed_frame[frame->length]; + decompress(frame, decompressed_frame); + const compressed_string_t* block_fmt = translate(", in %q\n"); + char decompressed_block[block_fmt->length]; + decompress(block_fmt, decompressed_block); + + // Print the traceback + mp_print_str(print, decompressed); for (int i = n - 3; i >= 0; i -= 3) { #if MICROPY_ENABLE_SOURCE_LINE - mp_printf(print, translate(" File \"%q\", line %d"), values[i], (int)values[i + 1]); + mp_printf(print, decompressed_frame, values[i], (int)values[i + 1]); #else - mp_printf(print, translate(" File \"%q\""), values[i]); + mp_printf(print, decompressed_frame, values[i]); #endif // the block name can be NULL if it's unknown qstr block = values[i + 2]; if (block == MP_QSTR_NULL) { mp_print_str(print, "\n"); } else { - mp_printf(print, translate(", in %q\n"), block); + mp_printf(print, decompressed_block, block); } } } @@ -34,6 +34,8 @@ #include "py/mpprint.h" #include "py/runtime0.h" +#include "supervisor/shared/translate.h" + // This is the definition of the opaque MicroPython object type. // All concrete objects have an encoding within this type and the // particular encoding is specified by MICROPY_OBJ_REPR. @@ -293,6 +295,13 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; #define MP_DEFINE_CONST_FUN_OBJ_KW(obj_name, n_args_min, fun_name) \ const mp_obj_fun_builtin_var_t obj_name = \ {{&mp_type_fun_builtin_var}, true, n_args_min, MP_OBJ_FUN_ARGS_MAX, .fun.kw = fun_name} +#define MP_DEFINE_CONST_PROP_GET(obj_name, fun_name) \ + const mp_obj_fun_builtin_fixed_t fun_name##_obj = {{&mp_type_fun_builtin_1}, .fun._1 = fun_name}; \ + const mp_obj_property_t obj_name = { \ + .base.type = &mp_type_property, \ + .proxy = {(mp_obj_t)&fun_name##_obj, \ + (mp_obj_t)&mp_const_none_obj, \ + (mp_obj_t)&mp_const_none_obj}, } // These macros are used to define constant map/dict objects // You can put "static" in front of the definition to make it local @@ -639,9 +648,9 @@ mp_obj_t mp_obj_new_complex(mp_float_t real, mp_float_t imag); mp_obj_t mp_obj_new_exception(const mp_obj_type_t *exc_type); mp_obj_t mp_obj_new_exception_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg); mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, const mp_obj_t *args); -mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg); -mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) -mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const char *fmt, va_list ap); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) +mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const compressed_string_t *msg); +mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const compressed_string_t *fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) +mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const compressed_string_t *fmt, va_list ap); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table); mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table); mp_obj_t mp_obj_new_fun_viper(size_t n_args, void *fun_data, mp_uint_t type_sig); diff --git a/py/objexcept.c b/py/objexcept.c index 68897d44d..8bd6245a4 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -114,7 +114,15 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin } else if (o->args->len == 1) { // try to provide a nice OSError error message if (o->base.type == &mp_type_OSError && MP_OBJ_IS_SMALL_INT(o->args->items[0])) { - const char* msg = mp_errno_to_str(o->args->items[0]); + const compressed_string_t* common = mp_common_errno_to_str(o->args->items[0]); + const char* msg; + char decompressed[50]; + if (common != NULL && common->length <= 50) { + decompress(common, decompressed); + msg = decompressed; + } else { + msg = mp_errno_to_str(o->args->items[0]); + } if (msg[0] != '\0') { mp_printf(print, "[Errno " INT_FMT "] %s", MP_OBJ_SMALL_INT_VALUE(o->args->items[0]), msg); return; @@ -311,7 +319,7 @@ mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, return exc_type->make_new(exc_type, n_args, 0, args); } -mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg) { +mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const compressed_string_t *msg) { return mp_obj_new_exception_msg_varg(exc_type, msg); } @@ -349,7 +357,7 @@ STATIC void exc_add_strn(void *data, const char *str, size_t len) { } -mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...) { +mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const compressed_string_t *fmt, ...) { va_list ap; va_start(ap, fmt); mp_obj_t exception = mp_obj_new_exception_msg_vlist(exc_type, fmt, ap); @@ -357,7 +365,7 @@ mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char return exception; } -mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const char *fmt, va_list ap) { +mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const compressed_string_t *fmt, va_list ap) { assert(fmt != NULL); // Check that the given type is an exception type @@ -365,7 +373,7 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha // Try to allocate memory for the message mp_obj_str_t *o_str = m_new_obj_maybe(mp_obj_str_t); - size_t o_str_alloc = strlen(fmt) + 1; + size_t o_str_alloc = fmt->length + 1; byte *o_str_buf = m_new_maybe(byte, o_str_alloc); bool used_emg_buf = false; @@ -391,15 +399,16 @@ mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, const cha } if (o_str_buf == NULL) { - // No memory for the string buffer: assume that the fmt string is in ROM - // and use that data as the data of the string - o_str->len = o_str_alloc - 1; // will be equal to strlen(fmt) - o_str->data = (const byte*)fmt; + // No memory for the string buffer: the string is compressed so don't add it. + o_str->len = 0; + o_str->data = NULL; } else { // We have some memory to format the string struct _exc_printer_t exc_pr = {!used_emg_buf, o_str_alloc, 0, o_str_buf}; mp_print_t print = {&exc_pr, exc_add_strn}; - mp_vprintf(&print, fmt, ap); + char fmt_decompressed[fmt->length]; + decompress(fmt, fmt_decompressed); + mp_vprintf(&print, fmt_decompressed, ap); exc_pr.buf[exc_pr.len] = '\0'; o_str->len = exc_pr.len; o_str->data = exc_pr.buf; diff --git a/py/parsenum.c b/py/parsenum.c index f6d419b91..6ef309b47 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -148,11 +148,11 @@ overflow: value_error: if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { mp_obj_t exc = mp_obj_new_exception_msg(&mp_type_ValueError, - "invalid syntax for integer"); + translate("invalid syntax for integer")); raise_exc(exc, lex); } else if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_NORMAL) { mp_obj_t exc = mp_obj_new_exception_msg_varg(&mp_type_ValueError, - "invalid syntax for integer with base %d", base); + translate("invalid syntax for integer with base %d"), base); raise_exc(exc, lex); } else { vstr_t vstr; @@ -328,7 +328,7 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool } #else if (imag || force_complex) { - raise_exc(mp_obj_new_exception_msg(&mp_type_ValueError, "complex values not supported"), lex); + raise_exc(mp_obj_new_exception_msg(&mp_type_ValueError, translate("complex values not supported")), lex); } #endif else { @@ -336,9 +336,9 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool } value_error: - raise_exc(mp_obj_new_exception_msg(&mp_type_ValueError, "invalid syntax for number"), lex); + raise_exc(mp_obj_new_exception_msg(&mp_type_ValueError, translate("invalid syntax for number")), lex); #else - raise_exc(mp_obj_new_exception_msg(&mp_type_ValueError, "decimal numbers not supported"), lex); + raise_exc(mp_obj_new_exception_msg(&mp_type_ValueError, translate("decimal numbers not supported")), lex); #endif } @@ -315,7 +315,7 @@ $(HEADER_BUILD)/qstrdefs.enum.h: $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrd # the lines in "" and then unwrap after the preprocessor is finished. $(HEADER_BUILD)/qstrdefs.generated.h: $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.preprocessed.h $(STEPECHO) "GEN $@" - $(PYTHON3) $(PY_SRC)/makeqstrdata.py --translation $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@ + $(PYTHON3) $(PY_SRC)/makeqstrdata.py --compression_filename $(HEADER_BUILD)/compression.generated.h --translation $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@ $(PY_BUILD)/qstr.o: $(HEADER_BUILD)/qstrdefs.generated.h @@ -104,7 +104,7 @@ const qstr_pool_t mp_qstr_const_pool = { { #ifndef NO_QSTR #define QDEF(id, str) str, -#define TRANSLATION(id, str) +#define TRANSLATION(id, length, compressed...) #include "genhdr/qstrdefs.generated.h" #undef TRANSLATION #undef QDEF diff --git a/py/runtime.c b/py/runtime.c index 20e3e578e..3d7a97f58 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1539,7 +1539,7 @@ NORETURN void m_malloc_fail(size_t num_bytes) { translate("memory allocation failed, allocating %u bytes"), (uint)num_bytes); } -NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg) { +NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const compressed_string_t *msg) { if (msg == NULL) { nlr_raise(mp_obj_new_exception(exc_type)); } else { @@ -1547,7 +1547,7 @@ NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg) { } } -NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...) { +NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const compressed_string_t *fmt, ...) { va_list argptr; va_start(argptr,fmt); mp_obj_t exception = mp_obj_new_exception_msg_vlist(exc_type, fmt, argptr); @@ -1555,27 +1555,27 @@ NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, nlr_raise(exception); } -NORETURN void mp_raise_AttributeError(const char *msg) { +NORETURN void mp_raise_AttributeError(const compressed_string_t *msg) { mp_raise_msg(&mp_type_AttributeError, msg); } -NORETURN void mp_raise_RuntimeError(const char *msg) { +NORETURN void mp_raise_RuntimeError(const compressed_string_t *msg) { mp_raise_msg(&mp_type_RuntimeError, msg); } -NORETURN void mp_raise_ImportError(const char *msg) { +NORETURN void mp_raise_ImportError(const compressed_string_t *msg) { mp_raise_msg(&mp_type_ImportError, msg); } -NORETURN void mp_raise_IndexError(const char *msg) { +NORETURN void mp_raise_IndexError(const compressed_string_t *msg) { mp_raise_msg(&mp_type_IndexError, msg); } -NORETURN void mp_raise_ValueError(const char *msg) { +NORETURN void mp_raise_ValueError(const compressed_string_t *msg) { mp_raise_msg(&mp_type_ValueError, msg); } -NORETURN void mp_raise_ValueError_varg(const char *fmt, ...) { +NORETURN void mp_raise_ValueError_varg(const compressed_string_t *fmt, ...) { va_list argptr; va_start(argptr,fmt); mp_obj_t exception = mp_obj_new_exception_msg_vlist(&mp_type_ValueError, fmt, argptr); @@ -1583,11 +1583,11 @@ NORETURN void mp_raise_ValueError_varg(const char *fmt, ...) { nlr_raise(exception); } -NORETURN void mp_raise_TypeError(const char *msg) { +NORETURN void mp_raise_TypeError(const compressed_string_t *msg) { mp_raise_msg(&mp_type_TypeError, msg); } -NORETURN void mp_raise_TypeError_varg(const char *fmt, ...) { +NORETURN void mp_raise_TypeError_varg(const compressed_string_t *fmt, ...) { va_list argptr; va_start(argptr,fmt); mp_obj_t exception = mp_obj_new_exception_msg_vlist(&mp_type_TypeError, fmt, argptr); @@ -1600,7 +1600,7 @@ NORETURN void mp_raise_OSError(int errno_) { } -NORETURN void mp_raise_NotImplementedError(const char *msg) { +NORETURN void mp_raise_NotImplementedError(const compressed_string_t *msg) { mp_raise_msg(&mp_type_NotImplementedError, msg); } diff --git a/py/runtime.h b/py/runtime.h index 95f89ee9e..9c7ffc02f 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -147,18 +147,18 @@ mp_obj_t mp_import_name(qstr name, mp_obj_t fromlist, mp_obj_t level); mp_obj_t mp_import_from(mp_obj_t module, qstr name); void mp_import_all(mp_obj_t module); -NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg); -NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...); -NORETURN void mp_raise_ValueError(const char *msg); -NORETURN void mp_raise_ValueError_varg(const char *fmt, ...); -NORETURN void mp_raise_TypeError(const char *msg); -NORETURN void mp_raise_TypeError_varg(const char *fmt, ...); -NORETURN void mp_raise_AttributeError(const char *msg); -NORETURN void mp_raise_RuntimeError(const char *msg); -NORETURN void mp_raise_ImportError(const char *msg); -NORETURN void mp_raise_IndexError(const char *msg); +NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const compressed_string_t *msg); +NORETURN void mp_raise_msg_varg(const mp_obj_type_t *exc_type, const compressed_string_t *fmt, ...); +NORETURN void mp_raise_ValueError(const compressed_string_t *msg); +NORETURN void mp_raise_ValueError_varg(const compressed_string_t *fmt, ...); +NORETURN void mp_raise_TypeError(const compressed_string_t *msg); +NORETURN void mp_raise_TypeError_varg(const compressed_string_t *fmt, ...); +NORETURN void mp_raise_AttributeError(const compressed_string_t *msg); +NORETURN void mp_raise_RuntimeError(const compressed_string_t *msg); +NORETURN void mp_raise_ImportError(const compressed_string_t *msg); +NORETURN void mp_raise_IndexError(const compressed_string_t *msg); NORETURN void mp_raise_OSError(int errno_); -NORETURN void mp_raise_NotImplementedError(const char *msg); +NORETURN void mp_raise_NotImplementedError(const compressed_string_t *msg); NORETURN void mp_raise_recursion_depth(void); #if MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG @@ -252,7 +252,7 @@ dispatch_loop: if (obj_shared == MP_OBJ_NULL) { local_name_error: { MARK_EXC_IP_SELECTIVE(); - mp_obj_t obj = mp_obj_new_exception_msg(&mp_type_NameError, "local variable referenced before assignment"); + mp_obj_t obj = mp_obj_new_exception_msg(&mp_type_NameError, translate("local variable referenced before assignment")); RAISE(obj); } } @@ -1139,7 +1139,7 @@ unwind_return: } } if (obj == MP_OBJ_NULL) { - obj = mp_obj_new_exception_msg(&mp_type_RuntimeError, "no active exception to reraise"); + obj = mp_obj_new_exception_msg(&mp_type_RuntimeError, translate("no active exception to reraise")); RAISE(obj); } } else { @@ -1281,7 +1281,7 @@ yield: } else #endif { - mp_obj_t obj = mp_obj_new_exception_msg(&mp_type_NotImplementedError, "byte code not implemented"); + mp_obj_t obj = mp_obj_new_exception_msg(&mp_type_NotImplementedError, translate("byte code not implemented")); nlr_pop(); fastn[0] = obj; return MP_VM_RETURN_EXCEPTION; |
