From 933add6cd833bb6ba6682ad2b6eb478871415ff5 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Tue, 31 Jul 2018 16:53:54 -0700 Subject: Support internationalisation. --- py/makeqstrdata.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++------ py/makeqstrdefs.py | 10 ++++++-- py/makeversionhdr.py | 1 - py/mkrules.mk | 8 +++---- py/modbuiltins.c | 16 +++++++------ py/modstruct.c | 9 ++++---- py/py.mk | 23 +++++++++++++++---- py/qstr.c | 2 ++ py/qstr.h | 6 ++--- 9 files changed, 107 insertions(+), 32 deletions(-) (limited to 'py') diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 3c0a60909..719bc62fa 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -9,6 +9,9 @@ from __future__ import print_function import re import sys +import collections +import gettext + # Python 2/3 compatibility: # - iterating through bytes is different # - codepoint2name lives in a different module @@ -59,6 +62,12 @@ def compute_hash(qstr, bytes_hash): # Make sure that valid hash is never zero, zero means "hash not computed" return (hash & ((1 << (8 * bytes_hash)) - 1)) or 1 +def translate(translation_file, i18ns): + with open(translation_file, "rb") as f: + table = gettext.GNUTranslations(f) + + return [(x, table.gettext(x)) for x in i18ns] + def qstr_escape(qst): def esc_char(m): c = ord(m.group(0)) @@ -73,6 +82,7 @@ def parse_input_headers(infiles): # read the qstrs in from the input files qcfgs = {} qstrs = {} + i18ns = set() for infile in infiles: with open(infile, 'rt') as f: for line in f: @@ -88,6 +98,12 @@ def parse_input_headers(infiles): qcfgs[match.group(1)] = value continue + + match = re.match(r'^TRANSLATE\("(.*)"\)$', line) + if match: + i18ns.add(match.group(1)) + continue + # is this a QSTR line? match = re.match(r'^Q\((.*)\)$', line) if not match: @@ -121,11 +137,11 @@ def parse_input_headers(infiles): order -= 100000 qstrs[ident] = (order, ident, qstr) - if not qcfgs: + if not qcfgs and qstrs: sys.stderr.write("ERROR: Empty preprocessor output - check for errors above\n") sys.exit(1) - return qcfgs, qstrs + return qcfgs, qstrs, i18ns def make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr): qbytes = bytes_cons(qstr, 'utf8') @@ -144,7 +160,7 @@ def make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr): qhash_str = ('\\x%02x' * cfg_bytes_hash) % tuple(((qhash >> (8 * i)) & 0xff) for i in range(cfg_bytes_hash)) return '(const byte*)"%s%s" "%s"' % (qhash_str, qlen_str, qdata) -def print_qstr_data(qcfgs, qstrs): +def print_qstr_data(qcfgs, qstrs, i18ns): # get config variables cfg_bytes_len = int(qcfgs['BYTES_IN_LEN']) cfg_bytes_hash = int(qcfgs['BYTES_IN_HASH']) @@ -156,14 +172,48 @@ def print_qstr_data(qcfgs, qstrs): # add NULL qstr with no hash or data print('QDEF(MP_QSTR_NULL, (const byte*)"%s%s" "")' % ('\\x00' * cfg_bytes_hash, '\\x00' * cfg_bytes_len)) + total_qstr_size = 0 # go through each qstr and print it out for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]): qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr) print('QDEF(MP_QSTR_%s, %s)' % (ident, qbytes)) + total_qstr_size += len(qstr) + + total_text_size = 0 + for original, translation in i18ns: + print("TRANSLATION(\"{}\", \"{}\")".format(original, translation)) + total_text_size += len(translation) -def do_work(infiles): - qcfgs, qstrs = parse_input_headers(infiles) - print_qstr_data(qcfgs, qstrs) + print() + print("// {} bytes worth of qstr".format(total_qstr_size)) + print("// {} bytes worth of translations".format(total_text_size)) + +def print_qstr_enums(qstrs): + # print out the starter of the generated C header file + print('// This file was automatically generated by makeqstrdata.py') + print('') + + # add NULL qstr with no hash or data + print('QENUM(MP_QSTR_NULL)') + + # go through each qstr and print it out + for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]): + print('QENUM(MP_QSTR_%s)' % (ident,)) if __name__ == "__main__": - do_work(sys.argv[1:]) + import argparse + + parser = argparse.ArgumentParser(description='Process QSTR definitions into headers for compilation') + parser.add_argument('infiles', metavar='N', type=str, nargs='+', + help='an integer for the accumulator') + parser.add_argument('--translation', default=None, type=str, + help='translations for i18n() items') + + args = parser.parse_args() + + qcfgs, qstrs, i18ns = parse_input_headers(args.infiles) + if args.translation: + translations = translate(args.translation, i18ns) + print_qstr_data(qcfgs, qstrs, translations) + else: + print_qstr_enums(qstrs) diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index 4253c1147..06cf9078a 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -73,10 +73,11 @@ def qstr_unescape(qstr): return qstr def process_file(f): - re_line = re.compile(r"#[line]*\s\d+\s\"([^\"]+)\"") + re_line = re.compile(r"#[line]*\s(\d+)\s\"([^\"]+)\"") re_qstr = re.compile(r'MP_QSTR_[_a-zA-Z0-9]+') output = [] last_fname = None + lineno = 0 for line in f: if line.isspace(): continue @@ -84,7 +85,9 @@ def process_file(f): if line.startswith(('# ', '#line')): m = re_line.match(line) assert m is not None - fname = m.group(1) + #print(m.groups()) + lineno = int(m.group(1)) + fname = m.group(2) if not fname.endswith(".c"): continue if fname != last_fname: @@ -96,6 +99,9 @@ def process_file(f): name = match.replace('MP_QSTR_', '') if name not in QSTRING_BLACK_LIST: output.append('Q(' + qstr_unescape(name) + ')') + for match in re.findall(r'translate\(\"([^\"]+)\"\)', line): + output.append('TRANSLATE("' + match + '")') + lineno += 1 write_out(last_fname, output) return "" diff --git a/py/makeversionhdr.py b/py/makeversionhdr.py index 89a604aeb..7ceeaeadd 100644 --- a/py/makeversionhdr.py +++ b/py/makeversionhdr.py @@ -100,7 +100,6 @@ def make_version_header(filename): # Only write the file if we need to if write_file: - print("GEN %s" % filename) with open(filename, 'w') as f: f.write(file_data) diff --git a/py/mkrules.mk b/py/mkrules.mk index 621d024a9..f1276f804 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -71,22 +71,22 @@ $(BUILD)/%.pp: %.c # the right .o's to get recompiled if the generated.h file changes. Adding # an order-only dependency to all of the .o's will cause the generated .h # to get built before we try to compile any of them. -$(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h +$(OBJ): | $(HEADER_BUILD)/qstrdefs.enum.h $(HEADER_BUILD)/mpversion.h # The logic for qstr regeneration is: # - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^) # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?) # - else, process all source files ($^) [this covers "make -B" which can set $? to empty] $(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(SRC_QSTR_PREPROCESSOR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h - $(ECHO) "GEN $@" + $(STEPECHO) "GEN $@" $(Q)grep -lE "(MP_QSTR|i18n)" $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) | xargs $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(SRC_QSTR_PREPROCESSOR) >$(HEADER_BUILD)/qstr.i.last; -$(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last +$(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last $(PY_SRC)/makeqstrdefs.py $(STEPECHO) "GEN $@" $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED) $(Q)touch $@ -$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split +$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split $(PY_SRC)/makeqstrdefs.py $(STEPECHO) "GEN $@" $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED) diff --git a/py/modbuiltins.c b/py/modbuiltins.c index eb1a17204..4b24e8811 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -35,6 +35,8 @@ #include "py/builtin.h" #include "py/stream.h" +#include "supervisor/shared/translate.h" + #if MICROPY_PY_BUILTINS_FLOAT #include #endif @@ -157,7 +159,7 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { str[3] = (c & 0x3F) | 0x80; len = 4; } else { - mp_raise_ValueError("chr() arg not in range(0x110000)"); + mp_raise_ValueError(translate("chr() arg not in range(0x110000)")); } return mp_obj_new_str_via_qstr((char*)str, len); #else @@ -166,7 +168,7 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { uint8_t str[1] = {ord}; return mp_obj_new_str_via_qstr((char*)str, 1); } else { - mp_raise_ValueError("chr() arg not in range(256)"); + mp_raise_ValueError(translate("chr() arg not in range(256)")); } #endif } @@ -280,7 +282,7 @@ STATIC mp_obj_t mp_builtin_min_max(size_t n_args, const mp_obj_t *args, mp_map_t if (default_elem != NULL) { best_obj = default_elem->value; } else { - mp_raise_ValueError("arg is an empty sequence"); + mp_raise_ValueError(translate("arg is an empty sequence")); } } return best_obj; @@ -345,10 +347,10 @@ STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) { } if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { - mp_raise_TypeError("ord expects a character"); + mp_raise_TypeError(translate("ord expects a character")); } else { mp_raise_TypeError_varg( - "ord() expected a character, but string of length %d found", (int)len); + translate("ord() expected a character, but string of length %d found"), (int)len); } } MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_ord_obj, mp_builtin_ord); @@ -358,7 +360,7 @@ STATIC mp_obj_t mp_builtin_pow(size_t n_args, const mp_obj_t *args) { case 2: return mp_binary_op(MP_BINARY_OP_POWER, args[0], args[1]); default: #if !MICROPY_PY_BUILTINS_POW3 - mp_raise_msg(&mp_type_NotImplementedError, "3-arg pow() not supported"); + mp_raise_msg(&mp_type_NotImplementedError, translate("3-arg pow() not supported")); #elif MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_MPZ return mp_binary_op(MP_BINARY_OP_MODULO, mp_binary_op(MP_BINARY_OP_POWER, args[0], args[1]), args[2]); #else @@ -512,7 +514,7 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_sum_obj, 1, 2, mp_builtin_sum); STATIC mp_obj_t mp_builtin_sorted(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) { if (n_args > 1) { - mp_raise_TypeError("must use keyword argument for key function"); + mp_raise_TypeError(translate("must use keyword argument for key function")); } mp_obj_t self = mp_type_list.make_new(&mp_type_list, 1, 0, args); mp_obj_list_sort(1, &self, kwargs); diff --git a/py/modstruct.c b/py/modstruct.c index 8617a8e0d..3f1b2f8b8 100644 --- a/py/modstruct.c +++ b/py/modstruct.c @@ -33,6 +33,7 @@ #include "py/objtuple.h" #include "py/binary.h" #include "py/parsenum.h" +#include "supervisor/shared/translate.h" #if MICROPY_PY_STRUCT @@ -141,7 +142,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) { // negative offsets are relative to the end of the buffer offset = bufinfo.len + offset; if (offset < 0) { - mp_raise_ValueError("buffer too small"); + mp_raise_ValueError(translate("buffer too small")); } } p += offset; @@ -149,7 +150,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) { // Check that the input buffer is big enough to unpack all the values if (p + total_sz > end_p) { - mp_raise_ValueError("buffer too small"); + mp_raise_ValueError(translate("buffer too small")); } for (size_t i = 0; i < num_items;) { @@ -230,7 +231,7 @@ STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) { // negative offsets are relative to the end of the buffer offset = (mp_int_t)bufinfo.len + offset; if (offset < 0) { - mp_raise_ValueError("buffer too small"); + mp_raise_ValueError(translate("buffer too small")); } } byte *p = (byte *)bufinfo.buf; @@ -240,7 +241,7 @@ STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) { // Check that the output buffer is big enough to hold all the values mp_int_t sz = MP_OBJ_SMALL_INT_VALUE(struct_calcsize(args[0])); if (p + sz > end_p) { - mp_raise_ValueError("buffer too small"); + mp_raise_ValueError(translate("buffer too small")); } struct_pack_into_internal(args[0], p, n_args - 3, &args[3]); diff --git a/py/py.mk b/py/py.mk index 8e1b86a13..ecd66531b 100644 --- a/py/py.mk +++ b/py/py.mk @@ -7,6 +7,8 @@ HEADER_BUILD = $(BUILD)/genhdr # file containing qstr defs for the core Python bit PY_QSTR_DEFS = $(PY_SRC)/qstrdefs.h +TRANSLATION := en_US + # If qstr autogeneration is not disabled we specify the output header # for all collected qstrings. ifneq ($(QSTR_AUTOGEN_DISABLE),1) @@ -288,21 +290,34 @@ FORCE: .PHONY: FORCE $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD) + $(STEPECHO) "GEN $@" $(Q)$(PYTHON) $(PY_SRC)/makeversionhdr.py $@ # mpconfigport.mk is optional, but changes to it may drastically change # overall config, so they need to be caught MPCONFIGPORT_MK = $(wildcard mpconfigport.mk) +$(HEADER_BUILD)/$(TRANSLATION).mo: $(TOP)/locale/$(TRANSLATION).po + $(Q)msgfmt -o $@ $^ + +$(HEADER_BUILD)/qstrdefs.preprocessed.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD) + $(STEPECHO) "GEN $@" + $(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $@ + # qstr data +$(HEADER_BUILD)/qstrdefs.enum.h: $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h + $(STEPECHO) "GEN $@" + $(PYTHON) $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@ + # Adding an order only dependency on $(HEADER_BUILD) causes $(HEADER_BUILD) to get # created before we run the script to generate the .h # Note: we need to protect the qstr names from the preprocessor, so we wrap # the lines in "" and then unwrap after the preprocessor is finished. -$(HEADER_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) $(PY_SRC)/makeqstrdata.py mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD) - $(ECHO) "GEN $@" - $(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $(HEADER_BUILD)/qstrdefs.preprocessed.h - $(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@ +$(HEADER_BUILD)/qstrdefs.generated.h: $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.preprocessed.h + $(STEPECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py --translation $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@ + +$(PY_BUILD)/qstr.o: $(HEADER_BUILD)/qstrdefs.generated.h # Force nlr code to always be compiled with space-saving optimisation so # that the function preludes are of a minimal and predictable form. diff --git a/py/qstr.c b/py/qstr.c index c68f3dfec..90581664c 100755 --- a/py/qstr.c +++ b/py/qstr.c @@ -104,7 +104,9 @@ const qstr_pool_t mp_qstr_const_pool = { { #ifndef NO_QSTR #define QDEF(id, str) str, +#define TRANSLATION(id, str) #include "genhdr/qstrdefs.generated.h" +#undef TRANSLATION #undef QDEF #endif }, diff --git a/py/qstr.h b/py/qstr.h index f4375ee0e..39b904fb1 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -38,9 +38,9 @@ // first entry in enum will be MP_QSTR_NULL=0, which indicates invalid/no qstr enum { #ifndef NO_QSTR -#define QDEF(id, str) id, -#include "genhdr/qstrdefs.generated.h" -#undef QDEF +#define QENUM(id) id, +#include "genhdr/qstrdefs.enum.h" +#undef QENUM #endif MP_QSTRnumber_of, // no underscore so it can't clash with any of the above }; -- cgit v1.2.3