Merge pull request #1097 from tannewt/i18n_only

Support internationalisation.

9 files changed, 107 insertions, 32 deletions

diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py
index 3c0a60909..719bc62fa 100644
--- a/py/makeqstrdata.py
+++ b/py/makeqstrdata.py
@@ -9,6 +9,9 @@ from __future__ import print_function
 import re
 import sys
 
+import collections
+import gettext
+
 # Python 2/3 compatibility:
 #   - iterating through bytes is different
 #   - codepoint2name lives in a different module
@@ -59,6 +62,12 @@ def compute_hash(qstr, bytes_hash):
     # Make sure that valid hash is never zero, zero means "hash not computed"
     return (hash & ((1 << (8 * bytes_hash)) - 1)) or 1
 
+def translate(translation_file, i18ns):
+    with open(translation_file, "rb") as f:
+        table = gettext.GNUTranslations(f)
+
+        return [(x, table.gettext(x)) for x in i18ns]
+
 def qstr_escape(qst):
     def esc_char(m):
         c = ord(m.group(0))
@@ -73,6 +82,7 @@ def parse_input_headers(infiles):
     # read the qstrs in from the input files
     qcfgs = {}
     qstrs = {}
+    i18ns = set()
     for infile in infiles:
         with open(infile, 'rt') as f:
             for line in f:
@@ -88,6 +98,12 @@ def parse_input_headers(infiles):
                     qcfgs[match.group(1)] = value
                     continue
 
+
+                match = re.match(r'^TRANSLATE\("(.*)"\)$', line)
+                if match:
+                    i18ns.add(match.group(1))
+                    continue
+
                 # is this a QSTR line?
                 match = re.match(r'^Q\((.*)\)$', line)
                 if not match:
@@ -121,11 +137,11 @@ def parse_input_headers(infiles):
                     order -= 100000
                 qstrs[ident] = (order, ident, qstr)
 
-    if not qcfgs:
+    if not qcfgs and qstrs:
         sys.stderr.write("ERROR: Empty preprocessor output - check for errors above\n")
         sys.exit(1)
 
-    return qcfgs, qstrs
+    return qcfgs, qstrs, i18ns
 
 def make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr):
     qbytes = bytes_cons(qstr, 'utf8')
@@ -144,7 +160,7 @@ def make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr):
     qhash_str = ('\\x%02x' * cfg_bytes_hash) % tuple(((qhash >> (8 * i)) & 0xff) for i in range(cfg_bytes_hash))
     return '(const byte*)"%s%s" "%s"' % (qhash_str, qlen_str, qdata)
 
-def print_qstr_data(qcfgs, qstrs):
+def print_qstr_data(qcfgs, qstrs, i18ns):
     # get config variables
     cfg_bytes_len = int(qcfgs['BYTES_IN_LEN'])
     cfg_bytes_hash = int(qcfgs['BYTES_IN_HASH'])
@@ -156,14 +172,48 @@ def print_qstr_data(qcfgs, qstrs):
     # add NULL qstr with no hash or data
     print('QDEF(MP_QSTR_NULL, (const byte*)"%s%s" "")' % ('\\x00' * cfg_bytes_hash, '\\x00' * cfg_bytes_len))
 
+    total_qstr_size = 0
     # go through each qstr and print it out
     for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):
         qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr)
         print('QDEF(MP_QSTR_%s, %s)' % (ident, qbytes))
+        total_qstr_size += len(qstr)
+
+    total_text_size = 0
+    for original, translation in i18ns:
+        print("TRANSLATION(\"{}\", \"{}\")".format(original, translation))
+        total_text_size += len(translation)
 
-def do_work(infiles):
-    qcfgs, qstrs = parse_input_headers(infiles)
-    print_qstr_data(qcfgs, qstrs)
+    print()
+    print("// {} bytes worth of qstr".format(total_qstr_size))
+    print("// {} bytes worth of translations".format(total_text_size))
+
+def print_qstr_enums(qstrs):
+    # print out the starter of the generated C header file
+    print('// This file was automatically generated by makeqstrdata.py')
+    print('')
+
+    # add NULL qstr with no hash or data
+    print('QENUM(MP_QSTR_NULL)')
+
+    # go through each qstr and print it out
+    for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):
+        print('QENUM(MP_QSTR_%s)' % (ident,))
 
 if __name__ == "__main__":
-    do_work(sys.argv[1:])
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Process QSTR definitions into headers for compilation')
+    parser.add_argument('infiles', metavar='N', type=str, nargs='+',
+                        help='an integer for the accumulator')
+    parser.add_argument('--translation', default=None, type=str,
+                        help='translations for i18n() items')
+
+    args = parser.parse_args()
+
+    qcfgs, qstrs, i18ns = parse_input_headers(args.infiles)
+    if args.translation:
+        translations = translate(args.translation, i18ns)
+        print_qstr_data(qcfgs, qstrs, translations)
+    else:
+        print_qstr_enums(qstrs)
diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py
index 4253c1147..06cf9078a 100644
--- a/py/makeqstrdefs.py
+++ b/py/makeqstrdefs.py
@@ -73,10 +73,11 @@ def qstr_unescape(qstr):
     return qstr
 
 def process_file(f):
-    re_line = re.compile(r"#[line]*\s\d+\s\"([^\"]+)\"")
+    re_line = re.compile(r"#[line]*\s(\d+)\s\"([^\"]+)\"")
     re_qstr = re.compile(r'MP_QSTR_[_a-zA-Z0-9]+')
     output = []
     last_fname = None
+    lineno = 0
     for line in f:
         if line.isspace():
             continue
@@ -84,7 +85,9 @@ def process_file(f):
         if line.startswith(('# ', '#line')):
             m = re_line.match(line)
             assert m is not None
-            fname = m.group(1)
+            #print(m.groups())
+            lineno = int(m.group(1))
+            fname = m.group(2)
             if not fname.endswith(".c"):
                 continue
             if fname != last_fname:
@@ -96,6 +99,9 @@ def process_file(f):
             name = match.replace('MP_QSTR_', '')
             if name not in QSTRING_BLACK_LIST:
                 output.append('Q(' + qstr_unescape(name) + ')')
+        for match in re.findall(r'translate\(\"([^\"]+)\"\)', line):
+            output.append('TRANSLATE("' + match + '")')
+        lineno += 1
 
     write_out(last_fname, output)
     return ""
diff --git a/py/makeversionhdr.py b/py/makeversionhdr.py
index 89a604aeb..7ceeaeadd 100644
--- a/py/makeversionhdr.py
+++ b/py/makeversionhdr.py
@@ -100,7 +100,6 @@ def make_version_header(filename):
 
     # Only write the file if we need to
     if write_file:
-        print("GEN %s" % filename)
         with open(filename, 'w') as f:
             f.write(file_data)
 
diff --git a/py/mkrules.mk b/py/mkrules.mk
index 621d024a9..f1276f804 100644
--- a/py/mkrules.mk
+++ b/py/mkrules.mk
@@ -71,22 +71,22 @@ $(BUILD)/%.pp: %.c
 # the right .o's to get recompiled if the generated.h file changes. Adding
 # an order-only dependency to all of the .o's will cause the generated .h
 # to get built before we try to compile any of them.
-$(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h
+$(OBJ): | $(HEADER_BUILD)/qstrdefs.enum.h $(HEADER_BUILD)/mpversion.h
 
 # The logic for qstr regeneration is:
 # - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^)
 # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?)
 # - else, process all source files ($^) [this covers "make -B" which can set $? to empty]
 $(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(SRC_QSTR_PREPROCESSOR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
-	$(ECHO) "GEN $@"
+	$(STEPECHO) "GEN $@"
 	$(Q)grep -lE "(MP_QSTR|i18n)" $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) | xargs $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(SRC_QSTR_PREPROCESSOR) >$(HEADER_BUILD)/qstr.i.last;
 
-$(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last
+$(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last $(PY_SRC)/makeqstrdefs.py
 	$(STEPECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
 	$(Q)touch $@
 
-$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split
+$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split $(PY_SRC)/makeqstrdefs.py
 	$(STEPECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
 
diff --git a/py/modbuiltins.c b/py/modbuiltins.c
index eb1a17204..4b24e8811 100644
--- a/py/modbuiltins.c
+++ b/py/modbuiltins.c
@@ -35,6 +35,8 @@
 #include "py/builtin.h"
 #include "py/stream.h"
 
+#include "supervisor/shared/translate.h"
+
 #if MICROPY_PY_BUILTINS_FLOAT
 #include <math.h>
 #endif
@@ -157,7 +159,7 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) {
         str[3] = (c & 0x3F) | 0x80;
         len = 4;
     } else {
-        mp_raise_ValueError("chr() arg not in range(0x110000)");
+        mp_raise_ValueError(translate("chr() arg not in range(0x110000)"));
     }
     return mp_obj_new_str_via_qstr((char*)str, len);
     #else
@@ -166,7 +168,7 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) {
         uint8_t str[1] = {ord};
         return mp_obj_new_str_via_qstr((char*)str, 1);
     } else {
-        mp_raise_ValueError("chr() arg not in range(256)");
+        mp_raise_ValueError(translate("chr() arg not in range(256)"));
     }
     #endif
 }
@@ -280,7 +282,7 @@ STATIC mp_obj_t mp_builtin_min_max(size_t n_args, const mp_obj_t *args, mp_map_t
             if (default_elem != NULL) {
                 best_obj = default_elem->value;
             } else {
-                mp_raise_ValueError("arg is an empty sequence");
+                mp_raise_ValueError(translate("arg is an empty sequence"));
             }
         }
         return best_obj;
@@ -345,10 +347,10 @@ STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) {
     }
 
     if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) {
-        mp_raise_TypeError("ord expects a character");
+        mp_raise_TypeError(translate("ord expects a character"));
     } else {
         mp_raise_TypeError_varg(
-            "ord() expected a character, but string of length %d found", (int)len);
+            translate("ord() expected a character, but string of length %d found"), (int)len);
     }
 }
 MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_ord_obj, mp_builtin_ord);
@@ -358,7 +360,7 @@ STATIC mp_obj_t mp_builtin_pow(size_t n_args, const mp_obj_t *args) {
         case 2: return mp_binary_op(MP_BINARY_OP_POWER, args[0], args[1]);
         default:
 #if !MICROPY_PY_BUILTINS_POW3
-            mp_raise_msg(&mp_type_NotImplementedError, "3-arg pow() not supported");
+            mp_raise_msg(&mp_type_NotImplementedError, translate("3-arg pow() not supported"));
 #elif MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_MPZ
             return mp_binary_op(MP_BINARY_OP_MODULO, mp_binary_op(MP_BINARY_OP_POWER, args[0], args[1]), args[2]);
 #else
@@ -512,7 +514,7 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_sum_obj, 1, 2, mp_builtin_sum);
 
 STATIC mp_obj_t mp_builtin_sorted(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) {
     if (n_args > 1) {
-        mp_raise_TypeError("must use keyword argument for key function");
+        mp_raise_TypeError(translate("must use keyword argument for key function"));
     }
     mp_obj_t self = mp_type_list.make_new(&mp_type_list, 1, 0, args);
     mp_obj_list_sort(1, &self, kwargs);
diff --git a/py/modstruct.c b/py/modstruct.c
index 8617a8e0d..3f1b2f8b8 100644
--- a/py/modstruct.c
+++ b/py/modstruct.c
@@ -33,6 +33,7 @@
 #include "py/objtuple.h"
 #include "py/binary.h"
 #include "py/parsenum.h"
+#include "supervisor/shared/translate.h"
 
 #if MICROPY_PY_STRUCT
 
@@ -141,7 +142,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
             // negative offsets are relative to the end of the buffer
             offset = bufinfo.len + offset;
             if (offset < 0) {
-                mp_raise_ValueError("buffer too small");
+                mp_raise_ValueError(translate("buffer too small"));
             }
         }
         p += offset;
@@ -149,7 +150,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
 
     // Check that the input buffer is big enough to unpack all the values
     if (p + total_sz > end_p) {
-        mp_raise_ValueError("buffer too small");
+        mp_raise_ValueError(translate("buffer too small"));
     }
 
     for (size_t i = 0; i < num_items;) {
@@ -230,7 +231,7 @@ STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) {
         // negative offsets are relative to the end of the buffer
         offset = (mp_int_t)bufinfo.len + offset;
         if (offset < 0) {
-            mp_raise_ValueError("buffer too small");
+            mp_raise_ValueError(translate("buffer too small"));
         }
     }
     byte *p = (byte *)bufinfo.buf;
@@ -240,7 +241,7 @@ STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) {
     // Check that the output buffer is big enough to hold all the values
     mp_int_t sz = MP_OBJ_SMALL_INT_VALUE(struct_calcsize(args[0]));
     if (p + sz > end_p) {
-        mp_raise_ValueError("buffer too small");
+        mp_raise_ValueError(translate("buffer too small"));
     }
 
     struct_pack_into_internal(args[0], p, n_args - 3, &args[3]);
diff --git a/py/py.mk b/py/py.mk
index 8e1b86a13..ecd66531b 100644
--- a/py/py.mk
+++ b/py/py.mk
@@ -7,6 +7,8 @@ HEADER_BUILD = $(BUILD)/genhdr
 # file containing qstr defs for the core Python bit
 PY_QSTR_DEFS = $(PY_SRC)/qstrdefs.h
 
+TRANSLATION := en_US
+
 # If qstr autogeneration is not disabled we specify the output header
 # for all collected qstrings.
 ifneq ($(QSTR_AUTOGEN_DISABLE),1)
@@ -288,21 +290,34 @@ FORCE:
 .PHONY: FORCE
 
 $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD)
+	$(STEPECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makeversionhdr.py $@
 
 # mpconfigport.mk is optional, but changes to it may drastically change
 # overall config, so they need to be caught
 MPCONFIGPORT_MK = $(wildcard mpconfigport.mk)
 
+$(HEADER_BUILD)/$(TRANSLATION).mo: $(TOP)/locale/$(TRANSLATION).po
+	$(Q)msgfmt -o $@ $^
+
+$(HEADER_BUILD)/qstrdefs.preprocessed.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD)
+	$(STEPECHO) "GEN $@"
+	$(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $@
+
 # qstr data
+$(HEADER_BUILD)/qstrdefs.enum.h: $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h
+	$(STEPECHO) "GEN $@"
+	$(PYTHON) $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
+
 # Adding an order only dependency on $(HEADER_BUILD) causes $(HEADER_BUILD) to get
 # created before we run the script to generate the .h
 # Note: we need to protect the qstr names from the preprocessor, so we wrap
 # the lines in "" and then unwrap after the preprocessor is finished.
-$(HEADER_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) $(PY_SRC)/makeqstrdata.py mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD)
-	$(ECHO) "GEN $@"
-	$(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $(HEADER_BUILD)/qstrdefs.preprocessed.h
-	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
+$(HEADER_BUILD)/qstrdefs.generated.h: $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.preprocessed.h
+	$(STEPECHO) "GEN $@"
+	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py --translation $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
+
+$(PY_BUILD)/qstr.o: $(HEADER_BUILD)/qstrdefs.generated.h
 
 # Force nlr code to always be compiled with space-saving optimisation so
 # that the function preludes are of a minimal and predictable form.
diff --git a/py/qstr.c b/py/qstr.c
index c68f3dfec..90581664c 100755
--- a/py/qstr.c
+++ b/py/qstr.c
@@ -104,7 +104,9 @@ const qstr_pool_t mp_qstr_const_pool = {
     {
 #ifndef NO_QSTR
 #define QDEF(id, str) str,
+#define TRANSLATION(id, str)
 #include "genhdr/qstrdefs.generated.h"
+#undef TRANSLATION
 #undef QDEF
 #endif
     },
diff --git a/py/qstr.h b/py/qstr.h
index f4375ee0e..39b904fb1 100644
--- a/py/qstr.h
+++ b/py/qstr.h
@@ -38,9 +38,9 @@
 // first entry in enum will be MP_QSTR_NULL=0, which indicates invalid/no qstr
 enum {
 #ifndef NO_QSTR
-#define QDEF(id, str) id,
-#include "genhdr/qstrdefs.generated.h"
-#undef QDEF
+#define QENUM(id) id,
+#include "genhdr/qstrdefs.enum.h"
+#undef QENUM
 #endif
     MP_QSTRnumber_of, // no underscore so it can't clash with any of the above
 };