From 16677ce311fd70162cc9f7cfe2ab97461df765fc Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 28 Jan 2015 14:07:11 +0000 Subject: py: Be more precise about unicode type and disabled unicode behaviour. --- py/lexer.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'py/lexer.c') diff --git a/py/lexer.c b/py/lexer.c index 13c3a3e9b..8a8875ed5 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -492,11 +492,19 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) { } } if (c != MP_LEXER_EOF) { + #if MICROPY_PY_BUILTINS_STR_UNICODE if (c < 0x110000 && !is_bytes) { vstr_add_char(&lex->vstr, c); } else if (c < 0x100 && is_bytes) { vstr_add_byte(&lex->vstr, c); - } else { + } + #else + // without unicode everything is just added as an 8-bit byte + if (c < 0x100) { + vstr_add_byte(&lex->vstr, c); + } + #endif + else { assert(!"TODO: Throw an error, invalid escape code probably"); } } -- cgit v1.2.3