From 511c1808699ef29908ab99180866c46dfeae4b95 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 1 Mar 2020 09:38:34 -0600 Subject: parse: push_result_token: throw an exception on too-long names Before this, such names would instead cause an assertion error inside qstr_from_strn. A simple reproducer is a python source file containing the letter "a" repeated 256 times --- py/parse.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'py/parse.c') diff --git a/py/parse.c b/py/parse.c index 911b891e0..864528470 100644 --- a/py/parse.c +++ b/py/parse.c @@ -477,6 +477,9 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) { mp_parse_node_t pn; mp_lexer_t *lex = parser->lexer; if (lex->tok_kind == MP_TOKEN_NAME) { + if(lex->vstr.len >= (1 << (8 * MICROPY_QSTR_BYTES_IN_LEN))) { + mp_raise_msg(&mp_type_SyntaxError, translate("Name too long")); + } qstr id = qstr_from_strn(lex->vstr.buf, lex->vstr.len); #if MICROPY_COMP_CONST // if name is a standalone identifier, look it up in the table of dynamic constants -- cgit v1.2.3 From 3a7a5ba6860c787cc691b5c828eff9a130f39526 Mon Sep 17 00:00:00 2001 From: Josh Klar Date: Sat, 10 Aug 2019 21:27:20 -0700 Subject: py: Implement partial PEP-498 (f-string) support This implements (most of) the PEP-498 spec for f-strings, with two exceptions: - raw f-strings (`fr` or `rf` prefixes) raise `NotImplementedError` - one special corner case does not function as specified in the PEP (more on that in a moment) This is implemented in the core as a syntax translation, brute-forcing all f-strings to run through `String.format`. For example, the statement `x='world'; print(f'hello {x}')` gets translated *at a syntax level* (injected into the lexer) to `x='world'; print('hello {}'.format(x))`. While this may lead to weird column results in tracebacks, it seemed like the fastest, most efficient, and *likely* most RAM-friendly option, despite being implemented under the hood with a completely separate `vstr_t`. Since [string concatenation of adjacent literals is implemented in the lexer](https://github.com/micropython/micropython/commit/534b7c368dc2af7720f3aaed0c936ef46d773957), two side effects emerge: - All strings with at least one f-string portion are concatenated into a single literal which *must* be run through `String.format()` wholesale, and: - Concatenation of a raw string with interpolation characters with an f-string will cause `IndexError`/`KeyError`, which is both different from CPython *and* different from the corner case mentioned in the PEP (which gave an example of the following:) ```python x = 10 y = 'hi' assert ('a' 'b' f'{x}' '{c}' f'str<{y:^4}>' 'd' 'e') == 'ab10{c}str< hi >de' ``` The above-linked commit detailed a pretty solid case for leaving string concatenation in the lexer rather than putting it in the parser, and undoing that decision would likely be disproportionately costly on resources for the sake of a probably-low-impact corner case. An alternative to become complaint with this corner case of the PEP would be to revert to string concatenation in the parser *only when an f-string is part of concatenation*, though I've done no investigation on the difficulty or costs of doing this. A decent set of tests is included. I've manually tested this on the `unix` port on Linux and on a Feather M4 Express (`atmel-samd`) and things seem sane. --- py/lexer.c | 132 ++++++++++++++++++++++++++++++++-- py/lexer.h | 10 +++ py/parse.c | 47 +++++++++--- tests/basics/string_pep498_fstring.py | 126 ++++++++++++++++++++++++++++++++ 4 files changed, 301 insertions(+), 14 deletions(-) create mode 100644 tests/basics/string_pep498_fstring.py (limited to 'py/parse.c') diff --git a/py/lexer.c b/py/lexer.c index 755fa625b..6894ada47 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -64,6 +64,10 @@ STATIC bool is_char_or3(mp_lexer_t *lex, byte c1, byte c2, byte c3) { return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3; } +STATIC bool is_char_or4(mp_lexer_t *lex, byte c1, byte c2, byte c3, byte c4) { + return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3 || lex->chr0 == c4; +} + STATIC bool is_char_following(mp_lexer_t *lex, byte c) { return lex->chr1 == c; } @@ -107,7 +111,9 @@ STATIC bool is_following_odigit(mp_lexer_t *lex) { STATIC bool is_string_or_bytes(mp_lexer_t *lex) { return is_char_or(lex, '\'', '\"') - || (is_char_or3(lex, 'r', 'u', 'b') && is_char_following_or(lex, '\'', '\"')) + || (is_char_or4(lex, 'r', 'u', 'b', 'f') && is_char_following_or(lex, '\'', '\"')) + || ((is_char_and(lex, 'r', 'f') || is_char_and(lex, 'f', 'r')) + && is_char_following_following_or(lex, '\'', '\"')) || ((is_char_and(lex, 'r', 'b') || is_char_and(lex, 'b', 'r')) && is_char_following_following_or(lex, '\'', '\"')); } @@ -121,6 +127,37 @@ STATIC bool is_tail_of_identifier(mp_lexer_t *lex) { return is_head_of_identifier(lex) || is_digit(lex); } +STATIC void swap_char_banks(mp_lexer_t *lex) { + if (lex->vstr_postfix_processing) { + unichar h0, h1, h2; + + h0 = lex->chr0; + h1 = lex->chr1; + h2 = lex->chr2; + + lex->chr0 = lex->vstr_postfix.len > 0 ? lex->vstr_postfix.buf[0] : 0; + lex->chr1 = lex->vstr_postfix.len > 1 ? lex->vstr_postfix.buf[1] : 0; + lex->chr2 = lex->vstr_postfix.len > 2 ? lex->vstr_postfix.buf[2] : 0; + lex->chr3 = h0; + lex->chr4 = h1; + lex->chr5 = h2; + + lex->vstr_postfix_idx = lex->vstr_postfix.len > 2 ? 3 : lex->vstr_postfix.len; + } else { + // blindly reset to the "backup" bank when done postfix processing + // this restores control to the mp_reader + lex->chr0 = lex->chr3; + lex->chr1 = lex->chr4; + lex->chr2 = lex->chr5; + lex->chr3 = 0; + lex->chr4 = 0; + lex->chr5 = 0; + + vstr_reset(&lex->vstr_postfix); + lex->vstr_postfix_idx = 0; + } +} + STATIC void next_char(mp_lexer_t *lex) { if (lex->chr0 == '\n') { // a new line @@ -136,7 +173,16 @@ STATIC void next_char(mp_lexer_t *lex) { lex->chr0 = lex->chr1; lex->chr1 = lex->chr2; - lex->chr2 = lex->reader.readbyte(lex->reader.data); + + if (lex->vstr_postfix_processing) { + if (lex->vstr_postfix_idx == lex->vstr_postfix.len) { + lex->chr2 = '\0'; + } else { + lex->chr2 = lex->vstr_postfix.buf[lex->vstr_postfix_idx++]; + } + } else { + lex->chr2 = lex->reader.readbyte(lex->reader.data); + } if (lex->chr1 == '\r') { // CR is a new line, converted to LF @@ -151,6 +197,11 @@ STATIC void next_char(mp_lexer_t *lex) { if (lex->chr2 == MP_LEXER_EOF && lex->chr1 != MP_LEXER_EOF && lex->chr1 != '\n') { lex->chr2 = '\n'; } + + if (lex->vstr_postfix_processing && lex->chr0 == '\0') { + lex->vstr_postfix_processing = false; + swap_char_banks(lex); + } } STATIC void indent_push(mp_lexer_t *lex, size_t indent) { @@ -270,7 +321,7 @@ STATIC bool get_hex(mp_lexer_t *lex, size_t num_digits, mp_uint_t *result) { return true; } -STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw) { +STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) { // get first quoting character char quote_char = '\''; if (is_char(lex, '\"')) { @@ -291,15 +342,67 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw) { } size_t n_closing = 0; + bool in_expression = false; + bool expression_eat = true; + while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) { if (is_char(lex, quote_char)) { n_closing += 1; vstr_add_char(&lex->vstr, CUR_CHAR(lex)); } else { n_closing = 0; + if (is_fstring && is_char(lex, '{')) { + vstr_add_char(&lex->vstr, CUR_CHAR(lex)); + in_expression = !in_expression; + expression_eat = in_expression; + + if (lex->vstr_postfix.len == 0) { + vstr_add_str(&lex->vstr_postfix, ".format("); + } + + next_char(lex); + continue; + } + + if (is_fstring && is_char(lex, '}')) { + vstr_add_char(&lex->vstr, CUR_CHAR(lex)); + + if (in_expression) { + in_expression = false; + vstr_add_char(&lex->vstr_postfix, ','); + } + + next_char(lex); + continue; + } + + if (in_expression) { + // throw errors for illegal chars inside f-string expressions + if (is_char(lex, '#')) { + lex->tok_kind = MP_TOKEN_FSTRING_COMMENT; + return; + } else if (is_char(lex, '\\')) { + lex->tok_kind = MP_TOKEN_FSTRING_BACKSLASH; + return; + } else if (is_char(lex, ':')) { + expression_eat = false; + } + + unichar c = CUR_CHAR(lex); + if (expression_eat) { + vstr_add_char(&lex->vstr_postfix, c); + } else { + vstr_add_char(&lex->vstr, c); + } + + next_char(lex); + continue; + } + if (is_char(lex, '\\')) { next_char(lex); unichar c = CUR_CHAR(lex); + if (is_raw) { // raw strings allow escaping of quotes, but the backslash is also emitted vstr_add_char(&lex->vstr, '\\'); @@ -430,6 +533,13 @@ STATIC bool skip_whitespace(mp_lexer_t *lex, bool stop_at_newline) { } void mp_lexer_to_next(mp_lexer_t *lex) { + if (lex->vstr_postfix.len && !lex->vstr_postfix_processing) { + // end format call injection + vstr_add_char(&lex->vstr_postfix, ')'); + lex->vstr_postfix_processing = true; + swap_char_banks(lex); + } + // start new token text vstr_reset(&lex->vstr); @@ -485,6 +595,7 @@ void mp_lexer_to_next(mp_lexer_t *lex) { do { // parse type codes bool is_raw = false; + bool is_fstring = false; mp_token_kind_t kind = MP_TOKEN_STRING; int n_char = 0; if (is_char(lex, 'u')) { @@ -503,6 +614,17 @@ void mp_lexer_to_next(mp_lexer_t *lex) { kind = MP_TOKEN_BYTES; n_char = 2; } + if (is_char_following(lex, 'f')) { + lex->tok_kind = MP_TOKEN_FSTRING_RAW; + break; + } + } else if (is_char(lex, 'f')) { + if (is_char_following(lex, 'r')) { + lex->tok_kind = MP_TOKEN_FSTRING_RAW; + break; + } + n_char = 1; + is_fstring = true; } // Set or check token kind @@ -522,13 +644,12 @@ void mp_lexer_to_next(mp_lexer_t *lex) { } // Parse the literal - parse_string_literal(lex, is_raw); + parse_string_literal(lex, is_raw, is_fstring); // Skip whitespace so we can check if there's another string following skip_whitespace(lex, true); } while (is_string_or_bytes(lex)); - } else if (is_head_of_identifier(lex)) { lex->tok_kind = MP_TOKEN_NAME; @@ -682,6 +803,7 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) { lex->num_indent_level = 1; lex->indent_level = m_new(uint16_t, lex->alloc_indent_level); vstr_init(&lex->vstr, 32); + vstr_init(&lex->vstr_postfix, 0); // store sentinel for first indentation level lex->indent_level[0] = 0; diff --git a/py/lexer.h b/py/lexer.h index a29709107..7fe271e84 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -44,6 +44,12 @@ typedef enum _mp_token_kind_t { MP_TOKEN_INVALID, MP_TOKEN_DEDENT_MISMATCH, MP_TOKEN_LONELY_STRING_OPEN, + MP_TOKEN_FSTRING_BACKSLASH, + MP_TOKEN_FSTRING_COMMENT, + MP_TOKEN_FSTRING_UNCLOSED, + MP_TOKEN_FSTRING_UNOPENED, + MP_TOKEN_FSTRING_EMPTY_EXP, + MP_TOKEN_FSTRING_RAW, MP_TOKEN_NEWLINE, MP_TOKEN_INDENT, @@ -150,6 +156,7 @@ typedef struct _mp_lexer_t { mp_reader_t reader; // stream source unichar chr0, chr1, chr2; // current cached characters from source + unichar chr3, chr4, chr5; // current cached characters from alt source size_t line; // current source line size_t column; // current source column @@ -165,6 +172,9 @@ typedef struct _mp_lexer_t { size_t tok_column; // token source column mp_token_kind_t tok_kind; // token kind vstr_t vstr; // token data + vstr_t vstr_postfix; // postfix to apply to string + bool vstr_postfix_processing; + uint16_t vstr_postfix_idx; } mp_lexer_t; mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader); diff --git a/py/parse.c b/py/parse.c index 864528470..d30d1b22f 100644 --- a/py/parse.c +++ b/py/parse.c @@ -924,6 +924,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { backtrack = false; } for (; i < n; ++i) { + //printf("--> inside for @L924\n"); uint16_t kind = rule_arg[i] & RULE_ARG_KIND_MASK; if (kind == RULE_ARG_TOK) { if (lex->tok_kind == (rule_arg[i] & RULE_ARG_ARG_MASK)) { @@ -1168,15 +1169,43 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { ) { syntax_error:; mp_obj_t exc; - if (lex->tok_kind == MP_TOKEN_INDENT) { - exc = mp_obj_new_exception_msg(&mp_type_IndentationError, - translate("unexpected indent")); - } else if (lex->tok_kind == MP_TOKEN_DEDENT_MISMATCH) { - exc = mp_obj_new_exception_msg(&mp_type_IndentationError, - translate("unindent does not match any outer indentation level")); - } else { - exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, - translate("invalid syntax")); + switch(lex->tok_kind) { + case MP_TOKEN_INDENT: + exc = mp_obj_new_exception_msg(&mp_type_IndentationError, + translate("unexpected indent")); + break; + case MP_TOKEN_DEDENT_MISMATCH: + exc = mp_obj_new_exception_msg(&mp_type_IndentationError, + translate("unindent does not match any outer indentation level")); + break; + case MP_TOKEN_FSTRING_BACKSLASH: + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + translate("f-string expression part cannot include a backslash")); + break; + case MP_TOKEN_FSTRING_COMMENT: + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + translate("f-string expression part cannot include a '#'")); + break; + case MP_TOKEN_FSTRING_UNCLOSED: + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + translate("f-string: expecting '}'")); + break; + case MP_TOKEN_FSTRING_UNOPENED: + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + translate("f-string: single '}' is not allowed")); + break; + case MP_TOKEN_FSTRING_EMPTY_EXP: + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + translate("f-string: empty expression not allowed")); + break; + case MP_TOKEN_FSTRING_RAW: + exc = mp_obj_new_exception_msg(&mp_type_NotImplementedError, + translate("raw f-strings are not implemented")); + break; + default: + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + translate("invalid syntax")); + break; } // add traceback to give info about file name and location // we don't have a 'block' name, so just pass the NULL qstr to indicate this diff --git a/tests/basics/string_pep498_fstring.py b/tests/basics/string_pep498_fstring.py new file mode 100644 index 000000000..f1ad1f19b --- /dev/null +++ b/tests/basics/string_pep498_fstring.py @@ -0,0 +1,126 @@ +# Tests against https://www.python.org/dev/peps/pep-0498/ + +assert f'no interpolation' == 'no interpolation' +assert f"no interpolation" == 'no interpolation' + +# Quoth the PEP: +# Backslashes may not appear anywhere within expressions. Comments, using the +# '#' character, are not allowed inside an expression +# +# CPython (3.7.4 on Linux) raises a SyntaxError here: +# >>> f'{#}' +# File "", line 1 +# SyntaxError: f-string expression part cannot include '#' +# >>> f'{\}' +# File "", line 1 +# SyntaxError: f-string expression part cannot include a backslash +# >>> f'{\\}' +# File "", line 1 +# SyntaxError: f-string expression part cannot include a backslash +# >>> f'{\#}' +# File "", line 1 +# SyntaxError: f-string expression part cannot include a backslash + +# Backslashes and comments allowed outside expression +assert f"\\" == "\\" +assert f'#' == '#' + +## But not inside +try: + eval("f'{\}'") +except SyntaxError: + pass +else: + raise AssertionError('f-string with backslash in expression did not raise SyntaxError') + +try: + eval("f'{#}'") +except SyntaxError: + pass +else: + raise AssertionError('f-string with \'#\' in expression did not raise SyntaxError') + +# Quoth the PEP: +# While scanning the string for expressions, any doubled braces '{{' or '}}' +# inside literal portions of an f-string are replaced by the corresponding +# single brace. Doubled literal opening braces do not signify the start of an +# expression. A single closing curly brace '}' in the literal portion of a +# string is an error: literal closing curly braces must be doubled '}}' in +# order to represent a single closing brace. +# +# CPython (3.7.4 on Linux) raises a SyntaxError for the last case: +# >>> f'{{}' +# File "", line 1 +# SyntaxError: f-string: single '}' is not allowed + +assert f'{{}}' == '{}' + +try: + eval("f'{{}'") +except ValueError: + pass +else: + raise RuntimeError('Expected ValueError for invalid f-string literal bracing') + +x = 1 +assert f'{x}' == '1' + +# Quoth the PEP: +# The expressions that are extracted from the string are evaluated in the +# context where the f-string appeared. This means the expression has full +# access to local and global variables. Any valid Python expression can be +# used, including function and method calls. Because the f-strings are +# evaluated where the string appears in the source code, there is no additional +# expressiveness available with f-strings. There are also no additional +# security concerns: you could have also just written the same expression, not +# inside of an f-string: + +def foo(): + return 20 + +assert f'result={foo()}' == 'result=20' +assert f'result={foo()}' == 'result={}'.format(foo()) +assert f'result={foo()}' == 'result={result}'.format(result=foo()) + + +# Quoth the PEP: +# Adjacent f-strings and regular strings are concatenated. Regular strings are +# concatenated at compile time, and f-strings are concatenated at run time. For +# example, the expression: +# +# >>> x = 10 +# >>> y = 'hi' +# >>> 'a' 'b' f'{x}' '{c}' f'str<{y:^4}>' 'd' 'e' +# +# yields the value: 'ab10{c}str< hi >de' +# +# Because strings are concatenated at lexer time rather than parser time in +# MicroPython for mostly RAM efficiency reasons (see +# https://github.com/micropython/micropython/commit/534b7c368dc2af7720f3aaed0c936ef46d773957), +# and because f-strings here are implemented as a syntax translation +# (f'{something}' => '{}'.format(something)), this particular functionality is unimplemented, +# and in the above example, the '{c}' portion will trigger a KeyError on String.format() + +x = 10 +y = 'hi' +assert (f'h' f'i') == 'hi' +assert (f'h' 'i') == 'hi' +assert ('h' f'i') == 'hi' +assert f'{x:^4}' == ' 10 ' +assert ('a' 'b' f'{x}' f'str<{y:^4}>' 'd' 'e') == 'ab10str< hi >de' + +# Other tests +assert f'{{{4*10}}}' == '{40}' + +try: + eval("fr''") +except NotImplementedError: + pass +else: + raise RuntimeError('expected raw f-string to raise NotImplementedError') +try: + eval("rf''") +except NotImplementedError: + pass +else: + raise RuntimeError('expected raw f-string to raise NotImplementedError') -- cgit v1.2.3 From 40bc05ee1eafcc2f0b3ed48f212a1526b65163bc Mon Sep 17 00:00:00 2001 From: Josh Klar Date: Thu, 26 Sep 2019 16:48:02 -0700 Subject: Address dpgeorge feedback - largely simplifications --- py/lexer.c | 26 +++++++++----------------- py/parse.c | 12 ++++++++++++ tests/basics/string_pep498_fstring.py | 13 ------------- 3 files changed, 21 insertions(+), 30 deletions(-) (limited to 'py/parse.c') diff --git a/py/lexer.c b/py/lexer.c index 6894ada47..c3f4e6506 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -129,29 +129,21 @@ STATIC bool is_tail_of_identifier(mp_lexer_t *lex) { STATIC void swap_char_banks(mp_lexer_t *lex) { if (lex->vstr_postfix_processing) { - unichar h0, h1, h2; - - h0 = lex->chr0; - h1 = lex->chr1; - h2 = lex->chr2; - - lex->chr0 = lex->vstr_postfix.len > 0 ? lex->vstr_postfix.buf[0] : 0; - lex->chr1 = lex->vstr_postfix.len > 1 ? lex->vstr_postfix.buf[1] : 0; - lex->chr2 = lex->vstr_postfix.len > 2 ? lex->vstr_postfix.buf[2] : 0; - lex->chr3 = h0; - lex->chr4 = h1; - lex->chr5 = h2; - - lex->vstr_postfix_idx = lex->vstr_postfix.len > 2 ? 3 : lex->vstr_postfix.len; + lex->chr3 = lex->chr0; + lex->chr4 = lex->chr1; + lex->chr5 = lex->chr2; + lex->chr0 = lex->vstr_postfix.buf[0]; + lex->chr1 = lex->vstr_postfix.buf[1]; + lex->chr2 = lex->vstr_postfix.buf[2]; + + lex->vstr_postfix_idx = 3; } else { // blindly reset to the "backup" bank when done postfix processing // this restores control to the mp_reader lex->chr0 = lex->chr3; lex->chr1 = lex->chr4; lex->chr2 = lex->chr5; - lex->chr3 = 0; - lex->chr4 = 0; - lex->chr5 = 0; + // willfully ignoring setting chr3-5 here - WARNING consider those garbage data now vstr_reset(&lex->vstr_postfix); lex->vstr_postfix_idx = 0; diff --git a/py/parse.c b/py/parse.c index d30d1b22f..f86ee25fe 100644 --- a/py/parse.c +++ b/py/parse.c @@ -1178,6 +1178,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { exc = mp_obj_new_exception_msg(&mp_type_IndentationError, translate("unindent does not match any outer indentation level")); break; +#if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_DETAILED case MP_TOKEN_FSTRING_BACKSLASH: exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, translate("f-string expression part cannot include a backslash")); @@ -1202,6 +1203,17 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { exc = mp_obj_new_exception_msg(&mp_type_NotImplementedError, translate("raw f-strings are not implemented")); break; +#else + case MP_TOKEN_FSTRING_BACKSLASH: + case MP_TOKEN_FSTRING_COMMENT: + case MP_TOKEN_FSTRING_UNCLOSED: + case MP_TOKEN_FSTRING_UNOPENED: + case MP_TOKEN_FSTRING_EMPTY_EXP: + case MP_TOKEN_FSTRING_RAW: + exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + translate("malformed f-string")); + break; +#endif default: exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, translate("invalid syntax")); diff --git a/tests/basics/string_pep498_fstring.py b/tests/basics/string_pep498_fstring.py index f1ad1f19b..f4e76c831 100644 --- a/tests/basics/string_pep498_fstring.py +++ b/tests/basics/string_pep498_fstring.py @@ -111,16 +111,3 @@ assert ('a' 'b' f'{x}' f'str<{y:^4}>' 'd' 'e') == 'ab10str< hi >de' # Other tests assert f'{{{4*10}}}' == '{40}' - -try: - eval("fr''") -except NotImplementedError: - pass -else: - raise RuntimeError('expected raw f-string to raise NotImplementedError') -try: - eval("rf''") -except NotImplementedError: - pass -else: - raise RuntimeError('expected raw f-string to raise NotImplementedError') -- cgit v1.2.3 From 473e9c5ffb081ae2750948099ac8288b039d4255 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Mon, 9 Mar 2020 09:02:47 -0500 Subject: f-strings: Make optional, defaulting to !CIRCUITPY_MINIMAL_BUILD This should reclaim *most* code space added to handle f-strings. However, there may be some small code growth as parse_string_literal takes a new parameter (which will always be 0, so hopefully the optimizer eliminates it) --- py/circuitpy_mpconfig.h | 3 +++ py/lexer.c | 33 ++++++++++++++++++++++++++++++++- py/lexer.h | 6 ++++++ py/mpconfig.h | 5 +++++ py/parse.c | 2 ++ 5 files changed, 48 insertions(+), 1 deletion(-) (limited to 'py/parse.c') diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index 61dd901ed..acff27bcf 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -187,6 +187,9 @@ typedef long mp_off_t; #if !defined(MICROPY_CPYTHON_COMPAT) #define MICROPY_CPYTHON_COMPAT (CIRCUITPY_FULL_BUILD) #endif +#if !defined(MICROPY_COMP_FSTRING_LITERAL) +#define MICROPY_COMP_FSTRING_LITERAL (MICROPY_CPYTHON_COMPAT) +#endif #define MICROPY_MODULE_WEAK_LINKS (CIRCUITPY_FULL_BUILD) #define MICROPY_PY_ALL_SPECIAL_METHODS (CIRCUITPY_FULL_BUILD) #define MICROPY_PY_BUILTINS_COMPLEX (CIRCUITPY_FULL_BUILD) diff --git a/py/lexer.c b/py/lexer.c index 80f8f043c..00cd59bca 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -64,9 +64,11 @@ STATIC bool is_char_or3(mp_lexer_t *lex, byte c1, byte c2, byte c3) { return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3; } +#if MICROPY_COMP_FSTRING_LITERAL STATIC bool is_char_or4(mp_lexer_t *lex, byte c1, byte c2, byte c3, byte c4) { return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3 || lex->chr0 == c4; } +#endif STATIC bool is_char_following(mp_lexer_t *lex, byte c) { return lex->chr1 == c; @@ -111,9 +113,13 @@ STATIC bool is_following_odigit(mp_lexer_t *lex) { STATIC bool is_string_or_bytes(mp_lexer_t *lex) { return is_char_or(lex, '\'', '\"') +#if MICROPY_COMP_FSTRING_LITERAL || (is_char_or4(lex, 'r', 'u', 'b', 'f') && is_char_following_or(lex, '\'', '\"')) || ((is_char_and(lex, 'r', 'f') || is_char_and(lex, 'f', 'r')) && is_char_following_following_or(lex, '\'', '\"')) +#else + || (is_char_or3(lex, 'r', 'u', 'b') && is_char_following_or(lex, '\'', '\"')) +#endif || ((is_char_and(lex, 'r', 'b') || is_char_and(lex, 'b', 'r')) && is_char_following_following_or(lex, '\'', '\"')); } @@ -127,6 +133,7 @@ STATIC bool is_tail_of_identifier(mp_lexer_t *lex) { return is_head_of_identifier(lex) || is_digit(lex); } +#if MICROPY_COMP_FSTRING_LITERAL STATIC void swap_char_banks(mp_lexer_t *lex) { if (lex->vstr_postfix_processing) { lex->chr3 = lex->chr0; @@ -149,6 +156,7 @@ STATIC void swap_char_banks(mp_lexer_t *lex) { lex->vstr_postfix_idx = 0; } } +#endif STATIC void next_char(mp_lexer_t *lex) { if (lex->chr0 == '\n') { @@ -166,13 +174,16 @@ STATIC void next_char(mp_lexer_t *lex) { lex->chr0 = lex->chr1; lex->chr1 = lex->chr2; +#if MICROPY_COMP_FSTRING_LITERAL if (lex->vstr_postfix_processing) { if (lex->vstr_postfix_idx == lex->vstr_postfix.len) { lex->chr2 = '\0'; } else { lex->chr2 = lex->vstr_postfix.buf[lex->vstr_postfix_idx++]; } - } else { + } else +#endif + { lex->chr2 = lex->reader.readbyte(lex->reader.data); } @@ -190,10 +201,12 @@ STATIC void next_char(mp_lexer_t *lex) { lex->chr2 = '\n'; } +#if MICROPY_COMP_FSTRING_LITERAL if (lex->vstr_postfix_processing && lex->chr0 == '\0') { lex->vstr_postfix_processing = false; swap_char_banks(lex); } +#endif } STATIC void indent_push(mp_lexer_t *lex, size_t indent) { @@ -334,8 +347,10 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) } size_t n_closing = 0; +#if MICROPY_COMP_FSTRING_LITERAL bool in_expression = false; bool expression_eat = true; +#endif while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) { if (is_char(lex, quote_char)) { @@ -343,6 +358,7 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) vstr_add_char(&lex->vstr, CUR_CHAR(lex)); } else { n_closing = 0; +#if MICROPY_COMP_FSTRING_LITERAL if (is_fstring && is_char(lex, '{')) { vstr_add_char(&lex->vstr, CUR_CHAR(lex)); in_expression = !in_expression; @@ -390,6 +406,7 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) next_char(lex); continue; } +#endif if (is_char(lex, '\\')) { next_char(lex); @@ -525,12 +542,14 @@ STATIC bool skip_whitespace(mp_lexer_t *lex, bool stop_at_newline) { } void mp_lexer_to_next(mp_lexer_t *lex) { +#if MICROPY_COMP_FSTRING_LITERAL if (lex->vstr_postfix.len && !lex->vstr_postfix_processing) { // end format call injection vstr_add_char(&lex->vstr_postfix, ')'); lex->vstr_postfix_processing = true; swap_char_banks(lex); } +#endif // start new token text vstr_reset(&lex->vstr); @@ -583,13 +602,19 @@ void mp_lexer_to_next(mp_lexer_t *lex) { // MP_TOKEN_END is used to indicate that this is the first string token lex->tok_kind = MP_TOKEN_END; +#if MICROPY_COMP_FSTRING_LITERAL bool saw_normal = false, saw_fstring = false; +#endif // Loop to accumulate string/bytes literals do { // parse type codes bool is_raw = false; +#if MICROPY_COMP_FSTRING_LITERAL bool is_fstring = false; +#else + const bool is_fstring = false; +#endif mp_token_kind_t kind = MP_TOKEN_STRING; int n_char = 0; if (is_char(lex, 'u')) { @@ -608,6 +633,7 @@ void mp_lexer_to_next(mp_lexer_t *lex) { kind = MP_TOKEN_BYTES; n_char = 2; } +#if MICROPY_COMP_FSTRING_LITERAL if (is_char_following(lex, 'f')) { lex->tok_kind = MP_TOKEN_FSTRING_RAW; break; @@ -619,8 +645,10 @@ void mp_lexer_to_next(mp_lexer_t *lex) { } n_char = 1; is_fstring = true; +#endif } +#if MICROPY_COMP_FSTRING_LITERAL if (is_fstring) { saw_fstring = true; } else { @@ -631,6 +659,7 @@ void mp_lexer_to_next(mp_lexer_t *lex) { // Can't concatenate f-string with normal string break; } +#endif // Set or check token kind if (lex->tok_kind == MP_TOKEN_END) { @@ -808,7 +837,9 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) { lex->num_indent_level = 1; lex->indent_level = m_new(uint16_t, lex->alloc_indent_level); vstr_init(&lex->vstr, 32); +#if MICROPY_COMP_FSTRING_LITERAL vstr_init(&lex->vstr_postfix, 0); +#endif // store sentinel for first indentation level lex->indent_level[0] = 0; diff --git a/py/lexer.h b/py/lexer.h index 7fe271e84..a3eaa2a7e 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -44,12 +44,14 @@ typedef enum _mp_token_kind_t { MP_TOKEN_INVALID, MP_TOKEN_DEDENT_MISMATCH, MP_TOKEN_LONELY_STRING_OPEN, +#if MICROPY_COMP_FSTRING_LITERAL MP_TOKEN_FSTRING_BACKSLASH, MP_TOKEN_FSTRING_COMMENT, MP_TOKEN_FSTRING_UNCLOSED, MP_TOKEN_FSTRING_UNOPENED, MP_TOKEN_FSTRING_EMPTY_EXP, MP_TOKEN_FSTRING_RAW, +#endif MP_TOKEN_NEWLINE, MP_TOKEN_INDENT, @@ -156,7 +158,9 @@ typedef struct _mp_lexer_t { mp_reader_t reader; // stream source unichar chr0, chr1, chr2; // current cached characters from source +#if MICROPY_COMP_FSTRING_LITERAL unichar chr3, chr4, chr5; // current cached characters from alt source +#endif size_t line; // current source line size_t column; // current source column @@ -172,9 +176,11 @@ typedef struct _mp_lexer_t { size_t tok_column; // token source column mp_token_kind_t tok_kind; // token kind vstr_t vstr; // token data +#if MICROPY_COMP_FSTRING_LITERAL vstr_t vstr_postfix; // postfix to apply to string bool vstr_postfix_processing; uint16_t vstr_postfix_idx; +#endif } mp_lexer_t; mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader); diff --git a/py/mpconfig.h b/py/mpconfig.h index 8f78ade5c..01572f546 100755 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -377,6 +377,11 @@ #define MICROPY_COMP_RETURN_IF_EXPR (0) #endif +// Whether to include parsing of f-string literals +#ifndef MICROPY_COMP_FSTRING_LITERAL +#define MICROPY_COMP_FSTRING_LITERAL (1) +#endif + /*****************************************************************************/ /* Internal debugging stuff */ diff --git a/py/parse.c b/py/parse.c index f86ee25fe..b8cfda2cb 100644 --- a/py/parse.c +++ b/py/parse.c @@ -1178,6 +1178,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { exc = mp_obj_new_exception_msg(&mp_type_IndentationError, translate("unindent does not match any outer indentation level")); break; +#if MICROPY_COMP_FSTRING_LITERAL #if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_DETAILED case MP_TOKEN_FSTRING_BACKSLASH: exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, @@ -1213,6 +1214,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, translate("malformed f-string")); break; +#endif #endif default: exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, -- cgit v1.2.3