From 1d410bb68b3630984804f12ebc4d8194410eb87f Mon Sep 17 00:00:00 2001 From: George Waters Date: Wed, 22 Jul 2020 14:06:04 -0400 Subject: Fix repl support for unicode Currently when a utf8 character that is bigger than 1 byte is typed in the repl, it isn't handled how it should be. If you try to move the cursor in any direction the text gets messed up. This fixes that. --- lib/mp-readline/readline.c | 89 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mp-readline/readline.c b/lib/mp-readline/readline.c index 432413d50..95a810e83 100644 --- a/lib/mp-readline/readline.c +++ b/lib/mp-readline/readline.c @@ -100,6 +100,7 @@ STATIC readline_t rl; int readline_process_char(int c) { size_t last_line_len = rl.line->len; + int cont_chars = 0; int redraw_step_back = 0; bool redraw_from_cursor = false; int redraw_step_forward = 0; @@ -178,6 +179,13 @@ int readline_process_char(int c) { int nspace = 1; #endif + // Check if we have moved into a UTF-8 continuation byte + while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos-nspace]) && + rl.cursor_pos-nspace > rl.orig_line_len) { + nspace++; + cont_chars++; + } + // do the backspace vstr_cut_out_bytes(rl.line, rl.cursor_pos - nspace, nspace); // set redraw parameters @@ -206,12 +214,37 @@ int readline_process_char(int c) { redraw_step_forward = compl_len; } #endif - } else if (32 <= c ) { + } else if (32 <= c && c <= 126) { // printable character vstr_ins_char(rl.line, rl.cursor_pos, c); // set redraw parameters redraw_from_cursor = true; redraw_step_forward = 1; + }else if (c >= 128) { + // utf-8 character + if (c >= 0xc0 && c < 0xf8) { + // First Code Point + vstr_ins_char(rl.line, rl.cursor_pos, c); + }else if (UTF8_IS_CONT(c)) { + char fcp = rl.line->buf[rl.cursor_pos]; + if (fcp >= 0xc0 && fcp < 0xf8) { + int need = (0xe5 >> ((fcp >> 3) & 0x6)) & 3; // From unicode.c L195 + cont_chars++; + while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos+cont_chars]) && + rl.cursor_pos+cont_chars < rl.line->len && cont_chars < need) { + cont_chars++; + } + vstr_ins_char(rl.line, rl.cursor_pos+cont_chars, c); + if (cont_chars == need) { + redraw_from_cursor = true; + redraw_step_forward = cont_chars+1; + } + }else{ + //ignore, for now (invalid first code point) + } + }else { + // ignore, invalid + } } } else if (rl.escape_seq == ESEQ_ESC) { switch (c) { @@ -237,6 +270,12 @@ up_arrow_key: #endif // up arrow if (rl.hist_cur + 1 < (int)READLINE_HIST_SIZE && MP_STATE_PORT(readline_hist)[rl.hist_cur + 1] != NULL) { + for (char *ch = rl.line->buf+rl.cursor_pos-1; ch > rl.line->buf+rl.orig_line_len; ch--) { + // printf("char: %d\n", ch); + if (UTF8_IS_CONT(*ch)) { + cont_chars++; + } + } // increase hist num rl.hist_cur += 1; // set line to history @@ -253,6 +292,12 @@ down_arrow_key: #endif // down arrow if (rl.hist_cur >= 0) { + for (char *ch = rl.line->buf+rl.cursor_pos-1; ch > rl.line->buf+rl.orig_line_len; ch--) { + // printf("char: %d\n", ch); + if (UTF8_IS_CONT(*ch)) { + cont_chars++; + } + } // decrease hist num rl.hist_cur -= 1; // set line to history @@ -272,6 +317,12 @@ right_arrow_key: // right arrow if (rl.cursor_pos < rl.line->len) { redraw_step_forward = 1; + // Check if we have moved into a UTF-8 continuation byte + while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos+redraw_step_forward]) && + rl.cursor_pos+redraw_step_forward < rl.line->len) { + redraw_step_forward++; + cont_chars++; + } } } else if (c == 'D') { #if MICROPY_REPL_EMACS_KEYS @@ -280,6 +331,12 @@ left_arrow_key: // left arrow if (rl.cursor_pos > rl.orig_line_len) { redraw_step_back = 1; + // Check if we have moved into a UTF-8 continuation byte + while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos-redraw_step_back]) && + rl.cursor_pos-redraw_step_back > rl.orig_line_len) { + redraw_step_back++; + cont_chars++; + } } } else if (c == 'H') { // home @@ -295,9 +352,21 @@ left_arrow_key: if (c == '~') { if (rl.escape_seq_buf[0] == '1' || rl.escape_seq_buf[0] == '7') { home_key: + for (char *ch = rl.line->buf+rl.cursor_pos-1; ch > rl.line->buf+rl.orig_line_len; ch--) { + // printf("char: %d\n", ch); + if (UTF8_IS_CONT(*ch)) { + cont_chars++; + } + } redraw_step_back = rl.cursor_pos - rl.orig_line_len; } else if (rl.escape_seq_buf[0] == '4' || rl.escape_seq_buf[0] == '8') { end_key: + for (char *ch = rl.line->buf+rl.cursor_pos-1; ch > rl.line->buf+rl.orig_line_len; ch--) { + // printf("char: %d\n", ch); + if (UTF8_IS_CONT(*ch)) { + cont_chars++; + } + } redraw_step_forward = rl.line->len - rl.cursor_pos; } else if (rl.escape_seq_buf[0] == '3') { // delete @@ -331,7 +400,7 @@ delete_key: // redraw command prompt, efficiently if (redraw_step_back > 0) { - mp_hal_move_cursor_back(redraw_step_back); + mp_hal_move_cursor_back(redraw_step_back-cont_chars); rl.cursor_pos -= redraw_step_back; } if (redraw_from_cursor) { @@ -339,10 +408,24 @@ delete_key: // erase old chars mp_hal_erase_line_from_cursor(last_line_len - rl.cursor_pos); } + // Check if we have moved into a UTF-8 continuation byte + // while (rl.cursor_pos+redraw_step_forward < rl.line->len && + // UTF8_IS_CONT(rl.line->buf[rl.cursor_pos]) && rl.cursor_pos > 0) { + // rl.cursor_pos--; + // redraw_step_forward++; + // } + + cont_chars = 0; + for (char *ch = rl.line->buf+rl.cursor_pos+redraw_step_forward; ch < rl.line->buf+rl.line->len; ch++) { + // printf("char: %d\n", ch); + if (UTF8_IS_CONT(*ch)) { + cont_chars++; + } + } // draw new chars mp_hal_stdout_tx_strn(rl.line->buf + rl.cursor_pos, rl.line->len - rl.cursor_pos); // move cursor forward if needed (already moved forward by length of line, so move it back) - mp_hal_move_cursor_back(rl.line->len - (rl.cursor_pos + redraw_step_forward)); + mp_hal_move_cursor_back(rl.line->len - (rl.cursor_pos + redraw_step_forward) - cont_chars); rl.cursor_pos += redraw_step_forward; } else if (redraw_step_forward > 0) { // draw over old chars to move cursor forwards -- cgit v1.2.3 From 71ce480dbbf8999b964214a5465640ffa038a0c2 Mon Sep 17 00:00:00 2001 From: George Waters Date: Sun, 2 Aug 2020 21:26:09 -0400 Subject: Refactor utf-8 code, reduce impact on code size --- lib/mp-readline/readline.c | 88 +++++++++++++++------------------------------- 1 file changed, 29 insertions(+), 59 deletions(-) (limited to 'lib') diff --git a/lib/mp-readline/readline.c b/lib/mp-readline/readline.c index 95a810e83..042694162 100644 --- a/lib/mp-readline/readline.c +++ b/lib/mp-readline/readline.c @@ -92,12 +92,21 @@ typedef struct _readline_t { int escape_seq; int hist_cur; size_t cursor_pos; + uint8_t utf8_cont_chars; char escape_seq_buf[1]; const char *prompt; } readline_t; STATIC readline_t rl; +int readline_count_cont_byte(char *start, char *end) { + int count = 0; + for (char *pos = start; pos < end; pos++) { + count += UTF8_IS_CONT(*pos); + } + return count; +} + int readline_process_char(int c) { size_t last_line_len = rl.line->len; int cont_chars = 0; @@ -180,8 +189,7 @@ int readline_process_char(int c) { #endif // Check if we have moved into a UTF-8 continuation byte - while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos-nspace]) && - rl.cursor_pos-nspace > rl.orig_line_len) { + while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos-nspace])) { nspace++; cont_chars++; } @@ -223,27 +231,23 @@ int readline_process_char(int c) { }else if (c >= 128) { // utf-8 character if (c >= 0xc0 && c < 0xf8) { - // First Code Point + // Lead code point vstr_ins_char(rl.line, rl.cursor_pos, c); + rl.utf8_cont_chars = 0; }else if (UTF8_IS_CONT(c)) { - char fcp = rl.line->buf[rl.cursor_pos]; - if (fcp >= 0xc0 && fcp < 0xf8) { - int need = (0xe5 >> ((fcp >> 3) & 0x6)) & 3; // From unicode.c L195 - cont_chars++; - while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos+cont_chars]) && - rl.cursor_pos+cont_chars < rl.line->len && cont_chars < need) { - cont_chars++; - } - vstr_ins_char(rl.line, rl.cursor_pos+cont_chars, c); - if (cont_chars == need) { + char lcp = rl.line->buf[rl.cursor_pos]; + // Check for valid lead code point + if (lcp >= 0xc0 && lcp < 0xf8) { + rl.utf8_cont_chars += 1; + vstr_ins_char(rl.line, rl.cursor_pos+rl.utf8_cont_chars, c); + // set redraw parameters if we have the entire character + uint8_t need = (0xe5 >> ((lcp >> 3) & 0x6)) & 3; // From unicode.c L195 + if (rl.utf8_cont_chars == need) { redraw_from_cursor = true; - redraw_step_forward = cont_chars+1; + redraw_step_forward = rl.utf8_cont_chars+1; + cont_chars = rl.utf8_cont_chars; } - }else{ - //ignore, for now (invalid first code point) } - }else { - // ignore, invalid } } } else if (rl.escape_seq == ESEQ_ESC) { @@ -270,12 +274,8 @@ up_arrow_key: #endif // up arrow if (rl.hist_cur + 1 < (int)READLINE_HIST_SIZE && MP_STATE_PORT(readline_hist)[rl.hist_cur + 1] != NULL) { - for (char *ch = rl.line->buf+rl.cursor_pos-1; ch > rl.line->buf+rl.orig_line_len; ch--) { - // printf("char: %d\n", ch); - if (UTF8_IS_CONT(*ch)) { - cont_chars++; - } - } + // Check for continuation characters through the cursor_pos + cont_chars = readline_count_cont_byte(rl.line->buf+rl.orig_line_len, rl.line->buf+rl.cursor_pos); // increase hist num rl.hist_cur += 1; // set line to history @@ -292,12 +292,8 @@ down_arrow_key: #endif // down arrow if (rl.hist_cur >= 0) { - for (char *ch = rl.line->buf+rl.cursor_pos-1; ch > rl.line->buf+rl.orig_line_len; ch--) { - // printf("char: %d\n", ch); - if (UTF8_IS_CONT(*ch)) { - cont_chars++; - } - } + // Check for continuation characters through the cursor_pos + cont_chars = readline_count_cont_byte(rl.line->buf+rl.orig_line_len, rl.line->buf+rl.cursor_pos); // decrease hist num rl.hist_cur -= 1; // set line to history @@ -321,7 +317,6 @@ right_arrow_key: while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos+redraw_step_forward]) && rl.cursor_pos+redraw_step_forward < rl.line->len) { redraw_step_forward++; - cont_chars++; } } } else if (c == 'D') { @@ -332,8 +327,7 @@ left_arrow_key: if (rl.cursor_pos > rl.orig_line_len) { redraw_step_back = 1; // Check if we have moved into a UTF-8 continuation byte - while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos-redraw_step_back]) && - rl.cursor_pos-redraw_step_back > rl.orig_line_len) { + while (UTF8_IS_CONT(rl.line->buf[rl.cursor_pos-redraw_step_back])) { redraw_step_back++; cont_chars++; } @@ -352,21 +346,9 @@ left_arrow_key: if (c == '~') { if (rl.escape_seq_buf[0] == '1' || rl.escape_seq_buf[0] == '7') { home_key: - for (char *ch = rl.line->buf+rl.cursor_pos-1; ch > rl.line->buf+rl.orig_line_len; ch--) { - // printf("char: %d\n", ch); - if (UTF8_IS_CONT(*ch)) { - cont_chars++; - } - } redraw_step_back = rl.cursor_pos - rl.orig_line_len; } else if (rl.escape_seq_buf[0] == '4' || rl.escape_seq_buf[0] == '8') { end_key: - for (char *ch = rl.line->buf+rl.cursor_pos-1; ch > rl.line->buf+rl.orig_line_len; ch--) { - // printf("char: %d\n", ch); - if (UTF8_IS_CONT(*ch)) { - cont_chars++; - } - } redraw_step_forward = rl.line->len - rl.cursor_pos; } else if (rl.escape_seq_buf[0] == '3') { // delete @@ -408,20 +390,8 @@ delete_key: // erase old chars mp_hal_erase_line_from_cursor(last_line_len - rl.cursor_pos); } - // Check if we have moved into a UTF-8 continuation byte - // while (rl.cursor_pos+redraw_step_forward < rl.line->len && - // UTF8_IS_CONT(rl.line->buf[rl.cursor_pos]) && rl.cursor_pos > 0) { - // rl.cursor_pos--; - // redraw_step_forward++; - // } - - cont_chars = 0; - for (char *ch = rl.line->buf+rl.cursor_pos+redraw_step_forward; ch < rl.line->buf+rl.line->len; ch++) { - // printf("char: %d\n", ch); - if (UTF8_IS_CONT(*ch)) { - cont_chars++; - } - } + // Check for continuation characters from the new cursor_pos to the EOL + cont_chars = readline_count_cont_byte(rl.line->buf+rl.cursor_pos+redraw_step_forward, rl.line->buf+rl.line->len); // draw new chars mp_hal_stdout_tx_strn(rl.line->buf + rl.cursor_pos, rl.line->len - rl.cursor_pos); // move cursor forward if needed (already moved forward by length of line, so move it back) -- cgit v1.2.3 From 41ccbbd4e92437a5d4153759e244067a869da277 Mon Sep 17 00:00:00 2001 From: George Waters Date: Sun, 2 Aug 2020 21:41:23 -0400 Subject: Fix failed unix build --- lib/mp-readline/readline.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/mp-readline/readline.c b/lib/mp-readline/readline.c index 042694162..070fb8676 100644 --- a/lib/mp-readline/readline.c +++ b/lib/mp-readline/readline.c @@ -58,6 +58,14 @@ STATIC char *str_dup_maybe(const char *str) { return s2; } +STATIC int count_cont_bytes(char *start, char *end) { + int count = 0; + for (char *pos = start; pos < end; pos++) { + count += UTF8_IS_CONT(*pos); + } + return count; +} + // By default assume terminal which implements VT100 commands... #ifndef MICROPY_HAL_HAS_VT100 #define MICROPY_HAL_HAS_VT100 (1) @@ -99,14 +107,6 @@ typedef struct _readline_t { STATIC readline_t rl; -int readline_count_cont_byte(char *start, char *end) { - int count = 0; - for (char *pos = start; pos < end; pos++) { - count += UTF8_IS_CONT(*pos); - } - return count; -} - int readline_process_char(int c) { size_t last_line_len = rl.line->len; int cont_chars = 0; @@ -275,7 +275,7 @@ up_arrow_key: // up arrow if (rl.hist_cur + 1 < (int)READLINE_HIST_SIZE && MP_STATE_PORT(readline_hist)[rl.hist_cur + 1] != NULL) { // Check for continuation characters through the cursor_pos - cont_chars = readline_count_cont_byte(rl.line->buf+rl.orig_line_len, rl.line->buf+rl.cursor_pos); + cont_chars = count_cont_bytes(rl.line->buf+rl.orig_line_len, rl.line->buf+rl.cursor_pos); // increase hist num rl.hist_cur += 1; // set line to history @@ -293,7 +293,7 @@ down_arrow_key: // down arrow if (rl.hist_cur >= 0) { // Check for continuation characters through the cursor_pos - cont_chars = readline_count_cont_byte(rl.line->buf+rl.orig_line_len, rl.line->buf+rl.cursor_pos); + cont_chars = count_cont_bytes(rl.line->buf+rl.orig_line_len, rl.line->buf+rl.cursor_pos); // decrease hist num rl.hist_cur -= 1; // set line to history @@ -391,7 +391,7 @@ delete_key: mp_hal_erase_line_from_cursor(last_line_len - rl.cursor_pos); } // Check for continuation characters from the new cursor_pos to the EOL - cont_chars = readline_count_cont_byte(rl.line->buf+rl.cursor_pos+redraw_step_forward, rl.line->buf+rl.line->len); + cont_chars = count_cont_bytes(rl.line->buf+rl.cursor_pos+redraw_step_forward, rl.line->buf+rl.line->len); // draw new chars mp_hal_stdout_tx_strn(rl.line->buf + rl.cursor_pos, rl.line->len - rl.cursor_pos); // move cursor forward if needed (already moved forward by length of line, so move it back) -- cgit v1.2.3 From 93829e57b22831fecbcceb2970753bc4668d5e70 Mon Sep 17 00:00:00 2001 From: George Waters Date: Mon, 3 Aug 2020 15:09:54 -0400 Subject: Try too make new utf-8 code smaller again --- lib/mp-readline/readline.c | 48 ++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/mp-readline/readline.c b/lib/mp-readline/readline.c index 070fb8676..78295bd09 100644 --- a/lib/mp-readline/readline.c +++ b/lib/mp-readline/readline.c @@ -222,32 +222,26 @@ int readline_process_char(int c) { redraw_step_forward = compl_len; } #endif - } else if (32 <= c && c <= 126) { + } else if (32 <= c) { // printable character - vstr_ins_char(rl.line, rl.cursor_pos, c); - // set redraw parameters - redraw_from_cursor = true; - redraw_step_forward = 1; - }else if (c >= 128) { - // utf-8 character - if (c >= 0xc0 && c < 0xf8) { - // Lead code point - vstr_ins_char(rl.line, rl.cursor_pos, c); + char lcp = rl.line->buf[rl.cursor_pos]; + uint8_t cont_need = 0; + if (!UTF8_IS_CONT(c)) { + // ASCII or Lead code point rl.utf8_cont_chars = 0; - }else if (UTF8_IS_CONT(c)) { - char lcp = rl.line->buf[rl.cursor_pos]; - // Check for valid lead code point - if (lcp >= 0xc0 && lcp < 0xf8) { - rl.utf8_cont_chars += 1; - vstr_ins_char(rl.line, rl.cursor_pos+rl.utf8_cont_chars, c); - // set redraw parameters if we have the entire character - uint8_t need = (0xe5 >> ((lcp >> 3) & 0x6)) & 3; // From unicode.c L195 - if (rl.utf8_cont_chars == need) { - redraw_from_cursor = true; - redraw_step_forward = rl.utf8_cont_chars+1; - cont_chars = rl.utf8_cont_chars; - } - } + lcp = c; + }else { + rl.utf8_cont_chars += 1; + } + if (lcp >= 0xc0 && lcp < 0xf8) { + cont_need = (0xe5 >> ((lcp >> 3) & 0x6)) & 3; // From unicode.c L195 + } + vstr_ins_char(rl.line, rl.cursor_pos+rl.utf8_cont_chars, c); + // set redraw parameters if we have the entire character + if (rl.utf8_cont_chars == cont_need) { + redraw_from_cursor = true; + redraw_step_forward = rl.utf8_cont_chars+1; + cont_chars = rl.utf8_cont_chars; } } } else if (rl.escape_seq == ESEQ_ESC) { @@ -274,7 +268,7 @@ up_arrow_key: #endif // up arrow if (rl.hist_cur + 1 < (int)READLINE_HIST_SIZE && MP_STATE_PORT(readline_hist)[rl.hist_cur + 1] != NULL) { - // Check for continuation characters through the cursor_pos + // Check for continuation characters cont_chars = count_cont_bytes(rl.line->buf+rl.orig_line_len, rl.line->buf+rl.cursor_pos); // increase hist num rl.hist_cur += 1; @@ -292,7 +286,7 @@ down_arrow_key: #endif // down arrow if (rl.hist_cur >= 0) { - // Check for continuation characters through the cursor_pos + // Check for continuation characters cont_chars = count_cont_bytes(rl.line->buf+rl.orig_line_len, rl.line->buf+rl.cursor_pos); // decrease hist num rl.hist_cur -= 1; @@ -390,7 +384,7 @@ delete_key: // erase old chars mp_hal_erase_line_from_cursor(last_line_len - rl.cursor_pos); } - // Check for continuation characters from the new cursor_pos to the EOL + // Check for continuation characters cont_chars = count_cont_bytes(rl.line->buf+rl.cursor_pos+redraw_step_forward, rl.line->buf+rl.line->len); // draw new chars mp_hal_stdout_tx_strn(rl.line->buf + rl.cursor_pos, rl.line->len - rl.cursor_pos); -- cgit v1.2.3 From 398be76bf6311f1667141051e38e3feced1d75fc Mon Sep 17 00:00:00 2001 From: George Waters Date: Sun, 9 Aug 2020 14:06:49 -0400 Subject: Count utf8 chars, not every byte for line --- lib/mp-readline/readline.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/mp-readline/readline.c b/lib/mp-readline/readline.c index 78295bd09..464916ca4 100644 --- a/lib/mp-readline/readline.c +++ b/lib/mp-readline/readline.c @@ -58,10 +58,12 @@ STATIC char *str_dup_maybe(const char *str) { return s2; } -STATIC int count_cont_bytes(char *start, char *end) { +STATIC size_t count_cont_bytes(char *start, char *end) { int count = 0; for (char *pos = start; pos < end; pos++) { - count += UTF8_IS_CONT(*pos); + if(UTF8_IS_CONT(*pos)) { + count++; + } } return count; } @@ -108,7 +110,7 @@ typedef struct _readline_t { STATIC readline_t rl; int readline_process_char(int c) { - size_t last_line_len = rl.line->len; + size_t last_line_len = utf8_charlen((byte *)rl.line->buf, rl.line->len); int cont_chars = 0; int redraw_step_back = 0; bool redraw_from_cursor = false; @@ -380,7 +382,7 @@ delete_key: rl.cursor_pos -= redraw_step_back; } if (redraw_from_cursor) { - if (rl.line->len < last_line_len) { + if (utf8_charlen((byte *)rl.line->buf, rl.line->len) < last_line_len) { // erase old chars mp_hal_erase_line_from_cursor(last_line_len - rl.cursor_pos); } -- cgit v1.2.3