From bc12eca461a317df842ce2e616afa97670cd0ce3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 1 Mar 2018 15:47:17 +1100 Subject: py/formatfloat: Fix rounding of %f format with edge-case FP values. Prior to this patch the %f formatting of some FP values could be off by up to 1, eg '%.0f' % 123 would return "122" (unix x64). Depending on the FP precision (single vs double) certain numbers would format correctly, but others wolud not. This patch should fix all cases of rounding for %f. --- tests/float/float_format.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/float/float_format.py (limited to 'tests/float/float_format.py') diff --git a/tests/float/float_format.py b/tests/float/float_format.py new file mode 100644 index 000000000..4d5ad1d69 --- /dev/null +++ b/tests/float/float_format.py @@ -0,0 +1,11 @@ +# test float formatting + +# general rounding +for val in (116, 1111, 1234, 5010, 11111): + print('%.0f' % val) + print('%.1f' % val) + print('%.3f' % val) + +# make sure rounding is done at the correct precision +for prec in range(8): + print(('%%.%df' % prec) % 6e-5) -- cgit v1.2.3 From 7b050fa76c6a763043739d40c82dde839d7f8fd9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 1 Mar 2018 16:02:59 +1100 Subject: py/formatfloat: Fix case where floats could render with a ":" character. Prior to this patch, some architectures (eg unix x86) could render floats with a ":" character in them, eg 1e+39 would come out as ":e+38" (":" is just after "9" in ASCII so this is like 10e+38). This patch fixes some of these cases. --- py/formatfloat.c | 2 +- tests/float/float_format.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'tests/float/float_format.py') diff --git a/py/formatfloat.c b/py/formatfloat.c index 22dd8aaac..60dcee6f5 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -258,7 +258,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch } // It can be that f was right on the edge of an entry in pos_pow needs to be reduced - if (f >= FPCONST(10.0)) { + if ((int)f >= 10) { e += 1; f *= FPCONST(0.1); } diff --git a/tests/float/float_format.py b/tests/float/float_format.py index 4d5ad1d69..cda395ce0 100644 --- a/tests/float/float_format.py +++ b/tests/float/float_format.py @@ -9,3 +9,7 @@ for val in (116, 1111, 1234, 5010, 11111): # make sure rounding is done at the correct precision for prec in range(8): print(('%%.%df' % prec) % 6e-5) + +# check certain cases that had a digit value of 10 render as a ":" character +print('%.2e' % float('9' * 51 + 'e-39')) +print('%.2e' % float('9' * 40 + 'e-21')) -- cgit v1.2.3 From 955ee6477f4b1d3a70bfe97a1e7727848bf2d06d Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 1 Mar 2018 17:00:02 +1100 Subject: py/formatfloat: Fix case where floats could render with negative digits. Prior to this patch, some architectures (eg unix x86) could render floats with "negative" digits, like ")". For example, '%.23e' % 1e-80 would come out as "1.0000000000000000/)/(,*0e-80". This patch fixes the known cases. --- py/formatfloat.c | 6 +++++- tests/float/float_format.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'tests/float/float_format.py') diff --git a/py/formatfloat.c b/py/formatfloat.c index 60dcee6f5..dc7fc1d1f 100644 --- a/py/formatfloat.c +++ b/py/formatfloat.c @@ -330,7 +330,11 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch // Print the digits of the mantissa for (int i = 0; i < num_digits; ++i, --dec) { int32_t d = (int32_t)f; - *s++ = '0' + d; + if (d < 0) { + *s++ = '0'; + } else { + *s++ = '0' + d; + } if (dec == 0 && prec > 0) { *s++ = '.'; } diff --git a/tests/float/float_format.py b/tests/float/float_format.py index cda395ce0..d43535cf2 100644 --- a/tests/float/float_format.py +++ b/tests/float/float_format.py @@ -13,3 +13,7 @@ for prec in range(8): # check certain cases that had a digit value of 10 render as a ":" character print('%.2e' % float('9' * 51 + 'e-39')) print('%.2e' % float('9' * 40 + 'e-21')) + +# check a case that would render negative digit values, eg ")" characters +# the string is converted back to a float to check for no illegal characters +float('%.23e' % 1e-80) -- cgit v1.2.3