From 84895f1a210d0037a86887f0f647570bdf40afa2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 27 Nov 2017 12:51:52 +1100 Subject: py/parsenum: Improve parsing of floating point numbers. This patch improves parsing of floating point numbers by converting all the digits (integer and fractional) together into a number 1 or greater, and then applying the correct power of 10 at the very end. In particular the multiple "multiply by 0.1" operations to build a fraction are now combined together and applied at the same time as the exponent, at the very end. This helps to retain precision during parsing of floats, and also includes a check that the number doesn't overflow during the parsing. One benefit is that a float will have the same value no matter where the decimal point is located, eg 1.23 == 123e-2. --- tests/float/float_parse.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/float/float_parse.py (limited to 'tests/float/float_parse.py') diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py new file mode 100644 index 000000000..448eff3bc --- /dev/null +++ b/tests/float/float_parse.py @@ -0,0 +1,22 @@ +# test parsing of floats + +inf = float('inf') + +# it shouldn't matter where the decimal point is if the exponent balances the value +print(float('1234') - float('0.1234e4')) +print(float('1.015625') - float('1015625e-6')) + +# very large integer part with a very negative exponent should cancel out +print(float('9' * 60 + 'e-60')) +print(float('9' * 60 + 'e-40')) +print(float('9' * 60 + 'e-20') == float('1e40')) + +# many fractional digits +print(float('.' + '9' * 70)) +print(float('.' + '9' * 70 + 'e20')) +print(float('.' + '9' * 70 + 'e-50') == float('1e-50')) + +# tiny fraction with large exponent +print(float('.' + '0' * 60 + '1e10') == float('1e-51')) +print(float('.' + '0' * 60 + '9e25')) +print(float('.' + '0' * 60 + '9e40')) -- cgit v1.2.3 From b75cb8392bdf5f8c12072eac3adbdaad53d1e8d2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 8 Feb 2018 14:02:50 +1100 Subject: py/parsenum: Fix parsing of floats that are close to subnormal. Prior to this patch, a float literal that was close to subnormal would have a loss of precision when parsed. The worst case was something like float('10000000000000000000e-326') which returned 0.0. --- py/parsenum.c | 14 ++++++++++++-- tests/float/float_parse.py | 5 +++++ tests/float/float_parse_doubleprec.py | 5 +++++ 3 files changed, 22 insertions(+), 2 deletions(-) (limited to 'tests/float/float_parse.py') diff --git a/py/parsenum.c b/py/parsenum.c index 98e773685..124489c66 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -172,10 +172,15 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool #if MICROPY_PY_BUILTINS_FLOAT // DEC_VAL_MAX only needs to be rough and is used to retain precision while not overflowing +// SMALL_NORMAL_VAL is the smallest power of 10 that is still a normal float #if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT #define DEC_VAL_MAX 1e20F +#define SMALL_NORMAL_VAL (1e-37F) +#define SMALL_NORMAL_EXP (-37) #elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE #define DEC_VAL_MAX 1e200 +#define SMALL_NORMAL_VAL (1e-307) +#define SMALL_NORMAL_EXP (-307) #endif const char *top = str + len; @@ -275,8 +280,13 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool exp_val = -exp_val; } - // apply the exponent - dec_val *= MICROPY_FLOAT_C_FUN(pow)(10, exp_val + exp_extra); + // apply the exponent, making sure it's not a subnormal value + exp_val += exp_extra; + if (exp_val < SMALL_NORMAL_EXP) { + exp_val -= SMALL_NORMAL_EXP; + dec_val *= SMALL_NORMAL_VAL; + } + dec_val *= MICROPY_FLOAT_C_FUN(pow)(10, exp_val); } // negate value if needed diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py index 448eff3bc..de4ea455f 100644 --- a/tests/float/float_parse.py +++ b/tests/float/float_parse.py @@ -20,3 +20,8 @@ print(float('.' + '9' * 70 + 'e-50') == float('1e-50')) print(float('.' + '0' * 60 + '1e10') == float('1e-51')) print(float('.' + '0' * 60 + '9e25')) print(float('.' + '0' * 60 + '9e40')) + +# ensure that accuracy is retained when value is close to a subnormal +print(float('1.00000000000000000000e-37')) +print(float('10.0000000000000000000e-38')) +print(float('100.000000000000000000e-39')) diff --git a/tests/float/float_parse_doubleprec.py b/tests/float/float_parse_doubleprec.py index 356601130..2ea7842f3 100644 --- a/tests/float/float_parse_doubleprec.py +++ b/tests/float/float_parse_doubleprec.py @@ -14,3 +14,8 @@ print(float('.' + '9' * 400 + 'e-100')) print(float('.' + '0' * 400 + '9e100')) print(float('.' + '0' * 400 + '9e200')) print(float('.' + '0' * 400 + '9e400')) + +# ensure that accuracy is retained when value is close to a subnormal +print(float('1.00000000000000000000e-307')) +print(float('10.0000000000000000000e-308')) +print(float('100.000000000000000000e-309')) -- cgit v1.2.3 From 6dad0885692f8d1e743873fb4be241f1fd1cb91a Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 26 Feb 2018 15:54:03 +1100 Subject: tests/float: Adjust float-parsing tests to pass with only a small error. Float parsing (both single and double precision) may have a relative error of order the floating point precision, so adjust tests to take this into account by not printing all of the digits of the answer. --- tests/float/float_parse.py | 5 ++--- tests/float/float_parse_doubleprec.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'tests/float/float_parse.py') diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py index de4ea455f..5eb16e79c 100644 --- a/tests/float/float_parse.py +++ b/tests/float/float_parse.py @@ -7,9 +7,8 @@ print(float('1234') - float('0.1234e4')) print(float('1.015625') - float('1015625e-6')) # very large integer part with a very negative exponent should cancel out -print(float('9' * 60 + 'e-60')) -print(float('9' * 60 + 'e-40')) -print(float('9' * 60 + 'e-20') == float('1e40')) +print('%.4e' % float('9' * 60 + 'e-60')) +print('%.4e' % float('9' * 60 + 'e-40')) # many fractional digits print(float('.' + '9' * 70)) diff --git a/tests/float/float_parse_doubleprec.py b/tests/float/float_parse_doubleprec.py index 2ea7842f3..dcc0dd592 100644 --- a/tests/float/float_parse_doubleprec.py +++ b/tests/float/float_parse_doubleprec.py @@ -11,9 +11,9 @@ print(float('.' + '9' * 400 + 'e100')) print(float('.' + '9' * 400 + 'e-100')) # tiny fraction with large exponent -print(float('.' + '0' * 400 + '9e100')) -print(float('.' + '0' * 400 + '9e200')) -print(float('.' + '0' * 400 + '9e400')) +print('%.14e' % float('.' + '0' * 400 + '9e100')) +print('%.14e' % float('.' + '0' * 400 + '9e200')) +print('%.14e' % float('.' + '0' * 400 + '9e400')) # ensure that accuracy is retained when value is close to a subnormal print(float('1.00000000000000000000e-307')) -- cgit v1.2.3 From d2c1db1e5cca03c9a34614af6d12045aefe3d719 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 11 May 2018 13:51:18 +1000 Subject: tests/float/float_parse: Allow test to run on 32-bit archs. Printing of uPy floats can differ by the floating-point precision on different architectures (eg 64-bit vs 32-bit x86), so it's not possible to using printing of floats in some parts of this test. Instead we can just check for equivalence with what is known to be the correct answer. --- tests/float/float_parse.py | 4 ++-- tests/run-tests | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'tests/float/float_parse.py') diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py index 5eb16e79c..ae6b114f0 100644 --- a/tests/float/float_parse.py +++ b/tests/float/float_parse.py @@ -17,8 +17,8 @@ print(float('.' + '9' * 70 + 'e-50') == float('1e-50')) # tiny fraction with large exponent print(float('.' + '0' * 60 + '1e10') == float('1e-51')) -print(float('.' + '0' * 60 + '9e25')) -print(float('.' + '0' * 60 + '9e40')) +print(float('.' + '0' * 60 + '9e25') == float('9e-36')) +print(float('.' + '0' * 60 + '9e40') == float('9e-21')) # ensure that accuracy is retained when value is close to a subnormal print(float('1.00000000000000000000e-37')) diff --git a/tests/run-tests b/tests/run-tests index 82f05bfa8..afefa264f 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -324,7 +324,6 @@ def run_tests(pyb, tests, args, base_path="."): skip_tests.add('basics/subclass_native_init.py')# native subclassing corner cases not support skip_tests.add('misc/rge_sm.py') # too large skip_tests.add('micropython/opt_level.py') # don't assume line numbers are stored - skip_tests.add('float/float_parse.py') # minor parsing artifacts with 32-bit floats # Some tests are known to fail on 64-bit machines if pyb is None and platform.architecture()[0] == '64bit': -- cgit v1.2.3 From 1ad0013decb42418ca667c2d9994198b537fa778 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 21 May 2018 13:05:40 +1000 Subject: tests: Add some tests for bigint hash, float hash and float parsing. Following outcome of recent fuzz testing and sanitizing by @jepler. --- tests/basics/builtin_hash_intbig.py | 3 +++ tests/float/builtin_float_hash.py | 2 ++ tests/float/float_parse.py | 6 ++++++ 3 files changed, 11 insertions(+) (limited to 'tests/float/float_parse.py') diff --git a/tests/basics/builtin_hash_intbig.py b/tests/basics/builtin_hash_intbig.py index 0092c0f3a..df51f72ab 100644 --- a/tests/basics/builtin_hash_intbig.py +++ b/tests/basics/builtin_hash_intbig.py @@ -8,3 +8,6 @@ class F: def __hash__(self): return 1 << 70 | 1 print(hash(F()) != 0) + +# this had a particular error with internal integer arithmetic of hash function +print(hash(6699999999999999999999999999999999999999999999999999999999999999999999) != 0) diff --git a/tests/float/builtin_float_hash.py b/tests/float/builtin_float_hash.py index dd184595f..7a7e37401 100644 --- a/tests/float/builtin_float_hash.py +++ b/tests/float/builtin_float_hash.py @@ -3,6 +3,7 @@ # these should hash to an integer with a specific value for val in ( '0.0', + '-0.0', '1.0', '2.0', '-12.0', @@ -15,6 +16,7 @@ for val in ( '0.1', '-0.1', '10.3', + '0.4e3', '1e16', 'inf', '-inf', diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py index ae6b114f0..4b026de1c 100644 --- a/tests/float/float_parse.py +++ b/tests/float/float_parse.py @@ -24,3 +24,9 @@ print(float('.' + '0' * 60 + '9e40') == float('9e-21')) print(float('1.00000000000000000000e-37')) print(float('10.0000000000000000000e-38')) print(float('100.000000000000000000e-39')) + +# very large exponent literal +print(float('1e4294967301')) +print(float('1e-4294967301')) +print(float('1e18446744073709551621')) +print(float('1e-18446744073709551621')) -- cgit v1.2.3