diff options
| -rw-r--r-- | .travis.yml | 12 | ||||
| -rw-r--r-- | ports/unix/Makefile | 10 | ||||
| -rw-r--r-- | ports/windows/.appveyor.yml | 2 | ||||
| -rwxr-xr-x | tests/run-tests | 67 |
4 files changed, 59 insertions, 32 deletions
diff --git a/.travis.yml b/.travis.yml index efb4ec35f..bd906cff3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,24 +78,24 @@ script: - echo -en 'travis_fold:end:qemu\\r' # run tests without coverage info - #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests) - #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --emit native) + #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --auto-jobs) + #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --auto-jobs --emit native) # run tests with coverage info - echo 'Test all' && echo -en 'travis_fold:start:test_all\\r' - - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests)) + - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs)) - echo -en 'travis_fold:end:test_all\\r' - echo 'Test threads' && echo -en 'travis_fold:start:test_threads\\r' - - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests -d thread)) + - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs -d thread)) - echo -en 'travis_fold:end:test_threads\\r' - echo 'Testing with native' && echo -en 'travis_fold:start:test_native\\r' - - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --emit native)) + - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs --emit native)) - echo -en 'travis_fold:end:test_native\\r' - (echo 'Testing with mpy' && echo -en 'travis_fold:start:test_mpy\\r') - - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --via-mpy -d basics float)) + - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests -j1 --via-mpy -d basics float)) - echo -en 'travis_fold:end:test_mpy\\r' - (echo 'Building docs' && echo -en 'travis_fold:start:build_docs\\r') diff --git a/ports/unix/Makefile b/ports/unix/Makefile index 5bdcf4e1a..dacf68019 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -189,7 +189,7 @@ include $(TOP)/py/mkrules.mk test: $(PROG) $(TOP)/tests/run-tests $(eval DIRNAME=ports/$(notdir $(CURDIR))) - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --auto-jobs # install micropython in /usr/local/bin TARGET = micropython @@ -254,10 +254,10 @@ coverage: coverage_test: coverage $(eval DIRNAME=ports/$(notdir $(CURDIR))) - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests -d thread - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --emit native - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --via-mpy -d basics float + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs -d thread + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs --emit native + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests -j1 --via-mpy -d basics float gcov -o build-coverage/py $(TOP)/py/*.c gcov -o build-coverage/extmod $(TOP)/extmod/*.c diff --git a/ports/windows/.appveyor.yml b/ports/windows/.appveyor.yml index a82cf5adc..9435ce6e6 100644 --- a/ports/windows/.appveyor.yml +++ b/ports/windows/.appveyor.yml @@ -22,7 +22,7 @@ test_script: - cmd: >- cd tests - %MICROPY_CPYTHON3% run-tests + %MICROPY_CPYTHON3% run-tests --auto-jobs skip_tags: true diff --git a/tests/run-tests b/tests/run-tests index f1035c435..4f05a4af6 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -6,6 +6,9 @@ import sys import platform import argparse import re +import threading +import multiprocessing +from multiprocessing.pool import ThreadPool from glob import glob # Tests require at least CPython 3.3. If your default python3 executable @@ -197,13 +200,27 @@ def run_micropython(pyb, args, test_file, is_special=False): def run_feature_check(pyb, args, base_path, test_file): return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True) +class ThreadSafeCounter: + def __init__(self, start=0): + self._value = start + self._lock = threading.Lock() -def run_tests(pyb, tests, args, base_path="."): - test_count = 0 - testcase_count = 0 - passed_count = 0 - failed_tests = [] - skipped_tests = [] + def add(self, to_add): + with self._lock: self._value += to_add + + def append(self, arg): + self.add([arg]) + + @property + def value(self): + return self._value + +def run_tests(pyb, tests, args, base_path=".", num_threads=1): + test_count = ThreadSafeCounter() + testcase_count = ThreadSafeCounter() + passed_count = ThreadSafeCounter() + failed_tests = ThreadSafeCounter([]) + skipped_tests = ThreadSafeCounter([]) skip_tests = set() skip_native = False @@ -354,7 +371,7 @@ def run_tests(pyb, tests, args, base_path="."): skip_tests.add('micropython/heapalloc_iter.py') # requires generators skip_tests.add('micropython/schedule.py') # native code doesn't check pending events - for test_file in tests: + def run_one_test(test_file): test_file = test_file.replace('\\', '/') test_basename = os.path.basename(test_file) test_name = os.path.splitext(test_basename)[0] @@ -377,7 +394,7 @@ def run_tests(pyb, tests, args, base_path="."): if skip_it: print("skip ", test_file) skipped_tests.append(test_name) - continue + return # get expected output test_file_expected = test_file + '.exp' @@ -405,7 +422,7 @@ def run_tests(pyb, tests, args, base_path="."): output_expected = output_expected.replace(b'\r\n', b'\n') if args.write_exp: - continue + return # run MicroPython output_mupy = run_micropython(pyb, args, test_file) @@ -413,16 +430,16 @@ def run_tests(pyb, tests, args, base_path="."): if output_mupy == b'SKIP\n': print("skip ", test_file) skipped_tests.append(test_name) - continue + return - testcase_count += len(output_expected.splitlines()) + testcase_count.add(len(output_expected.splitlines())) filename_expected = test_basename + ".exp" filename_mupy = test_basename + ".out" if output_expected == output_mupy: print("pass ", test_file) - passed_count += 1 + passed_count.add(1) rm_f(filename_expected) rm_f(filename_mupy) else: @@ -437,15 +454,22 @@ def run_tests(pyb, tests, args, base_path="."): print("FAIL ", test_file) failed_tests.append(test_name) - test_count += 1 + test_count.add(1) + + if num_threads > 1: + pool = ThreadPool(num_threads) + pool.map(run_one_test, tests) + else: + for test in tests: + run_one_test(test) - print("{} tests performed ({} individual testcases)".format(test_count, testcase_count)) - print("{} tests passed".format(passed_count)) + print("{} tests performed ({} individual testcases)".format(test_count.value, testcase_count.value)) + print("{} tests passed".format(passed_count.value)) - if len(skipped_tests) > 0: - print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests))) - if len(failed_tests) > 0: - print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests))) + if len(skipped_tests.value) > 0: + print("{} tests skipped: {}".format(len(skipped_tests.value), ' '.join(sorted(skipped_tests.value)))) + if len(failed_tests.value) > 0: + print("{} tests failed: {}".format(len(failed_tests.value), ' '.join(sorted(failed_tests.value)))) return False # all tests succeeded @@ -464,11 +488,14 @@ def main(): cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)') cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first') cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests') + cmd_parser.add_argument('-j', '--jobs', default=1, metavar='N', type=int, help='Number of tests to run simultaneously') + cmd_parser.add_argument('--auto-jobs', action='store_const', dest='jobs', const=multiprocessing.cpu_count(), help='Set the -j values to the CPU (thread) count') cmd_parser.add_argument('files', nargs='*', help='input test files') args = cmd_parser.parse_args() EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'minimal') if args.target in EXTERNAL_TARGETS: + args.jobs = 1 import pyboard pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) pyb.enter_raw_repl() @@ -507,7 +534,7 @@ def main(): # run-tests script itself. base_path = os.path.dirname(sys.argv[0]) or "." try: - res = run_tests(pyb, tests, args, base_path) + res = run_tests(pyb, tests, args, base_path, args.jobs) finally: if pyb: pyb.close() |