From 2d9484a9688666e84e1ae7f6dcee064e21e7e7bd Mon Sep 17 00:00:00 2001 From: Akos Kiss Date: Tue, 28 Mar 2017 15:48:09 +0200 Subject: [PATCH] Don't do duplicate builds during test runs (#1691) The `run-tests.py` test execution harness validates the correctness of JerryScript by building various configurations of jerry and running various test suites and subsets against it. However, until now, it always built jerry for each (build configuration, test suite) pair. This commit improves `run-tests.py` by skipping duplicate builds, sharing the same-built binaries across multiple test suite runs. The patch also improves on the style of how `Option`s are specified and fixes the name of the `jerry_tests-debug-cpointer_32bit` option. JerryScript-DCO-1.0-Signed-off-by: Akos Kiss akiss@inf.u-szeged.hu --- tools/run-tests.py | 175 +++++++++++++++++++++++++-------------------- 1 file changed, 96 insertions(+), 79 deletions(-) diff --git a/tools/run-tests.py b/tools/run-tests.py index 222c88aae..5c4d9b01d 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -32,90 +32,94 @@ class Options(object): if test_args is None: test_args = [] - self._build_args = build_args - self._name = name + self.build_args = build_args + self.name = name self.test_args = test_args - def get_build_args(self, outdir): - return self._build_args + ['--builddir=%s' % os.path.join(outdir, self._name)] - - def get_bin_dir_path(self, outdir): - return os.path.join(os.path.join(outdir, self._name), 'bin') - - def get_binary_path(self, outdir): - return os.path.join(self.get_bin_dir_path(outdir), 'jerry') +def get_binary_path(bin_dir_path): + return os.path.join(bin_dir_path, 'jerry') # Test options for unittests JERRY_UNITTESTS_OPTIONS = [ - Options('unittests', ['--unittests', '--error-messages=on', '--snapshot-save=on', '--snapshot-exec=on']), - Options('unittests-debug', ['--unittests', - '--debug', - '--error-messages=on', - '--snapshot-save=on', - '--snapshot-exec=on']) + Options('unittests', + ['--unittests', '--error-messages=on', '--snapshot-save=on', '--snapshot-exec=on']), + Options('unittests-debug', + ['--unittests', '--debug', '--error-messages=on', '--snapshot-save=on', '--snapshot-exec=on']) ] # Test options for jerry-tests JERRY_TESTS_OPTIONS = [ Options('jerry_tests'), - Options('jerry_tests-debug', ['--debug']), - Options('jerry_tests-debug', ['--debug', '--cpointer-32bit=on', '--mem-heap=1024']), - Options('jerry_tests-snapshot', ['--snapshot-save=on', '--snapshot-exec=on'], ['--snapshot']), - Options('jerry_tests-debug-snapshot', ['--debug', '--snapshot-save=on', '--snapshot-exec=on'], ['--snapshot']), - Options('jerry_tests-es2015-subset-debug', ['--debug', '--profile=es2015-subset']) + Options('jerry_tests-debug', + ['--debug']), + Options('jerry_tests-debug-cpointer_32bit', + ['--debug', '--cpointer-32bit=on', '--mem-heap=1024']), + Options('jerry_tests-snapshot', + ['--snapshot-save=on', '--snapshot-exec=on'], + ['--snapshot']), + Options('jerry_tests-debug-snapshot', + ['--debug', '--snapshot-save=on', '--snapshot-exec=on'], + ['--snapshot']), + Options('jerry_tests-es2015-subset-debug', + ['--debug', '--profile=es2015-subset']) ] # Test options for jerry-test-suite JERRY_TEST_SUITE_OPTIONS = JERRY_TESTS_OPTIONS[:] -JERRY_TEST_SUITE_OPTIONS.append(Options('jerry_test_suite-minimal', ['--profile=minimal'])) -JERRY_TEST_SUITE_OPTIONS.append(Options('jerry_test_suite-minimal-snapshot', - ['--profile=minimal', - '--snapshot-save=on', - '--snapshot-exec=on'], - ['--snapshot'])) -JERRY_TEST_SUITE_OPTIONS.append(Options('jerry_test_suite-minimal-debug', ['--debug', '--profile=minimal'])) -JERRY_TEST_SUITE_OPTIONS.append(Options('jerry_test_suite-minimal-debug-snapshot', - ['--debug', - '--profile=minimal', - '--snapshot-save=on', - '--snapshot-exec=on'], - ['--snapshot'])) -JERRY_TEST_SUITE_OPTIONS.append(Options('jerry_test_suite-es2015-subset', ['--profile=es2015-subset'])) -JERRY_TEST_SUITE_OPTIONS.append(Options('jerry_test_suite-es2015-subset-snapshot', - ['--profile=es2015-subset', - '--snapshot-save=on', - '--snapshot-exec=on'], - ['--snapshot'])) -JERRY_TEST_SUITE_OPTIONS.append(Options('jerry_test_suite-es2015-subset-debug-snapshot', - ['--debug', - '--profile=es2015-subset', - '--snapshot-save=on', - '--snapshot-exec=on'], - ['--snapshot'])) +JERRY_TEST_SUITE_OPTIONS.extend([ + Options('jerry_test_suite-minimal', + ['--profile=minimal']), + Options('jerry_test_suite-minimal-snapshot', + ['--profile=minimal', '--snapshot-save=on', '--snapshot-exec=on'], + ['--snapshot']), + Options('jerry_test_suite-minimal-debug', + ['--debug', '--profile=minimal']), + Options('jerry_test_suite-minimal-debug-snapshot', + ['--debug', '--profile=minimal', '--snapshot-save=on', '--snapshot-exec=on'], + ['--snapshot']), + Options('jerry_test_suite-es2015-subset', + ['--profile=es2015-subset']), + Options('jerry_test_suite-es2015-subset-snapshot', + ['--profile=es2015-subset', '--snapshot-save=on', '--snapshot-exec=on'], + ['--snapshot']), + Options('jerry_test_suite-es2015-subset-debug-snapshot', + ['--debug', '--profile=es2015-subset', '--snapshot-save=on', '--snapshot-exec=on'], + ['--snapshot']) +]) # Test options for test262 -TEST262_TEST_SUITE_OPTIONS = [Options('test262_tests')] +TEST262_TEST_SUITE_OPTIONS = [ + Options('test262_tests') +] # Test options for jerry-debugger DEBUGGER_TEST_OPTIONS = [ - Options('jerry_debugger_tests', ['--debug', '--jerry-debugger=on', '--jerry-libc=off']) + Options('jerry_debugger_tests', + ['--debug', '--jerry-debugger=on', '--jerry-libc=off']) ] # Test options for buildoption-test JERRY_BUILDOPTIONS = [ - Options('buildoption_test-lto', ['--lto=on']), - Options('buildoption_test-error_messages', ['--error-messages=on']), - Options('buildoption_test-all_in_one', ['--all-in-one=on']), - Options('buildoption_test-valgrind', ['--valgrind=on']), - Options('buildoption_test-valgrind_freya', ['--valgrind-freya=on']), - Options('buildoption_test-mem_stats', ['--mem-stats=on']), - Options('buildoption_test-show_opcodes', ['--show-opcodes=on']), - Options('buildoption_test-show_regexp_opcodes', ['--show-regexp-opcodes=on']), - Options('buildoption_test-compiler_default_libc', ['--jerry-libc=off']), - Options('buildoption_test-cpointer_32bit', ['--jerry-libc=off', - '--compile-flag=-m32', - '--cpointer-32bit=on', - '--system-allocator=on']), + Options('buildoption_test-lto', + ['--lto=on']), + Options('buildoption_test-error_messages', + ['--error-messages=on']), + Options('buildoption_test-all_in_one', + ['--all-in-one=on']), + Options('buildoption_test-valgrind', + ['--valgrind=on']), + Options('buildoption_test-valgrind_freya', + ['--valgrind-freya=on']), + Options('buildoption_test-mem_stats', + ['--mem-stats=on']), + Options('buildoption_test-show_opcodes', + ['--show-opcodes=on']), + Options('buildoption_test-show_regexp_opcodes', + ['--show-regexp-opcodes=on']), + Options('buildoption_test-compiler_default_libc', + ['--jerry-libc=off']), + Options('buildoption_test-cpointer_32bit', + ['--jerry-libc=off', '--compile-flag=-m32', '--cpointer-32bit=on', '--system-allocator=on']), ] def get_arguments(): @@ -154,9 +158,14 @@ def get_arguments(): return script_args -def create_binary(buildoptions, options): +BINARY_CACHE = {} + +def create_binary(job, options): build_cmd = [settings.BUILD_SCRIPT] - build_cmd.extend(buildoptions) + build_cmd.extend(job.build_args) + + build_dir_path = os.path.join(options.outdir, job.name) + build_cmd.append('--builddir=%s' % build_dir_path) if options.toolchain: build_cmd.append('--toolchain=%s' % options.toolchain) @@ -166,12 +175,20 @@ def create_binary(buildoptions, options): sys.stderr.write('Build command: %s\n' % ' '.join(build_cmd)) + binary_key = tuple(job.build_args) + if binary_key in BINARY_CACHE: + ret, build_dir_path = BINARY_CACHE[binary_key] + sys.stderr.write('(skipping: already built at %s with returncode %d)\n' % (build_dir_path, ret)) + return ret, os.path.join(build_dir_path, 'bin') + try: subprocess.check_output(build_cmd) + ret = 0 except subprocess.CalledProcessError as err: - return err.returncode + ret = err.returncode - return 0 + BINARY_CACHE[binary_key] = (ret, build_dir_path) + return ret, os.path.join(build_dir_path, 'bin') def run_check(runnable): sys.stderr.write('Test command: %s\n' % ' '.join(runnable)) @@ -186,7 +203,7 @@ def run_check(runnable): def run_jerry_debugger_tests(options): ret_build = ret_test = 0 for job in DEBUGGER_TEST_OPTIONS: - ret_build = create_binary(job.get_build_args(options.outdir), options) + ret_build, bin_dir_path = create_binary(job, options) if ret_build: break @@ -196,7 +213,7 @@ def run_jerry_debugger_tests(options): test_case_path = os.path.join(settings.DEBUGGER_TESTS_DIR, test_case) test_cmd = [ settings.DEBUGGER_TEST_RUNNER_SCRIPT, - job.get_binary_path(options.outdir), + get_binary_path(bin_dir_path), settings.DEBUGGER_CLIENT_SCRIPT, os.path.relpath(test_case_path, settings.PROJECT_DIR) ] @@ -211,18 +228,18 @@ def run_jerry_debugger_tests(options): def run_jerry_tests(options): ret_build = ret_test = 0 for job in JERRY_TESTS_OPTIONS: - ret_build = create_binary(job.get_build_args(options.outdir), options) + ret_build, bin_dir_path = create_binary(job, options) if ret_build: break test_cmd = [ settings.TEST_RUNNER_SCRIPT, - job.get_binary_path(options.outdir), + get_binary_path(bin_dir_path), settings.JERRY_TESTS_DIR ] skip_list = [] - if '--profile=es2015-subset' not in job.get_build_args(options.outdir): + if '--profile=es2015-subset' not in job.build_args: skip_list.append(r"es2015\/") if options.skip_list: @@ -241,15 +258,15 @@ def run_jerry_tests(options): def run_jerry_test_suite(options): ret_build = ret_test = 0 for job in JERRY_TEST_SUITE_OPTIONS: - ret_build = create_binary(job.get_build_args(options.outdir), options) + ret_build, bin_dir_path = create_binary(job, options) if ret_build: break - test_cmd = [settings.TEST_RUNNER_SCRIPT, job.get_binary_path(options.outdir)] + test_cmd = [settings.TEST_RUNNER_SCRIPT, get_binary_path(bin_dir_path)] - if '--profile=minimal' in job.get_build_args(options.outdir): + if '--profile=minimal' in job.build_args: test_cmd.append(settings.JERRY_TEST_SUITE_MINIMAL_LIST) - elif '--profile=es2015-subset' in job.get_build_args(options.outdir): + elif '--profile=es2015-subset' in job.build_args: test_cmd.append(settings.JERRY_TEST_SUITE_DIR) else: test_cmd.append(settings.JERRY_TEST_SUITE_ES51_LIST) @@ -267,13 +284,13 @@ def run_jerry_test_suite(options): def run_test262_test_suite(options): ret_build = ret_test = 0 for job in TEST262_TEST_SUITE_OPTIONS: - ret_build = create_binary(job.get_build_args(options.outdir), options) + ret_build, bin_dir_path = create_binary(job, options) if ret_build: break test_cmd = [ settings.TEST262_RUNNER_SCRIPT, - job.get_binary_path(options.outdir), + get_binary_path(bin_dir_path), settings.TEST262_TEST_SUITE_DIR ] @@ -287,20 +304,20 @@ def run_test262_test_suite(options): def run_unittests(options): ret_build = ret_test = 0 for job in JERRY_UNITTESTS_OPTIONS: - ret_build = create_binary(job.get_build_args(options.outdir), options) + ret_build, bin_dir_path = create_binary(job, options) if ret_build: break ret_test |= run_check([ settings.UNITTEST_RUNNER_SCRIPT, - job.get_bin_dir_path(options.outdir) + bin_dir_path ]) return ret_build | ret_test def run_buildoption_test(options): for job in JERRY_BUILDOPTIONS: - ret = create_binary(job.get_build_args(options.outdir), options) + ret, _ = create_binary(job, options) if ret: break