diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index 72642069f..b3c8e6261 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -79,16 +79,28 @@ jobs: name: Test262-ES2015-results path: build/tests/test262_tests_es2015/local/bin/test262.report - Conformance_Tests_ESNext: + Conformance_Tests_ESNext_A: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v1 - - name: Test262 - ESNext - run: $RUNNER --test262-esnext update + - name: Test262 - ESNext (built-ins,annexB,harness,intl402) + run: $RUNNER --test262-esnext update --test262-test-list built-ins,annexB,harness,intl402 - uses: actions/upload-artifact@v2 if: success() || failure() with: - name: Test262-ESNext-results + name: Test262-ESNext-results-A + path: build/tests/test262_tests_esnext/local/bin/test262.report + + Conformance_Tests_ESNext_B: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Test262 - ESNext (language) + run: $RUNNER --test262-esnext update --test262-test-list language + - uses: actions/upload-artifact@v2 + if: success() || failure() + with: + name: Test262-ESNext-results-B path: build/tests/test262_tests_esnext/local/bin/test262.report Unit_Tests: diff --git a/tools/pylint/pylintrc b/tools/pylint/pylintrc index bdcc915ea..76fd12e7d 100644 --- a/tools/pylint/pylintrc +++ b/tools/pylint/pylintrc @@ -316,7 +316,7 @@ max-locals=20 max-returns=6 # Maximum number of branch for function / method body -max-branches=20 +max-branches=25 # Maximum number of statements in function / method body max-statements=75 diff --git a/tools/run-tests.py b/tools/run-tests.py index beb9fa738..96d17d0e9 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -209,6 +209,8 @@ def get_arguments(): nargs='?', choices=['default', 'all', 'update'], help='Run test262 - ESnext. default: all tests except excludelist, ' + 'all: all tests, update: all tests and update excludelist') + parser.add_argument('--test262-test-list', metavar='LIST', + help='Add a comma separated list of tests or directories to run in test262 test suite') parser.add_argument('--unittests', action='store_true', help='Run unittests (including doctests)') parser.add_argument('--buildoption-test', action='store_true', @@ -222,6 +224,12 @@ def get_arguments(): script_args = parser.parse_args() + if script_args.test262_test_list and not \ + (script_args.test262 or script_args.test262_es2015 or script_args.test262_esnext): + print("--test262-test-list is only allowed with --test262 or --test262-es2015 or --test262-esnext\n") + parser.print_help() + sys.exit(1) + return script_args BINARY_CACHE = {} @@ -437,6 +445,10 @@ def run_test262_test_suite(options): if job.test_args: test_cmd.extend(job.test_args) + if options.test262_test_list: + test_cmd.append('--test262-test-list') + test_cmd.append(options.test262_test_list) + ret_test |= run_check(test_cmd, env=dict(TZ='America/Los_Angeles')) return ret_build | ret_test diff --git a/tools/runners/run-test-suite-test262.py b/tools/runners/run-test-suite-test262.py index a422afcab..e72bd2452 100755 --- a/tools/runners/run-test-suite-test262.py +++ b/tools/runners/run-test-suite-test262.py @@ -50,6 +50,8 @@ def get_arguments(): nargs='?', choices=['default', 'all', 'update'], help='Run test262 - ES.next. default: all tests except excludelist, ' + 'all: all tests, update: all tests and update excludelist') + parser.add_argument('--test262-test-list', metavar='LIST', + help='Add a comma separated list of tests or directories to run in test262 test suite') args = parser.parse_args() @@ -100,17 +102,21 @@ def prepare_test262_test_suite(args): def update_exclude_list(args): print("=== Summary - updating excludelist ===\n") + passing_tests = set() failing_tests = set() new_passing_tests = set() with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'r') as report_file: - summary_found = False for line in report_file: - if summary_found: - match = re.match(r" (\S*) in ", line) - if match: - failing_tests.add(match.group(1) + '.js') - elif line.startswith('Failed Tests'): - summary_found = True + match = re.match('(=== )?(.*) (?:failed|passed) in (?:non-strict|strict)', line) + if match: + (unexpected, test) = match.groups() + if unexpected: + failing_tests.add(test + '.js') + else: + passing_tests.add(test + '.js') + + # Tests pass in strict-mode but fail in non-strict-mode (or vice versa) should be considered as failures + passing_tests = passing_tests - failing_tests with open(args.excludelist_path, 'r+') as exclude_file: lines = exclude_file.readlines() @@ -125,8 +131,10 @@ def update_exclude_list(args): if test in failing_tests: failing_tests.remove(test) exclude_file.write(line) - else: + elif test in passing_tests: new_passing_tests.add(test) + else: + exclude_file.write(line) else: exclude_file.write(line) @@ -189,6 +197,9 @@ def main(args): if 'excludelist_path' in args and args.mode == 'default': test262_command.extend(['--exclude-list', args.excludelist_path]) + if args.test262_test_list: + test262_command.extend(args.test262_test_list.split(',')) + proc = subprocess.Popen(test262_command, universal_newlines=True, stdout=subprocess.PIPE, diff --git a/tools/runners/test262-harness.py b/tools/runners/test262-harness.py index b9b8c682b..ad84294ae 100755 --- a/tools/runners/test262-harness.py +++ b/tools/runners/test262-harness.py @@ -424,7 +424,7 @@ class TestResult(object): mode = self.case.get_mode() if self.has_unexpected_outcome(): if self.case.is_negative(): - print("=== %s was expected to fail in %s, but didn't ===" % (name, mode)) + print("=== %s passed in %s, but was expected to fail ===" % (name, mode)) print("--- expected error: %s ---\n" % self.case.get_negative_type()) else: if long_format: @@ -829,7 +829,7 @@ class TestSuite(object): if result.has_unexpected_outcome(): if result.case.is_negative(): self.logf.write( - "=== %s was expected to fail in %s, but didn't === \n" % (name, mode)) + "=== %s passed in %s, but was expected to fail === \n" % (name, mode)) self.logf.write("--- expected error: %s ---\n" % result.case.GetNegativeType()) result.write_output(self.logf) else: