archive-gh-me/jerryscript
1
0
Fork 0
mirror of https://github.com/jerryscript-project/jerryscript.git synced 2025-12-15 16:29:21 +00:00
Code Issues Projects Releases Wiki Activity

Improve test262 ES2015 runner (#3892)

Browse Source 
Changes:
 - Add options to the test runner to be able to update exclude list
 - Update the exclude list
 - Make Travis CI signal if the exclude list isn't up-to-date

JerryScript-DCO-1.0-Signed-off-by: Csaba Osztrogonác oszi@inf.u-szeged.hu
This commit is contained in:
Csaba Osztrogonác 2020-06-17 15:42:51 +02:00 committed by GitHub
parent b1912e7224
commit c55a0baaf8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 102 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -67,7 +67,7 @@ matrix:
- name: "Conformance Tests - ES2015"
env:
- OPTS="--test262-es2015"
- OPTS="--test262-es2015 update"
- name: "Unit Tests"
env:

2
tests/test262-es6-excludelist.xml
View File

@ -1,6 +1,5 @@
<?xml version="1.0" encoding="utf-8" ?>
<excludeList>
<test id="annexB/B.2.2.1.2.js"><reason></reason></test>
<test id="annexB/B.2.3.10.js"><reason></reason></test>
<test id="annexB/B.2.3.11.js"><reason></reason></test>
<test id="annexB/B.2.3.12.js"><reason></reason></test>
@ -416,7 +415,6 @@
<test id="language/statements/class/definition/this-check-ordering.js"><reason></reason></test>
<test id="language/statements/class/name-binding/in-extends-expression-assigned.js"><reason></reason></test>
<test id="language/statements/class/name-binding/in-extends-expression.js"><reason></reason></test>
<test id="language/statements/class/strict-mode/arguments-caller.js"><reason></reason></test>
<test id="language/statements/class/syntax/early-errors/class-body-static-method-get-propname-prototype.js"><reason></reason></test>
<test id="language/statements/continue/labeled-continue.js"><reason></reason></test>
<test id="language/statements/continue/nested-let-bound-for-loops-labeled-continue.js"><reason></reason></test>

7
tools/run-tests.py
View File

@ -194,8 +194,10 @@ def get_arguments():
help='Run jerry-tests')
parser.add_argument('--test262', action='store_true',
help='Run test262 - ES5.1')
parser.add_argument('--test262-es2015', action='store_true',
help='Run test262 - ES2015')
parser.add_argument('--test262-es2015', default=False, const='default',
nargs='?', choices=['default', 'all', 'update'],
help='Run test262 - ES2015. default: all tests except excludelist, ' +
'all: all tests, update: all tests and update excludelist')
parser.add_argument('--unittests', action='store_true',
help='Run unittests (including doctests)')
parser.add_argument('--buildoption-test', action='store_true',
@ -412,6 +414,7 @@ def run_test262_test_suite(options):
if '--profile=es.next' in job.build_args:
test_cmd.append('--es2015')
test_cmd.append(options.test262_es2015)
else:
test_cmd.append('--es51')

104
tools/runners/run-test-suite-test262.py
View File

@ -17,6 +17,7 @@
from __future__ import print_function
import argparse
import os
import re
import shutil
import subprocess
import sys
@ -41,8 +42,10 @@ def get_arguments():
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--es51', action='store_true',
help='Run test262 ES5.1 version')
group.add_argument('--es2015', action='store_true',
help='Run test262 ES2015 version')
group.add_argument('--es2015', default=False, const='default',
nargs='?', choices=['default', 'all', 'update'],
help='Run test262 - ES2015. default: all tests except excludelist, ' +
'all: all tests, update: all tests and update excludelist')
args = parser.parse_args()
@ -75,15 +78,11 @@ def prepare_test262_test_suite(args):
return return_code
if args.es2015:
shutil.copyfile(os.path.join('tests', 'test262-es6-excludelist.xml'),
os.path.join(args.test_dir, 'excludelist.xml'))
return_code = subprocess.call(['git', 'apply', os.path.join('..', '..', 'test262-es6.patch')],
cwd=args.test_dir)
if return_code:
print('Applying test262-es6.patch failed')
return return_code
else:
path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'bestPractice')
if os.path.isdir(path_to_remove):
@ -95,23 +94,109 @@ def prepare_test262_test_suite(args):
return 0
def prepare_exclude_list(args):
if args.es2015 == 'all' or args.es2015 == 'update':
return_code = subprocess.call(['git', 'checkout', 'excludelist.xml'], cwd=args.test_dir)
if return_code:
print('Reverting excludelist.xml failed')
return return_code
elif args.es2015 == 'default':
shutil.copyfile(os.path.join('tests', 'test262-es6-excludelist.xml'),
os.path.join(args.test_dir, 'excludelist.xml'))
return 0
def update_exclude_list(args):
assert args.es2015 == 'update', "Only --es2015 option supports updating excludelist"
print("=== Summary - updating excludelist ===\n")
failing_tests = set()
new_passing_tests = set()
with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'r') as report_file:
summary_found = False
for line in report_file:
if summary_found:
match = re.match(r" (\S*) ", line)
if match:
failing_tests.add(match.group(1) + '.js')
elif line.startswith('Failed Tests'):
summary_found = True
with open(os.path.join('tests', 'test262-es6-excludelist.xml'), 'r+') as exclude_file:
lines = exclude_file.readlines()
exclude_file.seek(0)
exclude_file.truncate()
exclude_file.write('<?xml version="1.0" encoding="utf-8" ?>\n')
exclude_file.write('<excludeList>\n')
for line in lines:
match = re.match(r" <test id=\"(\S*)\">", line)
if match:
test = match.group(1)
if test in failing_tests:
failing_tests.remove(test)
exclude_file.write(line)
else:
new_passing_tests.add(test)
if failing_tests:
print("New failing tests added to the excludelist")
for test in sorted(failing_tests):
exclude_file.write(' <test id="' + test + '"><reason></reason></test>\n')
print(" " + test)
print("")
exclude_file.write('</excludeList>\n')
if new_passing_tests:
print("New passing tests removed from the excludelist")
for test in sorted(new_passing_tests):
print(" " + test)
print("")
if failing_tests or new_passing_tests:
print("Excludelist was updated succesfully.")
return 1
print("Excludelist was already up-to-date.")
return 0
def main(args):
return_code = prepare_test262_test_suite(args)
if return_code:
return return_code
return_code = prepare_exclude_list(args)
if return_code:
return return_code
if sys.platform == 'win32':
original_timezone = util.get_timezone()
util.set_sighdl_to_reset_timezone(original_timezone)
util.set_timezone('Pacific Standard Time')
command = (args.runtime + ' ' + args.engine).strip()
if args.es2015:
try:
subprocess.check_output(["timeout", "--version"])
command = "timeout 3 " + command
except subprocess.CalledProcessError:
pass
kwargs = {}
if sys.version_info.major >= 3:
kwargs['errors'] = 'ignore'
proc = subprocess.Popen(get_platform_cmd_prefix() +
[os.path.join(args.test_dir, 'tools/packaging/test262.py'),
'--command', (args.runtime + ' ' + args.engine).strip(),
'--command', command,
'--tests', args.test_dir,
'--summary'],
universal_newlines=True,
stdout=subprocess.PIPE)
stdout=subprocess.PIPE,
**kwargs)
return_code = 1
with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'w') as output_file:
@ -140,6 +225,9 @@ def main(args):
if sys.platform == 'win32':
util.set_timezone(original_timezone)
if args.es2015 == 'update':
return_code = update_exclude_list(args)
return return_code