diff --git a/.travis.yml b/.travis.yml
index a037aba79..b43755c70 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -61,10 +61,14 @@ matrix:
apt:
packages: [gcc-multilib]
- - name: "Conformance Tests"
+ - name: "Conformance Tests - ES5.1"
env:
- OPTS="--test262"
+ - name: "Conformance Tests - ES2015"
+ env:
+ - OPTS="--test262-es2015"
+
- name: "Unit Tests"
env:
- OPTS="--unittests"
diff --git a/tests/test262-es6-excludelist.xml b/tests/test262-es6-excludelist.xml
new file mode 100644
index 000000000..b23fa565d
--- /dev/null
+++ b/tests/test262-es6-excludelist.xml
@@ -0,0 +1,648 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/test262-es6.patch b/tests/test262-es6.patch
new file mode 100644
index 000000000..c52d6bc18
--- /dev/null
+++ b/tests/test262-es6.patch
@@ -0,0 +1,15 @@
+diff --git a/tools/packaging/test262.py b/tools/packaging/test262.py
+index 921360a05e..27a2938e48 100755
+--- a/tools/packaging/test262.py
++++ b/tools/packaging/test262.py
+@@ -469,8 +469,8 @@ class TestSuite(object):
+ if self.ShouldRun(rel_path, tests):
+ basename = path.basename(full_path)[:-3]
+ name = rel_path.split(path.sep)[:-1] + [basename]
+- if EXCLUDE_LIST.count(basename) >= 1:
+- print 'Excluded: ' + basename
++ if rel_path in EXCLUDE_LIST:
++ print 'Excluded: ' + rel_path
+ else:
+ if not self.non_strict_only:
+ strict_case = TestCase(self, name, full_path, True)
diff --git a/tools/run-tests.py b/tools/run-tests.py
index 88605912f..637f71958 100755
--- a/tools/run-tests.py
+++ b/tools/run-tests.py
@@ -124,6 +124,11 @@ TEST262_TEST_SUITE_OPTIONS = [
Options('test262_tests-debug', OPTIONS_DEBUG)
]
+# Test options for test262-es2015
+TEST262_ES2015_TEST_SUITE_OPTIONS = [
+ Options('test262_tests_es2015', OPTIONS_PROFILE_ES2015 + ['--line-info=on', '--error-messages=on']),
+]
+
# Test options for jerry-debugger
DEBUGGER_TEST_OPTIONS = [
Options('jerry_debugger_tests',
@@ -213,7 +218,9 @@ def get_arguments():
parser.add_argument('--jerry-test-suite', action='store_true',
help='Run jerry-test-suite')
parser.add_argument('--test262', action='store_true',
- help='Run test262')
+ help='Run test262 - ES5.1')
+ parser.add_argument('--test262-es2015', action='store_true',
+ help='Run test262 - ES2015')
parser.add_argument('--unittests', action='store_true',
help='Run unittests (including doctests)')
parser.add_argument('--buildoption-test', action='store_true',
@@ -444,7 +451,14 @@ def run_jerry_test_suite(options):
def run_test262_test_suite(options):
ret_build = ret_test = 0
- for job in TEST262_TEST_SUITE_OPTIONS:
+
+ jobs = []
+ if options.test262:
+ jobs.extend(TEST262_TEST_SUITE_OPTIONS)
+ if options.test262_es2015:
+ jobs.extend(TEST262_ES2015_TEST_SUITE_OPTIONS)
+
+ for job in jobs:
ret_build, build_dir_path = create_binary(job, options)
if ret_build:
print("\n%sBuild failed%s\n" % (TERM_RED, TERM_NORMAL))
@@ -452,10 +466,15 @@ def run_test262_test_suite(options):
test_cmd = get_platform_cmd_prefix() + [
settings.TEST262_RUNNER_SCRIPT,
- get_binary_path(build_dir_path),
- settings.TEST262_TEST_SUITE_DIR
+ '--engine', get_binary_path(build_dir_path),
+ '--test-dir', settings.TEST262_TEST_SUITE_DIR
]
+ if '--profile=es2015-subset' in job.build_args:
+ test_cmd.append('--es2015')
+ else:
+ test_cmd.append('--es51')
+
if job.test_args:
test_cmd.extend(job.test_args)
@@ -520,7 +539,7 @@ def main(options):
Check(options.jerry_debugger, run_jerry_debugger_tests, options),
Check(options.jerry_tests, run_jerry_tests, options),
Check(options.jerry_test_suite, run_jerry_test_suite, options),
- Check(options.test262, run_test262_test_suite, options),
+ Check(options.test262 or options.test262_es2015, run_test262_test_suite, options),
Check(options.unittests, run_unittests, options),
Check(options.buildoption_test, run_buildoption_test, options),
]
diff --git a/tools/runners/run-test-suite-test262.py b/tools/runners/run-test-suite-test262.py
index 8040261ec..f3ab74d3e 100755
--- a/tools/runners/run-test-suite-test262.py
+++ b/tools/runners/run-test-suite-test262.py
@@ -15,6 +15,7 @@
# limitations under the License.
from __future__ import print_function
+import argparse
import os
import shutil
import subprocess
@@ -28,21 +29,76 @@ def get_platform_cmd_prefix():
return ['python2'] # The official test262.py isn't python3 compatible, but has python shebang.
-def run_test262_tests(runtime, engine, path_to_test262):
- if not os.path.isdir(os.path.join(path_to_test262, '.git')):
- return_code = subprocess.call(['git', 'clone', 'https://github.com/tc39/test262.git',
- '-b', 'es5-tests', path_to_test262])
+def get_arguments():
+ execution_runtime = os.environ.get('RUNTIME', '')
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--runtime', metavar='FILE', default=execution_runtime,
+ help='Execution runtime (e.g. qemu)')
+ parser.add_argument('--engine', metavar='FILE', required=True,
+ help='JerryScript binary to run tests with')
+ parser.add_argument('--test-dir', metavar='DIR', required=True,
+ help='Directory contains test262 test suite')
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument('--es51', action='store_true',
+ help='Run test262 ES5.1 version')
+ group.add_argument('--es2015', action='store_true',
+ help='Run test262 ES2015 version')
+
+ args = parser.parse_args()
+
+ if args.es2015:
+ args.test_dir = os.path.join(args.test_dir, 'es2015')
+ else:
+ args.test_dir = os.path.join(args.test_dir, 'es51')
+
+ return args
+
+
+def prepare_test262_test_suite(args):
+ if os.path.isdir(os.path.join(args.test_dir, '.git')):
+ return 0
+
+ return_code = subprocess.call(['git', 'clone', '--no-checkout',
+ 'https://github.com/tc39/test262.git', args.test_dir])
+ if return_code:
+ print('Cloning test262 repository failed.')
+ return return_code
+
+ if args.es2015:
+ git_hash = 'fd44cd73dfbce0b515a2474b7cd505d6176a9eb5'
+ else:
+ git_hash = 'es5-tests'
+
+ return_code = subprocess.call(['git', 'checkout', git_hash], cwd=args.test_dir)
+ if return_code:
+ print('Cloning test262 repository failed - invalid git revision.')
+ return return_code
+
+ if args.es2015:
+ shutil.copyfile(os.path.join('tests', 'test262-es6-excludelist.xml'),
+ os.path.join(args.test_dir, 'excludelist.xml'))
+
+ return_code = subprocess.call(['git', 'apply', os.path.join('..', '..', 'test262-es6.patch')],
+ cwd=args.test_dir)
if return_code:
- print('Cloning test262 repository failed.')
+ print('Applying test262-es6.patch failed')
return return_code
- path_to_remove = os.path.join(path_to_test262, 'test', 'suite', 'bestPractice')
- if os.path.isdir(path_to_remove):
- shutil.rmtree(path_to_remove)
+ else:
+ path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'bestPractice')
+ if os.path.isdir(path_to_remove):
+ shutil.rmtree(path_to_remove)
- path_to_remove = os.path.join(path_to_test262, 'test', 'suite', 'intl402')
- if os.path.isdir(path_to_remove):
- shutil.rmtree(path_to_remove)
+ path_to_remove = os.path.join(args.test_dir, 'test', 'suite', 'intl402')
+ if os.path.isdir(path_to_remove):
+ shutil.rmtree(path_to_remove)
+
+ return 0
+
+def main(args):
+ return_code = prepare_test262_test_suite(args)
+ if return_code:
+ return return_code
if sys.platform == 'win32':
original_timezone = util.get_timezone()
@@ -50,15 +106,15 @@ def run_test262_tests(runtime, engine, path_to_test262):
util.set_timezone('Pacific Standard Time')
proc = subprocess.Popen(get_platform_cmd_prefix() +
- [os.path.join(path_to_test262, 'tools/packaging/test262.py'),
- '--command', (runtime + ' ' + engine).strip(),
- '--tests', path_to_test262,
+ [os.path.join(args.test_dir, 'tools/packaging/test262.py'),
+ '--command', (args.runtime + ' ' + args.engine).strip(),
+ '--tests', args.test_dir,
'--summary'],
universal_newlines=True,
stdout=subprocess.PIPE)
return_code = 0
- with open(os.path.join(os.path.dirname(engine), 'test262.report'), 'w') as output_file:
+ with open(os.path.join(os.path.dirname(args.engine), 'test262.report'), 'w') as output_file:
counter = 0
summary_found = False
while True:
@@ -67,7 +123,7 @@ def run_test262_tests(runtime, engine, path_to_test262):
if not output:
break
output_file.write(output)
- if (counter % 100) == 0:
+ if not summary_found and (counter % 100) == 0:
print("\rExecuted approx %d tests..." % counter, end='')
if output.startswith('=== Summary ==='):
@@ -87,23 +143,5 @@ def run_test262_tests(runtime, engine, path_to_test262):
return return_code
-def main():
- if len(sys.argv) != 3:
- print ("This script performs test262 compliance testing of the specified engine.")
- print ("")
- print ("Usage:")
- print (" 1st parameter: JavaScript engine to be tested.")
- print (" 2nd parameter: path to the directory with official test262 testsuite.")
- print ("")
- print ("Example:")
- print (" ./run-test-suite-test262.py ")
- sys.exit(1)
-
- runtime = os.environ.get('RUNTIME', '')
- engine = sys.argv[1]
- path_to_test262 = sys.argv[2]
-
- sys.exit(run_test262_tests(runtime, engine, path_to_test262))
-
if __name__ == "__main__":
- main()
+ sys.exit(main(get_arguments()))