diff --git a/tools/run-tests.py b/tools/run-tests.py index 31a283a3a..40f48223f 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -283,7 +283,8 @@ def create_binary(job, options): return ret, build_dir_path def get_binary_path(build_dir_path): - return os.path.join(build_dir_path, 'local', 'bin', 'jerry') + executable_extension = '.exe' if sys.platform == 'win32' else '' + return os.path.join(build_dir_path, 'local', 'bin', 'jerry' + executable_extension) def hash_binary(bin_path): blocksize = 65536 @@ -428,7 +429,7 @@ def run_test262_test_suite(options): print("\n%sBuild failed%s\n" % (TERM_RED, TERM_NORMAL)) break - test_cmd = [ + test_cmd = get_platform_cmd_prefix() + [ settings.TEST262_RUNNER_SCRIPT, get_binary_path(build_dir_path), settings.TEST262_TEST_SUITE_DIR diff --git a/tools/runners/run-test-suite-test262.py b/tools/runners/run-test-suite-test262.py new file mode 100755 index 000000000..70fcda89e --- /dev/null +++ b/tools/runners/run-test-suite-test262.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +# Copyright JS Foundation and other contributors, http://js.foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import sys +import os +import subprocess +import shutil + +def get_platform_cmd_prefix(): + if sys.platform == 'win32': + return ['cmd', '/S', '/C'] + return ['python2'] # The official test262.py isn't python3 compatible, but has python shebang. + + +def run_test262_tests(runtime, engine, path_to_test262): + if not os.path.isdir(os.path.join(path_to_test262, '.git')): + return_code = subprocess.call(['git', 'clone', 'https://github.com/tc39/test262.git', + '-b', 'es5-tests', path_to_test262]) + if return_code: + print('Cloning test262 repository failed.') + return return_code + + path_to_remove = os.path.join(path_to_test262, 'test', 'suite', 'bestPractice') + if os.path.isdir(path_to_remove): + shutil.rmtree(path_to_remove) + + path_to_remove = os.path.join(path_to_test262, 'test', 'suite', 'intl402') + if os.path.isdir(path_to_remove): + shutil.rmtree(path_to_remove) + + proc = subprocess.Popen(get_platform_cmd_prefix() + + [os.path.join(path_to_test262, 'tools/packaging/test262.py'), + '--command', (runtime + ' ' + engine).strip(), + '--tests', path_to_test262, + '--summary'], + stdout=subprocess.PIPE) + + return_code = 0 + with open(os.path.join(os.path.dirname(engine), 'test262.report'), 'w') as output_file: + counter = 0 + summary_found = False + while True: + counter += 1 + output = proc.stdout.readline() + if not output: + break + output_file.write(output) + if (counter % 100) == 0: + print("\rExecuted approx %d tests..." % counter, end='') + + if output.startswith('=== Summary ==='): + summary_found = True + print('') + + if summary_found: + print(output, end='') + if ('Failed tests' in output) or ('Expected to fail but passed' in output): + return_code = 1 + + proc.wait() + return return_code + + +def main(): + if len(sys.argv) != 3: + print ("This script performs test262 compliance testing of the specified engine.") + print ("") + print ("Usage:") + print (" 1st parameter: JavaScript engine to be tested.") + print (" 2nd parameter: path to the directory with official test262 testsuite.") + print ("") + print ("Example:") + print (" ./run-test-suite-test262.py ") + sys.exit(1) + + runtime = os.environ.get('RUNTIME', '') + engine = sys.argv[1] + path_to_test262 = sys.argv[2] + + sys.exit(run_test262_tests(runtime, engine, path_to_test262)) + +if __name__ == "__main__": + main() diff --git a/tools/runners/run-test-suite-test262.sh b/tools/runners/run-test-suite-test262.sh deleted file mode 100755 index ea9267387..000000000 --- a/tools/runners/run-test-suite-test262.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash - -# Copyright JS Foundation and other contributors, http://js.foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ENGINE="$1" -PATH_TO_TEST262="$2" -OUTPUT_DIR=`dirname $ENGINE` -REPORT_PATH="${OUTPUT_DIR}/test262.report" -TIMEOUT="90s" -TIMEOUT_CMD=`which timeout` -if [ $? -ne 0 ] -then - TIMEOUT_CMD=`which gtimeout` -fi -if [ -z "${RUNTIME}" ] -then - COMMAND="${TIMEOUT_CMD} ${TIMEOUT} ${ENGINE}" -else - COMMAND="${TIMEOUT_CMD} ${TIMEOUT} ${RUNTIME} ${ENGINE}" -fi - -if [ $# -lt 2 ] -then - echo "This script performs parallel test262 compliance testing of the specified engine." - echo "" - echo "Usage:" - echo " 1st parameter: JavaScript engine to be tested." - echo " 2nd parameter: path to the directory with official test262 testsuite." - echo "" - echo "Example:" - echo " ./run-test-suite-test262.sh " - exit 1 -fi - -if [ ! -d "${PATH_TO_TEST262}/.git" ] -then - git clone https://github.com/tc39/test262.git -b es5-tests "${PATH_TO_TEST262}" -fi - -rm -rf "${PATH_TO_TEST262}/test/suite/bestPractice" -rm -rf "${PATH_TO_TEST262}/test/suite/intl402" - -echo "Starting test262 testing for ${ENGINE}. Running test262 may take several minutes." - - -function progress_monitor() { - NUM_LINES_GOTTEN=0 - (>&2 echo) - while read line - do - if [[ $((NUM_LINES_GOTTEN % 100)) == 0 ]] - then - (>&2 echo -ne "\rExecuted approx ${NUM_LINES_GOTTEN} tests...") - fi - echo "$line" - NUM_LINES_GOTTEN=$((NUM_LINES_GOTTEN + 1)) - done - (>&2 echo) - (>&2 echo) -} - -python2 "${PATH_TO_TEST262}"/tools/packaging/test262.py --command "${COMMAND}" \ - --tests="${PATH_TO_TEST262}" --summary \ - | progress_monitor > "${REPORT_PATH}" - -TEST262_EXIT_CODE=$? -if [ $TEST262_EXIT_CODE -ne 0 ] -then - echo -e "\nFailed to run test262\n" - echo "$0: see ${REPORT_PATH} for details about failures" - exit $TEST262_EXIT_CODE -fi - -grep -A3 "=== Summary ===" "${REPORT_PATH}" - -FAILURES=`sed -n '/Failed tests/,/^$/p' "${REPORT_PATH}"` - -EXIT_CODE=0 -if [ -n "$FAILURES" ] -then - echo -e "\n$FAILURES\n" - echo "$0: see ${REPORT_PATH} for details about failures" - EXIT_CODE=1 -fi - -FAILURES=`sed -n '/Expected to fail but passed/,/^$/p' "${REPORT_PATH}"` -if [ -n "$FAILURES" ] -then - echo -e "\n$FAILURES\n" - echo "$0: see ${REPORT_PATH} for details about failures" - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/tools/settings.py b/tools/settings.py index 89f225f8a..733f56054 100755 --- a/tools/settings.py +++ b/tools/settings.py @@ -35,6 +35,6 @@ MAGIC_STRINGS_SCRIPT = path.join(TOOLS_DIR, 'check-magic-strings.sh') PYLINT_SCRIPT = path.join(TOOLS_DIR, 'check-pylint.sh') SIGNED_OFF_SCRIPT = path.join(TOOLS_DIR, 'check-signed-off.sh') TEST_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-test-suite.sh') -TEST262_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-test-suite-test262.sh') +TEST262_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-test-suite-test262.py') VERA_SCRIPT = path.join(TOOLS_DIR, 'check-vera.sh') UNITTEST_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-unittests.py')