mirror of
https://github.com/jerryscript-project/jerryscript.git
synced 2025-12-15 16:29:21 +00:00
Make run-tests --jerry-tests --jerry-test-suite work on Windows too (#3260)
Changes: * Bash based runners/run-test-suite.sh runner replaced with a python runner * Common util functions moved to util.py * Fixed EOL issues in tests directory JerryScript-DCO-1.0-Signed-off-by: Csaba Osztrogonác oszi@inf.u-szeged.hu
This commit is contained in:
Csaba Osztrogonác
Dániel Bátyai
parent
525c35f148
commit
44b1b9855d
2
tests/.gitattributes
vendored
Normal file
2
tests/.gitattributes
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
*.js text eol=lf
|
||||
jerry/windows-line-ending.js text eol=crlf
|
||||
@ -1,22 +1,22 @@
|
||||
// Copyright JS Foundation and other contributors, http://js.foundation
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This test file should use CR-LF styled line endings to test if everything is
|
||||
// parsed correctly
|
||||
|
||||
|
||||
var value =
|
||||
5;
|
||||
|
||||
assert (value === 5);
|
||||
// Copyright JS Foundation and other contributors, http://js.foundation
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This test file should use CR-LF styled line endings to test if everything is
|
||||
// parsed correctly
|
||||
|
||||
|
||||
var value =
|
||||
5;
|
||||
|
||||
assert (value === 5);
|
||||
|
||||
@ -303,7 +303,7 @@ ignore-imports=no
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
max-args=6
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
|
||||
@ -327,7 +327,8 @@ def iterate_test_runner_jobs(jobs, options):
|
||||
else:
|
||||
tested_hashes[bin_hash] = build_dir_path
|
||||
|
||||
test_cmd = [settings.TEST_RUNNER_SCRIPT, bin_path]
|
||||
test_cmd = get_platform_cmd_prefix()
|
||||
test_cmd.extend([settings.TEST_RUNNER_SCRIPT, '--engine', bin_path])
|
||||
|
||||
yield job, ret_build, test_cmd
|
||||
|
||||
@ -377,6 +378,7 @@ def run_jerry_tests(options):
|
||||
if ret_build:
|
||||
break
|
||||
|
||||
test_cmd.append('--test-dir')
|
||||
test_cmd.append(settings.JERRY_TESTS_DIR)
|
||||
|
||||
if options.quiet:
|
||||
@ -385,9 +387,9 @@ def run_jerry_tests(options):
|
||||
skip_list = []
|
||||
|
||||
if '--profile=es2015-subset' in job.build_args:
|
||||
skip_list.append(r"es5.1\/")
|
||||
skip_list.append(os.path.join('es5.1', ''))
|
||||
else:
|
||||
skip_list.append(r"es2015\/")
|
||||
skip_list.append(os.path.join('es2015', ''))
|
||||
|
||||
if options.skip_list:
|
||||
skip_list.append(options.skip_list)
|
||||
@ -409,10 +411,13 @@ def run_jerry_test_suite(options):
|
||||
break
|
||||
|
||||
if '--profile=minimal' in job.build_args:
|
||||
test_cmd.append('--test-list')
|
||||
test_cmd.append(settings.JERRY_TEST_SUITE_MINIMAL_LIST)
|
||||
elif '--profile=es2015-subset' in job.build_args:
|
||||
test_cmd.append('--test-dir')
|
||||
test_cmd.append(settings.JERRY_TEST_SUITE_DIR)
|
||||
else:
|
||||
test_cmd.append('--test-list')
|
||||
test_cmd.append(settings.JERRY_TEST_SUITE_ES51_LIST)
|
||||
|
||||
if options.quiet:
|
||||
|
||||
@ -17,10 +17,10 @@
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import util
|
||||
|
||||
def get_platform_cmd_prefix():
|
||||
if sys.platform == 'win32':
|
||||
@ -28,16 +28,6 @@ def get_platform_cmd_prefix():
|
||||
return ['python2'] # The official test262.py isn't python3 compatible, but has python shebang.
|
||||
|
||||
|
||||
def set_timezone(timezone):
|
||||
assert sys.platform == 'win32', "set_timezone is Windows only function"
|
||||
subprocess.call(get_platform_cmd_prefix() + ['tzutil', '/s', timezone])
|
||||
|
||||
|
||||
def set_timezone_and_exit(timezone):
|
||||
set_timezone(timezone)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def run_test262_tests(runtime, engine, path_to_test262):
|
||||
if not os.path.isdir(os.path.join(path_to_test262, '.git')):
|
||||
return_code = subprocess.call(['git', 'clone', 'https://github.com/tc39/test262.git',
|
||||
@ -55,9 +45,9 @@ def run_test262_tests(runtime, engine, path_to_test262):
|
||||
shutil.rmtree(path_to_remove)
|
||||
|
||||
if sys.platform == 'win32':
|
||||
original_timezone = subprocess.check_output(get_platform_cmd_prefix() + ['tzutil', '/g'])
|
||||
set_timezone('Pacific Standard Time')
|
||||
signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(original_timezone))
|
||||
original_timezone = util.get_timezone()
|
||||
util.set_sighdl_to_reset_timezone(original_timezone)
|
||||
util.set_timezone('Pacific Standard Time')
|
||||
|
||||
proc = subprocess.Popen(get_platform_cmd_prefix() +
|
||||
[os.path.join(path_to_test262, 'tools/packaging/test262.py'),
|
||||
@ -92,7 +82,7 @@ def run_test262_tests(runtime, engine, path_to_test262):
|
||||
proc.wait()
|
||||
|
||||
if sys.platform == 'win32':
|
||||
set_timezone(original_timezone)
|
||||
util.set_timezone(original_timezone)
|
||||
|
||||
return return_code
|
||||
|
||||
|
||||
209
tools/runners/run-test-suite.py
Executable file
209
tools/runners/run-test-suite.py
Executable file
@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright JS Foundation and other contributors, http://js.foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import util
|
||||
|
||||
def get_arguments():
|
||||
execution_runtime = os.environ.get('RUNTIME')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-q', '--quiet', action='store_true',
|
||||
help='Only print out failing tests')
|
||||
parser.add_argument('--runtime', metavar='FILE', default=execution_runtime,
|
||||
help='Execution runtime (e.g. qemu)')
|
||||
parser.add_argument('--engine', metavar='FILE',
|
||||
help='JerryScript binary to run tests with')
|
||||
parser.add_argument('--test-list', metavar='FILE',
|
||||
help='File contains test paths to run')
|
||||
parser.add_argument('--skip-list', metavar='LIST',
|
||||
help='Add a comma separated list of patterns of the excluded JS-tests')
|
||||
parser.add_argument('--test-dir', metavar='DIR',
|
||||
help='Directory contains tests to run')
|
||||
parser.add_argument('--snapshot', action='store_true',
|
||||
help='Snapshot test')
|
||||
|
||||
script_args = parser.parse_args()
|
||||
if script_args.skip_list:
|
||||
script_args.skip_list = script_args.skip_list.split(',')
|
||||
else:
|
||||
script_args.skip_list = []
|
||||
|
||||
return script_args
|
||||
|
||||
|
||||
def get_tests(test_dir, test_list, skip_list):
|
||||
tests = []
|
||||
if test_dir:
|
||||
tests = []
|
||||
for root, _, files in os.walk(test_dir):
|
||||
tests.extend([os.path.join(root, test_file) for test_file in files if test_file.endswith('.js')])
|
||||
|
||||
if test_list:
|
||||
dirname = os.path.dirname(test_list)
|
||||
with open(test_list, "r") as test_list_fd:
|
||||
for test in test_list_fd:
|
||||
tests.append(os.path.normpath(os.path.join(dirname, test.rstrip())))
|
||||
|
||||
tests.sort()
|
||||
|
||||
def filter_tests(test):
|
||||
for skipped in skip_list:
|
||||
if skipped in test:
|
||||
return False
|
||||
return True
|
||||
|
||||
return filter(filter_tests, tests)
|
||||
|
||||
|
||||
def get_platform_cmd_prefix():
|
||||
if sys.platform == 'win32':
|
||||
return ['cmd', '/S', '/C']
|
||||
return []
|
||||
|
||||
|
||||
def execute_test_command(test_cmd):
|
||||
process = subprocess.Popen(test_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
|
||||
stdout = process.communicate()[0]
|
||||
return (process.returncode, stdout)
|
||||
|
||||
|
||||
def main(args):
|
||||
tests = get_tests(args.test_dir, args.test_list, args.skip_list)
|
||||
total = len(tests)
|
||||
if total == 0:
|
||||
print("No test to execute.")
|
||||
return 1
|
||||
|
||||
if sys.platform == 'win32':
|
||||
original_timezone = util.get_timezone()
|
||||
util.set_sighdl_to_reset_timezone(original_timezone)
|
||||
util.set_timezone('UTC')
|
||||
|
||||
if args.snapshot:
|
||||
passed = run_snapshot_tests(args, tests)
|
||||
else:
|
||||
passed = run_normal_tests(args, tests)
|
||||
|
||||
if sys.platform == 'win32':
|
||||
util.set_timezone(original_timezone)
|
||||
|
||||
failed = total - passed
|
||||
|
||||
summary_list = [os.path.relpath(args.engine)]
|
||||
if args.snapshot:
|
||||
summary_list.append('--snapshot')
|
||||
if args.test_dir:
|
||||
summary_list.append(os.path.relpath(args.test_dir))
|
||||
if args.test_list:
|
||||
summary_list.append(os.path.relpath(args.test_list))
|
||||
util.print_test_summary(' '.join(summary_list), total, passed, failed)
|
||||
|
||||
return bool(failed)
|
||||
|
||||
|
||||
def run_normal_tests(args, tests):
|
||||
test_cmd = get_platform_cmd_prefix()
|
||||
if args.runtime:
|
||||
test_cmd.append(args.runtime)
|
||||
test_cmd.append(args.engine)
|
||||
|
||||
total = len(tests)
|
||||
tested = 0
|
||||
passed = 0
|
||||
for test in tests:
|
||||
tested += 1
|
||||
test_path = os.path.relpath(test)
|
||||
is_expected_to_fail = os.path.join(os.path.sep, 'fail', '') in test
|
||||
(returncode, stdout) = execute_test_command(test_cmd + [test])
|
||||
|
||||
if bool(returncode) == is_expected_to_fail:
|
||||
passed += 1
|
||||
if not args.quiet:
|
||||
passed_string = 'PASS' + (' (XFAIL)' if is_expected_to_fail else '')
|
||||
util.print_test_result(tested, total, True, passed_string, test_path)
|
||||
else:
|
||||
passed_string = 'FAIL%s (%d)' % (' (XPASS)' if is_expected_to_fail else '', returncode)
|
||||
util.print_test_result(tested, total, False, passed_string, test_path)
|
||||
print("================================================")
|
||||
print(stdout)
|
||||
print("================================================")
|
||||
|
||||
return passed
|
||||
|
||||
|
||||
def run_snapshot_tests(args, tests):
|
||||
execute_snapshot_cmd = get_platform_cmd_prefix()
|
||||
generate_snapshot_cmd = get_platform_cmd_prefix()
|
||||
if args.runtime:
|
||||
execute_snapshot_cmd.append(args.runtime)
|
||||
generate_snapshot_cmd.append(args.runtime)
|
||||
|
||||
execute_snapshot_cmd.extend([args.engine, '--exec-snapshot', 'js.snapshot'])
|
||||
|
||||
# engine: jerry[.exe] -> snapshot generator: jerry-snapshot[.exe]
|
||||
engine = os.path.splitext(args.engine)
|
||||
generate_snapshot_cmd.append(engine[0] + '-snapshot' + engine[1])
|
||||
generate_snapshot_cmd.append('generate')
|
||||
|
||||
total = len(tests)
|
||||
tested = 0
|
||||
passed = 0
|
||||
for test in tests:
|
||||
tested += 1
|
||||
test_path = os.path.relpath(test)
|
||||
is_expected_to_fail = os.path.join(os.path.sep, 'fail', '') in test
|
||||
(returncode, stdout) = execute_test_command(generate_snapshot_cmd + [test])
|
||||
|
||||
if is_expected_to_fail or not returncode:
|
||||
if not args.quiet:
|
||||
passed_string = 'PASS' + (' (XFAIL)' if returncode else '')
|
||||
util.print_test_result(tested, total, True, passed_string, test_path, True)
|
||||
else:
|
||||
util.print_test_result(tested, total, False, 'FAIL (%d)' % (returncode), test_path, True)
|
||||
print("================================================")
|
||||
print(stdout)
|
||||
print("================================================")
|
||||
|
||||
if returncode:
|
||||
if is_expected_to_fail:
|
||||
passed += 1
|
||||
continue
|
||||
|
||||
(returncode, stdout) = execute_test_command(execute_snapshot_cmd)
|
||||
os.remove('js.snapshot')
|
||||
|
||||
if bool(returncode) == is_expected_to_fail:
|
||||
passed += 1
|
||||
if not args.quiet:
|
||||
passed_string = 'PASS' + (' (XFAIL)' if is_expected_to_fail else '')
|
||||
util.print_test_result(tested, total, True, passed_string, test_path, False)
|
||||
else:
|
||||
passed_string = 'FAIL%s (%d)' % (' (XPASS)' if is_expected_to_fail else '', returncode)
|
||||
util.print_test_result(tested, total, False, passed_string, test_path, False)
|
||||
print("================================================")
|
||||
print(stdout)
|
||||
print("================================================")
|
||||
|
||||
return passed
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(get_arguments()))
|
||||
@ -1,227 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright JS Foundation and other contributors, http://js.foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Usage:
|
||||
# ./tools/runners/run-test-suite.sh ENGINE TESTS [-q] [--skip-list=item1,item2] [--snapshot] ENGINE_ARGS....
|
||||
|
||||
TIMEOUT=${TIMEOUT:=5}
|
||||
TIMEOUT_CMD=`which timeout`
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
TIMEOUT_CMD=`which gtimeout`
|
||||
fi
|
||||
|
||||
ENGINE="$1"
|
||||
shift
|
||||
|
||||
TESTS="$1"
|
||||
shift
|
||||
|
||||
OUTPUT_DIR=`dirname $ENGINE`
|
||||
TESTS_BASENAME=`basename $TESTS`
|
||||
|
||||
TEST_FILES=$OUTPUT_DIR/$TESTS_BASENAME.files
|
||||
TEST_FAILED=$OUTPUT_DIR/$TESTS_BASENAME.failed
|
||||
TEST_PASSED=$OUTPUT_DIR/$TESTS_BASENAME.passed
|
||||
|
||||
TERM_NORMAL="\033[0m"
|
||||
TERM_RED="\033[1;31m"
|
||||
TERM_GREEN="\033[1;32m"
|
||||
|
||||
trap 'exit' SIGINT
|
||||
|
||||
VERBOSE=1
|
||||
if [[ "$1" == "-q" ]]
|
||||
then
|
||||
unset VERBOSE
|
||||
shift
|
||||
fi
|
||||
|
||||
if [[ "$1" =~ ^--skip-list=.* ]]
|
||||
then
|
||||
SKIP_LIST=${1#--skip-list=}
|
||||
SKIP_LIST=${SKIP_LIST//,/ }
|
||||
shift
|
||||
fi
|
||||
|
||||
if [ "$1" == "--snapshot" ]
|
||||
then
|
||||
TEST_FILES="$TEST_FILES.snapshot"
|
||||
TEST_FAILED="$TEST_FAILED.snapshot"
|
||||
TEST_PASSED="$TEST_PASSED.snapshot"
|
||||
IS_SNAPSHOT=true
|
||||
|
||||
SNAPSHOT_TOOL=${ENGINE}-snapshot
|
||||
if [ ! -x $SNAPSHOT_TOOL ]
|
||||
then
|
||||
echo "$0: $SNAPSHOT_TOOL: not an executable"
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
fi
|
||||
|
||||
ENGINE_ARGS="$@"
|
||||
|
||||
if [ ! -x $ENGINE ]
|
||||
then
|
||||
echo "$0: $ENGINE: not an executable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -d $TESTS ]
|
||||
then
|
||||
TESTS_DIR=$TESTS
|
||||
|
||||
( cd $TESTS; find . -name "*.js" ) | sort > $TEST_FILES
|
||||
elif [ -f $TESTS ]
|
||||
then
|
||||
TESTS_DIR=`dirname $TESTS`
|
||||
|
||||
grep -e '.js\s*$' $TESTS | sort > $TEST_FILES
|
||||
else
|
||||
echo "$0: $TESTS: not a test suite"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remove the skipped tests from list
|
||||
for TEST in ${SKIP_LIST}
|
||||
do
|
||||
( sed -i -r "/$TEST/d" $TEST_FILES )
|
||||
done
|
||||
|
||||
TOTAL=$(cat $TEST_FILES | wc -l)
|
||||
if [ "$TOTAL" -eq 0 ]
|
||||
then
|
||||
echo "$0: $TESTS: no test in test suite"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f $TEST_FAILED $TEST_PASSED
|
||||
|
||||
ROOT_DIR=""
|
||||
CURRENT_DIR=`pwd`
|
||||
PATH_STEP=2
|
||||
while true
|
||||
do
|
||||
TMP_ROOT_DIR=`(echo "$CURRENT_DIR"; echo "$0"; echo "$ENGINE"; echo "$TESTS") | cut -f1-$PATH_STEP -d/ | uniq -d`
|
||||
if [ -z "$TMP_ROOT_DIR" ]
|
||||
then
|
||||
break
|
||||
else
|
||||
ROOT_DIR="$TMP_ROOT_DIR"
|
||||
fi
|
||||
PATH_STEP=$((PATH_STEP+1))
|
||||
done
|
||||
if [ -n "$ROOT_DIR" ]
|
||||
then
|
||||
ROOT_DIR="$ROOT_DIR/"
|
||||
fi
|
||||
|
||||
tested=1
|
||||
failed=0
|
||||
passed=0
|
||||
|
||||
ENGINE_TEMP=`mktemp engine-out.XXXXXXXXXX`
|
||||
|
||||
for test in `cat $TEST_FILES`
|
||||
do
|
||||
if [[ $test =~ ^\.\/fail\/ ]]
|
||||
then
|
||||
error_code=1
|
||||
PASS="PASS (XFAIL)"
|
||||
else
|
||||
error_code=0
|
||||
PASS="PASS"
|
||||
fi
|
||||
|
||||
full_test=$TESTS_DIR/${test#./}
|
||||
|
||||
if [ "$IS_SNAPSHOT" == true ]
|
||||
then
|
||||
# Testing snapshot
|
||||
SNAPSHOT_TEMP=`mktemp $(basename -s .js $test).snapshot.XXXXXXXXXX`
|
||||
|
||||
$TIMEOUT_CMD $TIMEOUT $RUNTIME $SNAPSHOT_TOOL generate -o $SNAPSHOT_TEMP $full_test &> $ENGINE_TEMP
|
||||
status_code=$?
|
||||
comment=" (generate snapshot)"
|
||||
|
||||
if [ $status_code -eq 0 ]
|
||||
then
|
||||
test $VERBOSE && printf "[%4d/%4d] %bPASS: %s%s%b\n" "$tested" "$TOTAL" "$TERM_GREEN" "${full_test#$ROOT_DIR}" "$comment" "$TERM_NORMAL"
|
||||
|
||||
$TIMEOUT_CMD $TIMEOUT $RUNTIME $ENGINE $ENGINE_ARGS --exec-snapshot $SNAPSHOT_TEMP &> $ENGINE_TEMP
|
||||
status_code=$?
|
||||
comment=" (execute snapshot)"
|
||||
fi
|
||||
|
||||
rm -f $SNAPSHOT_TEMP
|
||||
else
|
||||
$TIMEOUT_CMD $TIMEOUT $RUNTIME $ENGINE $ENGINE_ARGS $full_test &> $ENGINE_TEMP
|
||||
status_code=$?
|
||||
comment=""
|
||||
fi
|
||||
|
||||
if [ $status_code -ne $error_code ]
|
||||
then
|
||||
printf "[%4d/%4d] %bFAIL (%d): %s%s%b\n" "$tested" "$TOTAL" "$TERM_RED" "$status_code" "${full_test#$ROOT_DIR}" "$comment" "$TERM_NORMAL"
|
||||
cat $ENGINE_TEMP
|
||||
|
||||
echo "$status_code: $test" >> $TEST_FAILED
|
||||
echo "============================================" >> $TEST_FAILED
|
||||
cat $ENGINE_TEMP >> $TEST_FAILED
|
||||
echo "============================================" >> $TEST_FAILED
|
||||
echo >> $TEST_FAILED
|
||||
echo >> $TEST_FAILED
|
||||
|
||||
failed=$((failed+1))
|
||||
else
|
||||
test $VERBOSE && printf "[%4d/%4d] %b%s: %s%s%b\n" "$tested" "$TOTAL" "$TERM_GREEN" "$PASS" "${full_test#$ROOT_DIR}" "$comment" "$TERM_NORMAL"
|
||||
echo "$test" >> $TEST_PASSED
|
||||
|
||||
passed=$((passed+1))
|
||||
fi
|
||||
|
||||
tested=$((tested+1))
|
||||
done
|
||||
|
||||
rm -f $ENGINE_TEMP
|
||||
|
||||
ratio=$(echo $passed*100/$TOTAL | bc)
|
||||
if [ $passed -eq $TOTAL ]
|
||||
then
|
||||
success_color=$TERM_GREEN
|
||||
else
|
||||
success_color=$TERM_RED
|
||||
fi
|
||||
|
||||
if [ "$IS_SNAPSHOT" == true ]
|
||||
then
|
||||
ENGINE_ARGS="--snapshot $ENGINE_ARGS"
|
||||
fi
|
||||
|
||||
echo -e "\n[summary] ${ENGINE#$ROOT_DIR} $ENGINE_ARGS ${TESTS#$ROOT_DIR}\n"
|
||||
echo -e "TOTAL: $TOTAL"
|
||||
echo -e "${TERM_GREEN}PASS: $passed${TERM_NORMAL}"
|
||||
echo -e "${TERM_RED}FAIL: $failed${TERM_NORMAL}\n"
|
||||
echo -e "${success_color}Success: $ratio%${TERM_NORMAL}\n"
|
||||
|
||||
if [ $failed -ne 0 ]
|
||||
then
|
||||
echo "$0: see $TEST_FAILED for details about failures"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@ -21,14 +21,16 @@ import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
TERM_NORMAL = '\033[0m'
|
||||
TERM_RED = '\033[1;31m'
|
||||
TERM_GREEN = '\033[1;32m'
|
||||
import util
|
||||
|
||||
|
||||
def get_arguments():
|
||||
runtime = os.environ.get('RUNTIME')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-q', '--quiet', action='store_true',
|
||||
help='Only print out failing tests')
|
||||
parser.add_argument('--runtime', metavar='FILE', default=runtime,
|
||||
help='Execution runtime (e.g. qemu)')
|
||||
parser.add_argument('path',
|
||||
help='Path of test binaries')
|
||||
|
||||
@ -54,39 +56,30 @@ def main(args):
|
||||
print("%s: no unit-* test to execute", args.path)
|
||||
return 1
|
||||
|
||||
test_cmd = [args.runtime] if args.runtime else []
|
||||
|
||||
tested = 0
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
runtime = os.environ.get('RUNTIME')
|
||||
test_cmd = [runtime] if runtime else []
|
||||
|
||||
for test in unittests:
|
||||
tested += 1
|
||||
testpath = os.path.relpath(test)
|
||||
test_path = os.path.relpath(test)
|
||||
try:
|
||||
subprocess.check_output(test_cmd + [test], stderr=subprocess.STDOUT, universal_newlines=True)
|
||||
passed += 1
|
||||
if not args.quiet:
|
||||
print("[%4d/%4d] %sPASS: %s%s" % (tested, total, TERM_GREEN, testpath, TERM_NORMAL))
|
||||
util.print_test_result(tested, total, True, 'PASS', test_path)
|
||||
except subprocess.CalledProcessError as err:
|
||||
failed += 1
|
||||
print("[%4d/%4d] %sFAIL (%d): %s%s" % (tested, total, TERM_RED, err.returncode, testpath, TERM_NORMAL))
|
||||
util.print_test_result(tested, total, False, 'FAIL (%d)' % err.returncode, test_path)
|
||||
print("================================================")
|
||||
print(err.output)
|
||||
print("================================================")
|
||||
|
||||
print("\n[summary] %s\n" % os.path.join(os.path.relpath(args.path), "unit-*"))
|
||||
print("TOTAL: %d" % total)
|
||||
print("%sPASS: %d%s" % (TERM_GREEN, passed, TERM_NORMAL))
|
||||
print("%sFAIL: %d%s\n" % (TERM_RED, failed, TERM_NORMAL))
|
||||
|
||||
success_color = TERM_GREEN if passed == total else TERM_RED
|
||||
print("%sSuccess: %d%%%s" % (success_color, passed*100/total, TERM_NORMAL))
|
||||
util.print_test_summary(os.path.join(os.path.relpath(args.path), "unit-*"), total, passed, failed)
|
||||
|
||||
if failed > 0:
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
67
tools/runners/util.py
Executable file
67
tools/runners/util.py
Executable file
@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright JS Foundation and other contributors, http://js.foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
TERM_NORMAL = '\033[0m'
|
||||
TERM_RED = '\033[1;31m'
|
||||
TERM_GREEN = '\033[1;32m'
|
||||
|
||||
|
||||
def set_timezone(timezone):
|
||||
assert sys.platform == 'win32', "set_timezone is Windows only function"
|
||||
subprocess.call(['cmd', '/S', '/C', 'tzutil', '/s', timezone])
|
||||
|
||||
|
||||
def set_timezone_and_exit(timezone):
|
||||
assert sys.platform == 'win32', "set_timezone_and_exit is Windows only function"
|
||||
set_timezone(timezone)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def get_timezone():
|
||||
assert sys.platform == 'win32', "get_timezone is Windows only function"
|
||||
return subprocess.check_output(['cmd', '/S', '/C', 'tzutil', '/g'])
|
||||
|
||||
|
||||
def set_sighdl_to_reset_timezone(timezone):
|
||||
assert sys.platform == 'win32', "install_signal_handler_to_restore_timezone is Windows only function"
|
||||
signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(timezone))
|
||||
|
||||
|
||||
def print_test_summary(summary_string, total, passed, failed):
|
||||
print("\n[summary] %s\n" % summary_string)
|
||||
print("TOTAL: %d" % total)
|
||||
print("%sPASS: %d%s" % (TERM_GREEN, passed, TERM_NORMAL))
|
||||
print("%sFAIL: %d%s\n" % (TERM_RED, failed, TERM_NORMAL))
|
||||
|
||||
success_color = TERM_GREEN if passed == total else TERM_RED
|
||||
print("%sSuccess: %d%%%s" % (success_color, passed*100/total, TERM_NORMAL))
|
||||
|
||||
|
||||
def print_test_result(tested, total, is_passed, passed_string, test_path, is_snapshot_generation=None):
|
||||
if is_snapshot_generation is None:
|
||||
snapshot_string = ''
|
||||
elif is_snapshot_generation:
|
||||
snapshot_string = ' (generate snapshot)'
|
||||
else:
|
||||
snapshot_string = ' (execute snapshot)'
|
||||
|
||||
color = TERM_GREEN if is_passed else TERM_RED
|
||||
print("[%4d/%4d] %s%s: %s%s%s" % (tested, total, color, passed_string, test_path, snapshot_string, TERM_NORMAL))
|
||||
@ -34,7 +34,7 @@ LICENSE_SCRIPT = path.join(TOOLS_DIR, 'check-license.py')
|
||||
MAGIC_STRINGS_SCRIPT = path.join(TOOLS_DIR, 'check-magic-strings.sh')
|
||||
PYLINT_SCRIPT = path.join(TOOLS_DIR, 'check-pylint.sh')
|
||||
SIGNED_OFF_SCRIPT = path.join(TOOLS_DIR, 'check-signed-off.sh')
|
||||
TEST_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-test-suite.sh')
|
||||
TEST_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-test-suite.py')
|
||||
TEST262_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-test-suite-test262.py')
|
||||
VERA_SCRIPT = path.join(TOOLS_DIR, 'check-vera.sh')
|
||||
UNITTEST_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-unittests.py')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user