mirror of
https://github.com/jerryscript-project/jerryscript.git
synced 2025-12-15 16:29:21 +00:00
Test 'test262' test suite on Travis CI (#1440)
JerryScript-DCO-1.0-Signed-off-by: László Langó llango.u-szeged@partner.samsung.com
This commit is contained in:
László Langó
GitHub
parent
2622e938d8
commit
cf7b7a1090
@ -8,8 +8,9 @@ env:
|
||||
- OPTS="--check-signed-off-travis --check-cppcheck --check-vera --check-license"
|
||||
- OPTS="--jerry-tests --jerry-test-suite"
|
||||
- OPTS="--jerry-tests --jerry-test-suite --toolchain=cmake/toolchain_linux_armv7l.cmake" TIMEOUT=300 INSTALL_QEMU_ARM=yes
|
||||
- OPTS=--buildoption-test
|
||||
- OPTS=--unittests
|
||||
- OPTS="--buildoption-test"
|
||||
- OPTS="--unittests"
|
||||
- OPTS="--test262"
|
||||
|
||||
matrix:
|
||||
include:
|
||||
@ -22,6 +23,7 @@ before_install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then tools/apt-get-install-deps.sh; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$INSTALL_QEMU_ARM" == "yes" ]]; then tools/apt-get-install-qemu-arm.sh; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then tools/brew-install-deps.sh; fi
|
||||
- if [[ "$OPTS" == "--test262" ]]; then sudo timedatectl set-timezone America/Los_Angeles; fi
|
||||
|
||||
install:
|
||||
|
||||
|
||||
@ -37,6 +37,7 @@ parser.add_argument('--jerry-tests', action='store_true', default=False, help='R
|
||||
parser.add_argument('--jerry-test-suite', action='store_true', default=False, help='Run jerry-test-suite')
|
||||
parser.add_argument('--unittests', action='store_true', default=False, help='Run unittests')
|
||||
parser.add_argument('--precommit', action='store_true', default=False, dest='all', help='Run all test')
|
||||
parser.add_argument('--test262', action='store_true', default=False, help='Run test262')
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
parser.print_help()
|
||||
@ -50,11 +51,13 @@ else:
|
||||
OUTPUT_DIR = path.join(PROJECT_DIR, script_args.outdir)
|
||||
|
||||
class Options:
|
||||
out_dir = ''
|
||||
build_args = []
|
||||
test_args = []
|
||||
def __init__(self, name = '', build_args = None, test_args = None):
|
||||
if build_args is None:
|
||||
build_args = []
|
||||
|
||||
if test_args is None:
|
||||
test_args = []
|
||||
|
||||
def __init__(self, name = '', build_args = [], test_args = []):
|
||||
self.out_dir = path.join(OUTPUT_DIR, name)
|
||||
self.build_args = build_args
|
||||
self.build_args.append('--builddir=%s' % self.out_dir)
|
||||
@ -63,15 +66,15 @@ class Options:
|
||||
|
||||
# Test options for unittests
|
||||
jerry_unittests_options = [
|
||||
Options('unittests', ['--unittests', '--error-messages=on', '--snapshot-save=on', '--snapshot-exec=on']),
|
||||
Options('unittests-debug', ['--unittests', '--debug', '--error-messages=on', '--snapshot-save=on', '--snapshot-exec=on']),
|
||||
Options('unittests', ['--unittests', '--error-messages=on', '--snapshot-save=on', '--snapshot-exec=on']),
|
||||
Options('unittests-debug', ['--unittests', '--debug', '--error-messages=on', '--snapshot-save=on', '--snapshot-exec=on']),
|
||||
]
|
||||
|
||||
# Test options for jerry-tests
|
||||
jerry_tests_options = [
|
||||
Options('jerry_tests'),
|
||||
Options('jerry_tests-snapshot', ['--snapshot-save=on', '--snapshot-exec=on'], ['--snapshot']),
|
||||
Options('jerry_tests-debug', ['--debug']),
|
||||
Options('jerry_tests-snapshot', ['--snapshot-save=on', '--snapshot-exec=on'], ['--snapshot']),
|
||||
Options('jerry_tests-debug-snapshot', ['--debug', '--snapshot-save=on', '--snapshot-exec=on'], ['--snapshot']),
|
||||
]
|
||||
|
||||
@ -82,6 +85,11 @@ jerry_test_suite_options.append(Options('jerry_test_suite-minimal-snapshot', ['-
|
||||
jerry_test_suite_options.append(Options('jerry_test_suite-minimal-debug', ['--debug', '--profile=minimal']))
|
||||
jerry_test_suite_options.append(Options('jerry_test_suite-minimal-debug-snapshot', ['--debug', '--profile=minimal', '--snapshot-save=on', '--snapshot-exec=on'], ['--snapshot']))
|
||||
|
||||
# Test options for test262
|
||||
test262_test_suite_options = [
|
||||
Options('test262_tests'),
|
||||
]
|
||||
|
||||
# Test options for buildoption-test
|
||||
jerry_buildoptions = [
|
||||
Options('buildoption_test-lto', ['--lto=on']),
|
||||
@ -163,6 +171,22 @@ def run_jerry_test_suite():
|
||||
|
||||
return ret_build | ret_test
|
||||
|
||||
def run_test262_test_suite():
|
||||
ret_build = ret_test = 0
|
||||
for job in test262_test_suite_options:
|
||||
ret_build = create_binary(job.build_args)
|
||||
if ret_build:
|
||||
break
|
||||
|
||||
test_cmd = [TEST262_RUNNER_SCRIPT, get_binary_path(job.out_dir), TEST262_TEST_SUITE_DIR]
|
||||
|
||||
if job.test_args:
|
||||
test_cmd.extend(job.test_args)
|
||||
|
||||
ret_test |= run_check(test_cmd)
|
||||
|
||||
return ret_build | ret_test
|
||||
|
||||
def run_unittests():
|
||||
ret_build = ret_test = 0
|
||||
for job in jerry_unittests_options:
|
||||
@ -209,6 +233,9 @@ def main():
|
||||
if not ret and (script_args.all or script_args.jerry_test_suite):
|
||||
ret = run_jerry_test_suite()
|
||||
|
||||
if not ret and (script_args.all or script_args.test262):
|
||||
ret = run_test262_test_suite()
|
||||
|
||||
if not ret and (script_args.all or script_args.unittests):
|
||||
ret = run_unittests()
|
||||
|
||||
|
||||
@ -16,10 +16,9 @@
|
||||
|
||||
ENGINE="$1"
|
||||
PATH_TO_TEST262="$2"
|
||||
REPORT_PREFIX="report"
|
||||
RUN_PIDS=""
|
||||
RESULT_OK=1
|
||||
TIMEOUT="5s"
|
||||
OUTPUT_DIR=`dirname $ENGINE`
|
||||
REPORT_PATH="${OUTPUT_DIR}/test262.report"
|
||||
TIMEOUT="90s"
|
||||
|
||||
if [ $# -lt 2 ]
|
||||
then
|
||||
@ -28,91 +27,44 @@ then
|
||||
echo "Usage:"
|
||||
echo " 1st parameter: JavaScript engine to be tested."
|
||||
echo " 2nd parameter: path to the directory with official test262 testsuite."
|
||||
echo " 3rd parameter: (optional) call this script with the '--sub-chapters' flag to print the detailed report."
|
||||
echo ""
|
||||
echo "Example:"
|
||||
echo " ./run-test-suite-test262.sh <engine> <test262_dir> --sub-chapters"
|
||||
echo " ./run-test-suite-test262.sh <engine> <test262_dir>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
rm "${REPORT_PREFIX}".* &> /dev/null
|
||||
|
||||
declare -a CHAPTER07_TO_TEST=("7.1" "7.2" "7.3" "7.4" "7.5" "7.6" "7.6.1" "7.7" \
|
||||
"7.8" "7.8.1" "7.8.2" "7.8.3" "7.8.4" "7.8.5" "7.9" "7.9.2")
|
||||
declare -a CHAPTER08_TO_TEST=("8.1" "8.2" "8.3" "8.4" "8.5" "8.6" "8.6.1" "8.6.2" "8.7" "8.7.1" "8.7.2" "8.8" "8.12" \
|
||||
"8.12.1" "8.12.3" "8.12.4" "8.12.5" "8.12.6" "8.12.7" "8.12.8" "8.12.9")
|
||||
declare -a CHAPTER09_TO_TEST=("9.1" "9.2" "9.3" "9.3.1" "9.4" "9.5" "9.6" "9.7" "9.8" "9.8.1" "9.9")
|
||||
declare -a CHAPTER10_TO_TEST=("10.1" "10.1.1" "10.2" "10.2.1" "10.2.2" "10.2.3" "10.3" "10.3.1" "10.4" "10.4.1" \
|
||||
"10.4.2" "10.4.3" "10.5" "10.6")
|
||||
declare -a CHAPTER11_TO_TEST=("11.1" "11.1.1" "11.1.2" "11.1.3" "11.1.4" "11.1.5" "11.1.6" "11.2" "11.2.1" "11.2.2" \
|
||||
"11.2.3" "11.2.4" "11.3" "11.3.1" "11.3.2" "11.4" "11.4.1" "11.4.2" "11.4.3" \
|
||||
"11.4.4" "11.4.5" "11.4.6" "11.4.7" "11.4.8" "11.4.9" "11.5" "11.5.1" "11.5.2" "11.5.3" \
|
||||
"11.6" "11.6.1" "11.6.2" "11.7" "11.7.1" "11.7.2" "11.7.3" "11.8" "11.8.1" "11.8.2" \
|
||||
"11.8.3" "11.8.4" "11.8.6" "11.8.7" "11.9" "11.9.1" "11.9.2" "11.9.4" "11.9.5" "11.10" \
|
||||
"11.11" "11.12" "11.13" "11.13.1" "11.13.2" "11.14")
|
||||
declare -a CHAPTER12_TO_TEST=("12.1" "12.2" "12.2.1" "12.3" "12.4" "12.5" "12.6" "12.6.1" "12.6.2" "12.6.3" "12.6.4" \
|
||||
"12.7" "12.8" "12.9" "12.10" "12.10.1" "12.11" "12.12" "12.13" "12.14" "12.14.1")
|
||||
declare -a CHAPTER13_TO_TEST=("13.1" "13.2" "13.2.1" "13.2.2" "13.2.3")
|
||||
declare -a CHAPTER14_TO_TEST=("14.1")
|
||||
declare -a CHAPTER14_TO_TEST=("12.6.4")
|
||||
declare -a CHAPTER15_TO_TEST=("15.1" "15.1.1" "15.1.2" "15.1.3" "15.1.4" "15.1.5" "15.2" "15.2.1" "15.2.2" "15.2.3" \
|
||||
"15.2.4" "15.2.5" "15.3" "15.3.1" "15.3.2" "15.3.3" "15.3.4" "15.3.5" "15.4" "15.4.1" \
|
||||
"15.4.2" "15.4.3" "15.4.4" "15.4.5" "15.5" "15.5.1" "15.5.2" "15.5.3" "15.5.4" "15.5.5" \
|
||||
"15.6" "15.6.1" "15.6.2" "15.6.3" "15.6.4" "15.6.5" "15.7" "15.7.1" "15.7.2" "15.7.3" \
|
||||
"15.7.4" "15.7.5" "15.8" "15.8.1" "15.8.2" "15.9" "15.9.1" "15.9.2" "15.9.3" "15.9.4" \
|
||||
"15.9.5" "15.9.6" "15.10" "15.10.1" "15.10.2" "15.10.3" "15.10.4" "15.10.5" "15.10.6" \
|
||||
"15.10.7" "15.11" "15.11.1" "15.11.2" "15.11.3" "15.11.4" "15.11.5" "15.11.6" "15.11.7" \
|
||||
"15.12" "15.12.1" "15.12.2" "15.12.3")
|
||||
declare -a FULL_CHAPTERS_TO_TEST=("ch06" "ch07" "ch08" "ch09" "ch10" "ch11" "ch12" "ch13" "ch14" "ch15")
|
||||
declare -a SUB_CHAPTERS_TO_TEST=("${CHAPTER07_TO_TEST[@]}" "${CHAPTER08_TO_TEST[@]}" "${CHAPTER09_TO_TEST[@]}" \
|
||||
"${CHAPTER10_TO_TEST[@]}" "${CHAPTER11_TO_TEST[@]}" "${CHAPTER12_TO_TEST[@]}" \
|
||||
"${CHAPTER13_TO_TEST[@]}" "${CHAPTER14_TO_TEST[@]}" "${CHAPTER15_TO_TEST[@]}")
|
||||
|
||||
declare -a CHAPTERS_TO_TEST=("${FULL_CHAPTERS_TO_TEST[@]}")
|
||||
|
||||
if [[ $* == *--sub-chapters* ]]
|
||||
if [ ! -d "${PATH_TO_TEST262}/.git" ]
|
||||
then
|
||||
declare -a CHAPTERS_TO_TEST=("${SUB_CHAPTERS_TO_TEST[@]}")
|
||||
git clone https://github.com/tc39/test262.git -b es5-tests "${PATH_TO_TEST262}"
|
||||
fi
|
||||
|
||||
function run_test262 () {
|
||||
ARG_ENGINE="$1"
|
||||
ARG_TEST262_PATH="$2"
|
||||
ARG_CHAPTER="$3"
|
||||
rm -rf "${PATH_TO_TEST262}/test/suite/bestPractice"
|
||||
rm -rf "${PATH_TO_TEST262}/test/suite/intl402"
|
||||
|
||||
"${ARG_TEST262_PATH}"/tools/packaging/test262.py --command "timeout ${TIMEOUT} ${ARG_ENGINE}" \
|
||||
--tests="${ARG_TEST262_PATH}" --full-summary "${ARG_CHAPTER}" \
|
||||
> "${REPORT_PREFIX}"."${ARG_CHAPTER}"
|
||||
}
|
||||
echo "Starting test262 testing for ${ENGINE}. Running test262 may take a several minutes."
|
||||
|
||||
function show_report_results () {
|
||||
ARG_CHAPTER="$1"
|
||||
"${PATH_TO_TEST262}"/tools/packaging/test262.py --command "timeout ${TIMEOUT} ${ENGINE}" \
|
||||
--tests="${PATH_TO_TEST262}" --summary \
|
||||
&> "${REPORT_PATH}"
|
||||
|
||||
echo ""
|
||||
echo "Chapter ${ARG_CHAPTER}:"
|
||||
grep -A3 "=== Summary ===" "${REPORT_PREFIX}"."${ARG_CHAPTER}"
|
||||
echo "==============="
|
||||
}
|
||||
grep -A3 "=== Summary ===" "${REPORT_PATH}"
|
||||
|
||||
echo "Starting test262 testing for ${ENGINE}."
|
||||
FAILURES=`sed -n '/Failed tests/,/^$/p' "${REPORT_PATH}"`
|
||||
|
||||
for TEST_NAME in "${CHAPTERS_TO_TEST[@]}"
|
||||
do
|
||||
run_test262 "${ENGINE}" "${PATH_TO_TEST262}" "$TEST_NAME" &
|
||||
RUN_PIDS="$RUN_PIDS $!";
|
||||
done
|
||||
EXIT_CODE=0
|
||||
if [ -n "$FAILURES" ]
|
||||
then
|
||||
echo -e "\n$FAILURES\n"
|
||||
echo "$0: see ${REPORT_PATH} for details about failures"
|
||||
EXIT_CODE=1
|
||||
fi
|
||||
|
||||
for RUN_PIDS in $RUN_PIDS
|
||||
do
|
||||
wait "$RUN_PIDS" || RESULT_OK=0
|
||||
done;
|
||||
#[ $RESULT_OK -eq 1 ] || exit 1
|
||||
FAILURES=`sed -n '/Expected to fail but passed/,/^$/p' "${REPORT_PATH}"`
|
||||
if [ -n "$FAILURES" ]
|
||||
then
|
||||
echo -e "\n$FAILURES\n"
|
||||
echo "$0: see ${REPORT_PATH} for details about failures"
|
||||
EXIT_CODE=1
|
||||
fi
|
||||
|
||||
echo "Testing is completed."
|
||||
|
||||
for TEST_NAME in "${CHAPTERS_TO_TEST[@]}"
|
||||
do
|
||||
echo "$TEST_NAME"
|
||||
show_report_results "$TEST_NAME"
|
||||
done
|
||||
exit $EXIT_CODE
|
||||
|
||||
@ -22,6 +22,7 @@ PROJECT_DIR = path.normpath(path.join(TOOLS_DIR, '..'))
|
||||
JERRY_TESTS_DIR = path.join(PROJECT_DIR, 'tests/jerry')
|
||||
JERRY_TEST_SUITE_DIR = path.join(PROJECT_DIR, 'tests/jerry-test-suite')
|
||||
JERRY_TEST_SUITE_MINIMAL_LIST = path.join(PROJECT_DIR, 'tests/jerry-test-suite/minimal-profile-list')
|
||||
TEST262_TEST_SUITE_DIR = path.join(PROJECT_DIR, 'tests/test262')
|
||||
|
||||
BUILD_SCRIPT = path.join(TOOLS_DIR, 'build.py')
|
||||
CPPCHECK_SCRIPT = path.join(TOOLS_DIR, 'check-cppcheck.sh')
|
||||
@ -29,4 +30,5 @@ SIGNED_OFF_SCRIPT = path.join(TOOLS_DIR, 'check-signed-off.sh')
|
||||
VERA_SCRIPT = path.join(TOOLS_DIR, 'check-vera.sh')
|
||||
LICENSE_SCRIPT = path.join(TOOLS_DIR, 'check-license.py')
|
||||
TEST_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-test-suite.sh')
|
||||
TEST262_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-test-suite-test262.sh')
|
||||
UNITTEST_RUNNER_SCRIPT = path.join(TOOLS_DIR, 'runners/run-unittests.sh')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user