From e3decffdd3dcea297a9e48146c0fffe381bcb371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Csaba=20Osztrogon=C3=A1c?= Date: Wed, 15 Jan 2020 09:14:51 +0100 Subject: [PATCH] Make run-test-suite.py handle crashes properly (#3512) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - Crashing jerry / jerry-snapshot is always "FAIL", only the 1 return code is accepted as expected fail. - "FAIL (XPASS)" string is only used if the test marked as expected fail, but the return code is 0. - Simple "FAIL" string is used if the test marked as expected fail, but the return code is not 0 or 1. (crash) JerryScript-DCO-1.0-Signed-off-by: Csaba Osztrogonác oszi@inf.u-szeged.hu --- tools/runners/run-test-suite.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/runners/run-test-suite.py b/tools/runners/run-test-suite.py index 4dfc7fc85..c92abaa18 100755 --- a/tools/runners/run-test-suite.py +++ b/tools/runners/run-test-suite.py @@ -138,13 +138,13 @@ def run_normal_tests(args, tests): is_expected_to_fail = os.path.join(os.path.sep, 'fail', '') in test (returncode, stdout) = execute_test_command(test_cmd + [test]) - if bool(returncode) == is_expected_to_fail: + if (returncode == 0 and not is_expected_to_fail) or (returncode == 1 and is_expected_to_fail): passed += 1 if not args.quiet: passed_string = 'PASS' + (' (XFAIL)' if is_expected_to_fail else '') util.print_test_result(tested, total, True, passed_string, test_path) else: - passed_string = 'FAIL%s (%d)' % (' (XPASS)' if is_expected_to_fail else '', returncode) + passed_string = 'FAIL%s (%d)' % (' (XPASS)' if returncode == 0 and is_expected_to_fail else '', returncode) util.print_test_result(tested, total, False, passed_string, test_path) print("================================================") print(stdout) @@ -176,7 +176,7 @@ def run_snapshot_tests(args, tests): is_expected_to_fail = os.path.join(os.path.sep, 'fail', '') in test (returncode, stdout) = execute_test_command(generate_snapshot_cmd + [test]) - if is_expected_to_fail or not returncode: + if (returncode == 0) or (returncode == 1 and is_expected_to_fail): if not args.quiet: passed_string = 'PASS' + (' (XFAIL)' if returncode else '') util.print_test_result(tested, total, True, passed_string, test_path, True) @@ -194,13 +194,13 @@ def run_snapshot_tests(args, tests): (returncode, stdout) = execute_test_command(execute_snapshot_cmd) os.remove('js.snapshot') - if bool(returncode) == is_expected_to_fail: + if (returncode == 0 and not is_expected_to_fail) or (returncode == 1 and is_expected_to_fail): passed += 1 if not args.quiet: passed_string = 'PASS' + (' (XFAIL)' if is_expected_to_fail else '') util.print_test_result(tested, total, True, passed_string, test_path, False) else: - passed_string = 'FAIL%s (%d)' % (' (XPASS)' if is_expected_to_fail else '', returncode) + passed_string = 'FAIL%s (%d)' % (' (XPASS)' if returncode == 0 and is_expected_to_fail else '', returncode) util.print_test_result(tested, total, False, passed_string, test_path, False) print("================================================") print(stdout)