From 73ab46ea5eb4a758c337e89fa45b8c6157cca7ae Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Mon, 11 Sep 2023 21:30:44 -0300 Subject: [PATCH] test: add option to rerun failed tests only --- test.py | 126 ++++++++++++++++++++++------ utils/tests/TestBase.py | 10 ++- utils/tests/gitlab-ci-scheduled.yml | 8 +- utils/tests/test-test.py | 21 ++--- 4 files changed, 125 insertions(+), 40 deletions(-) diff --git a/test.py b/test.py index 87e5c6b67..083610081 100755 --- a/test.py +++ b/test.py @@ -15,18 +15,23 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # -import os -import sys -import time import argparse -import subprocess -import threading -import signal -import shutil import fnmatch +import os +import shutil +import signal +import subprocess +import sys +import threading +import time +import xml.etree.ElementTree as ET + from utils import get_list_from_file +# Global variable +args = None + # imported from waflib Logs colors_lst={'USE':True,'BOLD':'\x1b[01;1m','RED':'\x1b[01;31m','GREEN':'\x1b[32m','YELLOW':'\x1b[33m','PINK':'\x1b[35m','BLUE':'\x1b[01;34m','CYAN':'\x1b[36m','GREY':'\x1b[37m','NORMAL':'\x1b[0m','cursor_on':'\x1b[?25h','cursor_off':'\x1b[?25l',} def get_color(cl): @@ -271,9 +276,8 @@ def node_to_text(test, f, test_type='Suite'): node_to_text(child, f, 'Case') def translate_to_text(results_file, text_file): - text_file += '.txt' + text_file += ('.txt' if '.txt' not in text_file else '') print('Writing results to text file \"%s\"...' % text_file, end='') - import xml.etree.ElementTree as ET et = ET.parse(results_file) with open(text_file, 'w', encoding='utf-8') as f: @@ -299,7 +303,7 @@ def translate_to_text(results_file, text_file): # since it will probably grow over time. # def translate_to_html(results_file, html_file): - html_file += '.html' + html_file += ('.html' if '.html' not in html_file else '') print('Writing results to html file %s...' % html_file, end='') with open(html_file, 'w', encoding='utf-8') as f: @@ -310,7 +314,6 @@ def translate_to_html(results_file, html_file): # # Read and parse the whole results file. # - import xml.etree.ElementTree as ET et = ET.parse(results_file) # @@ -391,7 +394,7 @@ def translate_to_html(results_file, html_file): f.write("Failure Details\n") # - # Now iterate through all of the test cases. + # Now iterate through all the test cases. # for case in suite.findall('Test'): @@ -502,7 +505,7 @@ def translate_to_html(results_file, html_file): f.write("Details\n") # - # Now iterate through all of the examples + # Now iterate through all the examples # for example in et.findall("Example"): @@ -1023,7 +1026,7 @@ class worker_thread(threading.Thread): # know. It will be something like "examples/udp/udp-echo" or # "examples/wireless/mixed-wireless.py" # - (job.returncode, standard_out, standard_err, et) = run_job_synchronously(job.shell_command, + (job.returncode, job.standard_out, job.standard_err, et) = run_job_synchronously(job.shell_command, job.cwd, args.valgrind, job.is_pyexample, job.build_path) else: # @@ -1035,7 +1038,7 @@ class worker_thread(threading.Thread): update_data = '--update-data' else: update_data = '' - (job.returncode, standard_out, standard_err, et) = run_job_synchronously(job.shell_command + + (job.returncode, job.standard_out, job.standard_err, et) = run_job_synchronously(job.shell_command + " --xml --tempdir=%s --out=%s %s" % (job.tempdir, job.tmp_file_name, update_data), job.cwd, args.valgrind, False) @@ -1044,13 +1047,42 @@ class worker_thread(threading.Thread): if args.verbose: print("returncode = %d" % job.returncode) print("---------- begin standard out ----------") - print(standard_out) + print(job.standard_out) print("---------- begin standard err ----------") - print(standard_err) + print(job.standard_err) print("---------- end standard err ----------") self.output_queue.put(job) +# +# This function loads the list of previously successful or skipped examples and test suites. +# +def load_previously_successful_tests(): + import glob + previously_run_tests_to_skip = {"test": [], "example": []} + previous_results = glob.glob(f"{TMP_OUTPUT_DIR}/*-results.xml") + if not previous_results: + print("No previous runs to rerun") + exit(-1) + latest_result_file = list(sorted(previous_results, key=lambda x: os.path.basename(x), reverse=True))[0] + + try: + previous_run_results = ET.parse(latest_result_file) + except ET.ParseError: + print(f"Failed to parse XML {latest_result_file}") + exit(-1) + + for test_type in ["Test", "Example"]: + if previous_run_results.find(test_type): + temp = list(map(lambda x: (x.find('Name').text, x.find('Result').text), + previous_run_results.findall(test_type) + ) + ) + temp = list(filter(lambda x: x[1] in ['PASS', 'SKIP'], temp)) + temp = [x[0] for x in temp] + previously_run_tests_to_skip[test_type.lower()] = temp + return previously_run_tests_to_skip + # # This is the main function that does the work of interacting with the # test-runner itself. @@ -1288,11 +1320,20 @@ def run_tests(): if not os.path.exists(testpy_output_dir): os.makedirs(testpy_output_dir) + # + # Load results from the latest results.xml, then use the list of + # failed tests to filter out (SKIP) successful tests + # + previously_run_tests_to_skip = {"test": [], "example": []} + if args.rerun_failed: + previously_run_tests_to_skip = load_previously_successful_tests() + # # Create the main output file and start filling it with XML. We need to # do this since the tests will just append individual results to this file. + # The file is created outside the directory that gets automatically deleted. # - xml_results_file = os.path.join(testpy_output_dir, "results.xml") + xml_results_file = os.path.join(TMP_OUTPUT_DIR, f"{date_and_time}-results.xml") with open(xml_results_file, 'w', encoding='utf-8') as f: f.write('\n') f.write('\n') @@ -1330,7 +1371,8 @@ def run_tests(): if isinstance(suites, bytes): suites = suites.decode() - suites_found = fnmatch.filter(suites.split('\n'), args.suite) + suites = suites.replace("\r\n", "\n") + suites_found = fnmatch.filter(suites.split("\n"), args.suite) if not suites_found: print('The test suite was not run because an unknown test suite name was requested.', file=sys.stderr) @@ -1440,7 +1482,7 @@ def run_tests(): # Dispatching will run with unlimited speed and the worker threads will # execute as fast as possible from the queue. # - # Note that we actually dispatch tests to be skipped, so all of the + # Note that we actually dispatch tests to be skipped, so all the # PASS, FAIL, CRASH and SKIP processing is done in the same place. # for test in suite_list: @@ -1472,6 +1514,10 @@ def run_tests(): job.set_is_skip(True) job.set_skip_reason("crashes valgrind") + if args.rerun_failed and test in previously_run_tests_to_skip["test"]: + job.is_skip = True + job.set_skip_reason("didn't fail in the previous run") + if args.verbose: print("Queue %s" % test) @@ -1538,6 +1584,10 @@ def run_tests(): job.set_is_skip(True) job.set_skip_reason("skip in valgrind runs") + if args.rerun_failed and name in previously_run_tests_to_skip["example"]: + job.is_skip = True + job.set_skip_reason("didn't fail in the previous run") + if args.verbose: print("Queue %s" % test) @@ -1713,6 +1763,7 @@ def run_tests(): crashed_testnames = [] valgrind_errors = 0 valgrind_testnames = [] + failed_jobs = [] for i in range(jobs): job = output_queue.get() if job.is_break: @@ -1729,10 +1780,12 @@ def run_tests(): skipped_tests = skipped_tests + 1 skipped_testnames.append(job.display_name + (" (%s)" % job.skip_reason)) else: + failed_jobs.append(job) if job.returncode == 0: status = "PASS" status_print = colors.GREEN + status + colors.NORMAL passed_tests = passed_tests + 1 + failed_jobs.pop() elif job.returncode == 1: failed_tests = failed_tests + 1 failed_testnames.append(job.display_name) @@ -1749,7 +1802,7 @@ def run_tests(): status = "CRASH" status_print = colors.PINK + status + colors.NORMAL - print("[%d/%d]" % (passed_tests + failed_tests + skipped_tests + crashed_tests, total_tests), end=' ') + print("[%d/%d]" % (i, total_tests), end=' ') if args.duration or args.constrain == "performance": print("%s (%.3f): %s %s" % (status_print, job.elapsed_time, kind, job.display_name)) else: @@ -1837,9 +1890,18 @@ def run_tests(): f.write(" %s\n" % job.skip_reason) f.write("\n") else: + failed_jobs.append(job) if job.returncode == 0 or job.returncode == 1 or job.returncode == 2: with open(xml_results_file, 'a', encoding='utf-8') as f_to, open(job.tmp_file_name, encoding='utf-8') as f_from: - f_to.write(f_from.read()) + contents = f_from.read() + if status == "VALGR": + pre = contents.find("") + len("") + post = contents.find("") + contents = contents[:pre] + "VALGR" + contents[post:] + f_to.write(contents) + et = ET.parse(job.tmp_file_name) + if et.find("Result").text in ["PASS", "SKIP"]: + failed_jobs.pop() else: with open(xml_results_file, 'a', encoding='utf-8') as f: f.write("\n") @@ -1884,8 +1946,18 @@ def run_tests(): if valgrind_testnames: valgrind_testnames.sort() print('List of VALGR failures:\n %s' % '\n '.join(map(str, valgrind_testnames))) + + if failed_jobs and args.verbose_failed: + for job in failed_jobs: + if job.standard_out or job.standard_err: + job_type = 'example' if (job.is_example or job.is_pyexample) else 'test suite' + print(f"===================== Begin of {job_type} '{job.display_name}' stdout =====================") + print(job.standard_out) + print(f"===================== Begin of {job_type} '{job.display_name}' stderr =====================") + print(job.standard_err) + print(f"===================== End of {job_type} '{job.display_name}' ==============================") # - # The last things to do are to translate the XML results file to "human + # The last things to do are to translate the XML results file to "human- # readable form" if the user asked for it (or make an XML file somewhere) # if len(args.html) + len(args.text) + len(args.xml): @@ -1898,7 +1970,7 @@ def run_tests(): translate_to_text(xml_results_file, args.text) if len(args.xml): - xml_file = args.xml + '.xml' + xml_file = args.xml + ('.xml' if ".xml" not in args.xml else '') print('Writing results to xml file %s...' % xml_file, end='') shutil.copyfile(xml_results_file, xml_file) print('done.') @@ -1993,6 +2065,9 @@ def main(argv): parser.add_argument("-v", "--verbose", action="store_true", default=False, help="print progress and informational messages") + parser.add_argument("--verbose-failed", action="store_true", default=False, + help="print progress and informational messages for failed jobs") + parser.add_argument("-w", "--web", "--html", action="store", type=str, dest="html", default="", metavar="HTML-FILE", help="write detailed test results into HTML-FILE.html") @@ -2007,6 +2082,9 @@ def main(argv): parser.add_argument("--jobs", action="store", type=int, dest="process_limit", default=0, help="limit number of worker threads") + parser.add_argument("--rerun-failed", action="store_true", dest="rerun_failed", default=False, + help="rerun failed tests") + global args args = parser.parse_args() signal.signal(signal.SIGINT, sigint_hook) diff --git a/utils/tests/TestBase.py b/utils/tests/TestBase.py index 0bf6ef3c2..dc58a3c75 100644 --- a/utils/tests/TestBase.py +++ b/utils/tests/TestBase.py @@ -115,7 +115,7 @@ class TestBaseClass: if self.options.cmds: print_cmds(cmds) return - + base_dir = os.sep.join(os.path.abspath(__file__).replace(os.path.pathsep, '/').split('/')[:-3]) final_return = 0 total_tests = len(cmds) passed = 0 @@ -131,7 +131,13 @@ class TestBaseClass: print("running test case: " + case_string) print_case_in_file(case_string, out) progress += 1 - ret = subprocess.call(cmd, shell=True, env=self.my_env, stdout=outstream, stderr=out) + ret = subprocess.call(cmd, + shell=True, + env=self.my_env, + stdout=outstream, + stderr=out, + cwd=base_dir + ) if not ret: passed += 1 else: diff --git a/utils/tests/gitlab-ci-scheduled.yml b/utils/tests/gitlab-ci-scheduled.yml index c6fbc9e85..df7d45350 100644 --- a/utils/tests/gitlab-ci-scheduled.yml +++ b/utils/tests/gitlab-ci-scheduled.yml @@ -139,7 +139,7 @@ daily-build-test-optimized-valgrind: dependencies: [] variables: MODE: optimized - VALGRIND_FLAG: -g + VALGRIND_FLAG: -g --verbose-failed FORCE_TESTS: Force tags: - nsnam @@ -157,7 +157,7 @@ weekly-build-test-debug-valgrind: dependencies: [] variables: MODE: debug - VALGRIND_FLAG: -g + VALGRIND_FLAG: -g --verbose-failed FORCE_TESTS: Force tags: - nsnam @@ -175,7 +175,7 @@ weekly-build-test-default-valgrind: dependencies: [] variables: MODE: default - VALGRIND_FLAG: -g + VALGRIND_FLAG: -g --verbose-failed FORCE_TESTS: Force tags: - nsnam @@ -193,7 +193,7 @@ weekly-build-test-optimized-valgrind: dependencies: [] variables: MODE: optimized - VALGRIND_FLAG: -g + VALGRIND_FLAG: -g --verbose-failed FORCE_TESTS: Force tags: - nsnam diff --git a/utils/tests/test-test.py b/utils/tests/test-test.py index 96a6197a5..3ffff07ed 100644 --- a/utils/tests/test-test.py +++ b/utils/tests/test-test.py @@ -19,7 +19,7 @@ # NOTE: Run this script with the Python3 interpreter if the python3 compatibility # of the ns-3 unit test runner needs to be tested. -# The following options of test.py are being tested for poratability by this script. +# The following options of test.py are being tested for portability by this script. # To see the options supported by this script, run with the -h option on the command line # # -h, --help show this help message and exit @@ -84,7 +84,8 @@ def main(argv): '--example=socket-options-ipv6', '-u', '--update-data', - '-f EXTENSIVE --fullness=EXTENSIVE' + '-f EXTENSIVE', + '--fullness=EXTENSIVE', '-g', '--grind', '-l', @@ -97,17 +98,17 @@ def main(argv): '--pyexample=first', '-r', '--retain', - '-s ns3-tcp-interoperability', - '--suite=ns3-tcp-interoperability', + '-s ns3-tcp-state', + '--suite=ns3-tcp-state', '-t t_opt.txt', - '--text=t_opt.txt && rm -rf t_opt.txt', + '--text=t_opt.txt && rm t_opt.txt', '-v', '--verbose', - '-w t_opt.html && rm -rf t_opt.html', - '--web=t_opt.html && rm -rf t_opt.html', - '--html=t_opt.html && rm -rf t_opt.html', - '-x t_opt.xml && rm -rf t_opt.xml', - '--xml=t_opt.xml && rm -rf t_opt.xml', + '-w t_opt.html && rm t_opt.html', + '--web=t_opt.html && rm t_opt.html', + '--html=t_opt.html && rm t_opt.html', + '-x t_opt.xml && rm t_opt.xml', + '--xml=t_opt.xml && rm t_opt.xml', ] configure_string = sys.executable + ' ns3 configure --enable-tests --enable-examples'