From 72e6105195c99d1c5486720c2b46585e63624c47 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 19 Mar 2022 12:19:50 -0300 Subject: [PATCH] build: CMake and ns3 fixes Includes: - print error message instead of forwarding posix signals in ns3 - supress printing of "Finished executing commands..." for ./ns3 run - fix ns3 typos and formatting issues - add verbose options and make doxygen/doxygen-no-build verbose - re-enable printing of build messages in ./ns3 run - refactor ns3 dry_run, quiet, jobs and verbose arguments - check if examples subdirectories have a CMakeLists.txt --- build-support/macros-and-definitions.cmake | 2 + examples/CMakeLists.txt | 3 + ns3 | 208 +++++++++++---------- utils/tests/test-ns3.py | 56 +++++- 4 files changed, 166 insertions(+), 103 deletions(-) diff --git a/build-support/macros-and-definitions.cmake b/build-support/macros-and-definitions.cmake index 54a1dee25..0a83d9fa1 100644 --- a/build-support/macros-and-definitions.cmake +++ b/build-support/macros-and-definitions.cmake @@ -949,6 +949,7 @@ macro(process_options) WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} DEPENDS update_doxygen_version run-print-introspected-doxygen assemble-introspected-command-line + USES_TERMINAL ) add_custom_target( @@ -956,6 +957,7 @@ macro(process_options) COMMAND ${DOXYGEN_EXECUTABLE} ${PROJECT_SOURCE_DIR}/doc/doxygen.conf WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} DEPENDS update_doxygen_version + USES_TERMINAL ) endif() diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index fd7f7f8dc..39508ec49 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -3,6 +3,9 @@ if(${ENABLE_EXAMPLES}) # Process subdirectories foreach(examplefolder ${examples_to_build}) + if(NOT (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${examplefolder}/CMakeLists.txt)) + continue() + endif() add_subdirectory(${examplefolder}) set(ns3-example-folders diff --git a/ns3 b/ns3 index ffd77ae52..59c49f0d4 100755 --- a/ns3 +++ b/ns3 @@ -1,7 +1,7 @@ #! /usr/bin/env python3 -import atexit import argparse +import atexit import glob import os import re @@ -13,6 +13,7 @@ ns3_path = os.path.dirname(os.path.abspath(__file__)) out_dir = os.sep.join([ns3_path, "build"]) lock_file = os.sep.join([ns3_path, ".lock-ns3_%s_build" % sys.platform]) +max_cpu_threads = max(1, os.cpu_count() - 1) print_buffer = "" run_verbose = True @@ -55,8 +56,36 @@ def on_off_condition(args, cmake_flag, option_name): return cmake_arg +def add_argument_to_subparsers(parsers: list, + arguments: list, + help_msg: str, + dest: str, + action="store_true", + default_value=None): + # Instead of copying and pasting repeated arguments for each parser, we add them here + for subparser in parsers: + subparser_name = subparser.prog.replace("ns3", "").strip() + destination = ("%s_%s" % (subparser_name, dest)) if subparser_name else dest + subparser.add_argument(*arguments, + help=help_msg, + action=action, + default=default_value, + dest=destination) + + def parse_args(argv): parser = argparse.ArgumentParser(description="ns-3 wrapper for the CMake build system", add_help=False) + + parser.add_argument('--help', + help="Print a summary of available commands", + action="store_true", default=None, dest="help") + # parser.add_argument('--docset', + # help=( + # 'Create Docset, without building. This requires the docsetutil tool from Xcode 9.2 or earlier.' + # 'See Bugzilla 2196 for more details.'), + # action="store_true", default=None, + # dest="docset_build") + sub_parser = parser.add_subparsers() parser_build = sub_parser.add_parser('build', @@ -65,15 +94,6 @@ def parse_args(argv): parser_build.add_argument('build', help='Build the entire project or the specified target and dependencies', action="store", nargs='*', default=None) - parser_build.add_argument('--dry-run', - help="Do not execute the commands", - action="store_true", default=None, dest="build_dry_run") - parser_build.add_argument('-v', '--verbose', - help="Print verbose build commands", - action="store_true", default=None, dest="build_verbose") - parser_build.add_argument('--quiet', - help="Don't print task lines, i.e. messages saying which tasks are being executed.", - action="store_true", default=None, dest="build_quiet") parser_configure = sub_parser.add_parser('configure', help='Try "./ns3 configure --help" for more configuration options') @@ -97,8 +117,8 @@ def parse_args(argv): # On-Off options # First positional is transformed into --enable-option --disable-option # Second positional is used for description "Enable %s" % second positional/"Disable %s" % second positional - # When an optional third positional is given, the second is used as is as the enable description - # and the third is used as is as the disable description + # When an optional third positional is given, the second is used as is as the 'enable' description + # and the third is used as is as the 'disable' description on_off_options = [ ("asserts", "the asserts regardless of the compile mode"), ("des-metrics", "Logging all events in a json file with the name of the executable " @@ -116,13 +136,13 @@ def parse_args(argv): ("tests", "the ns-3 tests"), ("sanitizers", "address, memory leaks and undefined behavior sanitizers"), ("static", "Build a single static library with all ns-3", - "Restore the shared libraries" + "Restore the shared libraries" ), ("sudo", "use of sudo to setup suid bits on ns3 executables."), ("verbose", "printing of additional build system messages"), ("warnings", "compiler warnings"), ("werror", "Treat compiler warnings as errors", - "Treat compiler warnings as warnings" + "Treat compiler warnings as warnings" ), ] for on_off_option in on_off_options: @@ -165,21 +185,12 @@ def parse_args(argv): parser_configure.add_argument('--prefix', help='Target output directory to install', action="store", default=None) - parser_configure.add_argument('--dry-run', - help="Do not execute the commands", - action="store_true", default=None, dest="configure_dry_run") - parser_configure.add_argument('--quiet', - help="Don't print task lines, i.e. messages saying which tasks are being executed.", - action="store_true", default=None, dest="configure_quiet") parser_configure.add_argument('--trace-performance', help="Generate a performance trace log for the CMake configuration", action="store_true", default=None, dest="trace_cmake_perf") parser_clean = sub_parser.add_parser('clean', help='Removes files created by ns3') parser_clean.add_argument('clean', action="store_true", default=False) - parser_clean.add_argument('--dry-run', - help="Do not execute the commands", - action="store_true", default=None, dest="clean_dry_run") parser_install = sub_parser.add_parser('install', help='Install ns-3') parser_install.add_argument('install', action="store_true", default=False) @@ -215,20 +226,10 @@ def parse_args(argv): parser_run.add_argument('--vis', '--visualize', help='Modify --run arguments to enable the visualizer', action="store_true", dest="visualize", default=None) - parser_run.add_argument('--dry-run', - help="Do not execute the commands", - action="store_true", default=None, dest="run_dry_run") parser_run.add_argument('--enable-sudo', help='Use sudo to setup suid bits on ns3 executables.', dest='enable_sudo', action='store_true', default=False) - parser_run.add_argument('-v', '--verbose', - help='Print which commands were executed', - dest='run_verbose', action='store_true', - default=False) - parser_run.add_argument('--quiet', - help="Don't print task lines, i.e. messages saying which tasks are being executed.", - action="store_true", default=None, dest="run_quiet") parser_shell = sub_parser.add_parser('shell', help='Try "./ns3 shell --help" for more runtime options') @@ -243,23 +244,6 @@ def parse_args(argv): choices=["manual", "models", "tutorial", "contributing", "sphinx", "doxygen-no-build", "doxygen", "all"], action="store", type=str, default=None) - parser_docs.add_argument('--quiet', - help="Don't print task lines, i.e. messages saying which tasks are being executed.", - action="store_true", default=None, dest="docs_quiet") - - parser.add_argument('-j', '--jobs', - help='Set number of parallel jobs', - action='store', type=int, dest="jobs", default=max(1, os.cpu_count() - 1)) - parser.add_argument('--dry-run', - help="Do not execute the commands", - action="store_true", default=None, dest="dry_run") - - parser.add_argument('--quiet', - help="Don't print task lines, i.e. messages saying which tasks are being executed.", - action="store_true", default=None, dest="quiet") - parser.add_argument('--help', - help="Print a summary of available commands", - action="store_true", default=None, dest="help") parser_show = sub_parser.add_parser('show', help='Try "./ns3 show --help" for more runtime options') @@ -267,19 +251,30 @@ def parse_args(argv): help='Print build profile type, ns-3 version or current configuration', choices=["profile", "version", "config"], action="store", type=str, default=None) - parser_show.add_argument('--dry-run', - help="Do not execute the commands", - action="store_true", default=None, dest="show_dry_run") - parser_show.add_argument('--quiet', - help="Don't print task lines, i.e. messages saying which tasks are being executed.", - action="store_true", default=None, dest="show_quiet") - # parser.add_argument('--docset', - # help=( - # 'Create Docset, without building. This requires the docsetutil tool from Xcode 9.2 or earlier.' - # 'See Bugzilla 2196 for more details.'), - # action="store_true", default=None, - # dest="docset_build") + add_argument_to_subparsers( + [parser, parser_build, parser_configure, parser_clean, parser_docs, parser_run, parser_show], + ["--dry-run"], + help_msg="Do not execute the commands.", + dest="dry_run") + + add_argument_to_subparsers([parser, parser_build, parser_run], + ['-j', '--jobs'], + help_msg="Set number of parallel jobs.", + dest="jobs", + action="store", + default_value=max_cpu_threads) + + add_argument_to_subparsers([parser, parser_build, parser_configure, parser_run, parser_show], + ["--quiet"], + help_msg="Don't print task lines, i.e. messages saying which tasks are being executed.", + dest="quiet") + + add_argument_to_subparsers([parser, parser_build, parser_configure, parser_docs, parser_run], + ['-v', '--verbose'], + help_msg='Print which commands were executed', + dest='verbose', + default_value=False) # Parse known arguments and separate from unknown arguments args, unknown_args = parser.parse_known_args(argv) @@ -298,7 +293,6 @@ def parse_args(argv): for subparsers_action in subparsers_actions: # get all subparsers and print help for choice, subparser in subparsers_action.choices.items(): - #print("Subparser '{}'".format(choice)) subcommand = subparser.format_usage()[:-1].replace("usage: ", " or: ") if len(subcommand) > 1: print(subcommand) @@ -308,11 +302,20 @@ def parse_args(argv): # Merge attributes attributes_to_merge = ["dry_run", "verbose", "quiet"] - filtered_attributes = list(filter(lambda x: x if ("disable" not in x and "enable" not in x) else None, args.__dir__())) + filtered_attributes = list( + filter(lambda x: x if ("disable" not in x and "enable" not in x) else None, args.__dir__())) for attribute in attributes_to_merge: - merging_attributes = list(map(lambda x: args.__getattribute__(x) if attribute in x else None, filtered_attributes)) + merging_attributes = list( + map(lambda x: args.__getattribute__(x) if attribute in x else None, filtered_attributes)) setattr(args, attribute, merging_attributes.count(True) > 0) + attributes_to_merge = ["jobs"] + filtered_attributes = list(filter(lambda x: x if ("disable" not in x and "enable" not in x) else 0, args.__dir__())) + for attribute in attributes_to_merge: + merging_attributes = list( + map(lambda x: int(args.__getattribute__(x)) if attribute in x else max_cpu_threads, filtered_attributes)) + setattr(args, attribute, min(merging_attributes)) + # If some positional options are not in args, set them to false. for option in ["clean", "configure", "docs", "install", "run", "shell", "uninstall", "show"]: if option not in args: @@ -347,7 +350,7 @@ def check_lock_data(output_directory): ns3_modules_bindings = [] ns3_modules = None - build_info = {"NS3_ENABLED_MODULES": None, + build_info = {"NS3_ENABLED_MODULES": [], "BUILD_PROFILE": None, "VERSION": None, "ENABLE_EXAMPLES": False, @@ -509,11 +512,15 @@ def configure_cmake(cmake, args, current_cmake_cache_folder, current_cmake_gener raise Exception("Unknown build type") else: if args.build_profile == "debug": - cmake_args.extend("-DCMAKE_BUILD_TYPE=debug -DNS3_ASSERT=ON -DNS3_LOG=ON -DNS3_WARNINGS_AS_ERRORS=ON".split()) + cmake_args.extend( + "-DCMAKE_BUILD_TYPE=debug -DNS3_ASSERT=ON -DNS3_LOG=ON -DNS3_WARNINGS_AS_ERRORS=ON".split()) elif args.build_profile == "default": - cmake_args.extend("-DCMAKE_BUILD_TYPE=default -DNS3_ASSERT=ON -DNS3_LOG=ON -DNS3_WARNINGS_AS_ERRORS=OFF".split()) + cmake_args.extend( + "-DCMAKE_BUILD_TYPE=default -DNS3_ASSERT=ON -DNS3_LOG=ON -DNS3_WARNINGS_AS_ERRORS=OFF".split()) else: - cmake_args.extend("-DCMAKE_BUILD_TYPE=release -DNS3_ASSERT=OFF -DNS3_LOG=OFF -DNS3_WARNINGS_AS_ERRORS=OFF".split()) + cmake_args.extend( + "-DCMAKE_BUILD_TYPE=release -DNS3_ASSERT=OFF -DNS3_LOG=OFF -DNS3_WARNINGS_AS_ERRORS=OFF".split() + ) cmake_args.append("-DNS3_NATIVE_OPTIMIZATIONS=%s" % on_off((args.build_profile == "optimized"))) options = (("ASSERT", "asserts"), @@ -720,21 +727,26 @@ def cmake_build(current_cmake_cache_folder, output, jobs, target=None, dry_run=F ) ) if not dry_run: - if output is not None: - kwargs = {"stdout": subprocess.PIPE, - "stderr": subprocess.PIPE - } - else: - kwargs = {"stdout": output} + # Assume quiet is not enabled, and print things normally + kwargs = {"stdout": None, + "stderr": None} proc_env = os.environ.copy() if build_verbose: + # If verbose is enabled, we print everything to the terminal + # and set the environment variable proc_env.update({"VERBOSE": "1"}) - kwargs["env"] = proc_env + + if output is not None: + # If quiet is enabled, we pipe the output of the + # build and only print in case of failure + kwargs["stdout"] = subprocess.PIPE + kwargs["stderr"] = subprocess.PIPE ret = subprocess.run(cmake_build_command.split(), cwd=current_cmake_cache_folder, - **kwargs + env=proc_env, + **kwargs, ) # Print errors in case compilation fails and output != None (quiet) @@ -874,7 +886,7 @@ def build_step(args, ns3_version, build_profile, output): - # There is one scenarios where we build everything: ./ns3 build + # There is one scenario where we build everything: ./ns3 build if "build" in args and len(args.build) == 0: cmake_build(current_cmake_cache_folder, jobs=args.jobs, @@ -1013,17 +1025,21 @@ def run_step(args, target_to_run, target_args): if not args.dry_run: try: - ret = subprocess.run(program_arguments, env=proc_env, cwd=working_dir, shell=use_shell) - - # Forward POSIX signal error codes - if ret.returncode < 0: - os.kill(os.getpid(), -ret.returncode) - - # Return in case of a positive error number - exit(ret.returncode) + subprocess.run(program_arguments, env=proc_env, cwd=working_dir, shell=use_shell, check=True) + except subprocess.CalledProcessError as e: + # Replace full path to binary to relative path + e.cmd[0] = os.path.relpath(target_to_run, ns3_path) + # Replace list of arguments with a single string + e.cmd = " ".join(e.cmd) + # Print error message and forward the return code + print(e) + exit(e.returncode) except KeyboardInterrupt: print("Process was interrupted by the user") + # Exit normally + exit(0) + # Debugging this with PyCharm is a no no. It refuses to work hanging indefinitely def sudo_command(command: list, password: str): @@ -1055,7 +1071,7 @@ def sudo_step(args, target_to_run, configure_post_build: set): raise Exception("Sudo is required by --enable-sudo, but it was not found") # We do this for specified targets if --enable-sudo was set in the run sub-parser - # And to all executables if set in the configure sub-parser + # And to all executables if set in the 'configure' sub-parser targets_to_sudo = configure_post_build if target_to_run: targets_to_sudo.add(target_to_run) @@ -1139,7 +1155,7 @@ def main(): enable_sudo = build_info["ENABLE_SUDO"] ns3_version = build_info["VERSION"] - # Docs options become cmake targets + # Docs subparser options become cmake targets if args.docs: args.build = [args.docs] if args.docs != "all" else ["sphinx", "doxygen"] if "doxygen" in args.build and (not build_info["ENABLE_EXAMPLES"] or not build_info["ENABLE_TESTS"]): @@ -1161,18 +1177,14 @@ def main(): run_only = False build_and_run = False if args.run: - # When running, default to not run_verbose - run_verbose = args.run_verbose - - # If not run_verbose, silence the rest of the script - if not run_verbose: - output = subprocess.DEVNULL + # Only print "Finished running..." if verbose is set + run_verbose = not (args.run_verbose is not True) # Check whether we are only running or we need to build first - if not args.no_build: - build_and_run = True - else: + if args.no_build: run_only = True + else: + build_and_run = True target_to_run = None target_args = [] current_cmake_cache_folder = None @@ -1197,7 +1209,7 @@ def main(): # We end things earlier if only checking the current project configuration return - # Check for changes in scratch sources and trigger a reconfigure if sources changed + # Check for changes in scratch sources and trigger a reconfiguration if sources changed if current_cmake_cache_folder: current_scratch_sources = glob.glob(os.path.join(ns3_path, "scratch", "**", "*.cc"), recursive=True) diff --git a/utils/tests/test-ns3.py b/utils/tests/test-ns3.py index 4eb3be7c3..300a4c54c 100644 --- a/utils/tests/test-ns3.py +++ b/utils/tests/test-ns3.py @@ -723,7 +723,7 @@ class NS3ConfigureTestCase(NS3BaseTestCase): # Run all cases and then check outputs return_code0, stdout0, stderr0 = run_ns3("--dry-run run scratch-simulator") - return_code1, stdout1, stderr1 = run_ns3("run scratch-simulator --verbose") + return_code1, stdout1, stderr1 = run_ns3("run scratch-simulator") return_code2, stdout2, stderr2 = run_ns3("--dry-run run scratch-simulator --no-build") return_code3, stdout3, stderr3 = run_ns3("run scratch-simulator --no-build") @@ -742,16 +742,18 @@ class NS3ConfigureTestCase(NS3BaseTestCase): self.assertIn(cmake_build_target_command(target="scratch_scratch-simulator"), stdout0) self.assertIn(scratch_path, stdout0) - # Case 1: run (should print all the commands of case 1 plus CMake output from build) - self.assertIn(cmake_build_target_command(target="scratch_scratch-simulator"), stdout1) + # Case 1: run (should print only make build message) + self.assertNotIn(cmake_build_target_command(target="scratch_scratch-simulator"), stdout1) self.assertIn("Built target", stdout1) - self.assertIn(scratch_path, stdout1) + self.assertNotIn(scratch_path, stdout1) # Case 2: dry-run + run-no-build (should print commands to run the target) + self.assertIn("The following commands would be executed:", stdout2) self.assertIn(scratch_path, stdout2) # Case 3: run-no-build (should print the target output only) - self.assertEqual("", stdout3) + self.assertNotIn("Finished executing the following commands:", stdout3) + self.assertNotIn(scratch_path, stdout3) def test_09_PropagationOfReturnCode(self): """! @@ -779,6 +781,40 @@ class NS3ConfigureTestCase(NS3BaseTestCase): ) self.assertNotEqual(return_code, 0) + # Cause a sigsegv + sigsegv_example = os.path.join(ns3_path, "scratch", "sigsegv.cc") + with open(sigsegv_example, "w") as f: + f.write(""" + int main (int argc, char *argv[]) + { + char *s = "hello world"; *s = 'H'; + return 0; + } + """) + return_code, stdout, stderr = run_ns3("run sigsegv") + self.assertEqual(return_code, 245) + self.assertIn("sigsegv-default' died with ", stdout) + + # Cause an abort + abort_example = os.path.join(ns3_path, "scratch", "abort.cc") + with open(abort_example, "w") as f: + f.write(""" + #include "ns3/core-module.h" + + using namespace ns3; + int main (int argc, char *argv[]) + { + NS_ABORT_IF(true); + return 0; + } + """) + return_code, stdout, stderr = run_ns3("run abort") + self.assertEqual(return_code, 250) + self.assertIn("abort-default' died with ", stdout) + + os.remove(sigsegv_example) + os.remove(abort_example) + def test_10_CheckConfig(self): """! Test passing 'show config' argument to ns3 to get the configuration table @@ -2003,6 +2039,16 @@ if __name__ == '__main__': suite.addTests(loader.loadTestsFromTestCase(NS3BuildBaseTestCase)) suite.addTests(loader.loadTestsFromTestCase(NS3ExpectedUseTestCase)) + # Generate a dictionary of test names and their objects + tests = dict(map(lambda x: (x._testMethodName, x), suite._tests)) + + # Filter tests by name + # name_to_search = "" + # tests_to_run = set(map(lambda x: x if name_to_search in x else None, tests.keys())) + # tests_to_remove = set(tests) - set(tests_to_run) + # for test_to_remove in tests_to_remove: + # suite._tests.remove(tests[test_to_remove]) + # Before running, check if ns3rc exists and save it ns3rc_script_bak = ns3rc_script + ".bak" if os.path.exists(ns3rc_script) and not os.path.exists(ns3rc_script_bak):