build: Bugfixes and refactoring of ns3 and CMake

Including:
- add missing command for introspected doxygen
- run get_version.sh before running doxygen
- use find_package(Doxygen) to get the doxygen executable
- silence python and sqlite find_package warnings
- return cmake returncode if configuration fails
- require GTK3 3.22
- link all libraries to print-introspected-doxygen
- replace shell with use_shell for variable name
- revert wrong changes to propagation of return codes and add test
- disable pch when ccache is found
- make --enable-sudo a post-build step and a runtime option
- add docs subparser
- add enable-sudo option
- refactor positional argument values
- fix --check option and add shell option
- replace --no-task-lines with --quiet
- replace --nowaf with --no-build
- replace --run --run-no-build with run (--no-build)
- replace ns3 documentation related arguments with targets
- document test-ns3.py
- export include directories used by ns3 libraries
- refactor CMake documentation dependency checking and behavior
- add --allow-run-as-root for running MPI examples on the CI
This commit is contained in:
Gabriel Ferreira
2022-01-08 23:09:57 -03:00
parent 7a95fb87be
commit ff3b511384
13 changed files with 970 additions and 357 deletions

392
ns3
View File

@@ -59,6 +59,16 @@ def parse_args(argv):
parser = argparse.ArgumentParser(description="ns-3 wrapper for the CMake build system")
sub_parser = parser.add_subparsers()
parser_build = sub_parser.add_parser('build',
help=('Accepts a list of targets to build,'
' or builds the entire project if no target is given'))
parser_build.add_argument('build',
help='Build the entire project or the specified target and dependencies',
action="store", nargs='*', default=None)
parser_build.add_argument('--dry-run',
help="Do not execute the commands",
action="store_true", default=None, dest="build_dry_run")
parser_configure = sub_parser.add_parser('configure',
help='Try "./ns3 configure --help" for more configuration options')
parser_configure.add_argument('configure',
@@ -83,7 +93,6 @@ def parse_args(argv):
parser_configure = on_off_argument(parser_configure, "des-metrics",
"Logging all events in a json file with the name of the executable "
"(which must call CommandLine::Parse(argc, argv)")
parser_configure = on_off_argument(parser_configure, "documentation", "documentation targets")
parser_configure = on_off_argument(parser_configure, "examples", "the ns-3 examples")
parser_configure = on_off_argument(parser_configure, "gcov", "code coverage analysis")
parser_configure = on_off_argument(parser_configure, "gtk", "GTK support in ConfigStore")
@@ -94,11 +103,12 @@ def parse_args(argv):
"address, memory leaks and undefined behavior sanitizers")
parser_configure = on_off_argument(parser_configure, "static", "Build a single static library with all ns-3",
"Restore the shared libraries")
parser_configure = on_off_argument(parser_configure, "sudo", "use of sudo to setup suid bits on ns3 executables.")
parser_configure = on_off_argument(parser_configure, "verbose", "printing of additional build system messages")
parser_configure = on_off_argument(parser_configure, "warnings", "compiler warnings")
parser_configure = on_off_argument(parser_configure, "werror", "Treat compiler warnings as errors",
"Treat compiler warnings as warnings")
parser_configure = on_off_argument(parser_configure, "visualizer", "the visualizer module")
parser_configure.add_argument('--enable-modules',
help='List of modules to build (e.g. core;network;internet)',
action="store", type=str, default=None)
@@ -140,16 +150,6 @@ def parse_args(argv):
help="Do not execute the commands",
action="store_true", default=None, dest="configure_dry_run")
parser_build = sub_parser.add_parser('build',
help=('Accepts a list of targets to build,'
' or builds the entire project if no target is given'))
parser_build.add_argument('build',
help='Build the entire project or the specified target and dependencies',
action="store", nargs='*', default=None)
parser_build.add_argument('--dry-run',
help="Do not execute the commands",
action="store_true", default=None, dest="build_dry_run")
parser_clean = sub_parser.add_parser('clean', help='Removes files created by waf and ns3')
parser_clean.add_argument('clean',
nargs="?",
@@ -167,6 +167,56 @@ def parse_args(argv):
nargs="?",
action="store", default=True)
parser_run = sub_parser.add_parser('run',
help='Try "./ns3 run --help" for more runtime options')
parser_run.add_argument('run',
help='Build and run executable. If --no-build is present, build step is skipped.',
type=str, default='')
parser_run.add_argument('--no-build',
help='Skip build step.',
action="store_true", default=False)
parser_run.add_argument('--command-template',
help=('Template of the command used to run the program given by run;'
' It should be a shell command string containing %s inside,'
' which will be replaced by the actual program.'),
type=str, default=None)
parser_run.add_argument('--cwd',
help='Set the working directory for a program.',
action="store", type=str, default=None)
parser_run.add_argument('--gdb',
help='Change the default command template to run programs with gdb',
action="store_true", default=None)
parser_run.add_argument('--valgrind',
help='Change the default command template to run programs with valgrind',
action="store_true", default=None)
parser_run.add_argument('--visualize',
help='Modify --run arguments to enable the visualizer',
action="store_true", default=None)
parser_run.add_argument('--dry-run',
help="Do not execute the commands",
action="store_true", default=None, dest="run_dry_run")
parser_run.add_argument('--enable-sudo',
help='Use sudo to setup suid bits on ns3 executables.',
dest='enable_sudo', action='store_true',
default=False)
parser_shell = sub_parser.add_parser('shell',
help='Try "./ns3 shell --help" for more runtime options')
parser_shell.add_argument('shell',
nargs="?",
help='Export necessary environment variables and open a shell',
action="store", default=True)
parser_docs = sub_parser.add_parser('docs',
help='Try "./ns3 docs --help" for more documentation options')
parser_docs.add_argument('docs',
help='Build project documentation',
choices=["manual", "models", "tutorial", "sphinx", "doxygen-no-build", "doxygen", "all"],
action="store", type=str, default=None)
parser.add_argument('-j', '--jobs',
help='Set number of parallel jobs',
action='store', type=int, dest="jobs", default=max(1, os.cpu_count() - 1))
parser.add_argument('--dry-run',
help="Do not execute the commands",
action="store_true", default=None, dest="dry_run")
@@ -174,69 +224,14 @@ def parse_args(argv):
help='Print the current configuration.',
action="store_true", default=None)
parser.add_argument('--cwd',
help='Set the working directory for a program.',
action="store", type=str, default=None)
parser.add_argument('--no-task-lines',
parser.add_argument('--quiet',
help="Don't print task lines, i.e. messages saying which tasks are being executed.",
action="store_true", default=None)
parser.add_argument('--run',
help=('Run a locally built program; argument can be a program name,'
' or a command starting with the program name.'),
type=str, default='')
parser.add_argument('--run-no-build',
help=(
'Run a locally built program without rebuilding the project; '
'argument can be a program name, or a command starting with the program name.'),
type=str, default='')
parser.add_argument('--visualize',
help='Modify --run arguments to enable the visualizer',
action="store_true", default=None)
parser.add_argument('--command-template',
help=('Template of the command used to run the program given by --run;'
' It should be a shell command string containing %s inside,'
' which will be replaced by the actual program.'),
type=str, default=None)
parser.add_argument('--pyrun',
help=('Run a python program using locally built ns3 python module;'
' argument is the path to the python program, optionally followed'
' by command-line options that are passed to the program.'),
type=str, default='')
parser.add_argument('--pyrun-no-build',
help=(
'Run a python program using locally built ns3 python module without rebuilding the project;'
' argument is the path to the python program, optionally followed'
' by command-line options that are passed to the program.'),
type=str, default='')
parser.add_argument('--gdb',
help='Change the default command template to run programs and unit tests with gdb',
action="store_true", default=None)
parser.add_argument('--valgrind',
help='Change the default command template to run programs and unit tests with valgrind',
action="store_true", default=None)
parser.add_argument('-j', '--jobs',
help='Set number of parallel jobs',
action='store', type=int, dest="jobs", default=max(1, os.cpu_count() - 1))
# parser.add_argument('--shell',
# help=('DEPRECATED (run ./waf shell)'),
# action="store_true", default=None)
# parser.add_argument('--enable-sudo',
# help=('Use sudo to setup suid bits on ns3 executables.'),
# dest='enable_sudo', action='store_true',
# default=None)
parser.add_argument('--check',
help='DEPRECATED (run ./test.py)',
action='store_true', default=None)
parser.add_argument('--doxygen',
help='Run doxygen to generate html documentation from source comments.',
action="store_true", default=None)
parser.add_argument('--doxygen-no-build',
help=('Run doxygen to generate html documentation from source comments, '
'but do not wait for ns-3 to finish the full build.'),
action="store_true", default=None)
# parser.add_argument('--docset',
# help=(
# 'Create Docset, without building. This requires the docsetutil tool from Xcode 9.2 or earlier.'
@@ -248,9 +243,16 @@ def parse_args(argv):
# Merge dry_runs
dry_run_args = [(args.__getattribute__(name) if name in args else None) for name in
["build_dry_run", "clean_dry_run", "configure_dry_run", "dry_run"]]
["build_dry_run", "clean_dry_run", "configure_dry_run", "dry_run", "run_dry_run"]]
args.dry_run = dry_run_args.count(True) > 0
# If some positional options are not in args, set them to false.
for option in ["clean", "configure", "docs", "install", "run", "shell", "uninstall"]:
if option not in args:
setattr(args, option, False)
if args.run and args.enable_sudo is None:
args.enable_sudo = True
return args
@@ -261,6 +263,7 @@ def check_build_profile(output_directory):
ns3_version = None
ns3_modules = None
ns3_modules_tests = []
enable_sudo = False
if output_directory and os.path.exists(c4che_path):
c4che_info = {}
exec(open(c4che_path).read(), globals(), c4che_info)
@@ -268,7 +271,9 @@ def check_build_profile(output_directory):
ns3_version = c4che_info["VERSION"]
ns3_modules = c4che_info["NS3_ENABLED_MODULES"]
ns3_modules_tests = [x + "-test" for x in ns3_modules]
return build_profile, ns3_version, ns3_modules + ns3_modules_tests if ns3_modules else None
if "ENABLE_SUDO" in c4che_info:
enable_sudo = c4che_info["ENABLE_SUDO"]
return build_profile, ns3_version, ns3_modules + ns3_modules_tests if ns3_modules else None, enable_sudo
def print_and_buffer(message):
@@ -414,7 +419,7 @@ def configure_cmake(cmake, args, current_cmake_cache_folder, current_cmake_gener
options = (("ASSERT", "asserts"),
("COVERAGE", "gcov"),
("DES_METRICS", "des_metrics"),
("DOCS", "documentation"),
("ENABLE_SUDO", "sudo"),
("EXAMPLES", "examples"),
("GTK3", "gtk"),
("LOG", "logs"),
@@ -423,6 +428,7 @@ def configure_cmake(cmake, args, current_cmake_cache_folder, current_cmake_gener
("STATIC", "static"),
("TESTS", "tests"),
("VERBOSE", "verbose"),
("VISUALIZER", "visualizer"),
("WARNINGS", "warnings"),
("WARNINGS_AS_ERRORS", "werror"),
)
@@ -450,10 +456,6 @@ def configure_cmake(cmake, args, current_cmake_cache_folder, current_cmake_gener
if args.prefix is not None:
cmake_args.append("-DCMAKE_INSTALL_PREFIX=%s" % args.prefix)
# Build and link visualizer
if args.visualize is not None:
cmake_args.append("-DNS3_VISUALIZER=%s" % on_off(args.visualize))
# Process enabled/disabled modules
if args.enable_modules:
cmake_args.append("-DNS3_ENABLED_MODULES=%s" % args.enable_modules)
@@ -478,7 +480,9 @@ def configure_cmake(cmake, args, current_cmake_cache_folder, current_cmake_gener
# Run cmake
if not dry_run:
subprocess.run(cmake_args, cwd=current_cmake_cache_folder, stdout=output)
ret = subprocess.run(cmake_args, cwd=current_cmake_cache_folder, stdout=output)
if ret.returncode != 0:
exit(ret.returncode)
def get_program_shortcuts(build_profile, ns3_version):
@@ -522,7 +526,7 @@ def cmake_build(current_cmake_cache_folder, output, jobs, target=None, dry_run=F
raise Exception("CMake was not found")
version = re.findall("version (.*)\n", subprocess.check_output([cmake, "--version"]).decode("utf-8"))[0]
# Older CMake versions do not accept the number of jobs directly
# Older CMake versions don't accept the number of jobs directly
jobs_part = ("-j %d" % jobs) if version >= "3.12.0" else ""
target_part = (" --target %s" % target) if target else ""
cmake_build_command = "cmake --build . %s%s" % (jobs_part, target_part)
@@ -654,14 +658,16 @@ def build_step(args,
output=output,
dry_run=args.dry_run
)
if "build" in args:
# We can exit early if only building
exit(0)
# If we are building specific targets, we build them one by one
if "build" in args:
non_executable_targets = ["doxygen",
non_executable_targets = ["docs",
"doxygen",
"doxygen-no-build",
"sphinx",
"manual",
"models",
"tutorial",
"install",
"uninstall",
"cmake-format"]
@@ -671,13 +677,21 @@ def build_step(args,
target = "lib" + target
elif target not in non_executable_targets:
target = get_target_to_build(target, ns3_version, build_profile)
else:
# Sphinx target should have the sphinx prefix
if target in ["manual", "models", "tutorial"]:
target = "sphinx_%s" % target
# Docs should build both doxygen and sphinx based docs
if target == "docs":
target = "sphinx"
args.build.append("doxygen")
cmake_build(current_cmake_cache_folder,
jobs=args.jobs,
target=target,
output=output,
dry_run=args.dry_run)
# We can also exit earlier in this case
exit(0)
# The remaining case is when we want to build something to run
if build_and_run:
@@ -692,44 +706,61 @@ def build_step(args,
def run_step(args, target_to_run, target_args):
libdir = "%s/lib" % out_dir
path_sep = ";" if sys.platform == "win32" else ":"
proc_env = {"PATH": os.getenv("PATH") + path_sep + libdir,
"PYTHON_PATH": "%s/bindings/python" % out_dir,
}
custom_env = {"PATH": libdir,
"PYTHONPATH": "%s/bindings/python" % out_dir,
}
if sys.platform != "win32":
proc_env["LD_LIBRARY_PATH"] = libdir
custom_env["LD_LIBRARY_PATH"] = libdir
proc_env = os.environ.copy()
for (key, value) in custom_env.items():
if key in proc_env:
proc_env[key] += path_sep + value
else:
proc_env[key] = value
# running from ns-3-dev (ns3_path) or cwd
working_dir = args.cwd if args.cwd else ns3_path
debugging_software = []
# running valgrind?
if args.valgrind:
debugging_software.append(shutil.which("valgrind"))
# running gdb?
if args.gdb:
debugging_software.extend([shutil.which("gdb"), "--args"])
# running with visualizer?
if args.visualize:
target_args.append("--SimulatorImplementationType=ns3::VisualSimulatorImpl")
working_dir = ns3_path
use_shell = False
# running test.py/check?
if args.check:
target_to_run = os.sep.join([ns3_path, "test.py"])
target_args = ["--nowaf", "--jobs=%d" % args.jobs]
target_args = ["--no-build", "--jobs=%d" % args.jobs]
elif args.shell:
target_to_run = "bash"
use_shell = True
else:
# running from ns-3-dev (ns3_path) or cwd
if args.cwd:
working_dir = args.cwd
# running with command template?
if args.command_template:
commands = (args.command_template % target_to_run).split()
target_to_run = commands[0]
target_args = commands[1:] + target_args
# running valgrind?
if args.valgrind:
debugging_software.append(shutil.which("valgrind"))
# running gdb?
if args.gdb:
debugging_software.extend([shutil.which("gdb"), "--args"])
# running with the visualizer?
if args.visualize:
target_args.append("--SimulatorImplementationType=ns3::VisualSimulatorImpl")
# running with command template?
if args.command_template:
commands = (args.command_template % target_to_run).split()
target_to_run = commands[0]
target_args = commands[1:] + target_args
if target_to_run in ["mpiexec", "mpirun"]:
target_args = ["--allow-run-as-root"] + target_args
program_arguments = [*debugging_software, target_to_run, *target_args]
if not run_only or args.dry_run:
exported_variables = "export "
for (variable, value) in proc_env.items():
for (variable, value) in custom_env.items():
if variable == "PATH":
value = "$PATH" + path_sep + libdir
exported_variables += "%s=%s " % (variable, value)
@@ -741,18 +772,99 @@ def run_step(args, target_to_run, target_args):
if not args.dry_run:
try:
subprocess.run(program_arguments, env=proc_env, cwd=working_dir)
ret = subprocess.run(program_arguments, env=proc_env, cwd=working_dir, shell=use_shell)
exit(ret.returncode)
except KeyboardInterrupt:
print("Process was interrupted by the user")
# Debugging this with PyCharm is a no no. It refuses to work hanging indefinitely
def sudo_command(command: list, password: str):
# Run command and feed the sudo password
proc = subprocess.Popen(['sudo', '-S', *command],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate(input=password.encode() + b'\n')
stdout, stderr = proc[0].decode(), proc[1].decode()
# Clean sudo password after each command
subprocess.Popen(["sudo", "-k"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
# Check if the password is wrong
if "try again" in stderr:
raise Exception("Incorrect sudo password")
return stdout, stderr
def sudo_step(args, target_to_run, configure_post_build: set):
# Check if sudo exists
sudo = shutil.which("sudo")
if not sudo:
raise Exception("Sudo is required by --enable-sudo, but it was not found")
# We do this for specified targets if --enable-sudo was set in the run sub-parser
# And to all executables if set in the configure sub-parser
targets_to_sudo = configure_post_build
if target_to_run:
targets_to_sudo.add(target_to_run)
password = os.getenv("SUDO_PASSWORD", None)
if not args.dry_run:
if password is None:
from getpass import getpass
password = getpass(prompt="Sudo password:")
import stat
for target in targets_to_sudo:
# Check if the file was already built
if not os.path.exists(target):
continue
# Check if we need to set anything
fstat = os.stat(target)
if (fstat.st_mode & stat.S_ISUID) == stat.S_ISUID:
continue
# Log commands
relative_path_to_target = os.path.relpath(target, ns3_path)
chown_command = "chown root {}".format(relative_path_to_target)
chmod_command = "chmod u+s {}".format(relative_path_to_target)
print_and_buffer("; ".join([chown_command, chmod_command]))
# Change permissions
if not args.dry_run:
out, err = sudo_command(chown_command.split(), password)
if len(out) > 0:
raise Exception("Failed to chown: ", relative_path_to_target)
out, err = sudo_command(chmod_command.split(), password)
if len(out) > 0:
raise Exception("Failed to chmod: ", relative_path_to_target)
return
def refuse_run_as_root():
# Check if the user is root and refuse to run
username = os.getenv("USER", "")
if username == "root":
raise Exception("Refusing to run as root. --enable-sudo will request your password when needed")
def main():
global out_dir, run_only
# Refuse to run with sudo
refuse_run_as_root()
# Parse arguments
args = parse_args(sys.argv[1:])
atexit.register(exit_handler, dry_run=args.dry_run)
output = subprocess.DEVNULL if args.no_task_lines else None
output = subprocess.DEVNULL if args.quiet else None
# no arguments were passed, so can't possibly be reconfiguring anything, then we refresh and rebuild
if len(sys.argv) == 1:
@@ -763,36 +875,37 @@ def main():
exec(open(lock_file).read(), globals())
# Clean project if needed
if "clean" in args and args.clean:
if args.clean:
clean_cmake_artifacts(dry_run=args.dry_run)
# We end things earlier when cleaning
return
# Doxygen-no-build requires print-introspected-doxygen, but it has no explicit dependencies,
# differently from the doxygen target
if args.doxygen:
args.build = ['doxygen']
if args.doxygen_no_build:
args.build = ['doxygen-no-build']
# Docs options become cmake targets
if args.docs:
args.build = [args.docs] if args.docs != "all" else ["sphinx", "doxygen"]
# Installation and uninstallation options become cmake targets
if "install" in args:
if args.install:
args.build = ['install']
if 'uninstall' in args:
if args.uninstall:
args.build = ['uninstall']
# Get build profile
build_profile, ns3_version, ns3_modules = check_build_profile(out_dir)
build_profile, ns3_version, ns3_modules, enable_sudo = check_build_profile(out_dir)
# Check if running something or reconfiguring ns-3
run_only = args.run_no_build or args.pyrun_no_build
build_and_run = args.run or args.pyrun
run_only = False
build_and_run = False
if args.run:
if not args.no_build:
build_and_run = True
else:
run_only = True
target_to_run = None
target_args = []
current_cmake_cache_folder = None
if not args.check and (run_only or build_and_run):
target_to_run = max(args.run_no_build, args.pyrun_no_build, args.run, args.pyrun)
target_to_run = args.run
if len(target_to_run) > 0:
# While testing a weird case appeared where the target to run is between quotes,
# so we remove in case they exist
@@ -812,7 +925,7 @@ def main():
# We end things earlier if only checking the current project configuration
return
if "configure" in args:
if args.configure:
configuration_step(current_cmake_cache_folder,
current_cmake_generator,
args,
@@ -869,22 +982,27 @@ def main():
target_path = os.sep.join(target_path)
return target_path
target_to_run = remove_overlapping_path(out_dir, target_to_run)
if not args.check and not args.shell and target_to_run:
target_to_run = remove_overlapping_path(out_dir, target_to_run)
# Waf doesn't add version prefix and build type suffix to the scratches, so we remove them
if current_cmake_cache_folder is None:
if "scratch" in target_to_run and run_only:
waf_target_to_run = target_to_run.replace(os.path.basename(target_to_run), run_only)
if os.path.exists(os.sep.join([out_dir, waf_target_to_run])):
target_to_run = waf_target_to_run
target_to_run = os.sep.join([out_dir, target_to_run])
# Waf doesn't add version prefix and build type suffix to the scratches, so we remove them
if current_cmake_cache_folder is None:
if "scratch" in target_to_run and run_only:
waf_target_to_run = target_to_run.replace(os.path.basename(target_to_run), args.run)
if os.path.exists(os.sep.join([out_dir, waf_target_to_run])):
target_to_run = waf_target_to_run
target_to_run = os.sep.join([out_dir, target_to_run])
# If we're only trying to run the target, we need to check if it actually exists first
if (run_only or build_and_run) and not os.path.exists(target_to_run):
raise Exception("Executable has not been built yet")
# If we're only trying to run the target, we need to check if it actually exists first
if (run_only or build_and_run) and not os.path.exists(target_to_run):
raise Exception("Executable has not been built yet")
# Setup program as sudo
if enable_sudo or (args.run and args.enable_sudo):
sudo_step(args, target_to_run, set(ns3_programs.values()) if enable_sudo else set())
# Finally, we try to run it
if args.check or run_only or build_and_run:
if args.check or args.shell or run_only or build_and_run:
run_step(args, target_to_run, target_args)
return