From ec9d26acd9d586a1131da9bb618aad00faf0a1b5 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Fri, 3 Feb 2023 22:30:38 -0300 Subject: [PATCH] bindings: package ns-3 as a pip wheel Includes: - Python examples to test Brite, Click and Openflow - CI jobs for manylinux packaging of pip wheel - Support for Linux distributions with lib and lib64 directories - Configuration of RPATH not to require setting LD_LIBRARY_PATH Signed-off-by: Gabriel Ferreira --- README.md | 46 ++ bindings/python/ns__init__.py | 402 +++++++++++------- build-support/macros-and-definitions.cmake | 27 +- .../pip-wheel/auditwheel-exclude-list.py | 11 + build-support/pip-wheel/ns/__init__.py | 10 + .../pip-wheel/visualizer/__init__.py | 11 + doc/manual/source/python.rst | 375 ++++++++++++++++ pyproject.toml | 16 + setup.cfg | 44 ++ setup.py | 35 ++ src/brite/examples/brite-generic-example.py | 101 +++++ src/brite/test/examples-to-run.py | 4 +- src/click/examples/nsclick-simple-lan.py | 79 ++++ src/click/test/examples-to-run.py | 4 +- src/config-store/CMakeLists.txt | 7 + src/openflow/examples/openflow-switch.py | 82 ++++ src/openflow/test/examples-to-run.py | 4 +- src/visualizer/CMakeLists.txt | 15 +- .../plugins/interface_statistics.py | 6 +- .../visualizer/plugins/ipv4_routing_table.py | 5 +- src/visualizer/visualizer/plugins/olsr.py | 5 +- .../visualizer/plugins/show_last_packets.py | 6 +- .../plugins/wifi_intrastructure_link.py | 6 +- utils/tests/gitlab-ci-cppyy.yml | 87 +++- 24 files changed, 1232 insertions(+), 156 deletions(-) create mode 100644 build-support/pip-wheel/auditwheel-exclude-list.py create mode 100644 build-support/pip-wheel/ns/__init__.py create mode 100644 build-support/pip-wheel/visualizer/__init__.py create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/brite/examples/brite-generic-example.py create mode 100644 src/click/examples/nsclick-simple-lan.py create mode 100644 src/openflow/examples/openflow-switch.py diff --git a/README.md b/README.md index f0ab483ed..a85353cc7 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,52 @@ trace file and a set of `simple-global-routing-xx-xx.pcap` binary pcap trace files, which can be read by `tcpdump -tt -r filename.pcap` The program source can be found in the examples/routing directory. + +## Running ns-3 from python + +If you do not plan to modify ns-3 upstream modules, you can get +a pre-built version of the ns-3 python bindings. + +```shell +pip install --user ns3 +``` + +If you do not have `pip`, check their documents +on [how to install it](https://pip.pypa.io/en/stable/installation/). + +After installing the `ns3` package, you can then create your simulation python script. +Below is a trivial demo script to get you started. + +```python +from ns import ns + +ns.LogComponentEnable("Simulator", ns.LOG_LEVEL_ALL) + +ns.Simulator.Stop(ns.Seconds(10)) +ns.Simulator.Run() +ns.Simulator.Destroy() +``` + +The simulation will take a while to start, while the bindings are loaded. +The script above will print the logging messages for the called commands. + +Use `help(ns)` to check the prototypes for all functions defined in the +ns3 namespace. To get more useful results, query specific classes of +interest and their functions e.g. `help(ns.Simulator)`. + +Smart pointers `Ptr<>` can be differentiated from objects by checking if +`__deref__` is listed in `dir(variable)`. To dereference the pointer, +use `variable.__deref__()`. + +Most ns-3 simulations are written in C++ and the documentation is +oriented towards C++ users. The ns-3 tutorial programs (first.cc, +second.cc, etc.) have Python equivalents, if you are looking for +some initial guidance on how to use the Python API. The Python +API may not be as full-featured as the C++ API, and an API guide +for what C++ APIs are supported or not from Python do not currently exist. +The project is looking for additional Python maintainers to improve +the support for future Python users. + ## Getting access to the ns-3 documentation Once you have verified that your build of ns-3 works by running diff --git a/bindings/python/ns__init__.py b/bindings/python/ns__init__.py index f43f639ce..f0f3c5d20 100644 --- a/bindings/python/ns__init__.py +++ b/bindings/python/ns__init__.py @@ -1,5 +1,4 @@ import builtins -from copy import copy from functools import lru_cache import glob import os.path @@ -11,7 +10,7 @@ DEFAULT_INCLUDE_DIR = sysconfig.get_config_var("INCLUDEDIR") DEFAULT_LIB_DIR = sysconfig.get_config_var("LIBDIR") -def find_ns3_lock(): +def find_ns3_lock() -> str: # Get the absolute path to this file path_to_this_init_file = os.path.dirname(os.path.abspath(__file__)) path_to_lock = path_to_this_init_file @@ -29,12 +28,14 @@ def find_ns3_lock(): if lock_file in os.listdir(path_to_lock): path_to_lock += os.sep + lock_file else: - path_to_lock = None + path_to_lock = "" return path_to_lock SYSTEM_LIBRARY_DIRECTORIES = (DEFAULT_LIB_DIR, - os.path.dirname(DEFAULT_LIB_DIR) + os.path.dirname(DEFAULT_LIB_DIR), + "/usr/lib64", + "/usr/lib" ) DYNAMIC_LIBRARY_EXTENSIONS = {"linux": "so", "win32": "dll", @@ -81,12 +82,14 @@ def _search_libraries() -> dict: # Search for the core library in the search paths libraries = [] for search_path in library_search_paths: - libraries += glob.glob("%s/**/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=True) + if os.path.exists(search_path): + libraries += glob.glob("%s/**/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=True) # Search system library directories (too slow for recursive search) for search_path in SYSTEM_LIBRARY_DIRECTORIES: - libraries += glob.glob("%s/**/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=False) - libraries += glob.glob("%s/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=False) + if os.path.exists(search_path): + libraries += glob.glob("%s/**/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=False) + libraries += glob.glob("%s/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=False) del search_path, library_search_paths @@ -120,8 +123,35 @@ def search_libraries(library_name: str) -> list: return matched_libraries -def extract_library_include_dirs(library_name: str, prefix: str) -> list: - library_path = "%s/lib/%s" % (prefix, library_name) +LIBRARY_AND_DEFINES = { + "libgsl": ["HAVE_GSL"], + "libxml2": ["HAVE_LIBXML2"], + "libsqlite3": ["HAVE_SQLITE3"], + "openflow": ["NS3_OPENFLOW", "ENABLE_OPENFLOW"], + "click": ["NS3_CLICK"] +} + + +def add_library_defines(library_name: str): + has_defines = list(filter(lambda x: x in library_name, LIBRARY_AND_DEFINES.keys())) + defines = "" + if len(has_defines): + for define in LIBRARY_AND_DEFINES[has_defines[0]]: + defines += (f""" + #ifndef {define} + #define {define} 1 + #endif + """) + return defines + + +def extract_linked_libraries(library_name: str, prefix: str) -> tuple: + lib = "" + for variant in ["lib", "lib64"]: + library_path = f"{prefix}/{variant}/{library_name}" + if os.path.exists(library_path): + lib = variant + break linked_libs = [] # First discover which 3rd-party libraries are used by the current module try: @@ -131,10 +161,17 @@ def extract_library_include_dirs(library_name: str, prefix: str) -> list: print("Failed to extract libraries used by {library} with exception:{exception}" .format(library=library_path, exception=e)) exit(-1) + return library_path, lib, list(map(lambda x: x.decode("utf-8"), linked_libs)) + + +def extract_library_include_dirs(library_name: str, prefix: str) -> tuple: + library_path, lib, linked_libs = extract_linked_libraries(library_name, prefix) linked_libs_include_dirs = set() + defines = add_library_defines(library_name) + # Now find these libraries and add a few include paths for them - for linked_library in map(lambda x: x.decode("utf-8"), linked_libs): + for linked_library in linked_libs: # Skip ns-3 modules if "libns3" in linked_library: continue @@ -148,15 +185,18 @@ def extract_library_include_dirs(library_name: str, prefix: str) -> list: "Failed to find {library}. Make sure its library directory is in LD_LIBRARY_PATH.".format( library=linked_library)) - # Get path with shortest length + # Get path with the shortest length linked_library_path = sorted(linked_library_path, key=lambda x: len(x))[0] # If library is part of the ns-3 build, continue without any new includes if prefix in linked_library_path: continue + # Add defines based in linked libraries found + defines += add_library_defines(linked_library) + # If it is part of the system directories, try to find it - system_include_dir = os.path.dirname(linked_library_path).replace("lib", "include") + system_include_dir = os.path.dirname(linked_library_path).replace(lib, "include") if os.path.exists(system_include_dir): linked_libs_include_dirs.add(system_include_dir) @@ -179,134 +219,214 @@ def extract_library_include_dirs(library_name: str, prefix: str) -> list: inc_path = os.path.join(lib_path, "include") if os.path.exists(inc_path): linked_libs_include_dirs.add(inc_path) - return list(linked_libs_include_dirs) + return list(linked_libs_include_dirs), defines + + +def find_ns3_from_lock_file(lock_file: str) -> (str, list, str): + # Load NS3_ENABLED_MODULES from the lock file inside the build directory + values = {} + + # If we find a lock file, load the ns-3 modules from it + # Should be the case when running from the source directory + exec(open(lock_file).read(), {}, values) + suffix = "-" + values["BUILD_PROFILE"] if values["BUILD_PROFILE"] != "release" else "" + modules = [module.replace("ns3-", "") for module in values["NS3_ENABLED_MODULES"]] + prefix = values["out_dir"] + libraries = {os.path.splitext(os.path.basename(x))[0]: x for x in os.listdir(os.path.join(prefix, "lib"))} + version = values["VERSION"] + + # Filter out test libraries and incorrect versions + def filter_in_matching_ns3_libraries(libraries_to_filter: dict, + modules_to_filter: list, + version: str, + suffix: str) -> dict: + suffix = [suffix[1:]] if len(suffix) > 1 else [] + filtered_in_modules = [] + for module in modules_to_filter: + filtered_in_modules += list(filter(lambda x: "-".join([version, module, *suffix]) in x, + libraries_to_filter.keys())) + for library in list(libraries_to_filter.keys()): + if library not in filtered_in_modules: + libraries_to_filter.pop(library) + return libraries_to_filter + + libraries = filter_in_matching_ns3_libraries(libraries, modules, version, suffix) + + # When we have the lock file, we assemble the correct library names + libraries_to_load = [] + for module in modules: + library_name = "libns{version}-{module}{suffix}".format( + version=version, + module=module, + suffix=suffix + ) + if library_name not in libraries: + raise Exception("Missing library %s\n" % library_name, + "Build all modules with './ns3 build'" + ) + libraries_to_load.append(libraries[library_name]) + return prefix, libraries_to_load, version + + +# Extract version and build suffix (if it exists) +def filter_module_name(library: str) -> str: + library = os.path.splitext(os.path.basename(library))[0] + components = library.split("-") + + # Remove version-related prefixes + if "libns3" in components[0]: + components.pop(0) + if "dev" == components[0]: + components.pop(0) + if "rc" in components[0]: + components.pop(0) + + # Drop build profile suffix and test libraries + if components[-1] in ["debug", "default", "optimized", "release", "relwithdebinfo", "minsizerel"]: + components.pop(-1) + return "-".join(components) + + +def extract_version(library: str, module: str) -> str: + library = os.path.basename(library) + return re.findall(r"libns([\d.|rc|\-dev]+)-", library)[0] + + +def get_newest_version(versions: list) -> str: + versions = list(sorted(versions)) + if "dev" in versions[0]: + return versions[0] + + # Check if there is a release of a possible candidate + try: + pos = versions.index(os.path.splitext(versions[-1])[0]) + except ValueError: + pos = None + + # Remove release candidates + if pos is not None: + return versions[pos] + else: + return versions[-1] + + +def find_ns3_from_search() -> (str, list, str): + libraries = search_libraries("ns3") + + if not libraries: + raise Exception("ns-3 libraries were not found.") + + # If there is a version with a hash by the end, we have a pip-installed library + pip_libraries = list(filter(lambda x: "python" in x, libraries)) + if pip_libraries: + # We drop non-pip libraries + libraries = pip_libraries + + # The prefix is the directory with the lib directory + # libns3-dev-core.so/../../ + prefix = os.path.dirname(os.path.dirname(libraries[0])) + + # Remove test libraries + libraries = list(filter(lambda x: "test" not in x, libraries)) + + # Filter out module names + modules = set([filter_module_name(library) for library in libraries]) + + def filter_in_newest_ns3_libraries(libraries_to_filter: list, modules_to_filter: list) -> tuple: + newest_version_found = "" + # Filter out older ns-3 libraries + for module in list(modules_to_filter): + # Filter duplicates of modules, while excluding test libraries + conflicting_libraries = list(filter(lambda x: module == filter_module_name(x), libraries_to_filter)) + + # Extract versions from conflicting libraries + conflicting_libraries_versions = list(map(lambda x: extract_version(x, module), conflicting_libraries)) + + # Get the newest version found for that library + newest_version = get_newest_version(conflicting_libraries_versions) + + # Check if the version found is the global newest version + if not newest_version_found: + newest_version_found = newest_version + else: + newest_version_found = get_newest_version([newest_version, newest_version_found]) + if newest_version != newest_version_found: + raise Exception("Incompatible versions of the ns-3 module '%s' were found: %s != %s." + % (module, newest_version, newest_version_found)) + + for conflicting_library in list(conflicting_libraries): + if "-".join([newest_version, module]) not in conflicting_library: + libraries.remove(conflicting_library) + conflicting_libraries.remove(conflicting_library) + + if len(conflicting_libraries) > 1: + raise Exception("There are multiple build profiles for module '%s'.\nDelete one to continue: %s" + % (module, ", ".join(conflicting_libraries))) + + return libraries_to_filter, newest_version_found + + # Get library base names + libraries, version = filter_in_newest_ns3_libraries(libraries, list(modules)) + return prefix, libraries, version def load_modules(): lock_file = find_ns3_lock() libraries_to_load = [] - if lock_file: - # Load NS3_ENABLED_MODULES from the lock file inside the build directory - values = {} + # Search for prefix to ns-3 build, modules and respective libraries plus version + ret = find_ns3_from_search() if not lock_file else find_ns3_from_lock_file(lock_file) - # If we find a lock file, load the ns-3 modules from it - # Should be the case when running from the source directory - exec(open(lock_file).read(), {}, values) - suffix = "-" + values["BUILD_PROFILE"] if values["BUILD_PROFILE"] != "release" else "" - modules = [module.replace("ns3-", "") for module in values["NS3_ENABLED_MODULES"]] - prefix = values["out_dir"] - libraries = {os.path.splitext(os.path.basename(x))[0]: x for x in os.listdir(os.path.join(prefix, "lib"))} - version = values["VERSION"] + # Unpack returned values + prefix, libraries, version = ret + prefix = os.path.abspath(prefix) - # Filter out test libraries and incorrect versions - def filter_in_matching_ns3_libraries(libraries_to_filter: dict, - modules_to_filter: list, - version: str, - suffix: str) -> dict: - suffix = [suffix[1:]] if len(suffix) > 1 else [] - filtered_in_modules = [] - for module in modules_to_filter: - filtered_in_modules += list(filter(lambda x: "-".join([version, module, *suffix]) in x, - libraries_to_filter.keys())) - for library in list(libraries_to_filter.keys()): - if library not in filtered_in_modules: - libraries_to_filter.pop(library) - return libraries_to_filter + # Sort libraries according to their dependencies + def sort_to_dependencies(libraries: list, prefix: str) -> list: + module_dependencies = {} + libraries = list(map(lambda x: os.path.basename(x), libraries)) + for ns3_library in libraries: + _, _, linked_libraries = extract_linked_libraries(ns3_library, prefix) + linked_libraries = list(filter(lambda x: "libns3" in x and ns3_library not in x, linked_libraries)) + linked_libraries = list(map(lambda x: os.path.basename(x), linked_libraries)) + module_dependencies[os.path.basename(ns3_library)] = linked_libraries - libraries = filter_in_matching_ns3_libraries(libraries, modules, version, suffix) - else: - libraries = search_libraries("ns3") + def modules_that_can_be_loaded(module_dependencies, pending_modules, current_modules): + modules = [] + for pending_module in pending_modules: + can_be_loaded = True + for dependency in module_dependencies[pending_module]: + if dependency not in current_modules: + can_be_loaded = False + break + if not can_be_loaded: + continue + modules.append(pending_module) + return modules - if not libraries: - raise Exception("ns-3 libraries were not found.") + def dependency_order(module_dependencies, pending_modules, current_modules, step_number=0, steps={}): + if len(pending_modules) == 0: + return steps + if step_number not in steps: + steps[step_number] = [] + for module in modules_that_can_be_loaded(module_dependencies, pending_modules, current_modules): + steps[step_number].append(module) + pending_modules.remove(module) + current_modules.append(module) + return dependency_order(module_dependencies, pending_modules, current_modules, step_number + 1, steps) - # The prefix is the directory with the lib directory - # libns3-dev-core.so/../../ - prefix = os.path.dirname(os.path.dirname(libraries[0])) + sorted_libraries = [] + for step in dependency_order(module_dependencies, list(module_dependencies.keys()), [], 0).values(): + sorted_libraries.extend(step) + return sorted_libraries - # Remove test libraries - libraries = list(filter(lambda x: "test" not in x, libraries)) + libraries_to_load = sort_to_dependencies(libraries, prefix) - # Extract version and build suffix (if it exists) - def filter_module_name(library): - library = os.path.splitext(os.path.basename(library))[0] - components = library.split("-") + # Extract library base names + libraries_to_load = [os.path.basename(x) for x in libraries_to_load] - # Remove version-related prefixes - if "libns3" in components[0]: - components.pop(0) - if "dev" == components[0]: - components.pop(0) - if "rc" in components[0]: - components.pop(0) - - # Drop build profile suffix and test libraries - if components[-1] in ["debug", "default", "optimized", "release", "relwithdebinfo"]: - components.pop(-1) - return "-".join(components) - - # Filter out module names - modules = set([filter_module_name(library) for library in libraries]) - - def extract_version(library: str, module: str) -> str: - library = os.path.basename(library) - return re.findall(r"libns([\d.|rc|\-dev]+)-", library)[0] - - def get_newest_version(versions: list) -> str: - versions = list(sorted(versions)) - if "dev" in versions[0]: - return versions[0] - - # Check if there is a release of a possible candidate - try: - pos = versions.index(os.path.splitext(versions[-1])[0]) - except ValueError: - pos = None - - # Remove release candidates - if pos is not None: - return versions[pos] - else: - return versions[-1] - - def filter_in_newest_ns3_libraries(libraries_to_filter: list, modules_to_filter: list) -> list: - newest_version_found = None - # Filter out older ns-3 libraries - for module in list(modules_to_filter): - # Filter duplicates of modules, while excluding test libraries - conflicting_libraries = list(filter(lambda x: module == filter_module_name(x), libraries_to_filter)) - - # Extract versions from conflicting libraries - conflicting_libraries_versions = list(map(lambda x: extract_version(x, module), conflicting_libraries)) - - # Get the newest version found for that library - newest_version = get_newest_version(conflicting_libraries_versions) - - # Check if the version found is the global newest version - if not newest_version_found: - newest_version_found = newest_version - else: - newest_version_found = get_newest_version([newest_version, newest_version_found]) - if newest_version != newest_version_found: - raise Exception("Incompatible versions of the ns-3 module '%s' were found: %s != %s." - % (module, newest_version, newest_version_found)) - - for conflicting_library in list(conflicting_libraries): - if "-".join([newest_version, module]) not in conflicting_library: - libraries.remove(conflicting_library) - conflicting_libraries.remove(conflicting_library) - num_libraries -= 1 - - if len(conflicting_libraries) > 1: - raise Exception("There are multiple build profiles for module '%s'.\nDelete one to continue: %s" - % (module, ", ".join(conflicting_libraries))) - - return libraries_to_filter - - # Get library base names - libraries = filter_in_newest_ns3_libraries(libraries, modules) - libraries_to_load = list(map(lambda x: os.path.basename(x), libraries)) + # Sort modules based on libraries + modules = list(map(lambda x: filter_module_name(x), libraries_to_load)) # Try to import Cppyy and warn the user in case it is not found try: @@ -324,35 +444,27 @@ def load_modules(): libcppyy.AddSmartPtrType('Ptr') # Import ns-3 libraries - prefix = os.path.abspath(prefix) - cppyy.add_library_path("%s/lib" % prefix) - cppyy.add_include_path("%s/include" % prefix) - - if lock_file: - # When we have the lock file, we assemble the correct library names - for module in modules: - library_name = "libns{version}-{module}{suffix}".format( - version=version, - module=module, - suffix=suffix - ) - if library_name not in libraries: - raise Exception("Missing library %s\n" % library_name, - "Build all modules with './ns3 build'" - ) - libraries_to_load.append(libraries[library_name]) + for variant in ["lib", "lib64"]: + path_to_lib = f"{prefix}/{variant}" + if not os.path.exists(path_to_lib): + continue + cppyy.add_library_path(path_to_lib) + del variant, path_to_lib + cppyy.add_include_path(f"{prefix}/include") known_include_dirs = set() # We then need to include all include directories for dependencies for library in libraries_to_load: - for linked_lib_include_dir in extract_library_include_dirs(library, prefix): + linked_lib_include_dirs, defines = extract_library_include_dirs(library, prefix) + cppyy.cppexec(defines) + for linked_lib_include_dir in linked_lib_include_dirs: if linked_lib_include_dir not in known_include_dirs: known_include_dirs.add(linked_lib_include_dir) if os.path.isdir(linked_lib_include_dir): cppyy.add_include_path(linked_lib_include_dir) for module in modules: - cppyy.include("ns3/%s-module.h" % module) + cppyy.include(f"ns3/{module}-module.h") # After including all headers, we finally load the modules for library in libraries_to_load: diff --git a/build-support/macros-and-definitions.cmake b/build-support/macros-and-definitions.cmake index d391909c4..7aac81d38 100644 --- a/build-support/macros-and-definitions.cmake +++ b/build-support/macros-and-definitions.cmake @@ -36,6 +36,9 @@ option(NS3_ENABLE_SUDO "Set executables ownership to root and enable the SUID flag" OFF ) +# a flag that controls some aspects related to pip packaging +option(NS3_PIP_PACKAGING "Control aspects related to pip wheel packaging" OFF) + # Replace default CMake messages (logging) with custom colored messages as early # as possible include(${PROJECT_SOURCE_DIR}/build-support/3rd-party/colored-messages.cmake) @@ -152,6 +155,25 @@ link_directories(${CMAKE_OUTPUT_DIRECTORY}/lib) include(GNUInstallDirs) include(build-support/custom-modules/ns3-cmake-package.cmake) +# Set RPATH not too need LD_LIBRARY_PATH after installing +set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib:$ORIGIN/:$ORIGIN/../lib") + +# Add the 64 suffix to the library path when manually requested with the +# -DNS3_USE_LIB64=ON flag. May be necessary depending on the target platform. +# This is used to properly build the manylinux pip wheel. +if(${NS3_USE_LIB64}) + link_directories(${CMAKE_OUTPUT_DIRECTORY}/lib64) + set(CMAKE_INSTALL_RPATH + "${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib64:$ORIGIN/:$ORIGIN/../lib64" + ) +endif() + +# cmake-format: off +# You are a wizard, Harry! +# source: https://gitlab.kitware.com/cmake/community/-/wikis/doc/cmake/RPATH-handling +# cmake-format: on +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + if(${XCODE}) # Is that so hard not to break people's CI, AAPL? Why would you output the # targets to a Debug/Release subfolder? Why? @@ -816,7 +838,7 @@ macro(process_options) find_package(Python3 COMPONENTS Interpreter Development) else() # cmake-format: off - set(Python_ADDITIONAL_VERSIONS 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9) + set(Python_ADDITIONAL_VERSIONS 3.6 3.7 3.8 3.9 3.10 3.11) # cmake-format: on find_package(PythonInterp) find_package(PythonLibs) @@ -924,6 +946,9 @@ macro(process_options) "Set NS3_BINDINGS_INSTALL_DIR=\"${SUGGESTED_BINDINGS_INSTALL_DIR}\" to install it to the default location." ) else() + if(${NS3_BINDINGS_INSTALL_DIR} STREQUAL "INSTALL_PREFIX") + set(NS3_BINDINGS_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}) + endif() install(FILES bindings/python/ns__init__.py DESTINATION ${NS3_BINDINGS_INSTALL_DIR}/ns RENAME __init__.py ) diff --git a/build-support/pip-wheel/auditwheel-exclude-list.py b/build-support/pip-wheel/auditwheel-exclude-list.py new file mode 100644 index 000000000..02ea3d24c --- /dev/null +++ b/build-support/pip-wheel/auditwheel-exclude-list.py @@ -0,0 +1,11 @@ +import os + +ns3_path = os.path.dirname(os.path.abspath(os.sep.join([__file__, "../../"]))) + +for variant in ["lib", "lib64"]: + lib_dir = os.path.abspath(os.path.join(ns3_path, f"build/{variant}")) + if not os.path.exists(lib_dir): + continue + for lib in os.listdir(lib_dir): + if "libns3" in lib: + print(f"--exclude {lib}", end=' ') diff --git a/build-support/pip-wheel/ns/__init__.py b/build-support/pip-wheel/ns/__init__.py new file mode 100644 index 000000000..fa5ccc03f --- /dev/null +++ b/build-support/pip-wheel/ns/__init__.py @@ -0,0 +1,10 @@ +# This is a stub module that loads the actual ns-3 +# bindings from nsnam.ns +import sys + +try: + import ns3.ns + sys.modules['ns'] = ns3.ns +except ModuleNotFoundError as e: + print("Install the ns3 package with pip install ns3.", file=sys.stderr) + exit(-1) diff --git a/build-support/pip-wheel/visualizer/__init__.py b/build-support/pip-wheel/visualizer/__init__.py new file mode 100644 index 000000000..449d517a0 --- /dev/null +++ b/build-support/pip-wheel/visualizer/__init__.py @@ -0,0 +1,11 @@ +# This is a stub module that loads the actual visualizer +# from nsnam.visualizer +import sys + +try: + import ns3.visualizer +except ModuleNotFoundError as e: + print("Install the ns3 package with pip install ns3.", file=sys.stderr) + exit(-1) + +from ns3.visualizer import start, register_plugin, set_bounds, add_initialization_hook diff --git a/doc/manual/source/python.rst b/doc/manual/source/python.rst index ca7e14fee..26f39cf52 100644 --- a/doc/manual/source/python.rst +++ b/doc/manual/source/python.rst @@ -101,6 +101,20 @@ Here is some example code that is written in Python and that runs |ns3|, which i Running Python Scripts ********************** +For users that want to change upstream modules in C++ and got a copy of +ns-3 by Git cloning the ns-3-dev repository, or downloaded the +ns3-allinone package, or is using bake, continue to the next section. + +`Note: models implemented in Python are not available from C++. If you want +your model to be available for both C++ and Python users, you must implement +it in C++.` + +For users that want to exclusively run simulation scenarios and implement +simple modules in python, jump to the `Using the pip wheel`_ section. + +Using the bindings from the ns-3 source +======================================= + The main prerequisite is to install `cppyy`. Depending on how you may manage Python extensions, the installation instructions may vary, but you can first check if it installed by seeing if the `cppyy` module can be @@ -165,6 +179,133 @@ To run your own Python script that calls |ns3| and that has this path, ``/path/t $ ./ns3 shell $ python3 /path/to/your/example/my-script.py + +Using the pip wheel +=================== + +Starting from ns-3.38, we provide a pip wheel for Python users using Linux. + +.. sourcecode:: bash + + $ pip install --user ns3 + +You can select a specific ns-3 version by specifying the wheel version. +Specifying a nonexistent version will result in an error message listing the available versions. + +.. sourcecode:: bash + + $ pip install --user ns3==3.37 + Defaulting to user installation because normal site-packages is not writeable + ERROR: Could not find a version that satisfies the requirement ns3==3.37 (from versions: 3.37.post415) + ERROR: No matching distribution found for ns3==3.37 + +You can also specify you want at least a specific version (e.g. which shipped a required feature). + +.. sourcecode:: bash + + $ pip install --user ns3>=3.37 + Defaulting to user installation because normal site-packages is not writeable + Requirement already satisfied: ns3==3.37.post415 in /home/username/.local/lib/python3.10/site-packages (3.37.post415) + Requirement already satisfied: cppyy in /home/username/.local/lib/python3.10/site-packages (from ns3==3.37.post415) (2.4.2) + Requirement already satisfied: cppyy-backend==1.14.10 in /home/username/.local/lib/python3.10/site-packages (from cppyy->ns3==3.37.post415) (1.14.10) + Requirement already satisfied: CPyCppyy==1.12.12 in /home/username/.local/lib/python3.10/site-packages (from cppyy->ns3==3.37.post415) (1.12.12) + Requirement already satisfied: cppyy-cling==6.27.1 in /home/username/.local/lib/python3.10/site-packages (from cppyy->ns3==3.37.post415) (6.27.1) + +To check if the pip wheel was installed, use the pip freeze command to list the installed packages, +then grep ns3 to filter the line of interest. + +.. sourcecode:: bash + + $ pip freeze | grep ns3 + ns3==3.37.post415 + +.. _ns3 wheel: https://pypi.org/project/ns3/#history + +The available versions are also listed on the Pypi page for the `ns3 wheel`_. + +After installing it, you can start using ns-3 right away. For example, using the following script. + +:: + + from ns import ns + + ns.cppyy.cppdef(""" + using namespace ns3; + + Callback,const Address&,const Address&> + make_sinktrace_callback(void(*func)(Ptr,Address,Address)) + { + return MakeCallback(func); + } + """) + + # Define the trace callback + def SinkTracer(packet: ns.Packet, src_address: ns.Address, dst_address: ns.Address) -> None: + print(f"At {ns.Simulator.Now().GetSeconds():.0f}s, '{dst_address}' received packet" + f" with {packet.__deref__().GetSerializedSize()} bytes from '{src_address}'") + + # Create two nodes + csmaNodes = ns.network.NodeContainer() + csmaNodes.Create(2) + + # Connect the two nodes + csma = ns.csma.CsmaHelper() + csma.SetChannelAttribute("DataRate", ns.core.StringValue("100Mbps")) + csma.SetChannelAttribute("Delay", ns.core.TimeValue(ns.core.NanoSeconds(6560))) + csmaDevices = csma.Install(csmaNodes) + + # Install the internet stack + stack = ns.internet.InternetStackHelper() + stack.Install(csmaNodes) + + # Assign Ipv4 addresses + address = ns.internet.Ipv4AddressHelper() + address.SetBase(ns.network.Ipv4Address("10.1.2.0"), ns.network.Ipv4Mask("255.255.255.0")) + csmaInterfaces = address.Assign(csmaDevices) + + # Setup applications + echoServer = ns.applications.UdpEchoServerHelper(9) + + serverApps = echoServer.Install(csmaNodes.Get(0)) + serverApps.Start(ns.core.Seconds(1.0)) + serverApps.Stop(ns.core.Seconds(10.0)) + + echoClient = ns.applications.UdpEchoClientHelper(csmaInterfaces.GetAddress(0).ConvertTo(), 9) + echoClient.SetAttribute("MaxPackets", ns.core.UintegerValue(10)) + echoClient.SetAttribute("Interval", ns.core.TimeValue(ns.core.Seconds(1.0))) + echoClient.SetAttribute("PacketSize", ns.core.UintegerValue(1024)) + + clientApps = echoClient.Install(csmaNodes.Get(1)) + clientApps.Start(ns.core.Seconds(2.0)) + clientApps.Stop(ns.core.Seconds(10.0)) + + # Populate routing tables + ns.internet.Ipv4GlobalRoutingHelper.PopulateRoutingTables() + + # Setup the trace callback + sinkTraceCallback = ns.cppyy.gbl.make_sinktrace_callback(SinkTracer) + serverApps.Get(0).__deref__().TraceConnectWithoutContext("RxWithAddresses", sinkTraceCallback); + + # Set the simulation duration to 11 seconds + ns.Simulator.Stop(ns.Seconds(11)) + + # Run the simulator + ns.Simulator.Run() + ns.Simulator.Destroy() + +Which should print: + +.. sourcecode:: bash + + At 2s, '04-07-00:00:00:00:09:00:00' received packet with 60 bytes from '04-07-0a:01:02:02:01:c0:00' + At 3s, '04-07-00:00:00:00:09:00:00' received packet with 60 bytes from '04-07-0a:01:02:02:01:c0:00' + At 4s, '04-07-00:00:00:00:09:00:00' received packet with 60 bytes from '04-07-0a:01:02:02:01:c0:00' + At 5s, '04-07-00:00:00:00:09:00:00' received packet with 60 bytes from '04-07-0a:01:02:02:01:c0:00' + At 6s, '04-07-00:00:00:00:09:00:00' received packet with 60 bytes from '04-07-0a:01:02:02:01:c0:00' + At 7s, '04-07-00:00:00:00:09:00:00' received packet with 60 bytes from '04-07-0a:01:02:02:01:c0:00' + At 8s, '04-07-00:00:00:00:09:00:00' received packet with 60 bytes from '04-07-0a:01:02:02:01:c0:00' + At 9s, '04-07-00:00:00:00:09:00:00' received packet with 60 bytes from '04-07-0a:01:02:02:01:c0:00' + Caveats ******* @@ -382,6 +523,240 @@ example. There is no structured documentation for the Python bindings like there is Doxygen for the C++ API, but the Doxygen can be consulted to understand how the C++ API works. +To inspect what function and classes are available, you can use +the ``dir`` function. Examples below: + +.. sourcecode:: bash + + >>> print(dir(ns.Simulator)) + ['Cancel', 'Destroy', 'GetContext', 'GetDelayLeft', 'GetEventCount', 'GetImplementation', 'GetMaximumSimulationTime', 'GetSystemId', 'IsExpired', 'IsFinished', 'NO_CONTEXT', 'Now', 'Remove', 'Run', 'Schedule', 'ScheduleDestroy', 'ScheduleNow', 'ScheduleWithContext', 'SetImplementation', 'SetScheduler', 'Stop', '__add__', '__assign__', '__bool__', '__class__', '__delattr__', '__destruct__', '__dict__', '__dir__', '__dispatch__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__invert__', '__le__', '__lt__', '__module__', '__mul__', '__ne__', '__neg__', '__new__', '__pos__', '__python_owns__', '__radd__', '__reduce__', '__reduce_ex__', '__repr__', '__reshape__', '__rmul__', '__rsub__', '__rtruediv__', '__setattr__', '__sizeof__', '__smartptr__', '__str__', '__sub__', '__subclasshook__', '__truediv__', '__weakref__'] + >>> print(dir(ns.DefaultSimulatorImpl)) + ['AggregateObject', 'Cancel', 'Destroy', 'Dispose', 'GetAggregateIterator', 'GetAttribute', 'GetAttributeFailSafe', 'GetContext', 'GetDelayLeft', 'GetEventCount', 'GetInstanceTypeId', 'GetMaximumSimulationTime', 'GetObject', 'GetReferenceCount', 'GetSystemId', 'GetTypeId', 'Initialize', 'IsExpired', 'IsFinished', 'IsInitialized', 'Now', 'PreEventHook', 'Ref', 'Remove', 'Run', 'Schedule', 'ScheduleDestroy', 'ScheduleNow', 'ScheduleWithContext', 'SetAttribute', 'SetAttributeFailSafe', 'SetScheduler', 'Stop', 'TraceConnect', 'TraceConnectWithoutContext', 'TraceDisconnect', 'TraceDisconnectWithoutContext', 'Unref', '__add__', '__assign__', '__bool__', '__class__', '__delattr__', '__destruct__', '__dict__', '__dir__', '__dispatch__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__invert__', '__le__', '__lt__', '__module__', '__mul__', '__ne__', '__neg__', '__new__', '__pos__', '__python_owns__', '__radd__', '__reduce__', '__reduce_ex__', '__repr__', '__reshape__', '__rmul__', '__rsub__', '__rtruediv__', '__setattr__', '__sizeof__', '__smartptr__', '__str__', '__sub__', '__subclasshook__', '__truediv__', '__weakref__'] + >>> print(dir(ns.Time)) + ['AUTO', 'As', 'Compare', 'D', 'FS', 'From', 'FromDouble', 'FromInteger', 'GetDays', 'GetDouble', 'GetFemtoSeconds', 'GetHours', 'GetInteger', 'GetMicroSeconds', 'GetMilliSeconds', 'GetMinutes', 'GetNanoSeconds', 'GetPicoSeconds', 'GetResolution', 'GetSeconds', 'GetTimeStep', 'GetYears', 'H', 'IsNegative', 'IsPositive', 'IsStrictlyNegative', 'IsStrictlyPositive', 'IsZero', 'LAST', 'MIN', 'MS', 'Max', 'Min', 'NS', 'PS', 'RoundTo', 'S', 'SetResolution', 'StaticInit', 'To', 'ToDouble', 'ToInteger', 'US', 'Y', '__add__', '__assign__', '__bool__', '__class__', '__delattr__', '__destruct__', '__dict__', '__dir__', '__dispatch__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__invert__', '__le__', '__lt__', '__module__', '__mul__', '__ne__', '__neg__', '__new__', '__pos__', '__python_owns__', '__radd__', '__reduce__', '__reduce_ex__', '__repr__', '__reshape__', '__rmul__', '__rsub__', '__rtruediv__', '__setattr__', '__sizeof__', '__smartptr__', '__str__', '__sub__', '__subclasshook__', '__truediv__', '__weakref__'] + + +To get more information about expected arguments, you can use the ``help`` +function. + +.. sourcecode:: bash + + >>> help(ns.DefaultSimulatorImpl) + class DefaultSimulatorImpl(SimulatorImpl) + | Method resolution order: + | DefaultSimulatorImpl + | SimulatorImpl + | Object + | SimpleRefCount + | ObjectBase + | cppyy.gbl.CPPInstance + | builtins.object + | + | Methods defined here: + | + | Cancel(...) + | void ns3::DefaultSimulatorImpl::Cancel(const ns3::EventId& id) + | + | Destroy(...) + | void ns3::DefaultSimulatorImpl::Destroy() + | + | GetContext(...) + | unsigned int ns3::DefaultSimulatorImpl::GetContext() + | + | GetDelayLeft(...) + | ns3::Time ns3::DefaultSimulatorImpl::GetDelayLeft(const ns3::EventId& id) + | + | GetEventCount(...) + | unsigned long ns3::DefaultSimulatorImpl::GetEventCount() + | + | GetMaximumSimulationTime(...) + | ns3::Time ns3::DefaultSimulatorImpl::GetMaximumSimulationTime() + | + | GetSystemId(...) + | unsigned int ns3::DefaultSimulatorImpl::GetSystemId() + | + | GetTypeId(...) + | static ns3::TypeId ns3::DefaultSimulatorImpl::GetTypeId() + | + | IsExpired(...) + | bool ns3::DefaultSimulatorImpl::IsExpired(const ns3::EventId& id) + | + | IsFinished(...) + | bool ns3::DefaultSimulatorImpl::IsFinished() + | + | Now(...) + | ns3::Time ns3::DefaultSimulatorImpl::Now() + | + | Remove(...) + | void ns3::DefaultSimulatorImpl::Remove(const ns3::EventId& id) + | + | Run(...) + | void ns3::DefaultSimulatorImpl::Run() + + +Pip wheel packaging +******************* + +This section is meant exclusively for ns-3 maintainers and ns-3 +users that want to redistribute their work as wheels for python. + +The packaging process is defined in the following GitLab job. +The job is split into blocks explained below. + +The manylinux image provides an old glibc compatible with most modern Linux +distributions, resulting on a pip wheel that is compatible across distributions. + +.. sourcecode:: yaml + + .manylinux-pip-wheel: + image: quay.io/pypa/manylinux_2_28_x86_64 + +Then we install the required toolchain and dependencies necessary for both +ns-3 (e.g. libxml2, gsl, sqlite, gtk, etc) and for the bindings and packaging +(e.g. setuptools, wheel, auditwheel, cmake-build-extension, cppyy). + +.. sourcecode:: yaml + + # Install minimal toolchain + - yum install -y libxml2-devel gsl-devel sqlite-devel gtk3-devel boost-devel + # Create Python venv + - $PYTHON -m venv ./venv + - . ./venv/bin/activate + # Upgrade the pip version to reuse the pre-build cppyy + - $PYTHON -m pip install pip --upgrade + - $PYTHON -m pip install setuptools setuptools_scm --upgrade + - $PYTHON -m pip install wheel auditwheel cmake-build-extension cppyy + +The project is then configured loading the configuration settings defined +in the ``ns-3-dev/setup.py`` file. + +.. sourcecode:: yaml + + # Configure and build wheel + - $PYTHON setup.py bdist_wheel build_ext "-DNS3_USE_LIB64=TRUE" + +At this point, we have a wheel that only works in the current system, +since external libraries are not shipped. + +Auditwheel needs to be called resolve and copy external libraries +that we need to ship along the ns-3 module libraries (e.g. libxml2, sqlite3, +gtk, gsl, etc). However, we need to prevent auditwheel from shipping copies of +the libraries built by the ns-3 project. A list of excluded libraries is generated +by the script ``ns-3-dev/build-support/pip-wheel/auditwheel-exclude-list.py``. + +.. sourcecode:: yaml + + - export EXCLUDE_INTERNAL_LIBRARIES=`$PYTHON ./build-support/pip-wheel/auditwheel-exclude-list.py` + # Bundle in shared libraries that were not explicitly packaged or depended upon + - $PYTHON -m auditwheel repair ./dist/*whl -L /lib64 $EXCLUDE_INTERNAL_LIBRARIES + + +At this point, we should have our final wheel ready, but we need to check if it works +before submitting it to Pypi servers. + +We first clean the environment and uninstall the packages previously installed. + +.. sourcecode:: yaml + + # Clean the build directory + - $PYTHON ./ns3 clean + # Clean up the environment + - deactivate + - rm -R ./venv + # Delete toolchain to check if required headers/libraries were really packaged + - yum remove -y libxml2-devel gsl-devel sqlite-devel gtk3-devel boost-devel + + +Then we can install our newly built wheel and test it. + +.. sourcecode:: yaml + + # Install wheel + - $PYTHON -m pip install ./wheelhouse/*whl + - $PYTHON -m pip install matplotlib numpy + # Test the bindings + - $PYTHON ./utils/python-unit-tests.py + - $PYTHON ./examples/realtime/realtime-udp-echo.py + - $PYTHON ./examples/routing/simple-routing-ping6.py + - $PYTHON ./examples/tutorial/first.py + - $PYTHON ./examples/tutorial/second.py + - $PYTHON ./examples/tutorial/third.py + - $PYTHON ./examples/wireless/wifi-ap.py + - $PYTHON ./examples/wireless/mixed-wired-wireless.py + - $PYTHON ./src/bridge/examples/csma-bridge.py + - $PYTHON ./src/brite/examples/brite-generic-example.py + - $PYTHON ./src/core/examples/sample-simulator.py + - $PYTHON ./src/core/examples/sample-rng-plot.py --not-blocking + - $PYTHON ./src/click/examples/nsclick-simple-lan.py + - $PYTHON ./src/flow-monitor/examples/wifi-olsr-flowmon.py + - $PYTHON ./src/flow-monitor/examples/flowmon-parse-results.py output.xml + - $PYTHON ./src/openflow/examples/openflow-switch.py + +If all programs finish normally, the bindings are working as expected, +and will be saved as an artifact. + +.. sourcecode:: yaml + + artifacts: + paths: + - wheelhouse/*.whl + +One can use ``gitlab-ci-local`` to build the pip wheels locally. After that, the wheels +will be stored in ``.gitlab-ci-local/artifacts/manylinux-pip-wheel-py3Lg10/wheelhouse`` +(for Python 3.10). + +The wheel names are based on the number of commits since the latest release. +For example, a wheel built 415 after the release 3.37 will be named +``ns3-3.37.post415-cp310-cp310-manylinux_2_28_x86_64.whl``. + +The wheel name (``ns3``) is defined in the ``/ns-3-dev/setup.cfg`` file, and that +name should match the build prefix specified in ``/ns-3-dev/setup.py`` file. + +The ``cp310-cp310`` indicates that this wheel is compatible from Python 3.10 and up to Python 3.10. + +The ``manylinux_2_28`` indicates that this is a manylinux wheel targeting glibc 2.28. + +The ``x86_64`` indicates that this is a 64-bit build targeting Intel/AMD processors. + +.. _Pypi: https://pypi.org/account/register/ +.. _Twine: https://twine.readthedocs.io/en/stable/ + +After packaging, we can either deploy that wheel locally or upload the wheel to Pypi for general availability. + +Local deployment +**************** + +To deploy a wheel locally, simply share the wheel file across the desired machines. +Then install the wheel and its dependencies running the following command: + +.. sourcecode:: bash + + $ pip install *.whl + +Publishing the pip wheel via Pypi +********************************* + +Publishing a pip wheel requires a `Pypi`_ account. + +After creating your account, install `Twine`_, an utility to upload the wheel to Pypi. + +Then run twine to upload the wheel to the Pypi servers. + +.. sourcecode:: bash + + $ twine upload .gitlab-ci-local/artifacts/manylinux-pip-wheel-py3Lg10/wheelhouse/*.whl + +Enter your Pypi username and password as requested. + +Your wheel should be up and running. Give it a try just to make sure. + +For the upstream pip wheel, try: + +.. sourcecode:: bash + + $ pip install ns3 + $ python3 -c "from ns import ns; print(ns.Simulator.Now())" + Historical Information ********************** diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..a18eb14ff --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = [ + "wheel", + "setuptools>=45", + "setuptools_scm[toml]>=6.0", + "cmake-build-extension>=0.4", + "cppyy==2.4.2", +] + +[tool.setuptools_scm] +version_scheme = "post-release" +local_scheme = "no-local-version" + +[tool.cibuildwheel] +build-frontend = "build" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..679a7d3d9 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,44 @@ +[metadata] +name = ns3 +description = ns-3 network simulator and visualizer +long_description = file: README.md +long_description_content_type = text/markdown +author = nsnam +author_email = webmaster@nsnam.org +license = GPL-2.0-only +platforms = any +url = https://www.nsnam.org/ + +project_urls = + Tracker = https://gitlab.com/nsnam/ns-3-dev/-/issues + Documentation = https://www.nsnam.org/docs/tutorial/html/index.html + Source = https://gitlab.com/nsnam/ns-3-dev + +keywords = + network-simulator + +classifiers = + Development Status :: 5 - Production/Stable + Operating System :: POSIX :: Linux + #Operating System :: MacOS + #Operating System :: Microsoft :: Windows + Intended Audience :: Education + Intended Audience :: Developers + Intended Audience :: Science/Research + Programming Language :: C++ + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: Implementation :: CPython + License :: OSI Approved :: GNU General Public License v2 (GPLv2) + +[options] +zip_safe = False +python_requires = >=3.6 +install_requires = + cppyy==2.4.2 + diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..8aa256955 --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +import cmake_build_extension +import setuptools +import sys +import sysconfig + +setuptools.setup( + cmdclass=dict(build_ext=cmake_build_extension.BuildExtension), + packages=['ns', 'visualizer'], + package_dir={ + 'ns': './build-support/pip-wheel/ns', + 'visualizer': './build-support/pip-wheel/visualizer' + }, + ext_modules=[ + cmake_build_extension.CMakeExtension( + name="BuildAndInstall", + install_prefix="ns3", + cmake_configure_options=[ + "-DCMAKE_BUILD_TYPE:STRING=release", + "-DNS3_ASSERT:BOOL=ON", + "-DNS3_LOG:BOOL=ON", + "-DNS3_WARNINGS_AS_ERRORS:BOOL=OFF", + "-DNS3_PYTHON_BINDINGS:BOOL=ON", + "-DNS3_BINDINGS_INSTALL_DIR:STRING=INSTALL_PREFIX", + "-DNS3_FETCH_OPTIONAL_COMPONENTS:BOOL=ON", + "-DNS3_PIP_PACKAGING:BOOL=ON", + "-DNS3_USE_LIB64:BOOL=ON", + # Make CMake find python components from the currently running python + # https://catherineh.github.io/programming/2021/11/16/python-binary-distributions-whls-with-c17-cmake-auditwheel-and-manylinux + f"-DPython3_LIBRARY_DIRS={sysconfig.get_config_var('LIBDIR')}", + f"-DPython3_INCLUDE_DIRS={sysconfig.get_config_var('INCLUDEPY')}", + f"-DPython3_EXECUTABLE={sys.executable}" + ] + ), + ], +) diff --git a/src/brite/examples/brite-generic-example.py b/src/brite/examples/brite-generic-example.py new file mode 100644 index 000000000..4c3eac2ec --- /dev/null +++ b/src/brite/examples/brite-generic-example.py @@ -0,0 +1,101 @@ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation; +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Modified by: Gabriel Ferreira +# + +from ns import ns + +ns.LogComponentEnable("BriteTopologyHelper", ns.LOG_LEVEL_ALL) + +# BRITE needs a configuration file to build its graph. By default, this +# example will use the TD_ASBarabasi_RTWaxman.conf file. There are many others +# which can be found in the BRITE/conf_files directory +confFile = "src/brite/examples/conf_files/TD_ASBarabasi_RTWaxman.conf" + +# Invoke the BriteTopologyHelper and pass in a BRITE +# configuration file and a seed file. This will use +# BRITE to build a graph from which we can build the ns-3 topology +bth = ns.BriteTopologyHelper(confFile) +bth.AssignStreams(3) + +p2p = ns.PointToPointHelper() + +stack = ns.InternetStackHelper() + +nixRouting = ns.Ipv4NixVectorHelper() +stack.SetRoutingHelper(nixRouting) + +address = ns.Ipv4AddressHelper() +address.SetBase("10.0.0.0", "255.255.255.252") + +bth.BuildBriteTopology(stack) +bth.AssignIpv4Addresses(address) + +print(f"Number of AS created {bth.GetNAs()}") + +# The BRITE topology generator generates a topology of routers. Here we create +# two subnetworks which we attach to router leaf nodes generated by BRITE +# use just one node +client = ns.NodeContainer() +server = ns.NodeContainer() + +client.Create(1) +stack.Install(client) + +# install client node on last leaf node of AS 0 +numLeafNodesInAsZero = bth.GetNLeafNodesForAs(0) +client.Add(bth.GetLeafNodeForAs(0, numLeafNodesInAsZero - 1)) + +server.Create(1) +stack.Install(server) + +# install server node on last leaf node on AS 1 +numLeafNodesInAsOne = bth.GetNLeafNodesForAs(1) +server.Add(bth.GetLeafNodeForAs(1, numLeafNodesInAsOne - 1)) + +p2p.SetDeviceAttribute("DataRate", ns.StringValue("5Mbps")) +p2p.SetChannelAttribute("Delay", ns.StringValue("2ms")) + +p2pClientDevices = p2p.Install(client) +p2pServerDevices = p2p.Install(server) + +address.SetBase("10.1.0.0", "255.255.0.0") +clientInterfaces = address.Assign(p2pClientDevices) + +address.SetBase("10.2.0.0", "255.255.0.0") +serverInterfaces = ns.Ipv4InterfaceContainer() +serverInterfaces = address.Assign(p2pServerDevices) + +echoServer = ns.UdpEchoServerHelper(9) +serverApps = echoServer.Install(server.Get(0)) +serverApps.Start(ns.Seconds(1.0)) +serverApps.Stop(ns.Seconds(5.0)) + +echoClient = ns.UdpEchoClientHelper(serverInterfaces.GetAddress(0).ConvertTo(), 9) +echoClient.SetAttribute("MaxPackets", ns.UintegerValue(1)) +echoClient.SetAttribute("Interval", ns.TimeValue(ns.Seconds(1.))) +echoClient.SetAttribute("PacketSize", ns.UintegerValue(1024)) + +clientApps = echoClient.Install(client.Get(0)) +clientApps.Start(ns.Seconds(2.0)) +clientApps.Stop(ns.Seconds(5.0)) + +asciiTrace = ns.AsciiTraceHelper() +p2p.EnableAsciiAll(asciiTrace.CreateFileStream("briteLeaves.tr")) + +# Run the simulator +ns.Simulator.Stop(ns.Seconds(6.0)) +ns.Simulator.Run() +ns.Simulator.Destroy() diff --git a/src/brite/test/examples-to-run.py b/src/brite/test/examples-to-run.py index b89d29a58..a69525170 100644 --- a/src/brite/test/examples-to-run.py +++ b/src/brite/test/examples-to-run.py @@ -16,4 +16,6 @@ cpp_examples = [ # (example_name, do_run). # # See test.py for more information. -python_examples = [] +python_examples = [ + ("brite-generic-example.py", "ENABLE_BRITE == True"), +] diff --git a/src/click/examples/nsclick-simple-lan.py b/src/click/examples/nsclick-simple-lan.py new file mode 100644 index 000000000..3a32e62f1 --- /dev/null +++ b/src/click/examples/nsclick-simple-lan.py @@ -0,0 +1,79 @@ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation; +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Authors: Lalith Suresh +# Modified by: Gabriel Ferreira +# + +import os.path + +from ns import ns + +ns.LogComponentEnable("Ipv4ClickRouting", ns.LOG_LEVEL_ALL) +ns.LogComponentEnable("Ipv4L3ClickProtocol", ns.LOG_LEVEL_ALL) + +clickConfigFolder = os.path.dirname(__file__) + +csmaNodes = ns.NodeContainer() +csmaNodes.Create(2) + +# Setup CSMA channel between the nodes +csma = ns.CsmaHelper() +csma.SetChannelAttribute("DataRate", ns.DataRateValue(ns.DataRate(5000000))) +csma.SetChannelAttribute("Delay", ns.TimeValue(ns.MilliSeconds(2))) +csmaDevices = csma.Install(csmaNodes) + +# Install normal internet stack on node B +internet = ns.InternetStackHelper() +internet.Install(csmaNodes.Get(1)) + +# Install Click on node A +clickinternet = ns.ClickInternetStackHelper() +clickinternet.SetClickFile(csmaNodes.Get(0), + clickConfigFolder + "/nsclick-lan-single-interface.click") +clickinternet.SetRoutingTableElement(csmaNodes.Get(0), "rt") +clickinternet.Install(csmaNodes.Get(0)) + +# Configure IP addresses for the nodes +ipv4 = ns.Ipv4AddressHelper() +ipv4.SetBase("172.16.1.0", "255.255.255.0") +ipv4.Assign(csmaDevices) + +# Configure traffic application and sockets +LocalAddress = ns.InetSocketAddress(ns.Ipv4Address.GetAny(), 50000).ConvertTo() +packetSinkHelper = ns.PacketSinkHelper("ns3::TcpSocketFactory", LocalAddress) +recvapp = packetSinkHelper.Install(csmaNodes.Get(1)) +recvapp.Start(ns.Seconds(5.0)) +recvapp.Stop(ns.Seconds(10.0)) + +onOffHelper = ns.OnOffHelper("ns3::TcpSocketFactory", ns.Address()) +onOffHelper.SetAttribute("OnTime", ns.StringValue("ns3::ConstantRandomVariable[Constant=1]")) +onOffHelper.SetAttribute("OffTime", ns.StringValue("ns3::ConstantRandomVariable[Constant=0]")) + +appcont = ns.ApplicationContainer() + +remoteAddress = ns.InetSocketAddress(ns.Ipv4Address("172.16.1.2"), 50000).ConvertTo() +onOffHelper.SetAttribute("Remote", ns.AddressValue(remoteAddress)) +appcont.Add(onOffHelper.Install(csmaNodes.Get(0))) + +appcont.Start(ns.Seconds(5.0)) +appcont.Stop(ns.Seconds(10.0)) + +# For tracing +csma.EnablePcap("nsclick-simple-lan", csmaDevices, False) + +ns.Simulator.Stop(ns.Seconds(20.0)) +ns.Simulator.Run() + +ns.Simulator.Destroy() diff --git a/src/click/test/examples-to-run.py b/src/click/test/examples-to-run.py index b5ad05946..342e58a9f 100644 --- a/src/click/test/examples-to-run.py +++ b/src/click/test/examples-to-run.py @@ -21,4 +21,6 @@ cpp_examples = [ # (example_name, do_run). # # See test.py for more information. -python_examples = [] +python_examples = [ + ("nsclick-simple-lan.py", "NSCLICK == True"), +] diff --git a/src/config-store/CMakeLists.txt b/src/config-store/CMakeLists.txt index 4a2931264..b62fba3e3 100644 --- a/src/config-store/CMakeLists.txt +++ b/src/config-store/CMakeLists.txt @@ -12,6 +12,13 @@ if(${GTK3_FOUND}) set(gtk_libraries ${GTK3_LIBRARIES} ) + if(${NS3_PIP_PACKAGING}) + # In case we are packaging ns-3, leave gtk symbols undefined and pray for + # the linker to find the correct libraries locally + set(gtk_libraries + -Wl,--allow-shlib-undefined + ) + endif() if(${GCC}) add_definitions(-Wno-parentheses) endif() diff --git a/src/openflow/examples/openflow-switch.py b/src/openflow/examples/openflow-switch.py new file mode 100644 index 000000000..d39abc41b --- /dev/null +++ b/src/openflow/examples/openflow-switch.py @@ -0,0 +1,82 @@ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation; +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Modified by: Gabriel Ferreira +# + +from ns import ns + +ns.LogComponentEnable("OpenFlowInterface", ns.LOG_LEVEL_ALL) +ns.LogComponentEnable("OpenFlowSwitchNetDevice", ns.LOG_LEVEL_ALL) + +terminals = ns.NodeContainer() +terminals.Create(4) + +csmaSwitch = ns.NodeContainer() +csmaSwitch.Create(1) + +csma = ns.CsmaHelper() +csma.SetChannelAttribute("DataRate", ns.DataRateValue(5000000)) +csma.SetChannelAttribute("Delay", ns.TimeValue(ns.MilliSeconds(2))) + +terminalDevices = ns.NetDeviceContainer() +switchDevices = ns.NetDeviceContainer() +for i in range(4): + container = ns.NodeContainer() + container.Add(terminals.Get(i)) + container.Add(csmaSwitch) + link = csma.Install(container) + terminalDevices.Add(link.Get(0)) + switchDevices.Add(link.Get(1)) + +switchNode = csmaSwitch.Get(0) +swtch = ns.OpenFlowSwitchHelper() +controller = ns.ofi.DropController() +# controller = ns.CreateObject("ns3::ofi::LearningController") +swtch.Install(switchNode, switchDevices, controller) +# controller->SetAttribute("ExpirationTime", TimeValue(timeout)) + + +internet = ns.InternetStackHelper() +internet.Install(terminals) + +ipv4 = ns.Ipv4AddressHelper() +ipv4.SetBase("10.1.1.0", "255.255.255.0") +ipv4.Assign(terminalDevices) + +port = 9 + +onoff = ns.OnOffHelper("ns3::UdpSocketFactory", ns.InetSocketAddress(ns.Ipv4Address("10.1.1.2"), port).ConvertTo()) +onoff.SetConstantRate(ns.DataRate("500kb/s")) + +app = onoff.Install(terminals.Get(0)) + +app.Start(ns.Seconds(1.0)) +app.Stop(ns.Seconds(10.0)) + +sink = ns.PacketSinkHelper("ns3::UdpSocketFactory", + ns.InetSocketAddress(ns.Ipv4Address.GetAny(), port).ConvertTo()) +app = sink.Install(terminals.Get(1)) +app.Start(ns.Seconds(0.0)) + +onoff.SetAttribute("Remote", ns.AddressValue(ns.InetSocketAddress(ns.Ipv4Address("10.1.1.1"), port).ConvertTo())) +app = onoff.Install(terminals.Get(3)) +app.Start(ns.Seconds(1.1)) +app.Stop(ns.Seconds(10.0)) + +app = sink.Install(terminals.Get(0)) +app.Start(ns.Seconds(0.0)) + +ns.Simulator.Run() +ns.Simulator.Destroy() diff --git a/src/openflow/test/examples-to-run.py b/src/openflow/test/examples-to-run.py index d5ab8e680..b03280950 100644 --- a/src/openflow/test/examples-to-run.py +++ b/src/openflow/test/examples-to-run.py @@ -16,4 +16,6 @@ cpp_examples = [ # (example_name, do_run). # # See test.py for more information. -python_examples = [] +python_examples = [ + ("openflow-switch.py", "ENABLE_OPENFLOW == True"), +] diff --git a/src/visualizer/CMakeLists.txt b/src/visualizer/CMakeLists.txt index 9030dacbb..30f01c739 100644 --- a/src/visualizer/CMakeLists.txt +++ b/src/visualizer/CMakeLists.txt @@ -1,10 +1,23 @@ +# Embedding python isn't a good option, so just leave symbols undefined and hope +# for the best https://github.com/pypa/manylinux/pull/1185 +# https://github.com/scikit-build/scikit-build/pull/47 +# https://9to5answer.com/can-gcc-not-complain-about-undefined-references +set(python_libraries + ${Python3_LIBRARIES} +) +if(${NS3_PIP_PACKAGING}) + set(python_libraries + -Wl,--allow-shlib-undefined + ) +endif() + build_lib( LIBNAME visualizer SOURCE_FILES model/pyviz.cc model/visual-simulator-impl.cc HEADER_FILES model/pyviz.h LIBRARIES_TO_LINK - ${Python3_LIBRARIES} + ${python_libraries} ${libcore} ${libinternet} ${libwifi} diff --git a/src/visualizer/visualizer/plugins/interface_statistics.py b/src/visualizer/visualizer/plugins/interface_statistics.py index f60e34ab6..dc2a7a4a1 100644 --- a/src/visualizer/visualizer/plugins/interface_statistics.py +++ b/src/visualizer/visualizer/plugins/interface_statistics.py @@ -1,5 +1,9 @@ from gi.repository import Gtk -from visualizer.base import InformationWindow + +try: + from nsnam.visualizer.base import InformationWindow +except ModuleNotFoundError: + from visualizer.base import InformationWindow NODE_STATISTICS_MEMORY = 10 diff --git a/src/visualizer/visualizer/plugins/ipv4_routing_table.py b/src/visualizer/visualizer/plugins/ipv4_routing_table.py index 724781beb..e14180290 100644 --- a/src/visualizer/visualizer/plugins/ipv4_routing_table.py +++ b/src/visualizer/visualizer/plugins/ipv4_routing_table.py @@ -1,6 +1,9 @@ from gi.repository import Gtk -from visualizer.base import InformationWindow +try: + from nsnam.visualizer.base import InformationWindow +except ModuleNotFoundError: + from visualizer.base import InformationWindow ## ShowIpv4RoutingTable class class ShowIpv4RoutingTable(InformationWindow): diff --git a/src/visualizer/visualizer/plugins/olsr.py b/src/visualizer/visualizer/plugins/olsr.py index 286bf1048..c43794895 100644 --- a/src/visualizer/visualizer/plugins/olsr.py +++ b/src/visualizer/visualizer/plugins/olsr.py @@ -1,7 +1,10 @@ from gi.repository import Gtk from gi.repository import Gdk -from visualizer.base import InformationWindow +try: + from nsnam.visualizer.base import InformationWindow +except ModuleNotFoundError: + from visualizer.base import InformationWindow ## ShowOlsrRoutingTable class class ShowOlsrRoutingTable(InformationWindow): diff --git a/src/visualizer/visualizer/plugins/show_last_packets.py b/src/visualizer/visualizer/plugins/show_last_packets.py index 7f9bb6289..90a71a667 100644 --- a/src/visualizer/visualizer/plugins/show_last_packets.py +++ b/src/visualizer/visualizer/plugins/show_last_packets.py @@ -3,7 +3,11 @@ from gi.repository import Gtk from ns import ns -from visualizer.base import InformationWindow +try: + from nsnam.visualizer.base import InformationWindow +except ModuleNotFoundError: + from visualizer.base import InformationWindow + from kiwi.ui.objectlist import ObjectList, Column ## ShowLastPackets class diff --git a/src/visualizer/visualizer/plugins/wifi_intrastructure_link.py b/src/visualizer/visualizer/plugins/wifi_intrastructure_link.py index 4504f076d..06ec82713 100644 --- a/src/visualizer/visualizer/plugins/wifi_intrastructure_link.py +++ b/src/visualizer/visualizer/plugins/wifi_intrastructure_link.py @@ -1,7 +1,11 @@ import math from ns import ns from gi.repository import GooCanvas -from visualizer.base import Link, transform_distance_canvas_to_simulation + +try: + from nsnam.visualizer.base import Link, transform_distance_canvas_to_simulation +except ModuleNotFoundError: + from visualizer.base import Link, transform_distance_canvas_to_simulation ## WifiLink class class WifiLink(Link): diff --git a/utils/tests/gitlab-ci-cppyy.yml b/utils/tests/gitlab-ci-cppyy.yml index f363ef3a2..66ed54087 100644 --- a/utils/tests/gitlab-ci-cppyy.yml +++ b/utils/tests/gitlab-ci-cppyy.yml @@ -25,7 +25,6 @@ cppyy-22.04: - ./ns3 run wifi-ap.py - ./ns3 run simple-routing-ping6.py - ./ns3 run realtime-udp-echo.py - - ./ns3 run bianchi11ax.py - ./ns3 run sample-simulator.py - ./ns3 run "sample-rng-plot.py --not-blocking" - ./ns3 run csma-bridge.py @@ -86,3 +85,89 @@ cppyy-18.04: - ./ns3 run third.py - ./ns3 run ./utils/python-unit-tests.py timeout: 9h + +.manylinux-pip-wheel: + image: quay.io/pypa/manylinux_2_28_x86_64 + only: + variables: + - $RELEASE == "manual" + script: + # Untar libraries (just to make CMake happy, but we are not going to actually link them) + # https://github.com/scikit-build/scikit-build/pull/47 + - tar -xvf /opt/_internal/static-libs-for-embedding-only.tar.xz -C /opt/_internal + # Install minimal toolchain + - yum install -y libxml2-devel gsl-devel sqlite-devel gtk3-devel boost-devel + # Create Python venv + - $PYTHON -m venv ./venv + - . ./venv/bin/activate + # Upgrade the pip version to reuse the pre-build cppyy + - $PYTHON -m pip install pip --upgrade + - $PYTHON -m pip install setuptools setuptools_scm wheel --upgrade + - $PYTHON -m pip install wheel auditwheel cmake-build-extension cppyy==2.4.2 + # Configure and build wheel + - $PYTHON setup.py bdist_wheel build_ext + - export EXCLUDE_INTERNAL_LIBRARIES=`$PYTHON ./build-support/pip-wheel/auditwheel-exclude-list.py` + # Bundle in shared libraries that were not explicitly packaged or depended upon + - $PYTHON -m auditwheel repair ./dist/*whl -L /lib64 $EXCLUDE_INTERNAL_LIBRARIES + # Clean the build directory + - $PYTHON ./ns3 clean + # Clean up the environment + - deactivate + - rm -R ./venv + # Delete toolchain to check if required headers/libraries were really packaged + - yum remove -y libxml2-devel gsl-devel sqlite-devel gtk3-devel boost-devel + # Install wheel + - $PYTHON -m pip install ./wheelhouse/*whl + - $PYTHON -m pip install matplotlib numpy + # Test the bindings + - $PYTHON ./utils/python-unit-tests.py + - $PYTHON ./examples/realtime/realtime-udp-echo.py + - $PYTHON ./examples/routing/simple-routing-ping6.py + - $PYTHON ./examples/tutorial/first.py + - $PYTHON ./examples/tutorial/second.py + - $PYTHON ./examples/tutorial/third.py + - $PYTHON ./examples/wireless/wifi-ap.py + - $PYTHON ./examples/wireless/mixed-wired-wireless.py + - $PYTHON ./src/bridge/examples/csma-bridge.py + - $PYTHON ./src/brite/examples/brite-generic-example.py + - $PYTHON ./src/core/examples/sample-simulator.py + - $PYTHON ./src/core/examples/sample-rng-plot.py --not-blocking + - $PYTHON ./src/click/examples/nsclick-simple-lan.py + - $PYTHON ./src/flow-monitor/examples/wifi-olsr-flowmon.py + - $PYTHON ./src/flow-monitor/examples/flowmon-parse-results.py output.xml + - $PYTHON ./src/openflow/examples/openflow-switch.py + timeout: 3h + artifacts: + paths: + - wheelhouse/*.whl + when: on_success + +manylinux-pip-wheel-py3.6: + extends: .manylinux-pip-wheel + variables: + PYTHON: 'python3.6' + +manylinux-pip-wheel-py3.7: + extends: .manylinux-pip-wheel + variables: + PYTHON: 'python3.7' + +manylinux-pip-wheel-py3.8: + extends: .manylinux-pip-wheel + variables: + PYTHON: 'python3.8' + +manylinux-pip-wheel-py3.9: + extends: .manylinux-pip-wheel + variables: + PYTHON: 'python3.9' + +manylinux-pip-wheel-py3.10: + extends: .manylinux-pip-wheel + variables: + PYTHON: 'python3.10' + +manylinux-pip-wheel-py3.11: + extends: .manylinux-pip-wheel + variables: + PYTHON: 'python3.11'