F5/unison
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bindings: package ns-3 as a pip wheel

Browse Source 
Includes:
- Python examples to test Brite, Click and Openflow
- CI jobs for manylinux packaging of pip wheel
- Support for Linux distributions with lib and lib64 directories
- Configuration of RPATH not to require setting LD_LIBRARY_PATH

Signed-off-by: Gabriel Ferreira <gabrielcarvfer@gmail.com>
This commit is contained in:
Gabriel Ferreira
2023-02-03 22:30:38 -03:00
parent 06845023d1
commit ec9d26acd9
24 changed files with 1232 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

402
bindings/python/ns__init__.py
View File

@@ -1,5 +1,4 @@
import builtins
from copy import copy
from functools import lru_cache
import glob
import os.path
@@ -11,7 +10,7 @@ DEFAULT_INCLUDE_DIR = sysconfig.get_config_var("INCLUDEDIR")
DEFAULT_LIB_DIR = sysconfig.get_config_var("LIBDIR")
def find_ns3_lock():
def find_ns3_lock() -> str:
# Get the absolute path to this file
path_to_this_init_file = os.path.dirname(os.path.abspath(__file__))
path_to_lock = path_to_this_init_file
@@ -29,12 +28,14 @@ def find_ns3_lock():
if lock_file in os.listdir(path_to_lock):
path_to_lock += os.sep + lock_file
else:
path_to_lock = None
path_to_lock = ""
return path_to_lock
SYSTEM_LIBRARY_DIRECTORIES = (DEFAULT_LIB_DIR,
os.path.dirname(DEFAULT_LIB_DIR)
os.path.dirname(DEFAULT_LIB_DIR),
"/usr/lib64",
"/usr/lib"
)
DYNAMIC_LIBRARY_EXTENSIONS = {"linux": "so",
"win32": "dll",
@@ -81,12 +82,14 @@ def _search_libraries() -> dict:
# Search for the core library in the search paths
libraries = []
for search_path in library_search_paths:
libraries += glob.glob("%s/**/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=True)
if os.path.exists(search_path):
libraries += glob.glob("%s/**/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=True)
# Search system library directories (too slow for recursive search)
for search_path in SYSTEM_LIBRARY_DIRECTORIES:
libraries += glob.glob("%s/**/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=False)
libraries += glob.glob("%s/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=False)
if os.path.exists(search_path):
libraries += glob.glob("%s/**/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=False)
libraries += glob.glob("%s/*.%s*" % (search_path, LIBRARY_EXTENSION), recursive=False)
del search_path, library_search_paths
@@ -120,8 +123,35 @@ def search_libraries(library_name: str) -> list:
return matched_libraries
def extract_library_include_dirs(library_name: str, prefix: str) -> list:
library_path = "%s/lib/%s" % (prefix, library_name)
LIBRARY_AND_DEFINES = {
"libgsl": ["HAVE_GSL"],
"libxml2": ["HAVE_LIBXML2"],
"libsqlite3": ["HAVE_SQLITE3"],
"openflow": ["NS3_OPENFLOW", "ENABLE_OPENFLOW"],
"click": ["NS3_CLICK"]
}
def add_library_defines(library_name: str):
has_defines = list(filter(lambda x: x in library_name, LIBRARY_AND_DEFINES.keys()))
defines = ""
if len(has_defines):
for define in LIBRARY_AND_DEFINES[has_defines[0]]:
defines += (f"""
#ifndef {define}
#define {define} 1
#endif
""")
return defines
def extract_linked_libraries(library_name: str, prefix: str) -> tuple:
lib = ""
for variant in ["lib", "lib64"]:
library_path = f"{prefix}/{variant}/{library_name}"
if os.path.exists(library_path):
lib = variant
break
linked_libs = []
# First discover which 3rd-party libraries are used by the current module
try:
@@ -131,10 +161,17 @@ def extract_library_include_dirs(library_name: str, prefix: str) -> list:
print("Failed to extract libraries used by {library} with exception:{exception}"
.format(library=library_path, exception=e))
exit(-1)
return library_path, lib, list(map(lambda x: x.decode("utf-8"), linked_libs))
def extract_library_include_dirs(library_name: str, prefix: str) -> tuple:
library_path, lib, linked_libs = extract_linked_libraries(library_name, prefix)
linked_libs_include_dirs = set()
defines = add_library_defines(library_name)
# Now find these libraries and add a few include paths for them
for linked_library in map(lambda x: x.decode("utf-8"), linked_libs):
for linked_library in linked_libs:
# Skip ns-3 modules
if "libns3" in linked_library:
continue
@@ -148,15 +185,18 @@ def extract_library_include_dirs(library_name: str, prefix: str) -> list:
"Failed to find {library}. Make sure its library directory is in LD_LIBRARY_PATH.".format(
library=linked_library))
# Get path with shortest length
# Get path with the shortest length
linked_library_path = sorted(linked_library_path, key=lambda x: len(x))[0]
# If library is part of the ns-3 build, continue without any new includes
if prefix in linked_library_path:
continue
# Add defines based in linked libraries found
defines += add_library_defines(linked_library)
# If it is part of the system directories, try to find it
system_include_dir = os.path.dirname(linked_library_path).replace("lib", "include")
system_include_dir = os.path.dirname(linked_library_path).replace(lib, "include")
if os.path.exists(system_include_dir):
linked_libs_include_dirs.add(system_include_dir)
@@ -179,134 +219,214 @@ def extract_library_include_dirs(library_name: str, prefix: str) -> list:
inc_path = os.path.join(lib_path, "include")
if os.path.exists(inc_path):
linked_libs_include_dirs.add(inc_path)
return list(linked_libs_include_dirs)
return list(linked_libs_include_dirs), defines
def find_ns3_from_lock_file(lock_file: str) -> (str, list, str):
# Load NS3_ENABLED_MODULES from the lock file inside the build directory
values = {}
# If we find a lock file, load the ns-3 modules from it
# Should be the case when running from the source directory
exec(open(lock_file).read(), {}, values)
suffix = "-" + values["BUILD_PROFILE"] if values["BUILD_PROFILE"] != "release" else ""
modules = [module.replace("ns3-", "") for module in values["NS3_ENABLED_MODULES"]]
prefix = values["out_dir"]
libraries = {os.path.splitext(os.path.basename(x))[0]: x for x in os.listdir(os.path.join(prefix, "lib"))}
version = values["VERSION"]
# Filter out test libraries and incorrect versions
def filter_in_matching_ns3_libraries(libraries_to_filter: dict,
modules_to_filter: list,
version: str,
suffix: str) -> dict:
suffix = [suffix[1:]] if len(suffix) > 1 else []
filtered_in_modules = []
for module in modules_to_filter:
filtered_in_modules += list(filter(lambda x: "-".join([version, module, *suffix]) in x,
libraries_to_filter.keys()))
for library in list(libraries_to_filter.keys()):
if library not in filtered_in_modules:
libraries_to_filter.pop(library)
return libraries_to_filter
libraries = filter_in_matching_ns3_libraries(libraries, modules, version, suffix)
# When we have the lock file, we assemble the correct library names
libraries_to_load = []
for module in modules:
library_name = "libns{version}-{module}{suffix}".format(
version=version,
module=module,
suffix=suffix
)
if library_name not in libraries:
raise Exception("Missing library %s\n" % library_name,
"Build all modules with './ns3 build'"
)
libraries_to_load.append(libraries[library_name])
return prefix, libraries_to_load, version
# Extract version and build suffix (if it exists)
def filter_module_name(library: str) -> str:
library = os.path.splitext(os.path.basename(library))[0]
components = library.split("-")
# Remove version-related prefixes
if "libns3" in components[0]:
components.pop(0)
if "dev" == components[0]:
components.pop(0)
if "rc" in components[0]:
components.pop(0)
# Drop build profile suffix and test libraries
if components[-1] in ["debug", "default", "optimized", "release", "relwithdebinfo", "minsizerel"]:
components.pop(-1)
return "-".join(components)
def extract_version(library: str, module: str) -> str:
library = os.path.basename(library)
return re.findall(r"libns([\d.|rc|\-dev]+)-", library)[0]
def get_newest_version(versions: list) -> str:
versions = list(sorted(versions))
if "dev" in versions[0]:
return versions[0]
# Check if there is a release of a possible candidate
try:
pos = versions.index(os.path.splitext(versions[-1])[0])
except ValueError:
pos = None
# Remove release candidates
if pos is not None:
return versions[pos]
else:
return versions[-1]
def find_ns3_from_search() -> (str, list, str):
libraries = search_libraries("ns3")
if not libraries:
raise Exception("ns-3 libraries were not found.")
# If there is a version with a hash by the end, we have a pip-installed library
pip_libraries = list(filter(lambda x: "python" in x, libraries))
if pip_libraries:
# We drop non-pip libraries
libraries = pip_libraries
# The prefix is the directory with the lib directory
# libns3-dev-core.so/../../
prefix = os.path.dirname(os.path.dirname(libraries[0]))
# Remove test libraries
libraries = list(filter(lambda x: "test" not in x, libraries))
# Filter out module names
modules = set([filter_module_name(library) for library in libraries])
def filter_in_newest_ns3_libraries(libraries_to_filter: list, modules_to_filter: list) -> tuple:
newest_version_found = ""
# Filter out older ns-3 libraries
for module in list(modules_to_filter):
# Filter duplicates of modules, while excluding test libraries
conflicting_libraries = list(filter(lambda x: module == filter_module_name(x), libraries_to_filter))
# Extract versions from conflicting libraries
conflicting_libraries_versions = list(map(lambda x: extract_version(x, module), conflicting_libraries))
# Get the newest version found for that library
newest_version = get_newest_version(conflicting_libraries_versions)
# Check if the version found is the global newest version
if not newest_version_found:
newest_version_found = newest_version
else:
newest_version_found = get_newest_version([newest_version, newest_version_found])
if newest_version != newest_version_found:
raise Exception("Incompatible versions of the ns-3 module '%s' were found: %s != %s."
% (module, newest_version, newest_version_found))
for conflicting_library in list(conflicting_libraries):
if "-".join([newest_version, module]) not in conflicting_library:
libraries.remove(conflicting_library)
conflicting_libraries.remove(conflicting_library)
if len(conflicting_libraries) > 1:
raise Exception("There are multiple build profiles for module '%s'.\nDelete one to continue: %s"
% (module, ", ".join(conflicting_libraries)))
return libraries_to_filter, newest_version_found
# Get library base names
libraries, version = filter_in_newest_ns3_libraries(libraries, list(modules))
return prefix, libraries, version
def load_modules():
lock_file = find_ns3_lock()
libraries_to_load = []
if lock_file:
# Load NS3_ENABLED_MODULES from the lock file inside the build directory
values = {}
# Search for prefix to ns-3 build, modules and respective libraries plus version
ret = find_ns3_from_search() if not lock_file else find_ns3_from_lock_file(lock_file)
# If we find a lock file, load the ns-3 modules from it
# Should be the case when running from the source directory
exec(open(lock_file).read(), {}, values)
suffix = "-" + values["BUILD_PROFILE"] if values["BUILD_PROFILE"] != "release" else ""
modules = [module.replace("ns3-", "") for module in values["NS3_ENABLED_MODULES"]]
prefix = values["out_dir"]
libraries = {os.path.splitext(os.path.basename(x))[0]: x for x in os.listdir(os.path.join(prefix, "lib"))}
version = values["VERSION"]
# Unpack returned values
prefix, libraries, version = ret
prefix = os.path.abspath(prefix)
# Filter out test libraries and incorrect versions
def filter_in_matching_ns3_libraries(libraries_to_filter: dict,
modules_to_filter: list,
version: str,
suffix: str) -> dict:
suffix = [suffix[1:]] if len(suffix) > 1 else []
filtered_in_modules = []
for module in modules_to_filter:
filtered_in_modules += list(filter(lambda x: "-".join([version, module, *suffix]) in x,
libraries_to_filter.keys()))
for library in list(libraries_to_filter.keys()):
if library not in filtered_in_modules:
libraries_to_filter.pop(library)
return libraries_to_filter
# Sort libraries according to their dependencies
def sort_to_dependencies(libraries: list, prefix: str) -> list:
module_dependencies = {}
libraries = list(map(lambda x: os.path.basename(x), libraries))
for ns3_library in libraries:
_, _, linked_libraries = extract_linked_libraries(ns3_library, prefix)
linked_libraries = list(filter(lambda x: "libns3" in x and ns3_library not in x, linked_libraries))
linked_libraries = list(map(lambda x: os.path.basename(x), linked_libraries))
module_dependencies[os.path.basename(ns3_library)] = linked_libraries
libraries = filter_in_matching_ns3_libraries(libraries, modules, version, suffix)
else:
libraries = search_libraries("ns3")
def modules_that_can_be_loaded(module_dependencies, pending_modules, current_modules):
modules = []
for pending_module in pending_modules:
can_be_loaded = True
for dependency in module_dependencies[pending_module]:
if dependency not in current_modules:
can_be_loaded = False
break
if not can_be_loaded:
continue
modules.append(pending_module)
return modules
if not libraries:
raise Exception("ns-3 libraries were not found.")
def dependency_order(module_dependencies, pending_modules, current_modules, step_number=0, steps={}):
if len(pending_modules) == 0:
return steps
if step_number not in steps:
steps[step_number] = []
for module in modules_that_can_be_loaded(module_dependencies, pending_modules, current_modules):
steps[step_number].append(module)
pending_modules.remove(module)
current_modules.append(module)
return dependency_order(module_dependencies, pending_modules, current_modules, step_number + 1, steps)
# The prefix is the directory with the lib directory
# libns3-dev-core.so/../../
prefix = os.path.dirname(os.path.dirname(libraries[0]))
sorted_libraries = []
for step in dependency_order(module_dependencies, list(module_dependencies.keys()), [], 0).values():
sorted_libraries.extend(step)
return sorted_libraries
# Remove test libraries
libraries = list(filter(lambda x: "test" not in x, libraries))
libraries_to_load = sort_to_dependencies(libraries, prefix)
# Extract version and build suffix (if it exists)
def filter_module_name(library):
library = os.path.splitext(os.path.basename(library))[0]
components = library.split("-")
# Extract library base names
libraries_to_load = [os.path.basename(x) for x in libraries_to_load]
# Remove version-related prefixes
if "libns3" in components[0]:
components.pop(0)
if "dev" == components[0]:
components.pop(0)
if "rc" in components[0]:
components.pop(0)
# Drop build profile suffix and test libraries
if components[-1] in ["debug", "default", "optimized", "release", "relwithdebinfo"]:
components.pop(-1)
return "-".join(components)
# Filter out module names
modules = set([filter_module_name(library) for library in libraries])
def extract_version(library: str, module: str) -> str:
library = os.path.basename(library)
return re.findall(r"libns([\d.|rc|\-dev]+)-", library)[0]
def get_newest_version(versions: list) -> str:
versions = list(sorted(versions))
if "dev" in versions[0]:
return versions[0]
# Check if there is a release of a possible candidate
try:
pos = versions.index(os.path.splitext(versions[-1])[0])
except ValueError:
pos = None
# Remove release candidates
if pos is not None:
return versions[pos]
else:
return versions[-1]
def filter_in_newest_ns3_libraries(libraries_to_filter: list, modules_to_filter: list) -> list:
newest_version_found = None
# Filter out older ns-3 libraries
for module in list(modules_to_filter):
# Filter duplicates of modules, while excluding test libraries
conflicting_libraries = list(filter(lambda x: module == filter_module_name(x), libraries_to_filter))
# Extract versions from conflicting libraries
conflicting_libraries_versions = list(map(lambda x: extract_version(x, module), conflicting_libraries))
# Get the newest version found for that library
newest_version = get_newest_version(conflicting_libraries_versions)
# Check if the version found is the global newest version
if not newest_version_found:
newest_version_found = newest_version
else:
newest_version_found = get_newest_version([newest_version, newest_version_found])
if newest_version != newest_version_found:
raise Exception("Incompatible versions of the ns-3 module '%s' were found: %s != %s."
% (module, newest_version, newest_version_found))
for conflicting_library in list(conflicting_libraries):
if "-".join([newest_version, module]) not in conflicting_library:
libraries.remove(conflicting_library)
conflicting_libraries.remove(conflicting_library)
num_libraries -= 1
if len(conflicting_libraries) > 1:
raise Exception("There are multiple build profiles for module '%s'.\nDelete one to continue: %s"
% (module, ", ".join(conflicting_libraries)))
return libraries_to_filter
# Get library base names
libraries = filter_in_newest_ns3_libraries(libraries, modules)
libraries_to_load = list(map(lambda x: os.path.basename(x), libraries))
# Sort modules based on libraries
modules = list(map(lambda x: filter_module_name(x), libraries_to_load))
# Try to import Cppyy and warn the user in case it is not found
try:
@@ -324,35 +444,27 @@ def load_modules():
libcppyy.AddSmartPtrType('Ptr')
# Import ns-3 libraries
prefix = os.path.abspath(prefix)
cppyy.add_library_path("%s/lib" % prefix)
cppyy.add_include_path("%s/include" % prefix)
if lock_file:
# When we have the lock file, we assemble the correct library names
for module in modules:
library_name = "libns{version}-{module}{suffix}".format(
version=version,
module=module,
suffix=suffix
)
if library_name not in libraries:
raise Exception("Missing library %s\n" % library_name,
"Build all modules with './ns3 build'"
)
libraries_to_load.append(libraries[library_name])
for variant in ["lib", "lib64"]:
path_to_lib = f"{prefix}/{variant}"
if not os.path.exists(path_to_lib):
continue
cppyy.add_library_path(path_to_lib)
del variant, path_to_lib
cppyy.add_include_path(f"{prefix}/include")
known_include_dirs = set()
# We then need to include all include directories for dependencies
for library in libraries_to_load:
for linked_lib_include_dir in extract_library_include_dirs(library, prefix):
linked_lib_include_dirs, defines = extract_library_include_dirs(library, prefix)
cppyy.cppexec(defines)
for linked_lib_include_dir in linked_lib_include_dirs:
if linked_lib_include_dir not in known_include_dirs:
known_include_dirs.add(linked_lib_include_dir)
if os.path.isdir(linked_lib_include_dir):
cppyy.add_include_path(linked_lib_include_dir)
for module in modules:
cppyy.include("ns3/%s-module.h" % module)
cppyy.include(f"ns3/{module}-module.h")
# After including all headers, we finally load the modules
for library in libraries_to_load: