From 9342082c537a1314e2c8aabbeba23bf956db375d Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Wed, 26 Jan 2022 01:53:28 -0300 Subject: [PATCH] bindings, build: fix bindings and visualizer build Includes: - scan python scripts - run python scripts from ns3 - replace visualizer file copy with configure_file to prevent cmake refreshes - replace ns__init__.py file copy with configure_file to prevent cmake refreshes - fix bindings scanning with cmake - pass include directories to modulegen for castxml consumption - add missing parameters of Recv in python-unit-tests.py - change apiscan targets from apiscan-module to libmodule-apiscan - change bindings targets from module-bingings to libmodule-bindings - scanning and bindings build tests - scan scratch python scripts - replace FindPython3 with FindPython to be compatible with CMake 3.10 - do not export private visual-simulator-impl.h - do not export udp-socket-impl.h - use .so suffix for bindings on Mac instead of .dylib --- CMakeLists.txt | 3 +- bindings/python/ns3modulescan-modular.py | 12 +- .../custom_modules/ns3_module_macros.cmake | 143 +++++++++++------- .../waf_workaround_buildstatus.cmake | 10 +- .../waf_workaround_c4cache.cmake | 5 +- buildsupport/macros_and_definitions.cmake | 77 +++++++--- buildsupport/pybindings_LP64_to_ILP32.py | 16 ++ examples/CMakeLists.txt | 1 + ns3 | 45 +++++- src/fd-net-device/CMakeLists.txt | 5 + src/internet/CMakeLists.txt | 1 - src/visualizer/CMakeLists.txt | 15 +- src/visualizer/model/visual-simulator-impl.cc | 2 + test.py | 1 + utils/python-unit-tests.py | 6 +- utils/tests/test-ns3.py | 90 ++++++++++- 16 files changed, 330 insertions(+), 102 deletions(-) create mode 100644 buildsupport/pybindings_LP64_to_ILP32.py diff --git a/CMakeLists.txt b/CMakeLists.txt index f50aaa62d..dd5597038 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,7 +87,7 @@ option(NS3_STATIC "Build a static ns-3 library and link it against executables" OFF ) option(NS3_VERBOSE "Print additional build system messages" OFF) -option(NS3_VISUALIZER "Build visualizer module" OFF) +option(NS3_VISUALIZER "Build visualizer module" ON) option(NS3_WARNINGS "Enable compiler warnings" ON) option(NS3_WARNINGS_AS_ERRORS "Treat warnings as errors. Requires NS3_WARNINGS=ON" ON @@ -144,6 +144,7 @@ generate_c4che_cachepy() generate_buildstatus() generate_fakewaflock() write_fakewaf_config() +write_header_to_modules_map() # Export package targets when installing ns3_cmake_package() diff --git a/bindings/python/ns3modulescan-modular.py b/bindings/python/ns3modulescan-modular.py index 813608e60..673e0a888 100644 --- a/bindings/python/ns3modulescan-modular.py +++ b/bindings/python/ns3modulescan-modular.py @@ -232,7 +232,7 @@ def ns3_module_scan(top_builddir, module_name, headers_map, output_file_name, cf #module_parser.add_post_scan_hook(post_scan_hook) castxml_options = dict( - include_paths=[top_builddir], + include_paths=[top_builddir, os.path.join(top_builddir, "include")], define_symbols={ #'NS3_ASSERT_ENABLE': None, #'NS3_LOG_ENABLE': None, @@ -256,6 +256,8 @@ def ns3_module_scan(top_builddir, module_name, headers_map, output_file_name, cf scan_header = os.path.join(os.path.dirname(output_file_name), "scan-header.h") if not os.path.exists(scan_header): scan_header = os.path.join(top_builddir, "ns3", "%s-module.h" % module_name) + if not os.path.exists(scan_header): + scan_header = os.path.join(top_builddir, "include", "ns3", "%s-module.h" % module_name) module_parser.parse_init([scan_header], None, whitelist_paths=[top_builddir], @@ -280,5 +282,11 @@ if __name__ == '__main__': if len(sys.argv) != 6: print("ns3modulescan-modular.py top_builddir module_path module_headers output_file_name cflags") sys.exit(1) - ns3_module_scan(sys.argv[1], sys.argv[2], eval(sys.argv[3]), sys.argv[4], sys.argv[5]) + if os.path.exists(sys.argv[3]): + import json + with open(sys.argv[3], "r") as f: + module_headers = json.load(f) + else: + module_headers = eval(sys.argv[3]) + ns3_module_scan(sys.argv[1], sys.argv[2], module_headers, sys.argv[4], sys.argv[5]) sys.exit(0) diff --git a/buildsupport/custom_modules/ns3_module_macros.cmake b/buildsupport/custom_modules/ns3_module_macros.cmake index b2b01eee0..fae3c2e95 100644 --- a/buildsupport/custom_modules/ns3_module_macros.cmake +++ b/buildsupport/custom_modules/ns3_module_macros.cmake @@ -30,7 +30,7 @@ # Hidden argument (this is not a function, so you don't really need to pass arguments explicitly) # deprecated_header_files = "list;of;deprecated;.h;files", copy won't get triggered if deprecated_header_files isn't set # ignore_pch = TRUE or FALSE, prevents the PCH from including undesired system libraries (e.g. custom GLIBC for DCE) - +# module_enabled_features = "list;of;enabled;features;for;this;module" (used by fd-net-device) macro( build_lib_impl @@ -42,6 +42,7 @@ macro( test_sources #deprecated_header_files #ignore_pch + #module_enabled_features ) # cmake-format: on @@ -215,24 +216,23 @@ macro( # Build lib examples if requested if(${ENABLE_EXAMPLES}) - if((EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/examples) - AND (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/examples/CMakeLists.txt) - ) - add_subdirectory(examples) - endif() - if((EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/example) - AND (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/example/CMakeLists.txt) - ) - add_subdirectory(example) - endif() + foreach(example_folder example;examples) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}/CMakeLists.txt) + add_subdirectory(${example_folder}) + endif() + scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}) + endif() + endforeach() endif() # Get architecture pair for python bindings - set(arch gcc_ILP32) - set(arch_flag) - if(${CMAKE_SIZEOF_VOID_P} EQUAL 8) + if((${CMAKE_SIZEOF_VOID_P} EQUAL 8) AND (NOT APPLE)) set(arch gcc_LP64) - set(arch_flag -m64) + set(arch_flags -m64) + else() + set(arch gcc_ILP32) + set(arch_flags) endif() # Add target to scan python bindings @@ -243,49 +243,63 @@ macro( ${PROJECT_SOURCE_DIR}/${folder}/${libname}/bindings ) file(MAKE_DIRECTORY ${bindings_output_folder}) - set(module_api ${bindings_output_folder}/modulegen__${arch}.py) + set(module_api_ILP32 ${bindings_output_folder}/modulegen__gcc_ILP32.py) + set(module_api_LP64 ${bindings_output_folder}/modulegen__gcc_LP64.py) set(modulescan_modular_command - ${Python3_EXECUTABLE} + ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/bindings/python/ns3modulescan-modular.py ) - # To build the header map for the module, we start by copying the headers - # and prepending the dictionary start - set(header_map "{\\\"${header_files};") + set(header_map "") + # We need a python map that takes header.h to module e.g. "ptr.h": "core" + foreach(header ${header_files}) + # header is a relative path to the current working directory + get_filename_component( + header_name ${CMAKE_CURRENT_SOURCE_DIR}/${header} NAME + ) + string(APPEND header_map "\"${header_name}\":\"${libname}\",") + endforeach() - # We then remove model/ helper/ prefixes e.g. - # {'model/angles.h;model/antenna-model.h;... -> - # {'angles.h;antenna-model.h;...) - string(REPLACE "model/" "" header_map "${header_map}") - string(REPLACE "helper/" "" header_map "${header_map}") - - # Now we replace list entry separators (;) with ending of the string quote - # ("), followed by the relative module e.g. - # {"model/angles.h;model/antenna-model.h;... -> {"angles.h" : "antenna", - # "antenna-model.h": "antenna", "...) - string(REPLACE ";" "\\\": \\\"${libname}\\\", \\\"" header_map - "${header_map}" + set(ns3-headers-to-module-map "${ns3-headers-to-module-map}${header_map}" + CACHE INTERNAL "Map connecting headers to their modules" ) - # We now remove the last character ("), which needs to be replaced with a - # (}), to close the dictionary e.g. "antenna-model.h" : "antenna", " -> - # "antenna-model.h" : "antenna" - string(LENGTH "${header_map}" header_map_len) - math(EXPR header_map_len "${header_map_len}-3") - string(SUBSTRING "${header_map}" 0 ${header_map_len} header_map) + # API scan needs the include directories to find a few headers (e.g. mpi.h) + get_target_property(include_dirs ${lib${libname}} INCLUDE_DIRECTORIES) + set(modulegen_include_dirs) + foreach(include_dir ${include_dirs}) + if(include_dir MATCHES "<") + # Skip CMake build and install interface includes + continue() + else() + # Append the include directory to a list + set(modulegen_include_dirs ${modulegen_include_dirs} -I${include_dir}) + endif() + endforeach() - # Append end of dictionary (}) - string(APPEND header_map "}") + set(module_to_generate_api ${module_api_ILP32}) + set(LP64toILP32) + if("${arch}" STREQUAL "gcc_LP64") + set(module_to_generate_api ${module_api_LP64}) + set(LP64toILP32 + ${Python_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/buildsupport/pybindings_LP64_to_ILP32.py + ${module_api_LP64} ${module_api_ILP32} + ) + endif() add_custom_target( - apiscan-${lib${libname}} - COMMAND ${modulescan_modular_command} ${CMAKE_OUTPUT_DIRECTORY} ${libname} - ${header_map} ${module_api} ${arch_flag} + ${lib${libname}}-apiscan + COMMAND + ${modulescan_modular_command} ${CMAKE_OUTPUT_DIRECTORY} ${libname} + ${PROJECT_BINARY_DIR}/header_map.json ${module_to_generate_api} + \"${arch_flags} ${modulegen_include_dirs}\" + COMMAND ${LP64toILP32} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${lib${libname}} ) - add_dependencies(apiscan-all apiscan-${lib${libname}}) + add_dependencies(apiscan-all ${lib${libname}}-apiscan) endif() # Build pybindings if requested and if bindings subfolder exists in @@ -316,10 +330,12 @@ macro( if((NOT EXISTS ${module_hdr}) OR (NOT EXISTS ${module_src})) # OR TRUE) # to # force # reprocessing - string(REPLACE ";" "," ENABLED_FEATURES "${ns3-libs}") + string(REPLACE ";" "," ENABLED_FEATURES + "${ns3-libs};${module_enabled_features}" + ) set(modulegen_modular_command GCC_RTTI_ABI_COMPLETE=True NS3_ENABLED_FEATURES="${ENABLED_FEATURES}" - ${Python3_EXECUTABLE} + ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/bindings/python/ns3modulegen-modular.py ) execute_process( @@ -354,11 +370,12 @@ macro( ${CMAKE_CURRENT_SOURCE_DIR}/bindings/scan-header.h ) endif() - add_library(${libname}-bindings SHARED "${python_module_files}") + set(bindings-name lib${libname}-bindings) + add_library(${bindings-name} SHARED "${python_module_files}") target_include_directories( - ${libname}-bindings PUBLIC ${Python3_INCLUDE_DIRS} - ${bindings_output_folder} + ${bindings-name} PUBLIC ${Python_INCLUDE_DIRS} ${bindings_output_folder} ) + target_compile_options(${bindings-name} PRIVATE -Wno-error) # If there is any, remove the "lib" prefix of libraries (search for # "set(lib${libname}") @@ -367,35 +384,45 @@ macro( string(REPLACE ";" "-bindings;" bindings_to_link "${ns_libraries_to_link};" ) # add -bindings suffix to all lib${name} - string(REPLACE "lib" "" bindings_to_link "${bindings_to_link}" - )# remove lib prefix from all lib${name}-bindings endif() target_link_libraries( - ${libname}-bindings + ${bindings-name} PUBLIC ${LIB_AS_NEEDED_PRE} ${lib${libname}} "${bindings_to_link}" "${libraries_to_link}" ${LIB_AS_NEEDED_POST} + PRIVATE ${Python_LIBRARIES} ) target_include_directories( - ${libname}-bindings PRIVATE ${PROJECT_SOURCE_DIR}/src/core/bindings + ${bindings-name} PRIVATE ${PROJECT_SOURCE_DIR}/src/core/bindings ) + set(suffix) + if(APPLE) + # Python doesn't like Apple's .dylib and will refuse to load bindings + # unless its an .so + set(suffix SUFFIX .so) + endif() + # Set binding library name and output folder set_target_properties( - ${libname}-bindings - PROPERTIES OUTPUT_NAME ${prefix}${libname_sub} PREFIX "" + ${bindings-name} + PROPERTIES OUTPUT_NAME ${prefix}${libname_sub} + PREFIX "" + ${suffix} LIBRARY_OUTPUT_DIRECTORY ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/ns ) set(ns3-python-bindings-modules - "${libname}-bindings;${ns3-python-bindings-modules}" + "${bindings-name};${ns3-python-bindings-modules}" CACHE INTERNAL "list of modules python bindings" ) # Make sure all bindings are built before building the visualizer module # that makes use of them - if(NOT (${name} STREQUAL visualizer)) - add_dependencies(${libvisualizer} ${libname}-bindings) + if(${ENABLE_VISUALIZER}) + if(NOT (${name} STREQUAL visualizer)) + add_dependencies(${libvisualizer} ${bindings-name}) + endif() endif() endif() diff --git a/buildsupport/custom_modules/waf_workaround_buildstatus.cmake b/buildsupport/custom_modules/waf_workaround_buildstatus.cmake index 704d62ada..ea051b90b 100644 --- a/buildsupport/custom_modules/waf_workaround_buildstatus.cmake +++ b/buildsupport/custom_modules/waf_workaround_buildstatus.cmake @@ -18,15 +18,17 @@ function(generate_buildstatus) # Build build-status.py file consumed by test.py set(buildstatus_contents "#! /usr/bin/env python3\n\n") - string(APPEND buildstatus_contents "ns3_runnable_programs = [") + string(APPEND buildstatus_contents "ns3_runnable_programs = [") foreach(executable ${ns3-execs}) - string(APPEND buildstatus_contents "'${executable}', ") + string(APPEND buildstatus_contents "'${executable}',\n") endforeach() string(APPEND buildstatus_contents "]\n\n") - string(APPEND buildstatus_contents "ns3_runnable_scripts = [") # missing - # support + string(APPEND buildstatus_contents "ns3_runnable_scripts = [") + foreach(executable ${ns3-execs-py}) + string(APPEND buildstatus_contents "'${executable}',\n") + endforeach() string(APPEND buildstatus_contents "]\n\n") file(WRITE ${CMAKE_OUTPUT_DIRECTORY}/build-status.py diff --git a/buildsupport/custom_modules/waf_workaround_c4cache.cmake b/buildsupport/custom_modules/waf_workaround_c4cache.cmake index 591fc4797..d31e365f1 100644 --- a/buildsupport/custom_modules/waf_workaround_c4cache.cmake +++ b/buildsupport/custom_modules/waf_workaround_c4cache.cmake @@ -65,6 +65,9 @@ function(generate_c4che_cachepy) cache_cmake_flag(NS3_BRITE "ENABLE_BRITE" cache_contents) cache_cmake_flag(NS3_ENABLE_SUDO "ENABLE_SUDO" cache_contents) cache_cmake_flag(NS3_PYTHON_BINDINGS "ENABLE_PYTHON_BINDINGS" cache_contents) + cache_cmake_flag( + NS3_SCAN_PYTHON_BINDINGS "ENABLE_SCAN_PYTHON_BINDINGS" cache_contents + ) string(APPEND cache_contents "EXAMPLE_DIRECTORIES = [") foreach(example_folder ${ns3-example-folders}) @@ -75,7 +78,7 @@ function(generate_c4che_cachepy) string(APPEND cache_contents "APPNAME = 'ns'\n") string(APPEND cache_contents "BUILD_PROFILE = '${build_profile}'\n") string(APPEND cache_contents "VERSION = '${NS3_VER}' \n") - string(APPEND cache_contents "PYTHON = ['${Python3_EXECUTABLE}']\n") + string(APPEND cache_contents "PYTHON = ['${Python_EXECUTABLE}']\n") mark_as_advanced(VALGRIND) find_program(VALGRIND valgrind) diff --git a/buildsupport/macros_and_definitions.cmake b/buildsupport/macros_and_definitions.cmake index 0788568e6..c9417dcfd 100644 --- a/buildsupport/macros_and_definitions.cmake +++ b/buildsupport/macros_and_definitions.cmake @@ -221,7 +221,9 @@ macro(clear_global_cached_variables) unset(ns3-contrib-libs CACHE) unset(ns3-example-folders CACHE) unset(ns3-execs CACHE) + unset(ns3-execs-py CACHE) unset(ns3-external-libs CACHE) + unset(ns3-headers-to-module-map CACHE) unset(ns3-libs CACHE) unset(ns3-libs-tests CACHE) unset(ns3-python-bindings-modules CACHE) @@ -232,7 +234,9 @@ macro(clear_global_cached_variables) ns3-contrib-libs ns3-example-folders ns3-execs + ns3-execs-py ns3-external-libs + ns3-headers-to-module-map ns3-libs ns3-libs-tests ns3-python-bindings-modules @@ -612,23 +616,28 @@ macro(process_options) endif() endif() - find_package(Python3 COMPONENTS Interpreter Development QUIET) + find_package(Python COMPONENTS Interpreter Development QUIET) + + # Check if python3 was found, and mark as not found if python2 is found + if(${Python_FOUND} AND (${Python_VERSION_MAJOR} LESS 3)) + set(Python_FOUND FALSE) + message( + STATUS: + "bindings: an incompatible version of Python was found, bindings will be disabled" + ) + endif() + set(ENABLE_PYTHON_BINDINGS OFF) if(${NS3_PYTHON_BINDINGS}) - if(NOT ${Python3_FOUND}) + if(NOT ${Python_FOUND}) message(FATAL_ERROR "NS3_PYTHON_BINDINGS requires Python3") endif() set(ENABLE_PYTHON_BINDINGS ON) - link_directories(${Python3_LIBRARY_DIRS}) - include_directories(${Python3_INCLUDE_DIRS}) - set(PYTHONDIR ${Python3_SITELIB}) - set(PYTHONARCHDIR ${Python3_SITEARCH}) - set(HAVE_PYEMBED TRUE) - set(HAVE_PYEXT TRUE) - set(HAVE_PYTHON_H TRUE) + set(destination_dir ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/ns) - file(COPY bindings/python/ns__init__.py DESTINATION ${destination_dir}) - file(RENAME ${destination_dir}/ns__init__.py ${destination_dir}/__init__.py) + configure_file( + bindings/python/ns__init__.py ${destination_dir}/__init__.py COPYONLY + ) endif() if(${NS3_SCAN_PYTHON_BINDINGS}) @@ -637,6 +646,15 @@ macro(process_options) add_custom_target(apiscan-all) endif() + set(ENABLE_VISUALIZER FALSE) + if(${NS3_VISUALIZER}) + if((NOT ${ENABLE_PYTHON_BINDINGS}) OR (NOT ${Python_FOUND})) + message(STATUS "Visualizer requires NS3_PYTHON_BINDINGS and Python3") + else() + set(ENABLE_VISUALIZER TRUE) + endif() + endif() + if(${NS3_COVERAGE} AND (NOT ${ENABLE_TESTS} OR NOT ${ENABLE_EXAMPLES})) message( FATAL_ERROR @@ -652,7 +670,7 @@ macro(process_options) # produce code coverage output add_custom_target( run_test_py - COMMAND ${Python3_EXECUTABLE} test.py --no-build + COMMAND ${Python_EXECUTABLE} test.py --no-build WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} DEPENDS all-test-targets ) @@ -685,14 +703,6 @@ macro(process_options) set_property(GLOBAL PROPERTY TARGET_MESSAGES OFF) endif() - set(ENABLE_VISUALIZER FALSE) - if(${NS3_VISUALIZER}) - if((NOT ${NS3_PYTHON_BINDINGS}) OR (NOT ${Python3_FOUND})) - message(FATAL_ERROR "Visualizer requires NS3_PYTHON_BINDINGS and Python3") - endif() - set(ENABLE_VISUALIZER TRUE) - endif() - mark_as_advanced(Boost_INCLUDE_DIR) find_package(Boost) if(${Boost_FOUND}) @@ -749,7 +759,7 @@ macro(process_options) add_custom_target( run-introspected-command-line COMMAND ${CMAKE_COMMAND} -E env NS_COMMANDLINE_INTROSPECTION=.. - ${Python3_EXECUTABLE} ./test.py --no-build --constrain=example + ${Python_EXECUTABLE} ./test.py --no-build --constrain=example WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} DEPENDS all-test-targets # all-test-targets only exists if ENABLE_TESTS is # set to ON @@ -1071,6 +1081,17 @@ function(set_runtime_outputdirectory target_name output_directory target_prefix) endif() endfunction(set_runtime_outputdirectory) +function(scan_python_examples path) + file(GLOB_RECURSE python_examples ${path}/*.py) + foreach(python_example ${python_examples}) + if(NOT (${python_example} MATCHES "examples-to-run")) + set(ns3-execs-py "${python_example};${ns3-execs-py}" + CACHE INTERNAL "list of python scripts" + ) + endif() + endforeach() +endfunction() + add_custom_target(copy_all_headers) function(copy_headers_before_building_lib libname outputdir headers visibility) foreach(header ${headers}) @@ -1437,6 +1458,20 @@ function(find_external_library_header_and_library name header_name library_name endif() endfunction() +function(write_header_to_modules_map) + if(${NS3_SCAN_PYTHON_BINDINGS}) + set(header_map ${ns3-headers-to-module-map}) + + # Trim last comma + string(LENGTH "${header_map}" header_map_len) + math(EXPR header_map_len "${header_map_len}-1") + string(SUBSTRING "${header_map}" 0 ${header_map_len} header_map) + + # Then write to header_map.json for consumption of pybindgen + file(WRITE ${PROJECT_BINARY_DIR}/header_map.json "{${header_map}}") + endif() +endfunction() + # Waf workaround scripts include(buildsupport/custom_modules/waf_workaround_c4cache.cmake) include(buildsupport/custom_modules/waf_workaround_buildstatus.cmake) diff --git a/buildsupport/pybindings_LP64_to_ILP32.py b/buildsupport/pybindings_LP64_to_ILP32.py new file mode 100644 index 000000000..60ff0719c --- /dev/null +++ b/buildsupport/pybindings_LP64_to_ILP32.py @@ -0,0 +1,16 @@ +#! /usr/bin/env python3 +# -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*- + +def lp64_to_ilp32(lp64path, ilp32path): + import re + lp64bindings = None + with open(lp64path, "r") as lp64file: + lp64bindings = lp64file.read() + with open(ilp32path, "w") as ilp32file: + ilp32bindings = re.sub("unsigned long(?!( long))", "unsigned long long", lp64bindings) + ilp32file.write(ilp32bindings) + +if __name__ == "__main__": + import sys + print(sys.argv) + exit(lp64_to_ilp32(sys.argv[1], sys.argv[2])) \ No newline at end of file diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 33ae91343..5fc72cd89 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -9,4 +9,5 @@ if(${ENABLE_EXAMPLES}) CACHE INTERNAL "list of example folders" ) endforeach() + scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR}) endif() diff --git a/ns3 b/ns3 index a2a61b84f..60c201bed 100755 --- a/ns3 +++ b/ns3 @@ -99,6 +99,7 @@ def parse_args(argv): parser_configure = on_off_argument(parser_configure, "gtk", "GTK support in ConfigStore") parser_configure = on_off_argument(parser_configure, "logs", "the logs regardless of the compile mode") parser_configure = on_off_argument(parser_configure, "mpi", "the MPI support for distributed simulation") + parser_configure = on_off_argument(parser_configure, "python-bindings", "python bindings") parser_configure = on_off_argument(parser_configure, "tests", "the ns-3 tests") parser_configure = on_off_argument(parser_configure, "sanitizers", "address, memory leaks and undefined behavior sanitizers") @@ -109,7 +110,6 @@ def parse_args(argv): parser_configure = on_off_argument(parser_configure, "warnings", "compiler warnings") parser_configure = on_off_argument(parser_configure, "werror", "Treat compiler warnings as errors", "Treat compiler warnings as warnings") - parser_configure = on_off_argument(parser_configure, "visualizer", "the visualizer module") parser_configure.add_argument('--enable-modules', help='List of modules to build (e.g. core;network;internet)', action="store", type=str, default=None) @@ -286,6 +286,8 @@ def check_build_profile(output_directory): ns3_version = None ns3_modules = None ns3_modules_tests = [] + ns3_modules_apiscan = [] + ns3_modules_bindings = [] enable_sudo = False if output_directory and os.path.exists(c4che_path): c4che_info = {} @@ -293,10 +295,17 @@ def check_build_profile(output_directory): build_profile = c4che_info["BUILD_PROFILE"] ns3_version = c4che_info["VERSION"] ns3_modules = c4che_info["NS3_ENABLED_MODULES"] - ns3_modules_tests = [x + "-test" for x in ns3_modules] if "ENABLE_SUDO" in c4che_info: enable_sudo = c4che_info["ENABLE_SUDO"] - return build_profile, ns3_version, ns3_modules + ns3_modules_tests if ns3_modules else None, enable_sudo + if ns3_modules: + if c4che_info["ENABLE_TESTS"]: + ns3_modules_tests = [x + "-test" for x in ns3_modules] + if c4che_info["ENABLE_PYTHON_BINDINGS"]: + ns3_modules_bindings = [x + "-bindings" for x in ns3_modules] + if "ENABLE_SCAN_PYTHON_BINDINGS" in c4che_info and c4che_info["ENABLE_SCAN_PYTHON_BINDINGS"]: + ns3_modules_apiscan = [x + "-apiscan" for x in ns3_modules] + ns3_modules = ns3_modules + ns3_modules_tests + ns3_modules_apiscan + ns3_modules_bindings + return build_profile, ns3_version, ns3_modules, enable_sudo def print_and_buffer(message): @@ -453,11 +462,11 @@ def configure_cmake(cmake, args, current_cmake_cache_folder, current_cmake_gener ("GTK3", "gtk"), ("LOG", "logs"), ("MPI", "mpi"), + ("PYTHON_BINDINGS", "python_bindings"), ("SANITIZE", "sanitizers"), ("STATIC", "static"), ("TESTS", "tests"), ("VERBOSE", "verbose"), - ("VISUALIZER", "visualizer"), ("WARNINGS", "warnings"), ("WARNINGS_AS_ERRORS", "werror"), ) @@ -553,6 +562,19 @@ def get_program_shortcuts(build_profile, ns3_version): if source_shortcut: ns3_program_map[shortcut_path + ".cc"] = program temp_path.pop(0) + + + if programs_dict["ns3_runnable_scripts"]: + scratch_scripts = glob.glob(os.path.join(ns3_path, "scratch", "*.py"), recursive=True) + programs_dict["ns3_runnable_scripts"].extend(scratch_scripts) + + for program in programs_dict["ns3_runnable_scripts"]: + temp_path = program.replace(ns3_path, "").split(os.sep) + program = program.strip() + while len(temp_path): + shortcut_path = os.sep.join(temp_path) + ns3_program_map[shortcut_path] = program + temp_path.pop(0) return ns3_program_map @@ -658,6 +680,9 @@ def reconfigure_cmake_to_force_refresh(cmake, current_cmake_cache_folder, output def get_target_to_build(program_path, ns3_version, build_profile): + if ".py" in program_path: + return None + build_profile_suffix = "" if build_profile in ["release"] else "-" + build_profile program_name = "".join(*re.findall("(.*)ns%s-(.*)%s" % (ns3_version, build_profile_suffix), program_path)) @@ -712,7 +737,8 @@ def build_step(args, # If we are building specific targets, we build them one by one if "build" in args: - non_executable_targets = ["check-version", + non_executable_targets = ["apiscan-all", + "check-version", "cmake-format", "docs", "doxygen", @@ -787,6 +813,11 @@ def run_step(args, target_to_run, target_args): target_to_run = "bash" use_shell = True else: + # running a python script? + if ".py" in target_to_run: + target_args = [target_to_run] + target_args + target_to_run = "python3" + # running from ns-3-dev (ns3_path) or cwd if args.cwd: working_dir = args.cwd @@ -1060,7 +1091,7 @@ def main(): target_path = os.sep.join(target_path) return target_path - if not args.check and not args.shell and target_to_run: + if not args.check and not args.shell and target_to_run and not ".py" in target_to_run: target_to_run = remove_overlapping_path(out_dir, target_to_run) # Waf doesn't add version prefix and build type suffix to the scratches, so we remove them @@ -1072,7 +1103,7 @@ def main(): target_to_run = os.sep.join([out_dir, target_to_run]) # If we're only trying to run the target, we need to check if it actually exists first - if (run_only or build_and_run) and not os.path.exists(target_to_run): + if (run_only or build_and_run) and ".py" not in target_to_run and not os.path.exists(target_to_run): raise Exception("Executable has not been built yet") # Setup program as sudo diff --git a/src/fd-net-device/CMakeLists.txt b/src/fd-net-device/CMakeLists.txt index 6016266ce..f301ba264 100644 --- a/src/fd-net-device/CMakeLists.txt +++ b/src/fd-net-device/CMakeLists.txt @@ -1,5 +1,7 @@ set(name fd-net-device) +set(module_enabled_features) # modulegen_customizations consumes this list + mark_as_advanced(ENABLE_THREADING) set(ENABLE_THREADING ${HAVE_PTHREAD_H}) @@ -57,6 +59,7 @@ endif() if(${ENABLE_FDNETDEV}) set(fd-net-device_creators) + list(APPEND module_enabled_features FdNetDevice) if(${ENABLE_EMUNETDEV}) set(emu_sources helper/emu-fd-net-device-helper.cc) @@ -74,6 +77,7 @@ if(${ENABLE_FDNETDEV}) ) list(APPEND fd-net-device_creators raw-sock-creator) + list(APPEND module_enabled_features EmuFdNetDevice) endif() if(${ENABLE_TAPNETDEV}) @@ -93,6 +97,7 @@ if(${ENABLE_FDNETDEV}) ) list(APPEND fd-net-device_creators tap-device-creator) + list(APPEND module_enabled_features TapFdNetDevice) endif() if(${ENABLE_NETMAP_EMU}) diff --git a/src/internet/CMakeLists.txt b/src/internet/CMakeLists.txt index 053c49e3e..88b52c8b9 100644 --- a/src/internet/CMakeLists.txt +++ b/src/internet/CMakeLists.txt @@ -251,7 +251,6 @@ set(header_files model/udp-header.h model/udp-l4-protocol.h model/udp-socket-factory.h - model/udp-socket-impl.h model/udp-socket.h model/windowed-filter.h ) diff --git a/src/visualizer/CMakeLists.txt b/src/visualizer/CMakeLists.txt index fb4d5f40a..654c3ca6b 100644 --- a/src/visualizer/CMakeLists.txt +++ b/src/visualizer/CMakeLists.txt @@ -2,12 +2,12 @@ set(name visualizer) set(source_files model/pyviz.cc model/visual-simulator-impl.cc) -set(header_files model/pyviz.h model/visual-simulator-impl.h) +set(header_files model/pyviz.h) -include_directories(${Python3_INCLUDE_DIRS}) +include_directories(${Python_INCLUDE_DIRS}) set(libraries_to_link - ${Python3_LIBRARIES} + ${Python_LIBRARIES} ${libcore} ${libinternet} ${libwifi} @@ -27,4 +27,11 @@ build_lib("${name}" "${source_files}" "${header_files}" "${libraries_to_link}" # move visualizer folder to build/bindings/python, which allows us to add only # PYTHONPATH=ns-3-dev/build/bindings/python -file(COPY visualizer DESTINATION ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/) +file(GLOB_RECURSE visualizer_files ${CMAKE_CURRENT_SOURCE_DIR}/visualizer/*) +foreach(file ${visualizer_files}) + string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_OUTPUT_DIRECTORY}/bindings/python" destination + ${file} + ) + configure_file(${file} ${destination} COPYONLY) +endforeach() diff --git a/src/visualizer/model/visual-simulator-impl.cc b/src/visualizer/model/visual-simulator-impl.cc index 0a068cee4..9f7040f1d 100644 --- a/src/visualizer/model/visual-simulator-impl.cc +++ b/src/visualizer/model/visual-simulator-impl.cc @@ -18,6 +18,8 @@ * Author: Gustavo Carneiro */ #include +#undef HAVE_PTHREAD_H +#undef HAVE_SYS_STAT_H #include "visual-simulator-impl.h" #include "ns3/default-simulator-impl.h" #include "ns3/log.h" diff --git a/test.py b/test.py index d5d555868..d57bbedd5 100755 --- a/test.py +++ b/test.py @@ -1120,6 +1120,7 @@ def run_tests(): if os.path.exists(build_status_file): ns3_runnable_programs = get_list_from_file(build_status_file, "ns3_runnable_programs") ns3_runnable_scripts = get_list_from_file(build_status_file, "ns3_runnable_scripts") + ns3_runnable_scripts = [os.path.basename(script) for script in ns3_runnable_scripts] else: print('The build status file was not found. You must do waf build before running test.py.', file=sys.stderr) sys.exit(2) diff --git a/utils/python-unit-tests.py b/utils/python-unit-tests.py index c48ceb45f..1f1758738 100644 --- a/utils/python-unit-tests.py +++ b/utils/python-unit-tests.py @@ -1,3 +1,5 @@ +#! /usr/bin/env python3 + # Copyright (C) 2008-2011 INESC Porto # This program is free software; you can redistribute it and/or modify @@ -25,6 +27,8 @@ import ns.mobility import ns.csma import ns.applications +UINT32_MAX = 0xFFFFFFFF + ## TestSimulator class class TestSimulator(unittest.TestCase): @@ -170,7 +174,7 @@ class TestSimulator(unittest.TestCase): @return none """ assert self._received_packet is None - self._received_packet = socket.Recv() + self._received_packet = socket.Recv(maxSize=UINT32_MAX, flags=0) sink = ns.network.Socket.CreateSocket(node, ns.core.TypeId.LookupByName("ns3::UdpSocketFactory")) sink.Bind(ns.network.InetSocketAddress(ns.network.Ipv4Address.GetAny(), 80)) diff --git a/utils/tests/test-ns3.py b/utils/tests/test-ns3.py index 942a56450..4ac870104 100644 --- a/utils/tests/test-ns3.py +++ b/utils/tests/test-ns3.py @@ -1138,8 +1138,8 @@ class NS3BuildBaseTestCase(NS3BaseTestCase): f.write(""" #include using namespace ns3; - int main () - { + int main () + { Simulator::Stop (Seconds (1.0)); Simulator::Run (); Simulator::Destroy (); @@ -1246,6 +1246,92 @@ class NS3BuildBaseTestCase(NS3BaseTestCase): stdout = stdout.replace("scratch_%s" % target_cmake, "") # remove build lines self.assertIn(target_to_run.split("/")[-1], stdout) + NS3BuildBaseTestCase.cleaned_once = False + + def test_10_PybindgenBindings(self): + """! + Test if cmake is calling pybindgen through modulegen to generate + the bindings source files correctly + @return None + """ + + # First we enable python bindings + return_code, stdout, stderr = run_ns3("configure --enable-examples --enable-tests --enable-python-bindings") + self.assertEqual(return_code, 0) + + # Then look for python bindings sources + core_bindings_generated_sources_path = os.path.join(ns3_path, "build", "src", "core", "bindings") + core_bindings_sources_path = os.path.join(ns3_path, "src", "core", "bindings") + core_bindings_path = os.path.join(ns3_path, "build", "bindings", "python", "ns") + core_bindings_header = os.path.join(core_bindings_generated_sources_path, "ns3module.h") + core_bindings_source = os.path.join(core_bindings_generated_sources_path, "ns3module.cc") + self.assertTrue(os.path.exists(core_bindings_header)) + self.assertTrue(os.path.exists(core_bindings_source)) + + # Then try to build the bindings for the core module + return_code, stdout, stderr = run_ns3("build core-bindings") + self.assertEqual(return_code, 0) + + # Then check if it was built + self.assertGreater(len(list(filter(lambda x: "_core" in x, os.listdir(core_bindings_path)))), 0) + + # Now enable python bindings scanning + return_code, stdout, stderr = run_ns3("configure -- -DNS3_SCAN_PYTHON_BINDINGS=ON") + self.assertEqual(return_code, 0) + + # Get the file status for the current scanned bindings + bindings_sources = os.listdir(core_bindings_sources_path) + bindings_sources = [os.path.join(core_bindings_sources_path, y) for y in bindings_sources] + timestamps = {} + [timestamps.update({x: os.stat(x)}) for x in bindings_sources] + + # Try to scan the bindings for the core module + return_code, stdout, stderr = run_ns3("build core-apiscan") + self.assertEqual(return_code, 0) + + # Check if they exist, are not empty and have a different timestamp + generated_python_files = ["callbacks_list.py", "modulegen__gcc_LP64.py"] + for binding_file in timestamps.keys(): + if os.path.basename(binding_file) in generated_python_files: + self.assertTrue(os.path.exists(binding_file)) + self.assertGreater(os.stat(binding_file).st_size, 0) + new_fstat = os.stat(binding_file) + self.assertNotEqual(timestamps[binding_file].st_mtime, new_fstat.st_mtime) + + # Then delete the old bindings sources + for f in os.listdir(core_bindings_generated_sources_path): + os.remove(os.path.join(core_bindings_generated_sources_path, f)) + + # Reconfigure to recreate the source files + return_code, stdout, stderr = run_ns3("configure") + self.assertEqual(return_code, 0) + + # Check again if they exist + self.assertTrue(os.path.exists(core_bindings_header)) + self.assertTrue(os.path.exists(core_bindings_source)) + + # Build the core bindings again + return_code, stdout, stderr = run_ns3("build core-bindings") + self.assertEqual(return_code, 0) + + # Then check if it was built + self.assertGreater(len(list(filter(lambda x: "_core" in x, os.listdir(core_bindings_path)))), 0) + + # We are on python anyways, so we can just try to load it from here + sys.path.insert(0, os.path.join(core_bindings_path, "..")) + try: + from ns import core + except ImportError: + self.assertTrue(True) + + # Check if ns3 can find and run the python example + return_code, stdout, stderr = run_ns3("run sample-simulator.py") + self.assertEqual(return_code, 0) + + # Check if test.py can find and run the python example + return_code, stdout, stderr = run_program("./test.py", "-p src/core/examples/sample-simulator.py", python=True) + self.assertEqual(return_code, 0) + class NS3ExpectedUseTestCase(NS3BaseTestCase): """!