From db4c10bb8c70e002b4bcce03ed6be87f43f1fc7f Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Mon, 4 Apr 2022 22:15:46 -0300 Subject: [PATCH] build: python bindings fixes Includes: - scan for python examples if bindings are enabled - do not scan python bindings from modules without a module/bindings directory - remove broken bindings if pybindgen generation fails - replace colored message with normal message for find_external_library - complement pybindgen to check if modules with no bindings folder was scanned and any bindings were generated --- .../custom-modules/ns3-module-macros.cmake | 30 +++++++------ build-support/macros-and-definitions.cmake | 21 ++++++--- examples/CMakeLists.txt | 3 +- utils/tests/test-ns3.py | 43 +++++++++++++------ 4 files changed, 64 insertions(+), 33 deletions(-) diff --git a/build-support/custom-modules/ns3-module-macros.cmake b/build-support/custom-modules/ns3-module-macros.cmake index a09060f94..1801f85b4 100644 --- a/build-support/custom-modules/ns3-module-macros.cmake +++ b/build-support/custom-modules/ns3-module-macros.cmake @@ -251,16 +251,16 @@ function(build_lib) endif() # Build lib examples if requested - if(${ENABLE_EXAMPLES}) - foreach(example_folder example;examples) - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}) + foreach(example_folder example;examples) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}) + if(${ENABLE_EXAMPLES}) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}/CMakeLists.txt) add_subdirectory(${example_folder}) endif() - scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}) endif() - endforeach() - endif() + scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}) + endif() + endforeach() # Get architecture pair for python bindings if((${CMAKE_SIZEOF_VOID_P} EQUAL 8) AND (NOT APPLE)) @@ -273,7 +273,8 @@ function(build_lib) # Add target to scan python bindings if(${ENABLE_SCAN_PYTHON_BINDINGS} - AND EXISTS ${CMAKE_HEADER_OUTPUT_DIRECTORY}/${BLIB_LIBNAME}-module.h + AND (EXISTS ${CMAKE_HEADER_OUTPUT_DIRECTORY}/${BLIB_LIBNAME}-module.h) + AND (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/bindings") ) set(bindings_output_folder ${PROJECT_SOURCE_DIR}/${FOLDER}/bindings) file(MAKE_DIRECTORY ${bindings_output_folder}) @@ -329,8 +330,8 @@ function(build_lib) # Build pybindings if requested and if bindings subfolder exists in # NS3/src/BLIB_LIBNAME - if(${ENABLE_PYTHON_BINDINGS} AND EXISTS - "${CMAKE_CURRENT_SOURCE_DIR}/bindings" + if(${ENABLE_PYTHON_BINDINGS} AND (EXISTS + "${CMAKE_CURRENT_SOURCE_DIR}/bindings") ) set(bindings_output_folder ${CMAKE_OUTPUT_DIRECTORY}/${FOLDER}/bindings) file(MAKE_DIRECTORY ${bindings_output_folder}) @@ -376,14 +377,19 @@ function(build_lib) RESULT_VARIABLE error_code ) if(${error_code} OR NOT (EXISTS ${module_hdr})) + # Delete broken bindings to make sure we will his this error again if + # nothing changed + if(EXISTS ${module_src}) + file(REMOVE ${module_src}) + endif() + if(EXISTS ${module_hdr}) + file(REMOVE ${module_hdr}) + endif() message( FATAL_ERROR "Something went wrong during processing of the python bindings of module ${BLIB_LIBNAME}." " Make sure you have the latest version of Pybindgen." ) - if(EXISTS ${module_src}) - file(REMOVE ${module_src}) - endif() endif() endif() diff --git a/build-support/macros-and-definitions.cmake b/build-support/macros-and-definitions.cmake index a96dd5f80..8dae33a04 100644 --- a/build-support/macros-and-definitions.cmake +++ b/build-support/macros-and-definitions.cmake @@ -1344,6 +1344,12 @@ function(set_runtime_outputdirectory target_name output_directory target_prefix) endfunction(set_runtime_outputdirectory) function(scan_python_examples path) + # Skip python examples search in case the bindings are disabled + if(NOT ${ENABLE_PYTHON_BINDINGS}) + return() + endif() + + # Search for python examples file(GLOB_RECURSE python_examples ${path}/*.py) foreach(python_example ${python_examples}) if(NOT (${python_example} MATCHES "examples-to-run")) @@ -2025,19 +2031,20 @@ function(find_external_library) set(${name}_LIBRARIES "${libraries}" PARENT_SCOPE) set(${name}_HEADER ${${name}_header_internal} PARENT_SCOPE) set(${name}_FOUND TRUE PARENT_SCOPE) - set(status_message "find_external_library: ${name} was found.") + if(NOT ${FIND_LIB_QUIET}) + message(STATUS "find_external_library: ${name} was found.") + endif() else() set(${name}_INCLUDE_DIRS PARENT_SCOPE) set(${name}_LIBRARIES PARENT_SCOPE) set(${name}_HEADER PARENT_SCOPE) set(${name}_FOUND FALSE PARENT_SCOPE) - set(status_message + if(NOT ${FIND_LIB_QUIET}) + message( + ${HIGHLIGHTED_STATUS} "find_external_library: ${name} was not found. Missing headers: \"${not_found_headers}\" and missing libraries: \"${not_found_libraries}\"." - ) - endif() - - if(NOT ${FIND_LIB_QUIET}) - message(${HIGHLIGHTED_STATUS} "${status_message}") + ) + endif() endif() endfunction() diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 39508ec49..224711434 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -13,5 +13,6 @@ if(${ENABLE_EXAMPLES}) CACHE INTERNAL "list of example folders" ) endforeach() - scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR}) endif() + +scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/utils/tests/test-ns3.py b/utils/tests/test-ns3.py index 300a4c54c..9d7e4c584 100644 --- a/utils/tests/test-ns3.py +++ b/utils/tests/test-ns3.py @@ -139,7 +139,7 @@ def get_headers_list(outdir=usual_outdir): return glob.glob(outdir + '/**/*.h', recursive=True) -def read_buildstatus_entry(entry): +def read_lock_entry(entry): """! Read interesting entries from the .lock-ns3 file @param entry: entry to read from .lock-ns3 @@ -156,7 +156,7 @@ def get_test_enabled(): Check if tests are enabled in the .lock-ns3 @return bool. """ - return read_buildstatus_entry("ENABLE_TESTS") + return read_lock_entry("ENABLE_TESTS") def get_enabled_modules(): @@ -164,7 +164,7 @@ def get_enabled_modules(): Check if tests are enabled in the .lock-ns3 @return list of enabled modules (prefixed with 'ns3-'). """ - return read_buildstatus_entry("NS3_ENABLED_MODULES") + return read_lock_entry("NS3_ENABLED_MODULES") class NS3UnusedSourcesTestCase(unittest.TestCase): @@ -199,7 +199,7 @@ class NS3UnusedSourcesTestCase(unittest.TestCase): unused_sources = set() for example_directory in self.directory_and_files.keys(): # Skip non-example directories - if os.sep+"examples" not in example_directory: + if os.sep + "examples" not in example_directory: continue # Open the examples CMakeLists.txt and read it @@ -265,7 +265,7 @@ class NS3UnusedSourcesTestCase(unittest.TestCase): for directory in self.directory_and_files.keys(): # Skip directories that are not utils is_module = "src" in directory or "contrib" in directory - if os.sep+"utils" not in directory or is_module: + if os.sep + "utils" not in directory or is_module: continue # We can be in one of the module subdirectories (helper, model, test, bindings, etc) @@ -1376,7 +1376,7 @@ class NS3BuildBaseTestCase(NS3BaseTestCase): pkg_check_modules(ns3 REQUIRED IMPORTED_TARGET ns3-core{version}) target_link_libraries(test PUBLIC PkgConfig::ns3) """.format(lib=("lib64" if lib64 else "lib"), - version="="+version if version else "" + version="=" + version if version else "" ) for import_type in [pkgconfig_import, find_package_import]: @@ -1476,6 +1476,10 @@ class NS3BuildBaseTestCase(NS3BaseTestCase): except Exception: self.skipTest("Pybindgen is not available") + # Check if the number of runnable python scripts is equal to 0 + python_scripts = read_lock_entry("ns3_runnable_scripts") + self.assertEqual(len(python_scripts), 0) + # First we enable python bindings return_code, stdout, stderr = run_ns3("configure -G \"Unix Makefiles\" --enable-examples --enable-tests --enable-python-bindings") self.assertEqual(return_code, 0) @@ -1538,7 +1542,7 @@ class NS3BuildBaseTestCase(NS3BaseTestCase): # Then check if it was built self.assertGreater(len(list(filter(lambda x: "_core" in x, os.listdir(core_bindings_path)))), 0) - # We are on python anyways, so we can just try to load it from here + # We are on python anyway, so we can just try to load it from here sys.path.insert(0, os.path.join(core_bindings_path, "..")) try: from ns import core @@ -1553,6 +1557,19 @@ class NS3BuildBaseTestCase(NS3BaseTestCase): return_code, stdout, stderr = run_program("./test.py", "-p src/core/examples/sample-simulator.py", python=True) self.assertEqual(return_code, 0) + # Since python examples do not require recompilation, + # test if we still can run the python examples after disabling examples + return_code, stdout, stderr = run_ns3("configure -G \"Unix Makefiles\" --disable-examples") + self.assertEqual(return_code, 0) + + # Check if the lock file has python runnable python scripts (it should) + python_scripts = read_lock_entry("ns3_runnable_scripts") + self.assertGreater(len(python_scripts), 0) + + # Try to run an example + return_code, stdout, stderr = run_ns3("run sample-simulator.py") + self.assertEqual(return_code, 0) + NS3BuildBaseTestCase.cleaned_once = False def test_11_AmbiguityCheck(self): @@ -1576,8 +1593,8 @@ class NS3BuildBaseTestCase(NS3BaseTestCase): return_code, stdout, stderr = run_ns3("build second") self.assertEqual(return_code, 1) self.assertIn( - 'Build target "second" is ambiguous. Try one of these: "scratch/second", "examples/tutorial/second"', - stdout + 'Build target "second" is ambiguous. Try one of these: "scratch/second", "examples/tutorial/second"', + stdout ) # Try to run scratch/second and succeed @@ -1798,7 +1815,7 @@ class NS3ExpectedUseTestCase(NS3BaseTestCase): doc_folder = os.path.abspath(os.sep.join([".", "doc"])) # For each sphinx doc target. - for target in ["contributing", "manual", "models", "tutorial"]: + for target in ["contributing", "manual", "models", "tutorial"]: # First we need to clean old docs, or it will not make any sense. doc_build_folder = os.sep.join([doc_folder, target, "build"]) doc_temp_folder = os.sep.join([doc_folder, target, "source-temp"]) @@ -1867,7 +1884,7 @@ class NS3ExpectedUseTestCase(NS3BaseTestCase): if sudo_password is None: self.skipTest("SUDO_PASSWORD environment variable was not specified") - enable_sudo = read_buildstatus_entry("ENABLE_SUDO") + enable_sudo = read_lock_entry("ENABLE_SUDO") self.assertFalse(enable_sudo is True) # First we run to ensure the program was built @@ -1907,7 +1924,7 @@ class NS3ExpectedUseTestCase(NS3BaseTestCase): self.assertEqual(return_code, 0) # Check if it was properly set in the buildstatus file - enable_sudo = read_buildstatus_entry("ENABLE_SUDO") + enable_sudo = read_lock_entry("ENABLE_SUDO") self.assertTrue(enable_sudo) # Remove old executables @@ -1936,7 +1953,7 @@ class NS3ExpectedUseTestCase(NS3BaseTestCase): @return None """ - # Command templates that are empty or do not have a %s should fail + # Command templates that are empty or do not have a '%s' should fail return_code0, stdout0, stderr0 = run_ns3('run sample-simulator --command-template') self.assertEqual(return_code0, 2) self.assertIn("argument --command-template: expected one argument", stderr0)