bindings, build: fix bindings and visualizer build

Includes: - scan python scripts - run python scripts from ns3 - replace visualizer file copy with configure_file to prevent cmake refreshes - replace ns__init__.py file copy with configure_file to prevent cmake refreshes - fix bindings scanning with cmake - pass include directories to modulegen for castxml consumption - add missing parameters of Recv in python-unit-tests.py - change apiscan targets from apiscan-module to libmodule-apiscan - change bindings targets from module-bingings to libmodule-bindings - scanning and bindings build tests - scan scratch python scripts - replace FindPython3 with FindPython to be compatible with CMake 3.10 - do not export private visual-simulator-impl.h - do not export udp-socket-impl.h - use .so suffix for bindings on Mac instead of .dylib
2022-01-26 01:53:28 -03:00
parent 192019ce94
commit 9342082c53
16 changed files with 330 additions and 102 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -87,7 +87,7 @@ option(NS3_STATIC "Build a static ns-3 library and link it against executables"
       OFF
 )
 option(NS3_VERBOSE "Print additional build system messages" OFF)
-option(NS3_VISUALIZER "Build visualizer module" OFF)
+option(NS3_VISUALIZER "Build visualizer module" ON)
 option(NS3_WARNINGS "Enable compiler warnings" ON)
 option(NS3_WARNINGS_AS_ERRORS
       "Treat warnings as errors. Requires NS3_WARNINGS=ON" ON
@@ -144,6 +144,7 @@ generate_c4che_cachepy()
 generate_buildstatus()
 generate_fakewaflock()
 write_fakewaf_config()
+write_header_to_modules_map()

 # Export package targets when installing
 ns3_cmake_package()
--- a/bindings/python/ns3modulescan-modular.py
+++ b/bindings/python/ns3modulescan-modular.py
@@ -232,7 +232,7 @@ def ns3_module_scan(top_builddir, module_name, headers_map, output_file_name, cf
    #module_parser.add_post_scan_hook(post_scan_hook)

    castxml_options = dict(
-        include_paths=[top_builddir],
+        include_paths=[top_builddir, os.path.join(top_builddir, "include")],
         define_symbols={
            #'NS3_ASSERT_ENABLE': None,
            #'NS3_LOG_ENABLE': None,
@@ -256,6 +256,8 @@ def ns3_module_scan(top_builddir, module_name, headers_map, output_file_name, cf
    scan_header = os.path.join(os.path.dirname(output_file_name), "scan-header.h")
    if not os.path.exists(scan_header):
        scan_header = os.path.join(top_builddir, "ns3", "%s-module.h" % module_name)
+    if not os.path.exists(scan_header):
+        scan_header = os.path.join(top_builddir, "include", "ns3", "%s-module.h" % module_name)

    module_parser.parse_init([scan_header],
                             None, whitelist_paths=[top_builddir],
@@ -280,5 +282,11 @@ if __name__ == '__main__':
    if len(sys.argv) != 6:
        print("ns3modulescan-modular.py top_builddir module_path module_headers output_file_name cflags")
        sys.exit(1)
-    ns3_module_scan(sys.argv[1], sys.argv[2], eval(sys.argv[3]), sys.argv[4], sys.argv[5])
+    if os.path.exists(sys.argv[3]):
+        import json
+        with open(sys.argv[3], "r") as f:
+            module_headers = json.load(f)
+    else:
+        module_headers = eval(sys.argv[3])
+    ns3_module_scan(sys.argv[1], sys.argv[2], module_headers, sys.argv[4], sys.argv[5])
    sys.exit(0)
--- a/buildsupport/custom_modules/ns3_module_macros.cmake
+++ b/buildsupport/custom_modules/ns3_module_macros.cmake
@@ -30,7 +30,7 @@
 # Hidden argument (this is not a function, so you don't really need to pass arguments explicitly)
 # deprecated_header_files = "list;of;deprecated;.h;files", copy won't get triggered if deprecated_header_files isn't set
 # ignore_pch = TRUE or FALSE, prevents the PCH from including undesired system libraries (e.g. custom GLIBC for DCE)
-
+# module_enabled_features = "list;of;enabled;features;for;this;module" (used by fd-net-device)

 macro(
  build_lib_impl
@@ -42,6 +42,7 @@ macro(
  test_sources
  #deprecated_header_files
  #ignore_pch
+  #module_enabled_features
 )
  # cmake-format: on

@@ -215,24 +216,23 @@ macro(

  # Build lib examples if requested
  if(${ENABLE_EXAMPLES})
-    if((EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/examples)
-       AND (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/examples/CMakeLists.txt)
-    )
-      add_subdirectory(examples)
+    foreach(example_folder example;examples)
+      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder})
+        if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}/CMakeLists.txt)
+          add_subdirectory(${example_folder})
        endif()
-    if((EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/example)
-       AND (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/example/CMakeLists.txt)
-    )
-      add_subdirectory(example)
+        scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR}/${example_folder})
      endif()
+    endforeach()
  endif()

  # Get architecture pair for python bindings
-  set(arch gcc_ILP32)
-  set(arch_flag)
-  if(${CMAKE_SIZEOF_VOID_P} EQUAL 8)
+  if((${CMAKE_SIZEOF_VOID_P} EQUAL 8) AND (NOT APPLE))
    set(arch gcc_LP64)
-    set(arch_flag -m64)
+    set(arch_flags -m64)
+  else()
+    set(arch gcc_ILP32)
+    set(arch_flags)
  endif()

  # Add target to scan python bindings
@@ -243,49 +243,63 @@ macro(
        ${PROJECT_SOURCE_DIR}/${folder}/${libname}/bindings
    )
    file(MAKE_DIRECTORY ${bindings_output_folder})
-    set(module_api ${bindings_output_folder}/modulegen__${arch}.py)
+    set(module_api_ILP32 ${bindings_output_folder}/modulegen__gcc_ILP32.py)
+    set(module_api_LP64 ${bindings_output_folder}/modulegen__gcc_LP64.py)

    set(modulescan_modular_command
-        ${Python3_EXECUTABLE}
+        ${Python_EXECUTABLE}
        ${PROJECT_SOURCE_DIR}/bindings/python/ns3modulescan-modular.py
    )

-    # To build the header map for the module, we start by copying the headers
-    # and prepending the dictionary start
-    set(header_map "{\\\"${header_files};")
+    set(header_map "")
+    # We need a python map that takes header.h to module e.g. "ptr.h": "core"
+    foreach(header ${header_files})
+      # header is a relative path to the current working directory
+      get_filename_component(
+        header_name ${CMAKE_CURRENT_SOURCE_DIR}/${header} NAME
+      )
+      string(APPEND header_map "\"${header_name}\":\"${libname}\",")
+    endforeach()

-    # We then remove model/ helper/ prefixes e.g.
-    # {'model/angles.h;model/antenna-model.h;... ->
-    # {'angles.h;antenna-model.h;...)
-    string(REPLACE "model/" "" header_map "${header_map}")
-    string(REPLACE "helper/" "" header_map "${header_map}")
-
-    # Now we replace list entry separators (;) with ending of the string quote
-    # ("), followed by the relative module e.g.
-    # {"model/angles.h;model/antenna-model.h;... -> {"angles.h" : "antenna",
-    # "antenna-model.h": "antenna", "...)
-    string(REPLACE ";" "\\\": \\\"${libname}\\\", \\\"" header_map
-                   "${header_map}"
+    set(ns3-headers-to-module-map "${ns3-headers-to-module-map}${header_map}"
+        CACHE INTERNAL "Map connecting headers to their modules"
    )

-    # We now remove the last character ("), which needs to be replaced with a
-    # (}), to close the dictionary e.g. "antenna-model.h" : "antenna", " ->
-    # "antenna-model.h" : "antenna"
-    string(LENGTH "${header_map}" header_map_len)
-    math(EXPR header_map_len "${header_map_len}-3")
-    string(SUBSTRING "${header_map}" 0 ${header_map_len} header_map)
+    # API scan needs the include directories to find a few headers (e.g. mpi.h)
+    get_target_property(include_dirs ${lib${libname}} INCLUDE_DIRECTORIES)
+    set(modulegen_include_dirs)
+    foreach(include_dir ${include_dirs})
+      if(include_dir MATCHES "<")
+        # Skip CMake build and install interface includes
+        continue()
+      else()
+        # Append the include directory to a list
+        set(modulegen_include_dirs ${modulegen_include_dirs} -I${include_dir})
+      endif()
+    endforeach()

-    # Append end of dictionary (})
-    string(APPEND header_map "}")
+    set(module_to_generate_api ${module_api_ILP32})
+    set(LP64toILP32)
+    if("${arch}" STREQUAL "gcc_LP64")
+      set(module_to_generate_api ${module_api_LP64})
+      set(LP64toILP32
+          ${Python_EXECUTABLE}
+          ${PROJECT_SOURCE_DIR}/buildsupport/pybindings_LP64_to_ILP32.py
+          ${module_api_LP64} ${module_api_ILP32}
+      )
+    endif()

    add_custom_target(
-      apiscan-${lib${libname}}
-      COMMAND ${modulescan_modular_command} ${CMAKE_OUTPUT_DIRECTORY} ${libname}
-              ${header_map} ${module_api} ${arch_flag}
+      ${lib${libname}}-apiscan
+      COMMAND
+        ${modulescan_modular_command} ${CMAKE_OUTPUT_DIRECTORY} ${libname}
+        ${PROJECT_BINARY_DIR}/header_map.json ${module_to_generate_api}
+        \"${arch_flags} ${modulegen_include_dirs}\"
+      COMMAND ${LP64toILP32}
      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
      DEPENDS ${lib${libname}}
    )
-    add_dependencies(apiscan-all apiscan-${lib${libname}})
+    add_dependencies(apiscan-all ${lib${libname}}-apiscan)
  endif()

  # Build pybindings if requested and if bindings subfolder exists in
@@ -316,10 +330,12 @@ macro(
    if((NOT EXISTS ${module_hdr}) OR (NOT EXISTS ${module_src})) # OR TRUE) # to
                                                                 # force
                                                                 # reprocessing
-      string(REPLACE ";" "," ENABLED_FEATURES "${ns3-libs}")
+      string(REPLACE ";" "," ENABLED_FEATURES
+                     "${ns3-libs};${module_enabled_features}"
+      )
      set(modulegen_modular_command
          GCC_RTTI_ABI_COMPLETE=True NS3_ENABLED_FEATURES="${ENABLED_FEATURES}"
-          ${Python3_EXECUTABLE}
+          ${Python_EXECUTABLE}
          ${PROJECT_SOURCE_DIR}/bindings/python/ns3modulegen-modular.py
      )
      execute_process(
@@ -354,11 +370,12 @@ macro(
           ${CMAKE_CURRENT_SOURCE_DIR}/bindings/scan-header.h
      )
    endif()
-    add_library(${libname}-bindings SHARED "${python_module_files}")
+    set(bindings-name lib${libname}-bindings)
+    add_library(${bindings-name} SHARED "${python_module_files}")
    target_include_directories(
-      ${libname}-bindings PUBLIC ${Python3_INCLUDE_DIRS}
-                                 ${bindings_output_folder}
+      ${bindings-name} PUBLIC ${Python_INCLUDE_DIRS} ${bindings_output_folder}
    )
+    target_compile_options(${bindings-name} PRIVATE -Wno-error)

    # If there is any, remove the "lib" prefix of libraries (search for
    # "set(lib${libname}")
@@ -367,35 +384,45 @@ macro(
      string(REPLACE ";" "-bindings;" bindings_to_link
                     "${ns_libraries_to_link};"
      ) # add -bindings suffix to all lib${name}
-      string(REPLACE "lib" "" bindings_to_link "${bindings_to_link}"
-      )# remove lib prefix from all lib${name}-bindings
    endif()
    target_link_libraries(
-      ${libname}-bindings
+      ${bindings-name}
      PUBLIC ${LIB_AS_NEEDED_PRE} ${lib${libname}} "${bindings_to_link}"
             "${libraries_to_link}" ${LIB_AS_NEEDED_POST}
+      PRIVATE ${Python_LIBRARIES}
    )
    target_include_directories(
-      ${libname}-bindings PRIVATE ${PROJECT_SOURCE_DIR}/src/core/bindings
+      ${bindings-name} PRIVATE ${PROJECT_SOURCE_DIR}/src/core/bindings
    )

+    set(suffix)
+    if(APPLE)
+      # Python doesn't like Apple's .dylib and will refuse to load bindings
+      # unless its an .so
+      set(suffix SUFFIX .so)
+    endif()
+
    # Set binding library name and output folder
    set_target_properties(
-      ${libname}-bindings
-      PROPERTIES OUTPUT_NAME ${prefix}${libname_sub} PREFIX ""
+      ${bindings-name}
+      PROPERTIES OUTPUT_NAME ${prefix}${libname_sub}
+                 PREFIX ""
+                 ${suffix}
                 LIBRARY_OUTPUT_DIRECTORY
                 ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/ns
    )

    set(ns3-python-bindings-modules
-        "${libname}-bindings;${ns3-python-bindings-modules}"
+        "${bindings-name};${ns3-python-bindings-modules}"
        CACHE INTERNAL "list of modules python bindings"
    )

    # Make sure all bindings are built before building the visualizer module
    # that makes use of them
+    if(${ENABLE_VISUALIZER})
      if(NOT (${name} STREQUAL visualizer))
-      add_dependencies(${libvisualizer} ${libname}-bindings)
+        add_dependencies(${libvisualizer} ${bindings-name})
+      endif()
    endif()
  endif()

--- a/buildsupport/custom_modules/waf_workaround_buildstatus.cmake
+++ b/buildsupport/custom_modules/waf_workaround_buildstatus.cmake
@@ -18,15 +18,17 @@
 function(generate_buildstatus)
  # Build build-status.py file consumed by test.py
  set(buildstatus_contents "#! /usr/bin/env python3\n\n")
-  string(APPEND buildstatus_contents "ns3_runnable_programs = [")

+  string(APPEND buildstatus_contents "ns3_runnable_programs = [")
  foreach(executable ${ns3-execs})
-    string(APPEND buildstatus_contents "'${executable}', ")
+    string(APPEND buildstatus_contents "'${executable}',\n")
  endforeach()
  string(APPEND buildstatus_contents "]\n\n")

-  string(APPEND buildstatus_contents "ns3_runnable_scripts = [") # missing
-                                                                 # support
+  string(APPEND buildstatus_contents "ns3_runnable_scripts = [")
+  foreach(executable ${ns3-execs-py})
+    string(APPEND buildstatus_contents "'${executable}',\n")
+  endforeach()
  string(APPEND buildstatus_contents "]\n\n")

  file(WRITE ${CMAKE_OUTPUT_DIRECTORY}/build-status.py
--- a/buildsupport/custom_modules/waf_workaround_c4cache.cmake
+++ b/buildsupport/custom_modules/waf_workaround_c4cache.cmake
@@ -65,6 +65,9 @@ function(generate_c4che_cachepy)
  cache_cmake_flag(NS3_BRITE "ENABLE_BRITE" cache_contents)
  cache_cmake_flag(NS3_ENABLE_SUDO "ENABLE_SUDO" cache_contents)
  cache_cmake_flag(NS3_PYTHON_BINDINGS "ENABLE_PYTHON_BINDINGS" cache_contents)
+  cache_cmake_flag(
+    NS3_SCAN_PYTHON_BINDINGS "ENABLE_SCAN_PYTHON_BINDINGS" cache_contents
+  )

  string(APPEND cache_contents "EXAMPLE_DIRECTORIES = [")
  foreach(example_folder ${ns3-example-folders})
@@ -75,7 +78,7 @@ function(generate_c4che_cachepy)
  string(APPEND cache_contents "APPNAME = 'ns'\n")
  string(APPEND cache_contents "BUILD_PROFILE = '${build_profile}'\n")
  string(APPEND cache_contents "VERSION = '${NS3_VER}' \n")
-  string(APPEND cache_contents "PYTHON = ['${Python3_EXECUTABLE}']\n")
+  string(APPEND cache_contents "PYTHON = ['${Python_EXECUTABLE}']\n")

  mark_as_advanced(VALGRIND)
  find_program(VALGRIND valgrind)
--- a/buildsupport/macros_and_definitions.cmake
+++ b/buildsupport/macros_and_definitions.cmake
@@ -221,7 +221,9 @@ macro(clear_global_cached_variables)
  unset(ns3-contrib-libs CACHE)
  unset(ns3-example-folders CACHE)
  unset(ns3-execs CACHE)
+  unset(ns3-execs-py CACHE)
  unset(ns3-external-libs CACHE)
+  unset(ns3-headers-to-module-map CACHE)
  unset(ns3-libs CACHE)
  unset(ns3-libs-tests CACHE)
  unset(ns3-python-bindings-modules CACHE)
@@ -232,7 +234,9 @@ macro(clear_global_cached_variables)
    ns3-contrib-libs
    ns3-example-folders
    ns3-execs
+    ns3-execs-py
    ns3-external-libs
+    ns3-headers-to-module-map
    ns3-libs
    ns3-libs-tests
    ns3-python-bindings-modules
@@ -612,23 +616,28 @@ macro(process_options)
    endif()
  endif()

-  find_package(Python3 COMPONENTS Interpreter Development QUIET)
+  find_package(Python COMPONENTS Interpreter Development QUIET)
+
+  # Check if python3 was found, and mark as not found if python2 is found
+  if(${Python_FOUND} AND (${Python_VERSION_MAJOR} LESS 3))
+    set(Python_FOUND FALSE)
+    message(
+      STATUS:
+      "bindings: an incompatible version of Python was found, bindings will be disabled"
+    )
+  endif()
+
  set(ENABLE_PYTHON_BINDINGS OFF)
  if(${NS3_PYTHON_BINDINGS})
-    if(NOT ${Python3_FOUND})
+    if(NOT ${Python_FOUND})
      message(FATAL_ERROR "NS3_PYTHON_BINDINGS requires Python3")
    endif()
    set(ENABLE_PYTHON_BINDINGS ON)
-    link_directories(${Python3_LIBRARY_DIRS})
-    include_directories(${Python3_INCLUDE_DIRS})
-    set(PYTHONDIR ${Python3_SITELIB})
-    set(PYTHONARCHDIR ${Python3_SITEARCH})
-    set(HAVE_PYEMBED TRUE)
-    set(HAVE_PYEXT TRUE)
-    set(HAVE_PYTHON_H TRUE)
+
    set(destination_dir ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/ns)
-    file(COPY bindings/python/ns__init__.py DESTINATION ${destination_dir})
-    file(RENAME ${destination_dir}/ns__init__.py ${destination_dir}/__init__.py)
+    configure_file(
+      bindings/python/ns__init__.py ${destination_dir}/__init__.py COPYONLY
+    )
  endif()

  if(${NS3_SCAN_PYTHON_BINDINGS})
@@ -637,6 +646,15 @@ macro(process_options)
    add_custom_target(apiscan-all)
  endif()

+  set(ENABLE_VISUALIZER FALSE)
+  if(${NS3_VISUALIZER})
+    if((NOT ${ENABLE_PYTHON_BINDINGS}) OR (NOT ${Python_FOUND}))
+      message(STATUS "Visualizer requires NS3_PYTHON_BINDINGS and Python3")
+    else()
+      set(ENABLE_VISUALIZER TRUE)
+    endif()
+  endif()
+
  if(${NS3_COVERAGE} AND (NOT ${ENABLE_TESTS} OR NOT ${ENABLE_EXAMPLES}))
    message(
      FATAL_ERROR
@@ -652,7 +670,7 @@ macro(process_options)
    # produce code coverage output
    add_custom_target(
      run_test_py
-      COMMAND ${Python3_EXECUTABLE} test.py --no-build
+      COMMAND ${Python_EXECUTABLE} test.py --no-build
      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
      DEPENDS all-test-targets
    )
@@ -685,14 +703,6 @@ macro(process_options)
    set_property(GLOBAL PROPERTY TARGET_MESSAGES OFF)
  endif()

-  set(ENABLE_VISUALIZER FALSE)
-  if(${NS3_VISUALIZER})
-    if((NOT ${NS3_PYTHON_BINDINGS}) OR (NOT ${Python3_FOUND}))
-      message(FATAL_ERROR "Visualizer requires NS3_PYTHON_BINDINGS and Python3")
-    endif()
-    set(ENABLE_VISUALIZER TRUE)
-  endif()
-
  mark_as_advanced(Boost_INCLUDE_DIR)
  find_package(Boost)
  if(${Boost_FOUND})
@@ -749,7 +759,7 @@ macro(process_options)
    add_custom_target(
      run-introspected-command-line
      COMMAND ${CMAKE_COMMAND} -E env NS_COMMANDLINE_INTROSPECTION=..
-              ${Python3_EXECUTABLE} ./test.py --no-build --constrain=example
+              ${Python_EXECUTABLE} ./test.py --no-build --constrain=example
      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
      DEPENDS all-test-targets # all-test-targets only exists if ENABLE_TESTS is
                               # set to ON
@@ -1071,6 +1081,17 @@ function(set_runtime_outputdirectory target_name output_directory target_prefix)
  endif()
 endfunction(set_runtime_outputdirectory)

+function(scan_python_examples path)
+  file(GLOB_RECURSE python_examples ${path}/*.py)
+  foreach(python_example ${python_examples})
+    if(NOT (${python_example} MATCHES "examples-to-run"))
+      set(ns3-execs-py "${python_example};${ns3-execs-py}"
+          CACHE INTERNAL "list of python scripts"
+      )
+    endif()
+  endforeach()
+endfunction()
+
 add_custom_target(copy_all_headers)
 function(copy_headers_before_building_lib libname outputdir headers visibility)
  foreach(header ${headers})
@@ -1437,6 +1458,20 @@ function(find_external_library_header_and_library name header_name library_name
  endif()
 endfunction()

+function(write_header_to_modules_map)
+  if(${NS3_SCAN_PYTHON_BINDINGS})
+    set(header_map ${ns3-headers-to-module-map})
+
+    # Trim last comma
+    string(LENGTH "${header_map}" header_map_len)
+    math(EXPR header_map_len "${header_map_len}-1")
+    string(SUBSTRING "${header_map}" 0 ${header_map_len} header_map)
+
+    # Then write to header_map.json for consumption of pybindgen
+    file(WRITE ${PROJECT_BINARY_DIR}/header_map.json "{${header_map}}")
+  endif()
+endfunction()
+
 # Waf workaround scripts
 include(buildsupport/custom_modules/waf_workaround_c4cache.cmake)
 include(buildsupport/custom_modules/waf_workaround_buildstatus.cmake)
--- a/buildsupport/pybindings_LP64_to_ILP32.py
+++ b/buildsupport/pybindings_LP64_to_ILP32.py
@@ -0,0 +1,16 @@
+#! /usr/bin/env python3
+# -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
+
+def lp64_to_ilp32(lp64path, ilp32path):
+    import re
+    lp64bindings = None
+    with open(lp64path, "r") as lp64file:
+        lp64bindings = lp64file.read()
+    with open(ilp32path, "w") as ilp32file:
+        ilp32bindings = re.sub("unsigned long(?!( long))", "unsigned long long", lp64bindings)
+        ilp32file.write(ilp32bindings)
+
+if __name__ == "__main__":
+    import sys
+    print(sys.argv)
+    exit(lp64_to_ilp32(sys.argv[1], sys.argv[2]))
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -9,4 +9,5 @@ if(${ENABLE_EXAMPLES})
        CACHE INTERNAL "list of example folders"
    )
  endforeach()
+  scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR})
 endif()
--- a/45
+++ b/45
@@ -99,6 +99,7 @@ def parse_args(argv):
    parser_configure = on_off_argument(parser_configure, "gtk", "GTK support in ConfigStore")
    parser_configure = on_off_argument(parser_configure, "logs", "the logs regardless of the compile mode")
    parser_configure = on_off_argument(parser_configure, "mpi", "the MPI support for distributed simulation")
+    parser_configure = on_off_argument(parser_configure, "python-bindings", "python bindings")
    parser_configure = on_off_argument(parser_configure, "tests", "the ns-3 tests")
    parser_configure = on_off_argument(parser_configure, "sanitizers",
                                       "address, memory leaks and undefined behavior sanitizers")
@@ -109,7 +110,6 @@ def parse_args(argv):
    parser_configure = on_off_argument(parser_configure, "warnings", "compiler warnings")
    parser_configure = on_off_argument(parser_configure, "werror", "Treat compiler warnings as errors",
                                       "Treat compiler warnings as warnings")
-    parser_configure = on_off_argument(parser_configure, "visualizer", "the visualizer module")
    parser_configure.add_argument('--enable-modules',
                                  help='List of modules to build (e.g. core;network;internet)',
                                  action="store", type=str, default=None)
@@ -286,6 +286,8 @@ def check_build_profile(output_directory):
    ns3_version = None
    ns3_modules = None
    ns3_modules_tests = []
+    ns3_modules_apiscan = []
+    ns3_modules_bindings = []
    enable_sudo = False
    if output_directory and os.path.exists(c4che_path):
        c4che_info = {}
@@ -293,10 +295,17 @@ def check_build_profile(output_directory):
        build_profile = c4che_info["BUILD_PROFILE"]
        ns3_version = c4che_info["VERSION"]
        ns3_modules = c4che_info["NS3_ENABLED_MODULES"]
-        ns3_modules_tests = [x + "-test" for x in ns3_modules]
        if "ENABLE_SUDO" in c4che_info:
            enable_sudo = c4che_info["ENABLE_SUDO"]
-    return build_profile, ns3_version, ns3_modules + ns3_modules_tests if ns3_modules else None, enable_sudo
+        if ns3_modules:
+            if c4che_info["ENABLE_TESTS"]:
+                ns3_modules_tests = [x + "-test" for x in ns3_modules]
+            if c4che_info["ENABLE_PYTHON_BINDINGS"]:
+                ns3_modules_bindings = [x + "-bindings" for x in ns3_modules]
+            if "ENABLE_SCAN_PYTHON_BINDINGS" in c4che_info and c4che_info["ENABLE_SCAN_PYTHON_BINDINGS"]:
+                ns3_modules_apiscan = [x + "-apiscan" for x in ns3_modules]
+            ns3_modules = ns3_modules + ns3_modules_tests + ns3_modules_apiscan + ns3_modules_bindings
+    return build_profile, ns3_version, ns3_modules, enable_sudo


 def print_and_buffer(message):
@@ -453,11 +462,11 @@ def configure_cmake(cmake, args, current_cmake_cache_folder, current_cmake_gener
               ("GTK3", "gtk"),
               ("LOG", "logs"),
               ("MPI", "mpi"),
+               ("PYTHON_BINDINGS", "python_bindings"),
               ("SANITIZE", "sanitizers"),
               ("STATIC", "static"),
               ("TESTS", "tests"),
               ("VERBOSE", "verbose"),
-               ("VISUALIZER", "visualizer"),
               ("WARNINGS", "warnings"),
               ("WARNINGS_AS_ERRORS", "werror"),
               )
@@ -553,6 +562,19 @@ def get_program_shortcuts(build_profile, ns3_version):
            if source_shortcut:
                ns3_program_map[shortcut_path + ".cc"] = program
            temp_path.pop(0)
+
+
+    if programs_dict["ns3_runnable_scripts"]:
+        scratch_scripts = glob.glob(os.path.join(ns3_path, "scratch", "*.py"), recursive=True)
+        programs_dict["ns3_runnable_scripts"].extend(scratch_scripts)
+
+    for program in programs_dict["ns3_runnable_scripts"]:
+        temp_path = program.replace(ns3_path, "").split(os.sep)
+        program = program.strip()
+        while len(temp_path):
+            shortcut_path = os.sep.join(temp_path)
+            ns3_program_map[shortcut_path] = program
+            temp_path.pop(0)
    return ns3_program_map


@@ -658,6 +680,9 @@ def reconfigure_cmake_to_force_refresh(cmake, current_cmake_cache_folder, output


 def get_target_to_build(program_path, ns3_version, build_profile):
+    if ".py" in program_path:
+        return None
+
    build_profile_suffix = "" if build_profile in ["release"] else "-" + build_profile
    program_name = "".join(*re.findall("(.*)ns%s-(.*)%s" % (ns3_version, build_profile_suffix), program_path))

@@ -712,7 +737,8 @@ def build_step(args,

    # If we are building specific targets, we build them one by one
    if "build" in args:
-        non_executable_targets = ["check-version",
+        non_executable_targets = ["apiscan-all",
+                                  "check-version",
                                  "cmake-format",
                                  "docs",
                                  "doxygen",
@@ -787,6 +813,11 @@ def run_step(args, target_to_run, target_args):
        target_to_run = "bash"
        use_shell = True
    else:
+        # running a python script?
+        if ".py" in target_to_run:
+            target_args = [target_to_run] + target_args
+            target_to_run = "python3"
+
        # running from ns-3-dev (ns3_path) or cwd
        if args.cwd:
            working_dir = args.cwd
@@ -1060,7 +1091,7 @@ def main():
        target_path = os.sep.join(target_path)
        return target_path

-    if not args.check and not args.shell and target_to_run:
+    if not args.check and not args.shell and target_to_run and not ".py" in target_to_run:
        target_to_run = remove_overlapping_path(out_dir, target_to_run)

        # Waf doesn't add version prefix and build type suffix to the scratches, so we remove them
@@ -1072,7 +1103,7 @@ def main():
        target_to_run = os.sep.join([out_dir, target_to_run])

        # If we're only trying to run the target, we need to check if it actually exists first
-        if (run_only or build_and_run) and not os.path.exists(target_to_run):
+        if (run_only or build_and_run) and ".py" not in target_to_run and not os.path.exists(target_to_run):
            raise Exception("Executable has not been built yet")

    # Setup program as sudo
--- a/src/fd-net-device/CMakeLists.txt
+++ b/src/fd-net-device/CMakeLists.txt
@@ -1,5 +1,7 @@
 set(name fd-net-device)

+set(module_enabled_features) # modulegen_customizations consumes this list
+
 mark_as_advanced(ENABLE_THREADING)
 set(ENABLE_THREADING ${HAVE_PTHREAD_H})

@@ -57,6 +59,7 @@ endif()

 if(${ENABLE_FDNETDEV})
  set(fd-net-device_creators)
+  list(APPEND module_enabled_features FdNetDevice)

  if(${ENABLE_EMUNETDEV})
    set(emu_sources helper/emu-fd-net-device-helper.cc)
@@ -74,6 +77,7 @@ if(${ENABLE_FDNETDEV})
    )

    list(APPEND fd-net-device_creators raw-sock-creator)
+    list(APPEND module_enabled_features EmuFdNetDevice)
  endif()

  if(${ENABLE_TAPNETDEV})
@@ -93,6 +97,7 @@ if(${ENABLE_FDNETDEV})
    )

    list(APPEND fd-net-device_creators tap-device-creator)
+    list(APPEND module_enabled_features TapFdNetDevice)
  endif()

  if(${ENABLE_NETMAP_EMU})
--- a/src/internet/CMakeLists.txt
+++ b/src/internet/CMakeLists.txt
@@ -251,7 +251,6 @@ set(header_files
    model/udp-header.h
    model/udp-l4-protocol.h
    model/udp-socket-factory.h
-    model/udp-socket-impl.h
    model/udp-socket.h
    model/windowed-filter.h
 )
--- a/src/visualizer/CMakeLists.txt
+++ b/src/visualizer/CMakeLists.txt
@@ -2,12 +2,12 @@ set(name visualizer)

 set(source_files model/pyviz.cc model/visual-simulator-impl.cc)

-set(header_files model/pyviz.h model/visual-simulator-impl.h)
+set(header_files model/pyviz.h)

-include_directories(${Python3_INCLUDE_DIRS})
+include_directories(${Python_INCLUDE_DIRS})

 set(libraries_to_link
-    ${Python3_LIBRARIES}
+    ${Python_LIBRARIES}
    ${libcore}
    ${libinternet}
    ${libwifi}
@@ -27,4 +27,11 @@ build_lib("${name}" "${source_files}" "${header_files}" "${libraries_to_link}"

 # move visualizer folder to build/bindings/python, which allows us to add only
 # PYTHONPATH=ns-3-dev/build/bindings/python
-file(COPY visualizer DESTINATION ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/)
+file(GLOB_RECURSE visualizer_files ${CMAKE_CURRENT_SOURCE_DIR}/visualizer/*)
+foreach(file ${visualizer_files})
+  string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}"
+                 "${CMAKE_OUTPUT_DIRECTORY}/bindings/python" destination
+                 ${file}
+  )
+  configure_file(${file} ${destination} COPYONLY)
+endforeach()
--- a/src/visualizer/model/visual-simulator-impl.cc
+++ b/src/visualizer/model/visual-simulator-impl.cc
@@ -18,6 +18,8 @@
 * Author: Gustavo Carneiro <gjcarneiro@gmail.com> <gjc@inescporto.pt>
 */
 #include <Python.h>
+#undef HAVE_PTHREAD_H
+#undef HAVE_SYS_STAT_H
 #include "visual-simulator-impl.h"
 #include "ns3/default-simulator-impl.h"
 #include "ns3/log.h"
--- a/test.py
+++ b/test.py
@@ -1120,6 +1120,7 @@ def run_tests():
    if os.path.exists(build_status_file):
        ns3_runnable_programs = get_list_from_file(build_status_file, "ns3_runnable_programs")
        ns3_runnable_scripts = get_list_from_file(build_status_file, "ns3_runnable_scripts")
+        ns3_runnable_scripts = [os.path.basename(script) for script in ns3_runnable_scripts]
    else:
        print('The build status file was not found.  You must do waf build before running test.py.', file=sys.stderr)
        sys.exit(2)
--- a/utils/python-unit-tests.py
+++ b/utils/python-unit-tests.py
@@ -1,3 +1,5 @@
+#! /usr/bin/env python3
+
 # Copyright (C) 2008-2011 INESC Porto

 # This program is free software; you can redistribute it and/or modify
@@ -25,6 +27,8 @@ import ns.mobility
 import ns.csma
 import ns.applications

+UINT32_MAX = 0xFFFFFFFF
+

 ## TestSimulator class
 class TestSimulator(unittest.TestCase):
@@ -170,7 +174,7 @@ class TestSimulator(unittest.TestCase):
            @return none
            """
            assert self._received_packet is None
-            self._received_packet = socket.Recv()
+            self._received_packet = socket.Recv(maxSize=UINT32_MAX, flags=0)

        sink = ns.network.Socket.CreateSocket(node, ns.core.TypeId.LookupByName("ns3::UdpSocketFactory"))
        sink.Bind(ns.network.InetSocketAddress(ns.network.Ipv4Address.GetAny(), 80))
--- a/utils/tests/test-ns3.py
+++ b/utils/tests/test-ns3.py
@@ -1246,6 +1246,92 @@ class NS3BuildBaseTestCase(NS3BaseTestCase):
            stdout = stdout.replace("scratch_%s" % target_cmake, "")  # remove build lines
            self.assertIn(target_to_run.split("/")[-1], stdout)

+        NS3BuildBaseTestCase.cleaned_once = False
+
+    def test_10_PybindgenBindings(self):
+        """!
+        Test if cmake is calling pybindgen through modulegen to generate
+        the bindings source files correctly
+        @return None
+        """
+
+        # First we enable python bindings
+        return_code, stdout, stderr = run_ns3("configure --enable-examples --enable-tests --enable-python-bindings")
+        self.assertEqual(return_code, 0)
+
+        # Then look for python bindings sources
+        core_bindings_generated_sources_path = os.path.join(ns3_path, "build", "src", "core", "bindings")
+        core_bindings_sources_path = os.path.join(ns3_path, "src", "core", "bindings")
+        core_bindings_path = os.path.join(ns3_path, "build", "bindings", "python", "ns")
+        core_bindings_header = os.path.join(core_bindings_generated_sources_path, "ns3module.h")
+        core_bindings_source = os.path.join(core_bindings_generated_sources_path, "ns3module.cc")
+        self.assertTrue(os.path.exists(core_bindings_header))
+        self.assertTrue(os.path.exists(core_bindings_source))
+
+        # Then try to build the bindings for the core module
+        return_code, stdout, stderr = run_ns3("build core-bindings")
+        self.assertEqual(return_code, 0)
+
+        # Then check if it was built
+        self.assertGreater(len(list(filter(lambda x: "_core" in x, os.listdir(core_bindings_path)))), 0)
+
+        # Now enable python bindings scanning
+        return_code, stdout, stderr = run_ns3("configure -- -DNS3_SCAN_PYTHON_BINDINGS=ON")
+        self.assertEqual(return_code, 0)
+
+        # Get the file status for the current scanned bindings
+        bindings_sources = os.listdir(core_bindings_sources_path)
+        bindings_sources = [os.path.join(core_bindings_sources_path, y) for y in bindings_sources]
+        timestamps = {}
+        [timestamps.update({x: os.stat(x)}) for x in bindings_sources]
+
+        # Try to scan the bindings for the core module
+        return_code, stdout, stderr = run_ns3("build core-apiscan")
+        self.assertEqual(return_code, 0)
+
+        # Check if they exist, are not empty and have a different timestamp
+        generated_python_files = ["callbacks_list.py", "modulegen__gcc_LP64.py"]
+        for binding_file in timestamps.keys():
+            if os.path.basename(binding_file) in generated_python_files:
+                self.assertTrue(os.path.exists(binding_file))
+                self.assertGreater(os.stat(binding_file).st_size, 0)
+                new_fstat = os.stat(binding_file)
+                self.assertNotEqual(timestamps[binding_file].st_mtime, new_fstat.st_mtime)
+
+        # Then delete the old bindings sources
+        for f in os.listdir(core_bindings_generated_sources_path):
+            os.remove(os.path.join(core_bindings_generated_sources_path, f))
+
+        # Reconfigure to recreate the source files
+        return_code, stdout, stderr = run_ns3("configure")
+        self.assertEqual(return_code, 0)
+
+        # Check again if they exist
+        self.assertTrue(os.path.exists(core_bindings_header))
+        self.assertTrue(os.path.exists(core_bindings_source))
+
+        # Build the core bindings again
+        return_code, stdout, stderr = run_ns3("build core-bindings")
+        self.assertEqual(return_code, 0)
+
+        # Then check if it was built
+        self.assertGreater(len(list(filter(lambda x: "_core" in x, os.listdir(core_bindings_path)))), 0)
+
+        # We are on python anyways, so we can just try to load it from here
+        sys.path.insert(0, os.path.join(core_bindings_path, ".."))
+        try:
+            from ns import core
+        except ImportError:
+            self.assertTrue(True)
+
+        # Check if ns3 can find and run the python example
+        return_code, stdout, stderr = run_ns3("run sample-simulator.py")
+        self.assertEqual(return_code, 0)
+
+        # Check if test.py can find and run the python example
+        return_code, stdout, stderr = run_program("./test.py", "-p src/core/examples/sample-simulator.py", python=True)
+        self.assertEqual(return_code, 0)
+

 class NS3ExpectedUseTestCase(NS3BaseTestCase):
    """!