bindings, build: fix bindings and visualizer build

Includes:
- scan python scripts
- run python scripts from ns3
- replace visualizer file copy with configure_file to prevent cmake refreshes
- replace ns__init__.py file copy with configure_file to prevent cmake refreshes
- fix bindings scanning with cmake
- pass include directories to modulegen for castxml consumption
- add missing parameters of Recv in python-unit-tests.py
- change apiscan targets from apiscan-module to libmodule-apiscan
- change bindings targets from module-bingings to libmodule-bindings
- scanning and bindings build tests
- scan scratch python scripts
- replace FindPython3 with FindPython to be compatible with CMake 3.10
- do not export private visual-simulator-impl.h
- do not export udp-socket-impl.h
- use .so suffix for bindings on Mac instead of .dylib

CMakeLists.txt

@@ -87,7 +87,7 @@ option(NS3_STATIC "Build a static ns-3 library and link it against executables"
        OFF
 )
 option(NS3_VERBOSE "Print additional build system messages" OFF)
-option(NS3_VISUALIZER "Build visualizer module" OFF)
+option(NS3_VISUALIZER "Build visualizer module" ON)
 option(NS3_WARNINGS "Enable compiler warnings" ON)
 option(NS3_WARNINGS_AS_ERRORS
        "Treat warnings as errors. Requires NS3_WARNINGS=ON" ON
@@ -144,6 +144,7 @@ generate_c4che_cachepy()
 generate_buildstatus()
 generate_fakewaflock()
 write_fakewaf_config()
+write_header_to_modules_map()
 
 # Export package targets when installing
 ns3_cmake_package()

bindings/python/ns3modulescan-modular.py

@@ -232,7 +232,7 @@ def ns3_module_scan(top_builddir, module_name, headers_map, output_file_name, cf
     #module_parser.add_post_scan_hook(post_scan_hook)
 
     castxml_options = dict(
-        include_paths=[top_builddir],
+        include_paths=[top_builddir, os.path.join(top_builddir, "include")],
          define_symbols={
             #'NS3_ASSERT_ENABLE': None,
             #'NS3_LOG_ENABLE': None,
@@ -256,6 +256,8 @@ def ns3_module_scan(top_builddir, module_name, headers_map, output_file_name, cf
     scan_header = os.path.join(os.path.dirname(output_file_name), "scan-header.h")
     if not os.path.exists(scan_header):
         scan_header = os.path.join(top_builddir, "ns3", "%s-module.h" % module_name)
+    if not os.path.exists(scan_header):
+        scan_header = os.path.join(top_builddir, "include", "ns3", "%s-module.h" % module_name)
 
     module_parser.parse_init([scan_header],
                              None, whitelist_paths=[top_builddir],
@@ -280,5 +282,11 @@ if __name__ == '__main__':
     if len(sys.argv) != 6:
         print("ns3modulescan-modular.py top_builddir module_path module_headers output_file_name cflags")
         sys.exit(1)
-    ns3_module_scan(sys.argv[1], sys.argv[2], eval(sys.argv[3]), sys.argv[4], sys.argv[5])
+    if os.path.exists(sys.argv[3]):
+        import json
+        with open(sys.argv[3], "r") as f:
+            module_headers = json.load(f)
+    else:
+        module_headers = eval(sys.argv[3])
+    ns3_module_scan(sys.argv[1], sys.argv[2], module_headers, sys.argv[4], sys.argv[5])
     sys.exit(0)

buildsupport/custom_modules/ns3_module_macros.cmake

@@ -30,7 +30,7 @@
 # Hidden argument (this is not a function, so you don't really need to pass arguments explicitly)
 # deprecated_header_files = "list;of;deprecated;.h;files", copy won't get triggered if deprecated_header_files isn't set
 # ignore_pch = TRUE or FALSE, prevents the PCH from including undesired system libraries (e.g. custom GLIBC for DCE)
-
+# module_enabled_features = "list;of;enabled;features;for;this;module" (used by fd-net-device)
 
 macro(
   build_lib_impl
@@ -42,6 +42,7 @@ macro(
   test_sources
   #deprecated_header_files
   #ignore_pch
+  #module_enabled_features
 )
   # cmake-format: on
 
@@ -215,24 +216,23 @@ macro(
 
   # Build lib examples if requested
   if(${ENABLE_EXAMPLES})
-    if((EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/examples)
+    foreach(example_folder example;examples)
-       AND (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/examples/CMakeLists.txt)
+      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder})
-    )
+        if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${example_folder}/CMakeLists.txt)
-      add_subdirectory(examples)
+          add_subdirectory(${example_folder})
-    endif()
+        endif()
-    if((EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/example)
+        scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR}/${example_folder})
-       AND (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/example/CMakeLists.txt)
+      endif()
-    )
+    endforeach()
-      add_subdirectory(example)
-    endif()
   endif()
 
   # Get architecture pair for python bindings
-  set(arch gcc_ILP32)
+  if((${CMAKE_SIZEOF_VOID_P} EQUAL 8) AND (NOT APPLE))
-  set(arch_flag)
-  if(${CMAKE_SIZEOF_VOID_P} EQUAL 8)
     set(arch gcc_LP64)
-    set(arch_flag -m64)
+    set(arch_flags -m64)
+  else()
+    set(arch gcc_ILP32)
+    set(arch_flags)
   endif()
 
   # Add target to scan python bindings
@@ -243,49 +243,63 @@ macro(
         ${PROJECT_SOURCE_DIR}/${folder}/${libname}/bindings
     )
     file(MAKE_DIRECTORY ${bindings_output_folder})
-    set(module_api ${bindings_output_folder}/modulegen__${arch}.py)
+    set(module_api_ILP32 ${bindings_output_folder}/modulegen__gcc_ILP32.py)
+    set(module_api_LP64 ${bindings_output_folder}/modulegen__gcc_LP64.py)
 
     set(modulescan_modular_command
-        ${Python3_EXECUTABLE}
+        ${Python_EXECUTABLE}
         ${PROJECT_SOURCE_DIR}/bindings/python/ns3modulescan-modular.py
     )
 
-    # To build the header map for the module, we start by copying the headers
+    set(header_map "")
-    # and prepending the dictionary start
+    # We need a python map that takes header.h to module e.g. "ptr.h": "core"
-    set(header_map "{\\\"${header_files};")
+    foreach(header ${header_files})
+      # header is a relative path to the current working directory
+      get_filename_component(
+        header_name ${CMAKE_CURRENT_SOURCE_DIR}/${header} NAME
+      )
+      string(APPEND header_map "\"${header_name}\":\"${libname}\",")
+    endforeach()
 
-    # We then remove model/ helper/ prefixes e.g.
+    set(ns3-headers-to-module-map "${ns3-headers-to-module-map}${header_map}"
-    # {'model/angles.h;model/antenna-model.h;... ->
+        CACHE INTERNAL "Map connecting headers to their modules"
-    # {'angles.h;antenna-model.h;...)
-    string(REPLACE "model/" "" header_map "${header_map}")
-    string(REPLACE "helper/" "" header_map "${header_map}")
-
-    # Now we replace list entry separators (;) with ending of the string quote
-    # ("), followed by the relative module e.g.
-    # {"model/angles.h;model/antenna-model.h;... -> {"angles.h" : "antenna",
-    # "antenna-model.h": "antenna", "...)
-    string(REPLACE ";" "\\\": \\\"${libname}\\\", \\\"" header_map
-                   "${header_map}"
     )
 
-    # We now remove the last character ("), which needs to be replaced with a
+    # API scan needs the include directories to find a few headers (e.g. mpi.h)
-    # (}), to close the dictionary e.g. "antenna-model.h" : "antenna", " ->
+    get_target_property(include_dirs ${lib${libname}} INCLUDE_DIRECTORIES)
-    # "antenna-model.h" : "antenna"
+    set(modulegen_include_dirs)
-    string(LENGTH "${header_map}" header_map_len)
+    foreach(include_dir ${include_dirs})
-    math(EXPR header_map_len "${header_map_len}-3")
+      if(include_dir MATCHES "<")
-    string(SUBSTRING "${header_map}" 0 ${header_map_len} header_map)
+        # Skip CMake build and install interface includes
+        continue()
+      else()
+        # Append the include directory to a list
+        set(modulegen_include_dirs ${modulegen_include_dirs} -I${include_dir})
+      endif()
+    endforeach()
 
-    # Append end of dictionary (})
+    set(module_to_generate_api ${module_api_ILP32})
-    string(APPEND header_map "}")
+    set(LP64toILP32)
+    if("${arch}" STREQUAL "gcc_LP64")
+      set(module_to_generate_api ${module_api_LP64})
+      set(LP64toILP32
+          ${Python_EXECUTABLE}
+          ${PROJECT_SOURCE_DIR}/buildsupport/pybindings_LP64_to_ILP32.py
+          ${module_api_LP64} ${module_api_ILP32}
+      )
+    endif()
 
     add_custom_target(
-      apiscan-${lib${libname}}
+      ${lib${libname}}-apiscan
-      COMMAND ${modulescan_modular_command} ${CMAKE_OUTPUT_DIRECTORY} ${libname}
+      COMMAND
-              ${header_map} ${module_api} ${arch_flag}
+        ${modulescan_modular_command} ${CMAKE_OUTPUT_DIRECTORY} ${libname}
+        ${PROJECT_BINARY_DIR}/header_map.json ${module_to_generate_api}
+        \"${arch_flags} ${modulegen_include_dirs}\"
+      COMMAND ${LP64toILP32}
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
       DEPENDS ${lib${libname}}
     )
-    add_dependencies(apiscan-all apiscan-${lib${libname}})
+    add_dependencies(apiscan-all ${lib${libname}}-apiscan)
   endif()
 
   # Build pybindings if requested and if bindings subfolder exists in
@@ -316,10 +330,12 @@ macro(
     if((NOT EXISTS ${module_hdr}) OR (NOT EXISTS ${module_src})) # OR TRUE) # to
                                                                  # force
                                                                  # reprocessing
-      string(REPLACE ";" "," ENABLED_FEATURES "${ns3-libs}")
+      string(REPLACE ";" "," ENABLED_FEATURES
+                     "${ns3-libs};${module_enabled_features}"
+      )
       set(modulegen_modular_command
           GCC_RTTI_ABI_COMPLETE=True NS3_ENABLED_FEATURES="${ENABLED_FEATURES}"
-          ${Python3_EXECUTABLE}
+          ${Python_EXECUTABLE}
           ${PROJECT_SOURCE_DIR}/bindings/python/ns3modulegen-modular.py
       )
       execute_process(
@@ -354,11 +370,12 @@ macro(
            ${CMAKE_CURRENT_SOURCE_DIR}/bindings/scan-header.h
       )
     endif()
-    add_library(${libname}-bindings SHARED "${python_module_files}")
+    set(bindings-name lib${libname}-bindings)
+    add_library(${bindings-name} SHARED "${python_module_files}")
     target_include_directories(
-      ${libname}-bindings PUBLIC ${Python3_INCLUDE_DIRS}
+      ${bindings-name} PUBLIC ${Python_INCLUDE_DIRS} ${bindings_output_folder}
-                                 ${bindings_output_folder}
     )
+    target_compile_options(${bindings-name} PRIVATE -Wno-error)
 
     # If there is any, remove the "lib" prefix of libraries (search for
     # "set(lib${libname}")
@@ -367,35 +384,45 @@ macro(
       string(REPLACE ";" "-bindings;" bindings_to_link
                      "${ns_libraries_to_link};"
       ) # add -bindings suffix to all lib${name}
-      string(REPLACE "lib" "" bindings_to_link "${bindings_to_link}"
-      )# remove lib prefix from all lib${name}-bindings
     endif()
     target_link_libraries(
-      ${libname}-bindings
+      ${bindings-name}
       PUBLIC ${LIB_AS_NEEDED_PRE} ${lib${libname}} "${bindings_to_link}"
              "${libraries_to_link}" ${LIB_AS_NEEDED_POST}
+      PRIVATE ${Python_LIBRARIES}
     )
     target_include_directories(
-      ${libname}-bindings PRIVATE ${PROJECT_SOURCE_DIR}/src/core/bindings
+      ${bindings-name} PRIVATE ${PROJECT_SOURCE_DIR}/src/core/bindings
     )
 
+    set(suffix)
+    if(APPLE)
+      # Python doesn't like Apple's .dylib and will refuse to load bindings
+      # unless its an .so
+      set(suffix SUFFIX .so)
+    endif()
+
     # Set binding library name and output folder
     set_target_properties(
-      ${libname}-bindings
+      ${bindings-name}
-      PROPERTIES OUTPUT_NAME ${prefix}${libname_sub} PREFIX ""
+      PROPERTIES OUTPUT_NAME ${prefix}${libname_sub}
+                 PREFIX ""
+                 ${suffix}
                  LIBRARY_OUTPUT_DIRECTORY
                  ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/ns
     )
 
     set(ns3-python-bindings-modules
-        "${libname}-bindings;${ns3-python-bindings-modules}"
+        "${bindings-name};${ns3-python-bindings-modules}"
         CACHE INTERNAL "list of modules python bindings"
     )
 
     # Make sure all bindings are built before building the visualizer module
     # that makes use of them
-    if(NOT (${name} STREQUAL visualizer))
+    if(${ENABLE_VISUALIZER})
-      add_dependencies(${libvisualizer} ${libname}-bindings)
+      if(NOT (${name} STREQUAL visualizer))
+        add_dependencies(${libvisualizer} ${bindings-name})
+      endif()
     endif()
   endif()
 

buildsupport/custom_modules/waf_workaround_buildstatus.cmake

@@ -18,15 +18,17 @@
 function(generate_buildstatus)
   # Build build-status.py file consumed by test.py
   set(buildstatus_contents "#! /usr/bin/env python3\n\n")
-  string(APPEND buildstatus_contents "ns3_runnable_programs = [")
 
+  string(APPEND buildstatus_contents "ns3_runnable_programs = [")
   foreach(executable ${ns3-execs})
-    string(APPEND buildstatus_contents "'${executable}', ")
+    string(APPEND buildstatus_contents "'${executable}',\n")
   endforeach()
   string(APPEND buildstatus_contents "]\n\n")
 
-  string(APPEND buildstatus_contents "ns3_runnable_scripts = [") # missing
+  string(APPEND buildstatus_contents "ns3_runnable_scripts = [")
-                                                                 # support
+  foreach(executable ${ns3-execs-py})
+    string(APPEND buildstatus_contents "'${executable}',\n")
+  endforeach()
   string(APPEND buildstatus_contents "]\n\n")
 
   file(WRITE ${CMAKE_OUTPUT_DIRECTORY}/build-status.py

buildsupport/custom_modules/waf_workaround_c4cache.cmake

@@ -65,6 +65,9 @@ function(generate_c4che_cachepy)
   cache_cmake_flag(NS3_BRITE "ENABLE_BRITE" cache_contents)
   cache_cmake_flag(NS3_ENABLE_SUDO "ENABLE_SUDO" cache_contents)
   cache_cmake_flag(NS3_PYTHON_BINDINGS "ENABLE_PYTHON_BINDINGS" cache_contents)
+  cache_cmake_flag(
+    NS3_SCAN_PYTHON_BINDINGS "ENABLE_SCAN_PYTHON_BINDINGS" cache_contents
+  )
 
   string(APPEND cache_contents "EXAMPLE_DIRECTORIES = [")
   foreach(example_folder ${ns3-example-folders})
@@ -75,7 +78,7 @@ function(generate_c4che_cachepy)
   string(APPEND cache_contents "APPNAME = 'ns'\n")
   string(APPEND cache_contents "BUILD_PROFILE = '${build_profile}'\n")
   string(APPEND cache_contents "VERSION = '${NS3_VER}' \n")
-  string(APPEND cache_contents "PYTHON = ['${Python3_EXECUTABLE}']\n")
+  string(APPEND cache_contents "PYTHON = ['${Python_EXECUTABLE}']\n")
 
   mark_as_advanced(VALGRIND)
   find_program(VALGRIND valgrind)

buildsupport/macros_and_definitions.cmake

@@ -221,7 +221,9 @@ macro(clear_global_cached_variables)
   unset(ns3-contrib-libs CACHE)
   unset(ns3-example-folders CACHE)
   unset(ns3-execs CACHE)
+  unset(ns3-execs-py CACHE)
   unset(ns3-external-libs CACHE)
+  unset(ns3-headers-to-module-map CACHE)
   unset(ns3-libs CACHE)
   unset(ns3-libs-tests CACHE)
   unset(ns3-python-bindings-modules CACHE)
@@ -232,7 +234,9 @@ macro(clear_global_cached_variables)
     ns3-contrib-libs
     ns3-example-folders
     ns3-execs
+    ns3-execs-py
     ns3-external-libs
+    ns3-headers-to-module-map
     ns3-libs
     ns3-libs-tests
     ns3-python-bindings-modules
@@ -612,23 +616,28 @@ macro(process_options)
     endif()
   endif()
 
-  find_package(Python3 COMPONENTS Interpreter Development QUIET)
+  find_package(Python COMPONENTS Interpreter Development QUIET)
+
+  # Check if python3 was found, and mark as not found if python2 is found
+  if(${Python_FOUND} AND (${Python_VERSION_MAJOR} LESS 3))
+    set(Python_FOUND FALSE)
+    message(
+      STATUS:
+      "bindings: an incompatible version of Python was found, bindings will be disabled"
+    )
+  endif()
+
   set(ENABLE_PYTHON_BINDINGS OFF)
   if(${NS3_PYTHON_BINDINGS})
-    if(NOT ${Python3_FOUND})
+    if(NOT ${Python_FOUND})
       message(FATAL_ERROR "NS3_PYTHON_BINDINGS requires Python3")
     endif()
     set(ENABLE_PYTHON_BINDINGS ON)
-    link_directories(${Python3_LIBRARY_DIRS})
+
-    include_directories(${Python3_INCLUDE_DIRS})
-    set(PYTHONDIR ${Python3_SITELIB})
-    set(PYTHONARCHDIR ${Python3_SITEARCH})
-    set(HAVE_PYEMBED TRUE)
-    set(HAVE_PYEXT TRUE)
-    set(HAVE_PYTHON_H TRUE)
     set(destination_dir ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/ns)
-    file(COPY bindings/python/ns__init__.py DESTINATION ${destination_dir})
+    configure_file(
-    file(RENAME ${destination_dir}/ns__init__.py ${destination_dir}/__init__.py)
+      bindings/python/ns__init__.py ${destination_dir}/__init__.py COPYONLY
+    )
   endif()
 
   if(${NS3_SCAN_PYTHON_BINDINGS})
@@ -637,6 +646,15 @@ macro(process_options)
     add_custom_target(apiscan-all)
   endif()
 
+  set(ENABLE_VISUALIZER FALSE)
+  if(${NS3_VISUALIZER})
+    if((NOT ${ENABLE_PYTHON_BINDINGS}) OR (NOT ${Python_FOUND}))
+      message(STATUS "Visualizer requires NS3_PYTHON_BINDINGS and Python3")
+    else()
+      set(ENABLE_VISUALIZER TRUE)
+    endif()
+  endif()
+
   if(${NS3_COVERAGE} AND (NOT ${ENABLE_TESTS} OR NOT ${ENABLE_EXAMPLES}))
     message(
       FATAL_ERROR
@@ -652,7 +670,7 @@ macro(process_options)
     # produce code coverage output
     add_custom_target(
       run_test_py
-      COMMAND ${Python3_EXECUTABLE} test.py --no-build
+      COMMAND ${Python_EXECUTABLE} test.py --no-build
       WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
       DEPENDS all-test-targets
     )
@@ -685,14 +703,6 @@ macro(process_options)
     set_property(GLOBAL PROPERTY TARGET_MESSAGES OFF)
   endif()
 
-  set(ENABLE_VISUALIZER FALSE)
-  if(${NS3_VISUALIZER})
-    if((NOT ${NS3_PYTHON_BINDINGS}) OR (NOT ${Python3_FOUND}))
-      message(FATAL_ERROR "Visualizer requires NS3_PYTHON_BINDINGS and Python3")
-    endif()
-    set(ENABLE_VISUALIZER TRUE)
-  endif()
-
   mark_as_advanced(Boost_INCLUDE_DIR)
   find_package(Boost)
   if(${Boost_FOUND})
@@ -749,7 +759,7 @@ macro(process_options)
     add_custom_target(
       run-introspected-command-line
       COMMAND ${CMAKE_COMMAND} -E env NS_COMMANDLINE_INTROSPECTION=..
-              ${Python3_EXECUTABLE} ./test.py --no-build --constrain=example
+              ${Python_EXECUTABLE} ./test.py --no-build --constrain=example
       WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
       DEPENDS all-test-targets # all-test-targets only exists if ENABLE_TESTS is
                                # set to ON
@@ -1071,6 +1081,17 @@ function(set_runtime_outputdirectory target_name output_directory target_prefix)
   endif()
 endfunction(set_runtime_outputdirectory)
 
+function(scan_python_examples path)
+  file(GLOB_RECURSE python_examples ${path}/*.py)
+  foreach(python_example ${python_examples})
+    if(NOT (${python_example} MATCHES "examples-to-run"))
+      set(ns3-execs-py "${python_example};${ns3-execs-py}"
+          CACHE INTERNAL "list of python scripts"
+      )
+    endif()
+  endforeach()
+endfunction()
+
 add_custom_target(copy_all_headers)
 function(copy_headers_before_building_lib libname outputdir headers visibility)
   foreach(header ${headers})
@@ -1437,6 +1458,20 @@ function(find_external_library_header_and_library name header_name library_name
   endif()
 endfunction()
 
+function(write_header_to_modules_map)
+  if(${NS3_SCAN_PYTHON_BINDINGS})
+    set(header_map ${ns3-headers-to-module-map})
+
+    # Trim last comma
+    string(LENGTH "${header_map}" header_map_len)
+    math(EXPR header_map_len "${header_map_len}-1")
+    string(SUBSTRING "${header_map}" 0 ${header_map_len} header_map)
+
+    # Then write to header_map.json for consumption of pybindgen
+    file(WRITE ${PROJECT_BINARY_DIR}/header_map.json "{${header_map}}")
+  endif()
+endfunction()
+
 # Waf workaround scripts
 include(buildsupport/custom_modules/waf_workaround_c4cache.cmake)
 include(buildsupport/custom_modules/waf_workaround_buildstatus.cmake)

buildsupport/pybindings_LP64_to_ILP32.py

@@ -0,0 +1,16 @@
+#! /usr/bin/env python3
+# -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
+
+def lp64_to_ilp32(lp64path, ilp32path):
+    import re
+    lp64bindings = None
+    with open(lp64path, "r") as lp64file:
+        lp64bindings = lp64file.read()
+    with open(ilp32path, "w") as ilp32file:
+        ilp32bindings = re.sub("unsigned long(?!( long))", "unsigned long long", lp64bindings)
+        ilp32file.write(ilp32bindings)
+
+if __name__ == "__main__":
+    import sys
+    print(sys.argv)
+    exit(lp64_to_ilp32(sys.argv[1], sys.argv[2]))

examples/CMakeLists.txt

@@ -9,4 +9,5 @@ if(${ENABLE_EXAMPLES})
         CACHE INTERNAL "list of example folders"
     )
   endforeach()
+  scan_python_examples(${CMAKE_CURRENT_SOURCE_DIR})
 endif()

ns3

@@ -99,6 +99,7 @@ def parse_args(argv):
     parser_configure = on_off_argument(parser_configure, "gtk", "GTK support in ConfigStore")
     parser_configure = on_off_argument(parser_configure, "logs", "the logs regardless of the compile mode")
     parser_configure = on_off_argument(parser_configure, "mpi", "the MPI support for distributed simulation")
+    parser_configure = on_off_argument(parser_configure, "python-bindings", "python bindings")
     parser_configure = on_off_argument(parser_configure, "tests", "the ns-3 tests")
     parser_configure = on_off_argument(parser_configure, "sanitizers",
                                        "address, memory leaks and undefined behavior sanitizers")
@@ -109,7 +110,6 @@ def parse_args(argv):
     parser_configure = on_off_argument(parser_configure, "warnings", "compiler warnings")
     parser_configure = on_off_argument(parser_configure, "werror", "Treat compiler warnings as errors",
                                        "Treat compiler warnings as warnings")
-    parser_configure = on_off_argument(parser_configure, "visualizer", "the visualizer module")
     parser_configure.add_argument('--enable-modules',
                                   help='List of modules to build (e.g. core;network;internet)',
                                   action="store", type=str, default=None)
@@ -286,6 +286,8 @@ def check_build_profile(output_directory):
     ns3_version = None
     ns3_modules = None
     ns3_modules_tests = []
+    ns3_modules_apiscan = []
+    ns3_modules_bindings = []
     enable_sudo = False
     if output_directory and os.path.exists(c4che_path):
         c4che_info = {}
@@ -293,10 +295,17 @@ def check_build_profile(output_directory):
         build_profile = c4che_info["BUILD_PROFILE"]
         ns3_version = c4che_info["VERSION"]
         ns3_modules = c4che_info["NS3_ENABLED_MODULES"]
-        ns3_modules_tests = [x + "-test" for x in ns3_modules]
         if "ENABLE_SUDO" in c4che_info:
             enable_sudo = c4che_info["ENABLE_SUDO"]
-    return build_profile, ns3_version, ns3_modules + ns3_modules_tests if ns3_modules else None, enable_sudo
+        if ns3_modules:
+            if c4che_info["ENABLE_TESTS"]:
+                ns3_modules_tests = [x + "-test" for x in ns3_modules]
+            if c4che_info["ENABLE_PYTHON_BINDINGS"]:
+                ns3_modules_bindings = [x + "-bindings" for x in ns3_modules]
+            if "ENABLE_SCAN_PYTHON_BINDINGS" in c4che_info and c4che_info["ENABLE_SCAN_PYTHON_BINDINGS"]:
+                ns3_modules_apiscan = [x + "-apiscan" for x in ns3_modules]
+            ns3_modules = ns3_modules + ns3_modules_tests + ns3_modules_apiscan + ns3_modules_bindings
+    return build_profile, ns3_version, ns3_modules, enable_sudo
 
 
 def print_and_buffer(message):
@@ -453,11 +462,11 @@ def configure_cmake(cmake, args, current_cmake_cache_folder, current_cmake_gener
                ("GTK3", "gtk"),
                ("LOG", "logs"),
                ("MPI", "mpi"),
+               ("PYTHON_BINDINGS", "python_bindings"),
                ("SANITIZE", "sanitizers"),
                ("STATIC", "static"),
                ("TESTS", "tests"),
                ("VERBOSE", "verbose"),
-               ("VISUALIZER", "visualizer"),
                ("WARNINGS", "warnings"),
                ("WARNINGS_AS_ERRORS", "werror"),
                )
@@ -553,6 +562,19 @@ def get_program_shortcuts(build_profile, ns3_version):
             if source_shortcut:
                 ns3_program_map[shortcut_path + ".cc"] = program
             temp_path.pop(0)
+
+
+    if programs_dict["ns3_runnable_scripts"]:
+        scratch_scripts = glob.glob(os.path.join(ns3_path, "scratch", "*.py"), recursive=True)
+        programs_dict["ns3_runnable_scripts"].extend(scratch_scripts)
+
+    for program in programs_dict["ns3_runnable_scripts"]:
+        temp_path = program.replace(ns3_path, "").split(os.sep)
+        program = program.strip()
+        while len(temp_path):
+            shortcut_path = os.sep.join(temp_path)
+            ns3_program_map[shortcut_path] = program
+            temp_path.pop(0)
     return ns3_program_map
 
 
@@ -658,6 +680,9 @@ def reconfigure_cmake_to_force_refresh(cmake, current_cmake_cache_folder, output
 
 
 def get_target_to_build(program_path, ns3_version, build_profile):
+    if ".py" in program_path:
+        return None
+
     build_profile_suffix = "" if build_profile in ["release"] else "-" + build_profile
     program_name = "".join(*re.findall("(.*)ns%s-(.*)%s" % (ns3_version, build_profile_suffix), program_path))
 
@@ -712,7 +737,8 @@ def build_step(args,
 
     # If we are building specific targets, we build them one by one
     if "build" in args:
-        non_executable_targets = ["check-version",
+        non_executable_targets = ["apiscan-all",
+                                  "check-version",
                                   "cmake-format",
                                   "docs",
                                   "doxygen",
@@ -787,6 +813,11 @@ def run_step(args, target_to_run, target_args):
         target_to_run = "bash"
         use_shell = True
     else:
+        # running a python script?
+        if ".py" in target_to_run:
+            target_args = [target_to_run] + target_args
+            target_to_run = "python3"
+
         # running from ns-3-dev (ns3_path) or cwd
         if args.cwd:
             working_dir = args.cwd
@@ -1060,7 +1091,7 @@ def main():
         target_path = os.sep.join(target_path)
         return target_path
 
-    if not args.check and not args.shell and target_to_run:
+    if not args.check and not args.shell and target_to_run and not ".py" in target_to_run:
         target_to_run = remove_overlapping_path(out_dir, target_to_run)
 
         # Waf doesn't add version prefix and build type suffix to the scratches, so we remove them
@@ -1072,7 +1103,7 @@ def main():
         target_to_run = os.sep.join([out_dir, target_to_run])
 
         # If we're only trying to run the target, we need to check if it actually exists first
-        if (run_only or build_and_run) and not os.path.exists(target_to_run):
+        if (run_only or build_and_run) and ".py" not in target_to_run and not os.path.exists(target_to_run):
             raise Exception("Executable has not been built yet")
 
     # Setup program as sudo

src/fd-net-device/CMakeLists.txt

@@ -1,5 +1,7 @@
 set(name fd-net-device)
 
+set(module_enabled_features) # modulegen_customizations consumes this list
+
 mark_as_advanced(ENABLE_THREADING)
 set(ENABLE_THREADING ${HAVE_PTHREAD_H})
 
@@ -57,6 +59,7 @@ endif()
 
 if(${ENABLE_FDNETDEV})
   set(fd-net-device_creators)
+  list(APPEND module_enabled_features FdNetDevice)
 
   if(${ENABLE_EMUNETDEV})
     set(emu_sources helper/emu-fd-net-device-helper.cc)
@@ -74,6 +77,7 @@ if(${ENABLE_FDNETDEV})
     )
 
     list(APPEND fd-net-device_creators raw-sock-creator)
+    list(APPEND module_enabled_features EmuFdNetDevice)
   endif()
 
   if(${ENABLE_TAPNETDEV})
@@ -93,6 +97,7 @@ if(${ENABLE_FDNETDEV})
     )
 
     list(APPEND fd-net-device_creators tap-device-creator)
+    list(APPEND module_enabled_features TapFdNetDevice)
   endif()
 
   if(${ENABLE_NETMAP_EMU})

src/internet/CMakeLists.txt

@@ -251,7 +251,6 @@ set(header_files
     model/udp-header.h
     model/udp-l4-protocol.h
     model/udp-socket-factory.h
-    model/udp-socket-impl.h
     model/udp-socket.h
     model/windowed-filter.h
 )

src/visualizer/CMakeLists.txt

@@ -2,12 +2,12 @@ set(name visualizer)
 
 set(source_files model/pyviz.cc model/visual-simulator-impl.cc)
 
-set(header_files model/pyviz.h model/visual-simulator-impl.h)
+set(header_files model/pyviz.h)
 
-include_directories(${Python3_INCLUDE_DIRS})
+include_directories(${Python_INCLUDE_DIRS})
 
 set(libraries_to_link
-    ${Python3_LIBRARIES}
+    ${Python_LIBRARIES}
     ${libcore}
     ${libinternet}
     ${libwifi}
@@ -27,4 +27,11 @@ build_lib("${name}" "${source_files}" "${header_files}" "${libraries_to_link}"
 
 # move visualizer folder to build/bindings/python, which allows us to add only
 # PYTHONPATH=ns-3-dev/build/bindings/python
-file(COPY visualizer DESTINATION ${CMAKE_OUTPUT_DIRECTORY}/bindings/python/)
+file(GLOB_RECURSE visualizer_files ${CMAKE_CURRENT_SOURCE_DIR}/visualizer/*)
+foreach(file ${visualizer_files})
+  string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}"
+                 "${CMAKE_OUTPUT_DIRECTORY}/bindings/python" destination
+                 ${file}
+  )
+  configure_file(${file} ${destination} COPYONLY)
+endforeach()

src/visualizer/model/visual-simulator-impl.cc

@@ -18,6 +18,8 @@
  * Author: Gustavo Carneiro <gjcarneiro@gmail.com> <gjc@inescporto.pt>
  */
 #include <Python.h>
+#undef HAVE_PTHREAD_H
+#undef HAVE_SYS_STAT_H
 #include "visual-simulator-impl.h"
 #include "ns3/default-simulator-impl.h"
 #include "ns3/log.h"

test.py

@@ -1120,6 +1120,7 @@ def run_tests():
     if os.path.exists(build_status_file):
         ns3_runnable_programs = get_list_from_file(build_status_file, "ns3_runnable_programs")
         ns3_runnable_scripts = get_list_from_file(build_status_file, "ns3_runnable_scripts")
+        ns3_runnable_scripts = [os.path.basename(script) for script in ns3_runnable_scripts]
     else:
         print('The build status file was not found.  You must do waf build before running test.py.', file=sys.stderr)
         sys.exit(2)

utils/python-unit-tests.py

@@ -1,3 +1,5 @@
+#! /usr/bin/env python3
+
 # Copyright (C) 2008-2011 INESC Porto
 
 # This program is free software; you can redistribute it and/or modify
@@ -25,6 +27,8 @@ import ns.mobility
 import ns.csma
 import ns.applications
 
+UINT32_MAX = 0xFFFFFFFF
+
 
 ## TestSimulator class
 class TestSimulator(unittest.TestCase):
@@ -170,7 +174,7 @@ class TestSimulator(unittest.TestCase):
             @return none
             """
             assert self._received_packet is None
-            self._received_packet = socket.Recv()
+            self._received_packet = socket.Recv(maxSize=UINT32_MAX, flags=0)
 
         sink = ns.network.Socket.CreateSocket(node, ns.core.TypeId.LookupByName("ns3::UdpSocketFactory"))
         sink.Bind(ns.network.InetSocketAddress(ns.network.Ipv4Address.GetAny(), 80))

utils/tests/test-ns3.py

@@ -1138,8 +1138,8 @@ class NS3BuildBaseTestCase(NS3BaseTestCase):
             f.write("""
             #include <ns3/core-module.h>
             using namespace ns3;
-            int main () 
+            int main ()
-            { 
+            {
                 Simulator::Stop (Seconds (1.0));
                 Simulator::Run ();
                 Simulator::Destroy ();
@@ -1246,6 +1246,92 @@ class NS3BuildBaseTestCase(NS3BaseTestCase):
             stdout = stdout.replace("scratch_%s" % target_cmake, "")  # remove build lines
             self.assertIn(target_to_run.split("/")[-1], stdout)
 
+        NS3BuildBaseTestCase.cleaned_once = False
+
+    def test_10_PybindgenBindings(self):
+        """!
+        Test if cmake is calling pybindgen through modulegen to generate
+        the bindings source files correctly
+        @return None
+        """
+
+        # First we enable python bindings
+        return_code, stdout, stderr = run_ns3("configure --enable-examples --enable-tests --enable-python-bindings")
+        self.assertEqual(return_code, 0)
+
+        # Then look for python bindings sources
+        core_bindings_generated_sources_path = os.path.join(ns3_path, "build", "src", "core", "bindings")
+        core_bindings_sources_path = os.path.join(ns3_path, "src", "core", "bindings")
+        core_bindings_path = os.path.join(ns3_path, "build", "bindings", "python", "ns")
+        core_bindings_header = os.path.join(core_bindings_generated_sources_path, "ns3module.h")
+        core_bindings_source = os.path.join(core_bindings_generated_sources_path, "ns3module.cc")
+        self.assertTrue(os.path.exists(core_bindings_header))
+        self.assertTrue(os.path.exists(core_bindings_source))
+
+        # Then try to build the bindings for the core module
+        return_code, stdout, stderr = run_ns3("build core-bindings")
+        self.assertEqual(return_code, 0)
+
+        # Then check if it was built
+        self.assertGreater(len(list(filter(lambda x: "_core" in x, os.listdir(core_bindings_path)))), 0)
+
+        # Now enable python bindings scanning
+        return_code, stdout, stderr = run_ns3("configure -- -DNS3_SCAN_PYTHON_BINDINGS=ON")
+        self.assertEqual(return_code, 0)
+
+        # Get the file status for the current scanned bindings
+        bindings_sources = os.listdir(core_bindings_sources_path)
+        bindings_sources = [os.path.join(core_bindings_sources_path, y) for y in bindings_sources]
+        timestamps = {}
+        [timestamps.update({x: os.stat(x)}) for x in bindings_sources]
+
+        # Try to scan the bindings for the core module
+        return_code, stdout, stderr = run_ns3("build core-apiscan")
+        self.assertEqual(return_code, 0)
+
+        # Check if they exist, are not empty and have a different timestamp
+        generated_python_files = ["callbacks_list.py", "modulegen__gcc_LP64.py"]
+        for binding_file in timestamps.keys():
+            if os.path.basename(binding_file) in generated_python_files:
+                self.assertTrue(os.path.exists(binding_file))
+                self.assertGreater(os.stat(binding_file).st_size, 0)
+                new_fstat = os.stat(binding_file)
+                self.assertNotEqual(timestamps[binding_file].st_mtime, new_fstat.st_mtime)
+
+        # Then delete the old bindings sources
+        for f in os.listdir(core_bindings_generated_sources_path):
+            os.remove(os.path.join(core_bindings_generated_sources_path, f))
+
+        # Reconfigure to recreate the source files
+        return_code, stdout, stderr = run_ns3("configure")
+        self.assertEqual(return_code, 0)
+
+        # Check again if they exist
+        self.assertTrue(os.path.exists(core_bindings_header))
+        self.assertTrue(os.path.exists(core_bindings_source))
+
+        # Build the core bindings again
+        return_code, stdout, stderr = run_ns3("build core-bindings")
+        self.assertEqual(return_code, 0)
+
+        # Then check if it was built
+        self.assertGreater(len(list(filter(lambda x: "_core" in x, os.listdir(core_bindings_path)))), 0)
+
+        # We are on python anyways, so we can just try to load it from here
+        sys.path.insert(0, os.path.join(core_bindings_path, ".."))
+        try:
+            from ns import core
+        except ImportError:
+            self.assertTrue(True)
+
+        # Check if ns3 can find and run the python example
+        return_code, stdout, stderr = run_ns3("run sample-simulator.py")
+        self.assertEqual(return_code, 0)
+
+        # Check if test.py can find and run the python example
+        return_code, stdout, stderr = run_program("./test.py", "-p src/core/examples/sample-simulator.py", python=True)
+        self.assertEqual(return_code, 0)
+
 
 class NS3ExpectedUseTestCase(NS3BaseTestCase):
     """!