From 1bf56b76975cb82e88a8d7c032198925eab28c9d Mon Sep 17 00:00:00 2001 From: "Peter D. Barnes, Jr" Date: Fri, 24 Jun 2022 10:33:55 -0700 Subject: [PATCH] utils: (merges !1000) Rename and refactor bench-simulator -> bench-scheduler Add --all option to benchmark all schedulers. --- doc/manual/source/events.rst | 2 +- doc/manual/source/utilities.rst | 94 +++--- src/core/model/scheduler.h | 2 +- utils/CMakeLists.txt | 6 +- ...{bench-simulator.cc => bench-scheduler.cc} | 290 ++++++++++++------ 5 files changed, 249 insertions(+), 145 deletions(-) rename utils/{bench-simulator.cc => bench-scheduler.cc} (57%) diff --git a/doc/manual/source/events.rst b/doc/manual/source/events.rst index 6db0a280e..0a7d6c373 100644 --- a/doc/manual/source/events.rst +++ b/doc/manual/source/events.rst @@ -315,7 +315,7 @@ store events in time order. Because event distributions vary by model there is no one best strategy for the priority queue, so |ns3| has several options with -differing tradeoffs. The example `utils/bench-simulator.c` can be used +differing tradeoffs. The example `utils/bench-scheduler.c` can be used to test the performance for a user-supplied event distribution. For modest execution times (less than an hour, say) the choice of priority queue is usually not significant; configuring the build type to optimized diff --git a/doc/manual/source/utilities.rst b/doc/manual/source/utilities.rst index 1129c4efc..731425033 100644 --- a/doc/manual/source/utilities.rst +++ b/doc/manual/source/utilities.rst @@ -33,7 +33,7 @@ search for specific information. To run it, simply open terminal and type -.. sourcecode:: bash +.. sourcecode:: $ ./ns3 run print-introspected-doxygen @@ -70,29 +70,45 @@ This will output the following:: -Bench-simulator -*************** +bench-scheduler +**************** This tool is used to benchmark the scheduler algorithms used in |ns3|. Command-line Arguments ++++++++++++++++++++++ -.. sourcecode:: bash +.. sourcecode:: - $ ./ns3 run "bench-simulator --help" + $ ./ns3 run "bench-scheduler --help" + bench-scheduler [Program Options] [General Arguments] + + Benchmark the simulator scheduler. + + Event intervals are taken from one of: + an exponential distribution, with mean 100 ns, + an ascii file, given by the --file="" argument, + or standard input, by the argument --file="-" + In the case of either --file form, the input is expected + to be ascii, giving the relative event times in ns. Program Options: - --cal: use CalendarSheduler [false] - --heap: use HeapScheduler [false] - --list: use ListSheduler [false] - --map: use MapScheduler (default) [true] - --debug: enable debugging output [false] - --pop: event population size (default 1E5) [100000] - --total: total number of events to run (default 1E6) [1000000] - --runs: number of runs (default 1) [1] - --file: file of relative event times [] - --prec: printed output precision [6] + --all: use all schedulers [false] + --cal: use CalendarSheduler [false] + --calrev: reverse ordering in the CalendarScheduler [false] + --heap: use HeapScheduler [false] + --list: use ListSheduler [false] + --map: use MapScheduler (default) [true] + --pri: use PriorityQueue [false] + --debug: enable debugging output [false] + --pop: event population size (default 1E5) [100000] + --total: total number of events to run (default 1E6) [1000000] + --runs: number of runs (default 1) [1] + --file: file of relative event times + --prec: printed output precision [6] + + General Arguments: + ... You can change the Scheduler being benchmarked by passing the appropriate flags, for example if you want to @@ -102,7 +118,7 @@ The default total number of events, runs or population size can be overridden by passing `--total=value`, `--runs=value` and `--pop=value` respectively. -If you want to use event distribution which is stored in a file, +If you want to use an event distribution which is stored in a file, you can pass the file option by `--file=FILE_NAME`. `--prec` can be used to change the output precision value and @@ -113,44 +129,42 @@ Invocation To run it, simply open the terminal and type -.. sourcecode:: bash +.. sourcecode:: - $ ./ns3 run bench-simulator + $ ./ns3 run bench-scheduler It will show something like this depending upon the scheduler being benchmarked:: - ns3-dev-bench-simulator-debug: - ns3-dev-bench-simulator-debug: scheduler: ns3::MapScheduler - ns3-dev-bench-simulator-debug: population: 100000 - ns3-dev-bench-simulator-debug: total events: 1000000 - ns3-dev-bench-simulator-debug: runs: 1 - ns3-dev-bench-simulator-debug: using default exponential distribution + bench-scheduler: Benchmark the simulator scheduler + Event population size: 100000 + Total events per run: 1000000 + Number of runs per scheduler: 1 + Event time distribution: default exponential - Run Inititialization: Simulation: + ns3::MapScheduler + Run # Initialization: Simulation: Time (s) Rate (ev/s) Per (s/ev) Time (s) Rate (ev/s) Per (s/ev) ----------- ----------- ----------- ----------- ----------- ----------- ----------- - (prime) 0.4 250000 4e-06 1.84 543478 1.84e-06 - 0 0.15 666667 1.5e-06 1.86 537634 1.86e-06 - + (prime) 0.09 1.11111e+06 9e-07 0.98 1.02041e+06 9.8e-07 + 0 0.04 2.5e+06 4e-07 0.98 1.02041e+06 9.8e-07 Suppose we had to benchmark `CalendarScheduler` instead, we would have written -.. sourcecode:: bash +.. sourcecode:: - $ ./ns3 run "bench-simulator --cal" + $ ./ns3 run "bench-scheduler --cal" And the output would look something like this:: - ns3-dev-bench-simulator-debug: - ns3-dev-bench-simulator-debug: scheduler: ns3::CalendarScheduler - ns3-dev-bench-simulator-debug: population: 100000 - ns3-dev-bench-simulator-debug: total events: 1000000 - ns3-dev-bench-simulator-debug: runs: 1 - ns3-dev-bench-simulator-debug: using default exponential distribution + bench-scheduler: Benchmark the simulator scheduler + Event population size: 100000 + Total events per run: 1000000 + Number of runs per scheduler: 1 + Event time distribution: default exponential - Run Inititialization: Simulation: + ns3::CalendarScheduler: insertion order: normal + Run # Initialization: Simulation: Time (s) Rate (ev/s) Per (s/ev) Time (s) Rate (ev/s) Per (s/ev) ----------- ----------- ----------- ----------- ----------- ----------- ----------- - (prime) 1.19 84033.6 1.19e-05 32.03 31220.7 3.203e-05 - 0 0.99 101010 9.9e-06 31.22 32030.7 3.122e-05 - ``` + (prime) 0.3 333333 3e-06 12.77 78308.5 1.277e-05 + 0 0.29 344828 2.9e-06 13.66 73206.4 1.366e-05 diff --git a/src/core/model/scheduler.h b/src/core/model/scheduler.h index 5cc0e5177..2289e7456 100644 --- a/src/core/model/scheduler.h +++ b/src/core/model/scheduler.h @@ -55,7 +55,7 @@ class EventImpl; * Which one is "best" depends in part on the characteristics * of the model being executed. For optimized production work common * practice is to benchmark each Scheduler on the model of interest. - * The utility program utils/bench-simulator.cc can do simple benchmarking + * The utility program utils/bench-scheduler.cc can do simple benchmarking * of each SchedulerImpl against an exponential or user-provided * event time distribution. * diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index eb5c4c3b3..24f93fae4 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -18,10 +18,10 @@ if(${ENABLE_TESTS} AND (test IN_LIST libs_to_build)) add_dependencies(all-test-targets test-runner) endif() -add_executable(bench-simulator bench-simulator.cc) -target_link_libraries(bench-simulator ${libcore}) +add_executable(bench-scheduler bench-scheduler.cc) +target_link_libraries(bench-scheduler ${libcore}) set_runtime_outputdirectory( - bench-simulator ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/utils/ "" + bench-scheduler ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/utils/ "" ) if(network IN_LIST libs_to_build) diff --git a/utils/bench-simulator.cc b/utils/bench-scheduler.cc similarity index 57% rename from utils/bench-simulator.cc rename to utils/bench-scheduler.cc index 4228879e2..ba76d9056 100644 --- a/utils/bench-simulator.cc +++ b/utils/bench-scheduler.cc @@ -28,25 +28,36 @@ using namespace ns3; - +/** Flag to write debugging output. */ bool g_debug = false; +/** Name of this program. */ std::string g_me; +/** Log to std::cout */ #define LOG(x) std::cout << x << std::endl - #define LOGME(x) LOG (g_me << x) - #define DEB(x) if (g_debug) { LOGME (x); } +/** Log with program name prefix. */ +#define LOGME(x) LOG (g_me << x) +/** Log debugging output. */ +#define DEB(x) if (g_debug) { LOGME (x); } -// Output field width +/** Output field width for numeric data. */ int g_fwidth = 6; -/// Bench class +/** + * Benchmark instance which can do a single run. + * + * The run is controlled by the event population size and + * total number of events, which are set at construction. + * + * The event distribution in time is set by SetRandomStream() + */ class Bench { public: /** - * constructor - * \param population the population - * \param total the total + * Constructor + * \param [in] population The number of events to keep in the scheduler. + * \param [in] total The total number of events to execute. */ Bench (const uint32_t population, const uint32_t total) : m_population (population), @@ -56,8 +67,10 @@ public: } /** - * Set random stream - * \param stream the random variable stream + * Set the event delay interval random stream. + * + * \param [in] stream The random variable stream to be used to generate + * delays for future events. */ void SetRandomStream (Ptr stream) { @@ -65,8 +78,9 @@ public: } /** - * Set population function - * \param population the population + * Set the number of events to populate the scheduler with. + * Each event executed schedules a new event, maintaining the population. + * \param [in] population The number of events to keep in the scheduler. */ void SetPopulation (const uint32_t population) { @@ -74,24 +88,28 @@ public: } /** - * Set total function - * \param total + * Set the total number of events to execute. + * \param [in] total The total number of events to execute. */ void SetTotal (const uint32_t total) { m_total = total; } - /// Run function + /** Run the benchmark as configure. */ void RunBench (void); + private: - /// callback function + /** + * Event function. This checks for completion (total number of events + * executed) and schedules a new event if not complete. + */ void Cb (void); - Ptr m_rand; ///< random variable - uint32_t m_population; ///< population - uint32_t m_total; ///< total - uint32_t m_count; ///< count + Ptr m_rand; /**< Stream for event delays. */ + uint32_t m_population; /**< Event population size. */ + uint32_t m_total; /**< Total number of events to execute. */ + uint32_t m_count; /**< Count of events executed so far. */ }; void @@ -125,8 +143,9 @@ Bench::RunBench (void) std::setw (g_fwidth) << (m_population / init) << std::setw (g_fwidth) << (init / m_population) << std::setw (g_fwidth) << simu << - std::setw (g_fwidth) << (m_count / simu) << - std::setw (g_fwidth) << (simu / m_count)); + std::setw (g_fwidth) << (m_count / simu) + << (simu / m_count) + ); } @@ -135,6 +154,7 @@ Bench::Cb (void) { if (m_count >= m_total) { + Simulator::Stop (); return; } DEB ("event at " << Simulator::Now ().GetSeconds () << "s"); @@ -145,6 +165,17 @@ Bench::Cb (void) } +/** + * Create a RandomVariableStream to generate next event delays. + * + * If the \p filename parameter is empty a default exponential time + * distribution will be used, with mean delay of 100 ns. + * + * If the \p filename is `-` standard input will be used. + * + * \param [in] filename The delay interval source file name. + * \returns The RandomVariableStream. + */ Ptr GetRandomStream (std::string filename) { @@ -152,7 +183,7 @@ GetRandomStream (std::string filename) if (filename == "") { - LOGME ("using default exponential distribution"); + LOG (" Event time distribution: default exponential"); Ptr erv = CreateObject (); erv->SetAttribute ("Mean", DoubleValue (100)); stream = erv; @@ -163,12 +194,12 @@ GetRandomStream (std::string filename) if (filename == "-") { - LOGME ("using event distribution from stdin"); + LOG (" Event time distribution: from stdin"); input = &std::cin; } else { - LOGME ("using event distribution from " << filename); + LOG (" Event time distribution: from " << filename); input = new std::ifstream (filename.c_str ()); } @@ -189,8 +220,8 @@ GetRandomStream (std::string filename) *input >> line; } } - LOGME ("found " << nsValues.size () << " entries"); - Ptr drv = CreateObject (); + LOG (" Found " << nsValues.size () << " entries"); + auto drv = CreateObject (); drv->SetValueArray (&nsValues[0], nsValues.size ()); stream = drv; } @@ -198,97 +229,57 @@ GetRandomStream (std::string filename) return stream; } - - -int main (int argc, char *argv[]) +/** + * Perform the runs for a single scheduler type. + * + * This will create and set the scheduler, then execute a priming run + * followed by the number of data runs requsted. + * + * Output will be in the form of a table showing performance for each run. + * + * \param [in] factory Factory pre-configured to create the desired Scheduler. + * \param [in] pop The event population size. + * \param [in] total The total number of events to execute. + * \param [in] runs The number of replications. + * \param [in] eventStream The random stream of event delays. + * \param [in] calRev For the CalendarScheduler, whether to set the Reverse attribute. + */ +void +Run (ObjectFactory & factory, uint32_t pop, uint32_t total, uint32_t runs, + Ptr eventStream, bool calRev) { - - bool schedCal = false; - bool schedHeap = false; - bool schedList = false; - bool schedMap = true; - bool schedPriorityQueue = false; - - uint32_t pop = 100000; - uint32_t total = 1000000; - uint32_t runs = 1; - std::string filename = ""; - bool calRev = false; - - CommandLine cmd (__FILE__); - cmd.Usage ("Benchmark the simulator scheduler.\n" - "\n" - "Event intervals are taken from one of:\n" - " an exponential distribution, with mean 100 ns,\n" - " an ascii file, given by the --file=\"\" argument,\n" - " or standard input, by the argument --file=\"-\"\n" - "In the case of either --file form, the input is expected\n" - "to be ascii, giving the relative event times in ns."); - cmd.AddValue ("cal", "use CalendarSheduler", schedCal); - cmd.AddValue ("calrev", "reverse ordering in the CalendarScheduler", calRev); - cmd.AddValue ("heap", "use HeapScheduler", schedHeap); - cmd.AddValue ("list", "use ListSheduler", schedList); - cmd.AddValue ("map", "use MapScheduler (default)", schedMap); - cmd.AddValue ("pri", "use PriorityQueue", schedPriorityQueue); - cmd.AddValue ("debug", "enable debugging output", g_debug); - cmd.AddValue ("pop", "event population size (default 1E5)", pop); - cmd.AddValue ("total", "total number of events to run (default 1E6)", total); - cmd.AddValue ("runs", "number of runs (default 1)", runs); - cmd.AddValue ("file", "file of relative event times", filename); - cmd.AddValue ("prec", "printed output precision", g_fwidth); - cmd.Parse (argc, argv); - g_me = cmd.GetName () + ": "; - g_fwidth += 6; // 5 extra chars in '2.000002e+07 ': . e+0 _ - - ObjectFactory factory ("ns3::MapScheduler"); - if (schedCal) - { - factory.SetTypeId ("ns3::CalendarScheduler"); - factory.Set ("Reverse", BooleanValue (calRev)); - } - if (schedHeap) - { - factory.SetTypeId ("ns3::HeapScheduler"); - } - if (schedList) - { - factory.SetTypeId ("ns3::ListScheduler"); - } - if (schedPriorityQueue) - { - factory.SetTypeId ("ns3::PriorityQueueScheduler"); - } - Simulator::SetScheduler (factory); - LOGME (std::setprecision (g_fwidth - 6)); - DEB ("debugging is ON"); - - std::string order; - if (schedCal) + std::string schedType = factory.GetTypeId ().GetName (); + if (schedType == "ns3::CalendarScheduler") { - order = ": insertion order: " + std::string (calRev ? "reverse" : "normal"); + schedType += ": insertion order: " + + std::string (calRev ? "reverse" : "normal"); } - LOGME ("scheduler: " << factory.GetTypeId ().GetName () << order); - LOGME ("population: " << pop); - LOGME ("total events: " << total); - LOGME ("runs: " << runs); + + DEB ("scheduler: " << schedType); + DEB ("population: " << pop); + DEB ("total events: " << total); + DEB ("runs: " << runs); Bench *bench = new Bench (pop, total); - bench->SetRandomStream (GetRandomStream (filename)); + bench->SetRandomStream (eventStream); // table header LOG (""); + LOG (schedType); LOG (std::left << std::setw (g_fwidth) << "Run #" << std::left << std::setw (3 * g_fwidth) << "Initialization:" << - std::left << std::setw (3 * g_fwidth) << "Simulation:"); + std::left << "Simulation:" + ); LOG (std::left << std::setw (g_fwidth) << "" << std::left << std::setw (g_fwidth) << "Time (s)" << std::left << std::setw (g_fwidth) << "Rate (ev/s)" << std::left << std::setw (g_fwidth) << "Per (s/ev)" << std::left << std::setw (g_fwidth) << "Time (s)" << std::left << std::setw (g_fwidth) << "Rate (ev/s)" << - std::left << std::setw (g_fwidth) << "Per (s/ev)" ); + std::left << "Per (s/ev)" + ); LOG (std::setfill ('-') << std::right << std::setw (g_fwidth) << " " << std::right << std::setw (g_fwidth) << " " << @@ -317,5 +308,104 @@ int main (int argc, char *argv[]) LOG (""); Simulator::Destroy (); delete bench; +} + + +int main (int argc, char *argv[]) +{ + + bool allSched = false; + bool schedCal = false; + bool schedHeap = false; + bool schedList = false; + bool schedMap = false; // default scheduler + bool schedPQ = false; + + uint32_t pop = 100000; + uint32_t total = 1000000; + uint32_t runs = 1; + std::string filename = ""; + bool calRev = false; + + CommandLine cmd (__FILE__); + cmd.Usage ("Benchmark the simulator scheduler.\n" + "\n" + "Event intervals are taken from one of:\n" + " an exponential distribution, with mean 100 ns,\n" + " an ascii file, given by the --file=\"\" argument,\n" + " or standard input, by the argument --file=\"-\"\n" + "In the case of either --file form, the input is expected\n" + "to be ascii, giving the relative event times in ns."); + cmd.AddValue ("all", "use all schedulers", allSched); + cmd.AddValue ("cal", "use CalendarSheduler", schedCal); + cmd.AddValue ("calrev", "reverse ordering in the CalendarScheduler", calRev); + cmd.AddValue ("heap", "use HeapScheduler", schedHeap); + cmd.AddValue ("list", "use ListSheduler", schedList); + cmd.AddValue ("map", "use MapScheduler (default)", schedMap); + cmd.AddValue ("pri", "use PriorityQueue", schedPQ); + cmd.AddValue ("debug", "enable debugging output", g_debug); + cmd.AddValue ("pop", "event population size (default 1E5)", pop); + cmd.AddValue ("total", "total number of events to run (default 1E6)", total); + cmd.AddValue ("runs", "number of runs (default 1)", runs); + cmd.AddValue ("file", "file of relative event times", filename); + cmd.AddValue ("prec", "printed output precision", g_fwidth); + cmd.Parse (argc, argv); + + g_me = cmd.GetName () + ": "; + g_fwidth += 6; // 5 extra chars in '2.000002e+07 ': . e+0 _ + + LOG (std::setprecision (g_fwidth - 6)); // prints blank line + LOGME (" Benchmark the simulator scheduler"); + LOG (" Event population size: " << pop); + LOG (" Total events per run: " << total); + LOG (" Number of runs per scheduler: " << runs); + DEB ("debugging is ON"); + + if (allSched) + { + schedCal = schedHeap = schedList = schedMap = schedPQ = true; + } + // Set the default case if nothing else is set + if (! (schedCal || schedHeap || schedList || schedMap || schedPQ)) + { + schedMap = true; + } + + Ptr eventStream = GetRandomStream (filename); + + + ObjectFactory factory ("ns3::MapScheduler"); + if (schedCal) + { + factory.SetTypeId ("ns3::CalendarScheduler"); + factory.Set ("Reverse", BooleanValue (calRev)); + Run (factory, pop, total, runs, eventStream, calRev); + if (allSched) + { + factory.Set ("Reverse", BooleanValue (!calRev)); + Run (factory, pop, total, runs, eventStream, !calRev); + } + } + if (schedHeap) + { + factory.SetTypeId ("ns3::HeapScheduler"); + Run (factory, pop, total, runs, eventStream, calRev); + } + if (schedList) + { + factory.SetTypeId ("ns3::ListScheduler"); + Run (factory, pop, total, runs, eventStream, calRev); + } + if (schedMap) + { + factory.SetTypeId ("ns3::MapScheduler"); + Run (factory, pop, total, runs, eventStream, calRev); + } + if (schedPQ) + { + factory.SetTypeId ("ns3::PriorityQueueScheduler"); + Run (factory, pop, total, runs, eventStream, calRev); + } + return 0; }