diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 371f534ad..bbfc8b2a2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -27,6 +27,8 @@ Release 3-dev - (wifi) Implement 802.11be Multi-link discovery and setup - (lr-wpan) !1072 - Adds support for association (network bootstrap) - (lr-wpan) Adds support for PAN Id compression (IEEE 802.15.4-2006 (7.5.6.1)) +- (utils) `utils/bench-simulator` has been moved to `utils/bench-scheduler` to better reflect what it actually tests +- (utils) `utils/bench-scheduler` has been enhanced to test multiple schedulers. ### Bugs fixed diff --git a/doc/manual/source/utilities.rst b/doc/manual/source/utilities.rst index 731425033..f606a0f19 100644 --- a/doc/manual/source/utilities.rst +++ b/doc/manual/source/utilities.rst @@ -131,40 +131,52 @@ To run it, simply open the terminal and type .. sourcecode:: - $ ./ns3 run bench-scheduler + $ ./ns3 run bench-scheduler -- --runs=5 It will show something like this depending upon the scheduler being benchmarked:: bench-scheduler: Benchmark the simulator scheduler Event population size: 100000 Total events per run: 1000000 - Number of runs per scheduler: 1 + Number of runs per scheduler: 5 Event time distribution: default exponential - ns3::MapScheduler + ns3::MapScheduler (default) Run # Initialization: Simulation: Time (s) Rate (ev/s) Per (s/ev) Time (s) Rate (ev/s) Per (s/ev) ----------- ----------- ----------- ----------- ----------- ----------- ----------- - (prime) 0.09 1.11111e+06 9e-07 0.98 1.02041e+06 9.8e-07 - 0 0.04 2.5e+06 4e-07 0.98 1.02041e+06 9.8e-07 + prime 0.01 1e+06 1e-06 5.51 1.81488e+06 5.51e-07 + 0 0 inf 0 6.25 1.6e+06 6.25e-07 + 1 0 inf 0 6.52 1.53374e+06 6.52e-07 + 2 0.01 1e+06 1e-06 7.28 1.37363e+06 7.28e-07 + 3 0 inf 0 7.72 1.29534e+06 7.72e-07 + 4 0.01 1e+06 1e-06 8.16 1.22549e+06 8.16e-07 + average 0.004 nan 4e-07 7.186 1.40564e+06 7.186e-07 + stdev 0.00489898 nan 4.89898e-07 0.715866 141302 7.15866e-08 Suppose we had to benchmark `CalendarScheduler` instead, we would have written .. sourcecode:: - $ ./ns3 run "bench-scheduler --cal" + $ ./ns3 run bench-scheduler -- --runs=5 --cal" And the output would look something like this:: bench-scheduler: Benchmark the simulator scheduler - Event population size: 100000 - Total events per run: 1000000 - Number of runs per scheduler: 1 + Event population size: 10000 + Total events per run: 10000000 + Number of runs per scheduler: 5 Event time distribution: default exponential ns3::CalendarScheduler: insertion order: normal Run # Initialization: Simulation: Time (s) Rate (ev/s) Per (s/ev) Time (s) Rate (ev/s) Per (s/ev) ----------- ----------- ----------- ----------- ----------- ----------- ----------- - (prime) 0.3 333333 3e-06 12.77 78308.5 1.277e-05 - 0 0.29 344828 2.9e-06 13.66 73206.4 1.366e-05 + prime 0.01 1e+06 1e-06 8.14 1.2285e+06 8.14e-07 + 0 0.01 1e+06 1e-06 17.14 583431 1.714e-06 + 1 0.02 500000 2e-06 23.33 428633 2.333e-06 + 2 0.02 500000 2e-06 33.2 301205 3.32e-06 + 3 0.03 333333 3e-06 42.98 232666 4.298e-06 + 4 0.05 200000 5e-06 57.1 175131 5.71e-06 + average 0.026 506667 2.6e-06 34.75 344213 3.475e-06 + stdev 0.0135647 271129 1.35647e-06 14.214 146446 1.4214e-06 diff --git a/utils/bench-scheduler.cc b/utils/bench-scheduler.cc index ba76d9056..6764e432d 100644 --- a/utils/bench-scheduler.cc +++ b/utils/bench-scheduler.cc @@ -18,6 +18,7 @@ * Author: Mathieu Lacage */ +#include // sqrt #include #include #include @@ -59,7 +60,7 @@ public: * \param [in] population The number of events to keep in the scheduler. * \param [in] total The total number of events to execute. */ - Bench (const uint32_t population, const uint32_t total) + Bench (const uint64_t population, const uint64_t total) : m_population (population), m_total (total), m_count (0) @@ -82,7 +83,7 @@ public: * Each event executed schedules a new event, maintaining the population. * \param [in] population The number of events to keep in the scheduler. */ - void SetPopulation (const uint32_t population) + void SetPopulation (const uint64_t population) { m_population = population; } @@ -91,13 +92,26 @@ public: * Set the total number of events to execute. * \param [in] total The total number of events to execute. */ - void SetTotal (const uint32_t total) + void SetTotal (const uint64_t total) { m_total = total; } - /** Run the benchmark as configure. */ - void RunBench (void); + /** The output. */ + struct Result + { + double init; /**< Time (s) for initialization. */ + double simu; /**< Time (s) for simulation. */ + uint64_t pop; /**< Event population. */ + uint64_t events; /**< Number of events executed. */ + }; + + /** + * Run the benchmark as configured. + * + * \returns The Result. + */ + Result Run (void); private: /** @@ -107,46 +121,39 @@ private: void Cb (void); Ptr m_rand; /**< Stream for event delays. */ - uint32_t m_population; /**< Event population size. */ - uint32_t m_total; /**< Total number of events to execute. */ - uint32_t m_count; /**< Count of events executed so far. */ -}; + uint64_t m_population; /**< Event population size. */ + uint64_t m_total; /**< Total number of events to execute. */ + uint64_t m_count; /**< Count of events executed so far. */ -void -Bench::RunBench (void) +}; // class Bench + +Bench::Result +Bench::Run (void) { - SystemWallClockMs time; + SystemWallClockMs timer; double init, simu; DEB ("initializing"); m_count = 0; - - time.Start (); - for (uint32_t i = 0; i < m_population; ++i) + timer.Start (); + for (uint64_t i = 0; i < m_population; ++i) { Time at = NanoSeconds (m_rand->GetValue ()); Simulator::Schedule (at, &Bench::Cb, this); } - init = time.End (); - init /= 1000; + init = timer.End () / 1000.0; DEB ("initialization took " << init << "s"); DEB ("running"); - time.Start (); + timer.Start (); Simulator::Run (); - simu = time.End (); - simu /= 1000; + simu = timer.End () / 1000.0; DEB ("run took " << simu << "s"); - LOG (std::setw (g_fwidth) << init << - std::setw (g_fwidth) << (m_population / init) << - std::setw (g_fwidth) << (init / m_population) << - std::setw (g_fwidth) << simu << - std::setw (g_fwidth) << (m_count / simu) - << (simu / m_count) - ); + Simulator::Destroy (); + return Result {init, simu, m_population, m_count}; } void @@ -165,6 +172,233 @@ Bench::Cb (void) } +/** Benchmark which performs an ensemble of runs. */ +class BenchSuite +{ +public: + + /** + * Perform the runs for a single scheduler type. + * + * This will create and set the scheduler, then execute a priming run + * followed by the number of data runs requested. + * + * Output will be in the form of a table showing performance for each run. + * + * \param [in] factory Factory pre-configured to create the desired Scheduler. + * \param [in] pop The event population size. + * \param [in] total The total number of events to execute. + * \param [in] runs The number of replications. + * \param [in] eventStream The random stream of event delays. + * \param [in] calRev For the CalendarScheduler, whether the Reverse attribute was set. + */ + BenchSuite (ObjectFactory & factory, + uint64_t pop, uint64_t total, uint64_t runs, + Ptr eventStream, + bool calRev); + + /** Write the results to \c LOG() */ + void Log () const; + +private: + + /** Print the table header. */ + void Header () const; + + /** Statistics from a single phase, init or run. */ + struct PhaseResult + { + double time; /**< Phase run time time (s). */ + double rate; /**< Phase event rate (events/s). */ + double period; /**< Phase period (s/event). */ + }; + + /** Results from initialization and execution of a single run. */ + struct Result + { + PhaseResult init; /**< Initialization phase results. */ + PhaseResult run; /**< Run (simulation) phase results. */ + /** + * Construct from the individual run result. + * + * \param [in] r The result from a single run. + * \returns The run result. + */ + static Result Bench (Bench::Result r); + + /** + * Log this result. + * + * \tparam T The type of the label. + * \param label The label for the line. + */ + template + void Log(T label) const; + }; // struct Result + + std::string m_scheduler; /**< Descriptive string for the scheduler. */ + std::vector m_results; /**< Store for the run results. */ + +}; // BenchSuite + +/* static */ +BenchSuite::Result +BenchSuite::Result::Bench (Bench::Result r) +{ + return Result { {r.init, r.pop / r.init, r.init / r.pop }, + {r.simu, r.events / r.simu, r.simu / r.events} + }; +} + +template +void +BenchSuite::Result::Log(T label) const +{ + // Need std::left for string labels + + LOG (std::left << std::setw (g_fwidth) << label << + std::setw (g_fwidth) << init.time << + std::setw (g_fwidth) << init.rate << + std::setw (g_fwidth) << init.period << + std::setw (g_fwidth) << run.time << + std::setw (g_fwidth) << run.rate << + std::setw (g_fwidth) << run.period + ); +} + +BenchSuite::BenchSuite (ObjectFactory & factory, + uint64_t pop, uint64_t total, uint64_t runs, + Ptr eventStream, + bool calRev) +{ + Simulator::SetScheduler (factory); + + m_scheduler = factory.GetTypeId ().GetName (); + if (m_scheduler == "ns3::CalendarScheduler") + { + m_scheduler += ": insertion order: " + + std::string (calRev ? "reverse" : "normal"); + } + if (m_scheduler == "ns3::MapScheduler") + { + m_scheduler += " (default)"; + } + + Bench bench (pop, total); + bench.SetRandomStream (eventStream); + bench.SetPopulation (pop); + bench.SetTotal (total); + + m_results.reserve (runs); + Header (); + + // Prime + DEB ("priming"); + auto prime = bench.Run (); + Result::Bench (prime).Log ("prime"); + + // Perform the actual runs + for (uint64_t i = 0; i < runs; i++) + { + auto run = bench.Run (); + m_results.push_back (Result::Bench (run)); + m_results.back ().Log (i); + } + + Simulator::Destroy (); + +} // BenchSuite::Run + +void +BenchSuite::Header () const +{ + // table header + LOG (""); + LOG (m_scheduler); + LOG (std::left << std::setw (g_fwidth) << "Run #" << + std::left << std::setw (3 * g_fwidth) << "Initialization:" << + std::left << "Simulation:" + ); + LOG (std::left << std::setw (g_fwidth) << "" << + std::left << std::setw (g_fwidth) << "Time (s)" << + std::left << std::setw (g_fwidth) << "Rate (ev/s)" << + std::left << std::setw (g_fwidth) << "Per (s/ev)" << + std::left << std::setw (g_fwidth) << "Time (s)" << + std::left << std::setw (g_fwidth) << "Rate (ev/s)" << + std::left << "Per (s/ev)" + ); + LOG (std::setfill ('-') << + std::right << std::setw (g_fwidth) << " " << + std::right << std::setw (g_fwidth) << " " << + std::right << std::setw (g_fwidth) << " " << + std::right << std::setw (g_fwidth) << " " << + std::right << std::setw (g_fwidth) << " " << + std::right << std::setw (g_fwidth) << " " << + std::right << std::setw (g_fwidth) << " " << + std::setfill (' ') + ); + +} + +void +BenchSuite::Log () const +{ + if (m_results.size () < 2) + { + LOG (""); + return; + } + + // Average the results + + // See Welford's online algorithm for these expressions, + // which avoid subtracting large numbers. + // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + + uint64_t n {0}; // number of samples + Result average {m_results[0]}; // average + Result moment2 { {0, 0, 0}, // 2nd moment, to calculate stdev + {0, 0, 0} }; + + for ( ; n < m_results.size (); ++n) + { + double deltaPre, deltaPost; + const auto & run = m_results[n]; + uint64_t count = n + 1; + +#define ACCUMULATE(phase, field) \ + deltaPre = run.phase.field - average.phase.field; \ + average.phase.field += deltaPre / count ; \ + deltaPost = run.phase.field - average.phase.field; \ + moment2.phase.field += deltaPre * deltaPost + + ACCUMULATE (init, time); + ACCUMULATE (init, rate); + ACCUMULATE (init, period); + ACCUMULATE (run, time); + ACCUMULATE (run, rate); + ACCUMULATE (run, period); + +#undef ACCUMULATE + } + + auto stdev = Result { + { std::sqrt (moment2.init.time / n), + std::sqrt (moment2.init.rate / n), + std::sqrt (moment2.init.period / n)}, + { std::sqrt (moment2.run.time / n), + std::sqrt (moment2.run.rate / n), + std::sqrt (moment2.run.period / n)} + }; + + average.Log ("average"); + stdev.Log ("stdev"); + + LOG (""); + +} // BenchSuite::Log() + + /** * Create a RandomVariableStream to generate next event delays. * @@ -184,7 +418,7 @@ GetRandomStream (std::string filename) if (filename == "") { LOG (" Event time distribution: default exponential"); - Ptr erv = CreateObject (); + auto erv = CreateObject (); erv->SetAttribute ("Mean", DoubleValue (100)); stream = erv; } @@ -229,87 +463,6 @@ GetRandomStream (std::string filename) return stream; } -/** - * Perform the runs for a single scheduler type. - * - * This will create and set the scheduler, then execute a priming run - * followed by the number of data runs requsted. - * - * Output will be in the form of a table showing performance for each run. - * - * \param [in] factory Factory pre-configured to create the desired Scheduler. - * \param [in] pop The event population size. - * \param [in] total The total number of events to execute. - * \param [in] runs The number of replications. - * \param [in] eventStream The random stream of event delays. - * \param [in] calRev For the CalendarScheduler, whether to set the Reverse attribute. - */ -void -Run (ObjectFactory & factory, uint32_t pop, uint32_t total, uint32_t runs, - Ptr eventStream, bool calRev) -{ - Simulator::SetScheduler (factory); - - std::string schedType = factory.GetTypeId ().GetName (); - if (schedType == "ns3::CalendarScheduler") - { - schedType += ": insertion order: " - + std::string (calRev ? "reverse" : "normal"); - } - - DEB ("scheduler: " << schedType); - DEB ("population: " << pop); - DEB ("total events: " << total); - DEB ("runs: " << runs); - - Bench *bench = new Bench (pop, total); - bench->SetRandomStream (eventStream); - - // table header - LOG (""); - LOG (schedType); - LOG (std::left << std::setw (g_fwidth) << "Run #" << - std::left << std::setw (3 * g_fwidth) << "Initialization:" << - std::left << "Simulation:" - ); - LOG (std::left << std::setw (g_fwidth) << "" << - std::left << std::setw (g_fwidth) << "Time (s)" << - std::left << std::setw (g_fwidth) << "Rate (ev/s)" << - std::left << std::setw (g_fwidth) << "Per (s/ev)" << - std::left << std::setw (g_fwidth) << "Time (s)" << - std::left << std::setw (g_fwidth) << "Rate (ev/s)" << - std::left << "Per (s/ev)" - ); - LOG (std::setfill ('-') << - std::right << std::setw (g_fwidth) << " " << - std::right << std::setw (g_fwidth) << " " << - std::right << std::setw (g_fwidth) << " " << - std::right << std::setw (g_fwidth) << " " << - std::right << std::setw (g_fwidth) << " " << - std::right << std::setw (g_fwidth) << " " << - std::right << std::setw (g_fwidth) << " " << - std::setfill (' ') - ); - - // prime - DEB ("priming"); - std::cout << std::left << std::setw (g_fwidth) << "(prime)"; - bench->RunBench (); - - bench->SetPopulation (pop); - bench->SetTotal (total); - for (uint32_t i = 0; i < runs; i++) - { - std::cout << std::setw (g_fwidth) << i; - - bench->RunBench (); - } - - LOG (""); - Simulator::Destroy (); - delete bench; -} - int main (int argc, char *argv[]) { @@ -321,9 +474,9 @@ int main (int argc, char *argv[]) bool schedMap = false; // default scheduler bool schedPQ = false; - uint32_t pop = 100000; - uint32_t total = 1000000; - uint32_t runs = 1; + uint64_t pop = 100000; + uint64_t total = 1000000; + uint64_t runs = 1; std::string filename = ""; bool calRev = false; @@ -335,7 +488,9 @@ int main (int argc, char *argv[]) " an ascii file, given by the --file=\"\" argument,\n" " or standard input, by the argument --file=\"-\"\n" "In the case of either --file form, the input is expected\n" - "to be ascii, giving the relative event times in ns."); + "to be ascii, giving the relative event times in ns.\n" + "\n" + "If no scheduler is specified the MapScheduler will be run."); cmd.AddValue ("all", "use all schedulers", allSched); cmd.AddValue ("cal", "use CalendarSheduler", schedCal); cmd.AddValue ("calrev", "reverse ordering in the CalendarScheduler", calRev); @@ -344,9 +499,9 @@ int main (int argc, char *argv[]) cmd.AddValue ("map", "use MapScheduler (default)", schedMap); cmd.AddValue ("pri", "use PriorityQueue", schedPQ); cmd.AddValue ("debug", "enable debugging output", g_debug); - cmd.AddValue ("pop", "event population size (default 1E5)", pop); - cmd.AddValue ("total", "total number of events to run (default 1E6)", total); - cmd.AddValue ("runs", "number of runs (default 1)", runs); + cmd.AddValue ("pop", "event population size", pop); + cmd.AddValue ("total", "total number of events to run", total); + cmd.AddValue ("runs", "number of runs", runs); cmd.AddValue ("file", "file of relative event times", filename); cmd.AddValue ("prec", "printed output precision", g_fwidth); cmd.Parse (argc, argv); @@ -371,7 +526,7 @@ int main (int argc, char *argv[]) schedMap = true; } - Ptr eventStream = GetRandomStream (filename); + auto eventStream = GetRandomStream (filename); ObjectFactory factory ("ns3::MapScheduler"); @@ -379,32 +534,38 @@ int main (int argc, char *argv[]) { factory.SetTypeId ("ns3::CalendarScheduler"); factory.Set ("Reverse", BooleanValue (calRev)); - Run (factory, pop, total, runs, eventStream, calRev); + BenchSuite (factory, pop, total, runs, eventStream, calRev).Log (); if (allSched) { factory.Set ("Reverse", BooleanValue (!calRev)); - Run (factory, pop, total, runs, eventStream, !calRev); + BenchSuite (factory, pop, total, runs, eventStream, !calRev).Log (); } } if (schedHeap) { factory.SetTypeId ("ns3::HeapScheduler"); - Run (factory, pop, total, runs, eventStream, calRev); + BenchSuite (factory, pop, total, runs, eventStream, calRev).Log (); } if (schedList) { factory.SetTypeId ("ns3::ListScheduler"); - Run (factory, pop, total, runs, eventStream, calRev); + auto listTotal = total; + if (allSched) + { + LOG ("Running List scheduler with 1/10 total events"); + listTotal /= 10; + } + BenchSuite (factory, pop, listTotal, runs, eventStream, calRev).Log (); } if (schedMap) { factory.SetTypeId ("ns3::MapScheduler"); - Run (factory, pop, total, runs, eventStream, calRev); + BenchSuite (factory, pop, total, runs, eventStream, calRev).Log (); } if (schedPQ) { factory.SetTypeId ("ns3::PriorityQueueScheduler"); - Run (factory, pop, total, runs, eventStream, calRev); + BenchSuite (factory, pop, total, runs, eventStream, calRev).Log (); } return 0;