F5/unison
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

utils: update bench-scheduler to report average and stdev, use 64-bit counters

Browse Source
This commit is contained in:
Peter Barnes
2022-10-04 23:09:25 +00:00
parent ef2af4c536
commit 41fed49054
3 changed files with 309 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASE_NOTES.md
View File

@@ -27,6 +27,8 @@ Release 3-dev
- (wifi) Implement 802.11be Multi-link discovery and setup
- (lr-wpan) !1072 - Adds support for association (network bootstrap)
- (lr-wpan) Adds support for PAN Id compression (IEEE 802.15.4-2006 (7.5.6.1))
- (utils) `utils/bench-simulator` has been moved to `utils/bench-scheduler` to better reflect what it actually tests
- (utils) `utils/bench-scheduler` has been enhanced to test multiple schedulers.
### Bugs fixed

34
doc/manual/source/utilities.rst
View File

@@ -131,40 +131,52 @@ To run it, simply open the terminal and type
.. sourcecode::
$ ./ns3 run bench-scheduler
$ ./ns3 run bench-scheduler -- --runs=5
It will show something like this depending upon the scheduler being benchmarked::
bench-scheduler: Benchmark the simulator scheduler
Event population size: 100000
Total events per run: 1000000
Number of runs per scheduler: 1
Number of runs per scheduler: 5
Event time distribution: default exponential
ns3::MapScheduler
ns3::MapScheduler (default)
Run # Initialization: Simulation:
Time (s) Rate (ev/s) Per (s/ev) Time (s) Rate (ev/s) Per (s/ev)
----------- ----------- ----------- ----------- ----------- ----------- -----------
(prime) 0.09 1.11111e+06 9e-07 0.98 1.02041e+06 9.8e-07
0 0.04 2.5e+06 4e-07 0.98 1.02041e+06 9.8e-07
prime 0.01 1e+06 1e-06 5.51 1.81488e+06 5.51e-07
0 0 inf 0 6.25 1.6e+06 6.25e-07
1 0 inf 0 6.52 1.53374e+06 6.52e-07
2 0.01 1e+06 1e-06 7.28 1.37363e+06 7.28e-07
3 0 inf 0 7.72 1.29534e+06 7.72e-07
4 0.01 1e+06 1e-06 8.16 1.22549e+06 8.16e-07
average 0.004 nan 4e-07 7.186 1.40564e+06 7.186e-07
stdev 0.00489898 nan 4.89898e-07 0.715866 141302 7.15866e-08
Suppose we had to benchmark `CalendarScheduler` instead, we would have written
.. sourcecode::
$ ./ns3 run "bench-scheduler --cal"
$ ./ns3 run bench-scheduler -- --runs=5 --cal"
And the output would look something like this::
bench-scheduler: Benchmark the simulator scheduler
Event population size: 100000
Total events per run: 1000000
Number of runs per scheduler: 1
Event population size: 10000
Total events per run: 10000000
Number of runs per scheduler: 5
Event time distribution: default exponential
ns3::CalendarScheduler: insertion order: normal
Run # Initialization: Simulation:
Time (s) Rate (ev/s) Per (s/ev) Time (s) Rate (ev/s) Per (s/ev)
----------- ----------- ----------- ----------- ----------- ----------- -----------
(prime) 0.3 333333 3e-06 12.77 78308.5 1.277e-05
0 0.29 344828 2.9e-06 13.66 73206.4 1.366e-05
prime 0.01 1e+06 1e-06 8.14 1.2285e+06 8.14e-07
0 0.01 1e+06 1e-06 17.14 583431 1.714e-06
1 0.02 500000 2e-06 23.33 428633 2.333e-06
2 0.02 500000 2e-06 33.2 301205 3.32e-06
3 0.03 333333 3e-06 42.98 232666 4.298e-06
4 0.05 200000 5e-06 57.1 175131 5.71e-06
average 0.026 506667 2.6e-06 34.75 344213 3.475e-06
stdev 0.0135647 271129 1.35647e-06 14.214 146446 1.4214e-06

407
utils/bench-scheduler.cc
View File

@@ -18,6 +18,7 @@
* Author: Mathieu Lacage <mathieu.lacage@sophia.inria.fr>
*/
#include <cmath> // sqrt
#include <iomanip>
#include <iostream>
#include <fstream>
@@ -59,7 +60,7 @@ public:
* \param [in] population The number of events to keep in the scheduler.
* \param [in] total The total number of events to execute.
*/
Bench (const uint32_t population, const uint32_t total)
Bench (const uint64_t population, const uint64_t total)
: m_population (population),
m_total (total),
m_count (0)
@@ -82,7 +83,7 @@ public:
* Each event executed schedules a new event, maintaining the population.
* \param [in] population The number of events to keep in the scheduler.
*/
void SetPopulation (const uint32_t population)
void SetPopulation (const uint64_t population)
{
m_population = population;
}
@@ -91,13 +92,26 @@ public:
* Set the total number of events to execute.
* \param [in] total The total number of events to execute.
*/
void SetTotal (const uint32_t total)
void SetTotal (const uint64_t total)
{
m_total = total;
}
/** Run the benchmark as configure. */
void RunBench (void);
/** The output. */
struct Result
{
double init; /**< Time (s) for initialization. */
double simu; /**< Time (s) for simulation. */
uint64_t pop; /**< Event population. */
uint64_t events; /**< Number of events executed. */
};
/**
* Run the benchmark as configured.
*
* \returns The Result.
*/
Result Run (void);
private:
/**
@@ -107,46 +121,39 @@ private:
void Cb (void);
Ptr<RandomVariableStream> m_rand; /**< Stream for event delays. */
uint32_t m_population; /**< Event population size. */
uint32_t m_total; /**< Total number of events to execute. */
uint32_t m_count; /**< Count of events executed so far. */
};
uint64_t m_population; /**< Event population size. */
uint64_t m_total; /**< Total number of events to execute. */
uint64_t m_count; /**< Count of events executed so far. */
void
Bench::RunBench (void)
}; // class Bench
Bench::Result
Bench::Run (void)
{
SystemWallClockMs time;
SystemWallClockMs timer;
double init, simu;
DEB ("initializing");
m_count = 0;
time.Start ();
for (uint32_t i = 0; i < m_population; ++i)
timer.Start ();
for (uint64_t i = 0; i < m_population; ++i)
{
Time at = NanoSeconds (m_rand->GetValue ());
Simulator::Schedule (at, &Bench::Cb, this);
}
init = time.End ();
init /= 1000;
init = timer.End () / 1000.0;
DEB ("initialization took " << init << "s");
DEB ("running");
time.Start ();
timer.Start ();
Simulator::Run ();
simu = time.End ();
simu /= 1000;
simu = timer.End () / 1000.0;
DEB ("run took " << simu << "s");
LOG (std::setw (g_fwidth) << init <<
std::setw (g_fwidth) << (m_population / init) <<
std::setw (g_fwidth) << (init / m_population) <<
std::setw (g_fwidth) << simu <<
std::setw (g_fwidth) << (m_count / simu)
<< (simu / m_count)
);
Simulator::Destroy ();
return Result {init, simu, m_population, m_count};
}
void
@@ -165,6 +172,233 @@ Bench::Cb (void)
}
/** Benchmark which performs an ensemble of runs. */
class BenchSuite
{
public:
/**
* Perform the runs for a single scheduler type.
*
* This will create and set the scheduler, then execute a priming run
* followed by the number of data runs requested.
*
* Output will be in the form of a table showing performance for each run.
*
* \param [in] factory Factory pre-configured to create the desired Scheduler.
* \param [in] pop The event population size.
* \param [in] total The total number of events to execute.
* \param [in] runs The number of replications.
* \param [in] eventStream The random stream of event delays.
* \param [in] calRev For the CalendarScheduler, whether the Reverse attribute was set.
*/
BenchSuite (ObjectFactory & factory,
uint64_t pop, uint64_t total, uint64_t runs,
Ptr<RandomVariableStream> eventStream,
bool calRev);
/** Write the results to \c LOG() */
void Log () const;
private:
/** Print the table header. */
void Header () const;
/** Statistics from a single phase, init or run. */
struct PhaseResult
{
double time; /**< Phase run time time (s). */
double rate; /**< Phase event rate (events/s). */
double period; /**< Phase period (s/event). */
};
/** Results from initialization and execution of a single run. */
struct Result
{
PhaseResult init; /**< Initialization phase results. */
PhaseResult run; /**< Run (simulation) phase results. */
/**
* Construct from the individual run result.
*
* \param [in] r The result from a single run.
* \returns The run result.
*/
static Result Bench (Bench::Result r);
/**
* Log this result.
*
* \tparam T The type of the label.
* \param label The label for the line.
*/
template <typename T>
void Log(T label) const;
}; // struct Result
std::string m_scheduler; /**< Descriptive string for the scheduler. */
std::vector<Result> m_results; /**< Store for the run results. */
}; // BenchSuite
/* static */
BenchSuite::Result
BenchSuite::Result::Bench (Bench::Result r)
{
return Result { {r.init, r.pop / r.init, r.init / r.pop },
{r.simu, r.events / r.simu, r.simu / r.events}
};
}
template <typename T>
void
BenchSuite::Result::Log(T label) const
{
// Need std::left for string labels
LOG (std::left << std::setw (g_fwidth) << label <<
std::setw (g_fwidth) << init.time <<
std::setw (g_fwidth) << init.rate <<
std::setw (g_fwidth) << init.period <<
std::setw (g_fwidth) << run.time <<
std::setw (g_fwidth) << run.rate <<
std::setw (g_fwidth) << run.period
);
}
BenchSuite::BenchSuite (ObjectFactory & factory,
uint64_t pop, uint64_t total, uint64_t runs,
Ptr<RandomVariableStream> eventStream,
bool calRev)
{
Simulator::SetScheduler (factory);
m_scheduler = factory.GetTypeId ().GetName ();
if (m_scheduler == "ns3::CalendarScheduler")
{
m_scheduler += ": insertion order: "
+ std::string (calRev ? "reverse" : "normal");
}
if (m_scheduler == "ns3::MapScheduler")
{
m_scheduler += " (default)";
}
Bench bench (pop, total);
bench.SetRandomStream (eventStream);
bench.SetPopulation (pop);
bench.SetTotal (total);
m_results.reserve (runs);
Header ();
// Prime
DEB ("priming");
auto prime = bench.Run ();
Result::Bench (prime).Log ("prime");
// Perform the actual runs
for (uint64_t i = 0; i < runs; i++)
{
auto run = bench.Run ();
m_results.push_back (Result::Bench (run));
m_results.back ().Log (i);
}
Simulator::Destroy ();
} // BenchSuite::Run
void
BenchSuite::Header () const
{
// table header
LOG ("");
LOG (m_scheduler);
LOG (std::left << std::setw (g_fwidth) << "Run #" <<
std::left << std::setw (3 * g_fwidth) << "Initialization:" <<
std::left << "Simulation:"
);
LOG (std::left << std::setw (g_fwidth) << "" <<
std::left << std::setw (g_fwidth) << "Time (s)" <<
std::left << std::setw (g_fwidth) << "Rate (ev/s)" <<
std::left << std::setw (g_fwidth) << "Per (s/ev)" <<
std::left << std::setw (g_fwidth) << "Time (s)" <<
std::left << std::setw (g_fwidth) << "Rate (ev/s)" <<
std::left << "Per (s/ev)"
);
LOG (std::setfill ('-') <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::setfill (' ')
);
}
void
BenchSuite::Log () const
{
if (m_results.size () < 2)
{
LOG ("");
return;
}
// Average the results
// See Welford's online algorithm for these expressions,
// which avoid subtracting large numbers.
// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
uint64_t n {0}; // number of samples
Result average {m_results[0]}; // average
Result moment2 { {0, 0, 0}, // 2nd moment, to calculate stdev
{0, 0, 0} };
for ( ; n < m_results.size (); ++n)
{
double deltaPre, deltaPost;
const auto & run = m_results[n];
uint64_t count = n + 1;
#define ACCUMULATE(phase, field) \
deltaPre = run.phase.field - average.phase.field; \
average.phase.field += deltaPre / count ; \
deltaPost = run.phase.field - average.phase.field; \
moment2.phase.field += deltaPre * deltaPost
ACCUMULATE (init, time);
ACCUMULATE (init, rate);
ACCUMULATE (init, period);
ACCUMULATE (run, time);
ACCUMULATE (run, rate);
ACCUMULATE (run, period);
#undef ACCUMULATE
}
auto stdev = Result {
{ std::sqrt (moment2.init.time / n),
std::sqrt (moment2.init.rate / n),
std::sqrt (moment2.init.period / n)},
{ std::sqrt (moment2.run.time / n),
std::sqrt (moment2.run.rate / n),
std::sqrt (moment2.run.period / n)}
};
average.Log ("average");
stdev.Log ("stdev");
LOG ("");
} // BenchSuite::Log()
/**
* Create a RandomVariableStream to generate next event delays.
*
@@ -184,7 +418,7 @@ GetRandomStream (std::string filename)
if (filename == "")
{
LOG (" Event time distribution: default exponential");
Ptr<ExponentialRandomVariable> erv = CreateObject<ExponentialRandomVariable> ();
auto erv = CreateObject<ExponentialRandomVariable> ();
erv->SetAttribute ("Mean", DoubleValue (100));
stream = erv;
}
@@ -229,87 +463,6 @@ GetRandomStream (std::string filename)
return stream;
}
/**
* Perform the runs for a single scheduler type.
*
* This will create and set the scheduler, then execute a priming run
* followed by the number of data runs requsted.
*
* Output will be in the form of a table showing performance for each run.
*
* \param [in] factory Factory pre-configured to create the desired Scheduler.
* \param [in] pop The event population size.
* \param [in] total The total number of events to execute.
* \param [in] runs The number of replications.
* \param [in] eventStream The random stream of event delays.
* \param [in] calRev For the CalendarScheduler, whether to set the Reverse attribute.
*/
void
Run (ObjectFactory & factory, uint32_t pop, uint32_t total, uint32_t runs,
Ptr<RandomVariableStream> eventStream, bool calRev)
{
Simulator::SetScheduler (factory);
std::string schedType = factory.GetTypeId ().GetName ();
if (schedType == "ns3::CalendarScheduler")
{
schedType += ": insertion order: "
+ std::string (calRev ? "reverse" : "normal");
}
DEB ("scheduler: " << schedType);
DEB ("population: " << pop);
DEB ("total events: " << total);
DEB ("runs: " << runs);
Bench *bench = new Bench (pop, total);
bench->SetRandomStream (eventStream);
// table header
LOG ("");
LOG (schedType);
LOG (std::left << std::setw (g_fwidth) << "Run #" <<
std::left << std::setw (3 * g_fwidth) << "Initialization:" <<
std::left << "Simulation:"
);
LOG (std::left << std::setw (g_fwidth) << "" <<
std::left << std::setw (g_fwidth) << "Time (s)" <<
std::left << std::setw (g_fwidth) << "Rate (ev/s)" <<
std::left << std::setw (g_fwidth) << "Per (s/ev)" <<
std::left << std::setw (g_fwidth) << "Time (s)" <<
std::left << std::setw (g_fwidth) << "Rate (ev/s)" <<
std::left << "Per (s/ev)"
);
LOG (std::setfill ('-') <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::right << std::setw (g_fwidth) << " " <<
std::setfill (' ')
);
// prime
DEB ("priming");
std::cout << std::left << std::setw (g_fwidth) << "(prime)";
bench->RunBench ();
bench->SetPopulation (pop);
bench->SetTotal (total);
for (uint32_t i = 0; i < runs; i++)
{
std::cout << std::setw (g_fwidth) << i;
bench->RunBench ();
}
LOG ("");
Simulator::Destroy ();
delete bench;
}
int main (int argc, char *argv[])
{
@@ -321,9 +474,9 @@ int main (int argc, char *argv[])
bool schedMap = false; // default scheduler
bool schedPQ = false;
uint32_t pop = 100000;
uint32_t total = 1000000;
uint32_t runs = 1;
uint64_t pop = 100000;
uint64_t total = 1000000;
uint64_t runs = 1;
std::string filename = "";
bool calRev = false;
@@ -335,7 +488,9 @@ int main (int argc, char *argv[])
" an ascii file, given by the --file=\"<filename>\" argument,\n"
" or standard input, by the argument --file=\"-\"\n"
"In the case of either --file form, the input is expected\n"
"to be ascii, giving the relative event times in ns.");
"to be ascii, giving the relative event times in ns.\n"
"\n"
"If no scheduler is specified the MapScheduler will be run.");
cmd.AddValue ("all", "use all schedulers", allSched);
cmd.AddValue ("cal", "use CalendarSheduler", schedCal);
cmd.AddValue ("calrev", "reverse ordering in the CalendarScheduler", calRev);
@@ -344,9 +499,9 @@ int main (int argc, char *argv[])
cmd.AddValue ("map", "use MapScheduler (default)", schedMap);
cmd.AddValue ("pri", "use PriorityQueue", schedPQ);
cmd.AddValue ("debug", "enable debugging output", g_debug);
cmd.AddValue ("pop", "event population size (default 1E5)", pop);
cmd.AddValue ("total", "total number of events to run (default 1E6)", total);
cmd.AddValue ("runs", "number of runs (default 1)", runs);
cmd.AddValue ("pop", "event population size", pop);
cmd.AddValue ("total", "total number of events to run", total);
cmd.AddValue ("runs", "number of runs", runs);
cmd.AddValue ("file", "file of relative event times", filename);
cmd.AddValue ("prec", "printed output precision", g_fwidth);
cmd.Parse (argc, argv);
@@ -371,7 +526,7 @@ int main (int argc, char *argv[])
schedMap = true;
}
Ptr<RandomVariableStream> eventStream = GetRandomStream (filename);
auto eventStream = GetRandomStream (filename);
ObjectFactory factory ("ns3::MapScheduler");
@@ -379,32 +534,38 @@ int main (int argc, char *argv[])
{
factory.SetTypeId ("ns3::CalendarScheduler");
factory.Set ("Reverse", BooleanValue (calRev));
Run (factory, pop, total, runs, eventStream, calRev);
BenchSuite (factory, pop, total, runs, eventStream, calRev).Log ();
if (allSched)
{
factory.Set ("Reverse", BooleanValue (!calRev));
Run (factory, pop, total, runs, eventStream, !calRev);
BenchSuite (factory, pop, total, runs, eventStream, !calRev).Log ();
}
}
if (schedHeap)
{
factory.SetTypeId ("ns3::HeapScheduler");
Run (factory, pop, total, runs, eventStream, calRev);
BenchSuite (factory, pop, total, runs, eventStream, calRev).Log ();
}
if (schedList)
{
factory.SetTypeId ("ns3::ListScheduler");
Run (factory, pop, total, runs, eventStream, calRev);
auto listTotal = total;
if (allSched)
{
LOG ("Running List scheduler with 1/10 total events");
listTotal /= 10;
}
BenchSuite (factory, pop, listTotal, runs, eventStream, calRev).Log ();
}
if (schedMap)
{
factory.SetTypeId ("ns3::MapScheduler");
Run (factory, pop, total, runs, eventStream, calRev);
BenchSuite (factory, pop, total, runs, eventStream, calRev).Log ();
}
if (schedPQ)
{
factory.SetTypeId ("ns3::PriorityQueueScheduler");
Run (factory, pop, total, runs, eventStream, calRev);
BenchSuite (factory, pop, total, runs, eventStream, calRev).Log ();
}
return 0;