diff --git a/src/mpi/model/hybrid-simulator-impl.cc b/src/mpi/model/hybrid-simulator-impl.cc index 6d7d4b48b..786301a9f 100644 --- a/src/mpi/model/hybrid-simulator-impl.cc +++ b/src/mpi/model/hybrid-simulator-impl.cc @@ -17,6 +17,13 @@ * Author: Songyuan Bai */ +/** + * \file + * \ingroup mtp + * \ingroup mpi + * Implementation of classes ns3::HybridSimulatorImpl + */ + #include "hybrid-simulator-impl.h" #include "granted-time-window-mpi-interface.h" diff --git a/src/mpi/model/hybrid-simulator-impl.h b/src/mpi/model/hybrid-simulator-impl.h index fb596c5e2..1813e16d2 100644 --- a/src/mpi/model/hybrid-simulator-impl.h +++ b/src/mpi/model/hybrid-simulator-impl.h @@ -17,6 +17,13 @@ * Author: Songyuan Bai */ +/** + * \file + * \ingroup mtp + * \ingroup mpi + * Declaration of classes ns3::HybridSimulatorImpl + */ + #ifndef NS3_HYBRID_SIMULATOR_IMPL_H #define NS3_HYBRID_SIMULATOR_IMPL_H @@ -33,6 +40,10 @@ namespace ns3 { +/** + * @brief + * Implementation of the hybrid simulator + */ class HybridSimulatorImpl : public SimulatorImpl { public: @@ -68,6 +79,12 @@ class HybridSimulatorImpl : public SimulatorImpl // Inherited from Object virtual void DoDispose(); + /** + * @brief Whether LPs on the current local process is finished. + * + * @return true if all finished + * @return false if not all finished + */ bool IsLocalFinished() const; /** Are all parallel instances completed. */ @@ -78,6 +95,17 @@ class HybridSimulatorImpl : public SimulatorImpl uint32_t m_systemCount; /**< MPI communicator size. */ Time m_smallestTime; /**< End of current window. */ + /** + * @brief Automatically divides the to-be-simulated topology + * + * This method is called at the beginning of MultithreadedSimulatorImpl::Run. + * It will set each node a systemId. Then it creates logical processes according + * to the number of partitions, and transfer old events to newly created logical + * processes. + * + * If manual partition is enabled by calling MtpInterface::Enable with two parameters, + * this method will not be called. + */ void Partition(); uint32_t m_maxThreads; diff --git a/src/mtp/model/logical-process.cc b/src/mtp/model/logical-process.cc index 7198a365a..5ba1106db 100644 --- a/src/mtp/model/logical-process.cc +++ b/src/mtp/model/logical-process.cc @@ -17,6 +17,12 @@ * Author: Songyuan Bai */ +/** + * \file + * \ingroup mtp + * Implementation of classes ns3::LogicalProcess + */ + #include "logical-process.h" #include "mtp-interface.h" @@ -76,7 +82,7 @@ LogicalProcess::CalculateLookAhead() if (m_systemId == 0) { - m_lookAhead = TimeStep(0); // No lookahead for public LP + m_lookAhead = TimeStep(0); // No lookahead for the public LP } else { @@ -85,6 +91,8 @@ LogicalProcess::CalculateLookAhead() for (auto iter = c.Begin(); iter != c.End(); ++iter) { #ifdef NS3_MPI + // for hybrid simulation, the left 16-bit indicates local system ID, + // and the right 16-bit indicates global system ID (MPI rank) if (((*iter)->GetSystemId() >> 16) != m_systemId) { continue; diff --git a/src/mtp/model/logical-process.h b/src/mtp/model/logical-process.h index 3bb4248a3..03de0f8ce 100644 --- a/src/mtp/model/logical-process.h +++ b/src/mtp/model/logical-process.h @@ -17,6 +17,12 @@ * Author: Songyuan Bai */ +/** + * \file + * \ingroup mtp + * Declaration of classes ns3::LogicalProcess + */ + #ifndef LOGICAL_PROCESS_H #define LOGICAL_PROCESS_H @@ -36,40 +42,96 @@ namespace ns3 { +/** + * @brief + * Implementation of the logical process (LP) used by the multhreaded simulator. + */ class LogicalProcess { public: + /** Default constructor */ LogicalProcess(); + + /** Destructor */ ~LogicalProcess(); + /** + * Enable this logical process object by giving it a unique systemId, + * and let it know the total number of systems. + * + * @param systemId + * @param systemCount + */ void Enable(const uint32_t systemId, const uint32_t systemCount); + + /** + * @brief Calculate the lookahead value. + */ void CalculateLookAhead(); + + /** + * @brief Receive events sent by other logical processes in the previous round. + */ void ReceiveMessages(); + + /** + * @brief Process all events in the current round. + */ void ProcessOneRound(); + /** + * @brief Get the execution time of the last round. + * + * This method is called by MtpInterfaceused to determine the priority of each LP. + * + * @return The execution tiem of the last round + */ inline uint64_t GetExecutionTime() const { return m_executionTime; } + /** + * @brief Get the pending event count of the next round. + * + * This method is called by MtpInterfaceused to determine the priority of each LP. + * + * @return Number of pending events of the next round + */ inline uint64_t GetPendingEventCount() const { return m_pendingEventCount; } + /** + * @brief Get the future event list (scheduler) + * + * @return The event list + */ inline Ptr GetPendingEvents() const { return m_events; } - // mapped from MultithreadedSimulatorImpl + /** + * @brief Invoke an event immediately at the current time. + * + * This method is called when another thread wants to process an event of an LP + * that does not belongs to it. It is used at the very beginning of the simulation + * when the main thread will invoke events of newly allocated LP, whose timestamps + * are zero. + * + * @param ev The event to be invoked now + */ + void InvokeNow(const Scheduler::Event& ev); + + // The following methods are mapped from MultithreadedSimulatorImpl EventId Schedule(const Time& delay, EventImpl* event); void ScheduleAt(const uint32_t context, const Time& time, EventImpl* event); void ScheduleWithContext(LogicalProcess* remote, const uint32_t context, const Time& delay, EventImpl* event); - void InvokeNow(const Scheduler::Event& ev); // cross context immediate invocation void Remove(const EventId& id); void Cancel(const EventId& id); bool IsExpired(const EventId& id) const; diff --git a/src/mtp/model/mtp-interface.cc b/src/mtp/model/mtp-interface.cc index 138b00c94..180ff927d 100644 --- a/src/mtp/model/mtp-interface.cc +++ b/src/mtp/model/mtp-interface.cc @@ -17,6 +17,12 @@ * Author: Songyuan Bai */ +/** + * \file + * \ingroup mtp + * Implementation of classes ns3::MtpInterface + */ + #include "mtp-interface.h" #include "ns3/assert.h" diff --git a/src/mtp/model/mtp-interface.h b/src/mtp/model/mtp-interface.h index b16dab427..9279a28db 100644 --- a/src/mtp/model/mtp-interface.h +++ b/src/mtp/model/mtp-interface.h @@ -17,6 +17,12 @@ * Author: Songyuan Bai */ +/** + * \file + * \ingroup mtp + * Declaration of classes ns3::MtpInterface + */ + #ifndef MTP_INTERFACE_H #define MTP_INTERFACE_H @@ -32,86 +38,284 @@ namespace ns3 { +/** + * @brief + * Implementation of the interface for multithreaded parallel simulation. + */ class MtpInterface { public: + /** + * @brief + * Implementation of the critical section based on spln lock via + * atomic store & exchange. + */ class CriticalSection { public: + /** Default constructor, using a globally shared atomic variable */ inline CriticalSection() + : m_spinLock(&g_inCriticalSection) { - while (g_inCriticalSection.exchange(true, std::memory_order_acquire)) - ; + while (m_spinLock->exchange(true, std::memory_order_acquire)) + { + }; } + /** + * @brief Construct a new critical section object using a custom + * atomic variable. + * + * @param lock Custom boolean atomic variable act as a spin lock + */ + inline CriticalSection(std::atomic* lock) + : m_spinLock(lock) + { + while (m_spinLock->exchange(true, std::memory_order_acquire)) + { + }; + } + + /** Destructor */ inline ~CriticalSection() { - g_inCriticalSection.store(false, std::memory_order_release); + m_spinLock->store(false, std::memory_order_release); } + + private: + std::atomic* m_spinLock; }; - static void Enable(); // auto topology partition - static void Enable(const uint32_t threadCount); // auto partition, specify thread count - static void Enable(const uint32_t threadCount, const uint32_t systemCount); // manual partition - static void EnableNew(const uint32_t newSystemCount); // add LPs for dynamic added node + /** + * @brief Enable the multithreaded simulation, the number of threads + * will be automatically chosen and the partition is also automatic. + */ + static void Enable(); + + /** + * @brief Enable the multithreaded simulation, the number of threads + * will be manually set and the partition is automatic. + * + * @param threadCount The number of threads to be used. + */ + static void Enable(const uint32_t threadCount); + + /** + * @brief Enable the multithreaded simulation, the number of threads + * will be manually set and the partition is also done manually (by + * assigning each node a systemId). + * + * @param threadCount The number of threads to be used. + * @param systemCount The number of partitions. + */ + static void Enable(const uint32_t threadCount, const uint32_t systemCount); + + /** + * @brief Create new LPs and enable them. + * + * This method can be used to dynamically create LPs for dynamically + * created nodes. After this operation, newly added LP must set their + * scheduler before running. + * + * @param newSystemCount The number of newly to-be-created LPs. + */ + static void EnableNew(const uint32_t newSystemCount); + + /** + * @brief Create new LPs and enable them, while adjusting number of + * threads the simulator will use. + * + * This method is called after the automatic partition. Before the + * automatic partition, there is only one LP, and we do not know the + * number of threads to be used since it is related to the number of + * LPs. Therefore, we have to adjust the number of threads and create + * new LPs simultaneously. + * + * @param threadCount + * @param newSystemCount + */ static void EnableNew(const uint32_t threadCount, const uint32_t newSystemCount); + + /** + * @brief Disable the multithreaded simulation and free the memory + * space of LPs and threads. + * + * This method is called by the multithreaded simulator and you do + * not have to call it manually. + */ static void Disable(); + + /** + * @brief Running the LPs and threads. + * + * This method is called by Simulator::Run. + */ static void Run(); + + /** + * @brief Preparation before running the LPs and threads. + * + * This method is called by MtpInterface::Run. It will actually create + * threads and prepare them to process LPs. + */ static void RunBefore(); + + /** + * @brief Process all events of all LPs in the current round. + * + * This method is called by MtpInterface::Run. + */ static void ProcessOneRound(); + + /** + * @brief Calculate the global smallest time to determine the next + * time window of each LP. + * + * This method is called by MtpInterface::Run. + */ static void CalculateSmallestTime(); + + /** + * @brief Post actions after all LPs are finished. + * + * This method is called by MtpInterface::Run. It will let threads know + * that we have done everything, and terminates them. + */ static void RunAfter(); + + /** + * @brief Whether this interface is enabled. + * + * @return true if it is enabled + * @return false if it is not enabled + */ static bool isEnabled(); + + /** + * @brief Whether the topology is already partitioned. + * + * This method is called by the constructor of the multithreaded simulator + * to check whether user has already manually partitioned the topology. + * + * @return true if it is partitioned + * @return false if it is not partitioned + */ static bool isPartitioned(); + + /** + * @brief Calculate the lookahead value of every LP. + * + * This method is called by MtpInterface::RunBefore. + */ static void CalculateLookAhead(); - // get current thread's executing logical process + /** + * @brief Get the running logical process of the current thread. + * + * @return The curretly running logical process of the + * current thread + */ inline static LogicalProcess* GetSystem() { return static_cast(pthread_getspecific(g_key)); } + /** + * @brief Get the a logical process based on its ID. + * + * @param systemId The given ID of the logical process to be got + * @return The corresponding logical process + */ inline static LogicalProcess* GetSystem(const uint32_t systemId) { return &g_systems[systemId]; } - // set current thread's executing logical process + /** + * @brief Set the running logical process of the current thread. + * + * @param systemId The given ID of the logical process to be set + */ inline static void SetSystem(const uint32_t systemId) { pthread_setspecific(g_key, &g_systems[systemId]); } + /** + * @brief Get the total number of logical processes. + * + * @return The total number of logical processes, including + * the public LP (whose ID is zero) + */ inline static uint32_t GetSize() { return g_systemCount + 1; } + /** + * @brief Get how many rounds are passed since the simulation starts. + * + * @return The number of rounds + */ inline static uint32_t GetRound() { return g_round; } + /** + * @brief Get the smallest timestamp of every to-be-processed event + * of every LP. + * + * The smalles timestamp is used to calculate LBTS. + * + * @return The smallest timestamp. + */ inline static Time GetSmallestTime() { return g_smallestTime; } + /** + * @brief Set the smallest timestamp of every LP. + * + * This method is called by the hybrid simulator, where global MPI + * communication may resulting in a smaller timestamp than the local + * smallest timestamp, so we have to update the current smallest timestamp. + * + * @param smallestTime The new smallest timestamp + */ inline static void SetSmallestTime(const Time smallestTime) { g_smallestTime = smallestTime; } + /** + * @brief Get the timestamp of the next global event. + * + * The next global event's timestamp is also used to calculate LBTS. + * + * @return The timestamp of the next global event + */ inline static Time GetNextPublicTime() { return g_nextPublicTime; } + /** + * @brief Whether all LPs are finished all rounds (or terminated by + * Simulator::Stop). + * + * @return true if all finished + * @return false if not all finished + */ inline static bool isFinished() { return g_globalFinished; } + /** + * @brief Schedule a global event right after the current round is finished. + */ template < typename FUNC, typename std::enable_if>::value, int>::type, @@ -126,6 +330,9 @@ class MtpInterface MakeEvent(f, std::forward(args)...)); } + /** + * @brief Schedule a global event right after the current round is finished. + */ template inline static void ScheduleGlobal(void (*f)(Us...), Ts&&... args) { @@ -136,13 +343,34 @@ class MtpInterface } private: + /** + * @brief The actual function each thread will run. + * + * In this function, each thread repeatedly get the next unprocessed LP, + * execute it and wait until all LPs are processed. + */ static void* ThreadFunc(void* arg); - // determine logical process priority + /** + * @brief Determine logical process priority by execution time. + */ static bool SortByExecutionTime(const uint32_t& i, const uint32_t& j); + + /** + * @brief Determine logical process priority by event count. + */ static bool SortByEventCount(const uint32_t& i, const uint32_t& j); + + /** + * @brief Determine logical process priority by pending event count. + */ static bool SortByPendingEventCount(const uint32_t& i, const uint32_t& j); + + /** + * @brief Determine logical process priority by simulation time. + */ static bool SortBySimulationTime(const uint32_t& i, const uint32_t& j); + static bool (*g_sortFunc)(const uint32_t&, const uint32_t&); static GlobalValue g_sortMethod; static GlobalValue g_sortPeriod; diff --git a/src/mtp/model/multithreaded-simulator-impl.cc b/src/mtp/model/multithreaded-simulator-impl.cc index b91cc1ff5..492d33230 100644 --- a/src/mtp/model/multithreaded-simulator-impl.cc +++ b/src/mtp/model/multithreaded-simulator-impl.cc @@ -17,6 +17,12 @@ * Author: Songyuan Bai */ +/** + * \file + * \ingroup mtp + * Implementation of classes ns3::MultithreadedSimulatorImpl + */ + #include "multithreaded-simulator-impl.h" #include "mtp-interface.h" diff --git a/src/mtp/model/multithreaded-simulator-impl.h b/src/mtp/model/multithreaded-simulator-impl.h index 7949fe2f0..006c68db0 100644 --- a/src/mtp/model/multithreaded-simulator-impl.h +++ b/src/mtp/model/multithreaded-simulator-impl.h @@ -17,6 +17,12 @@ * Author: Songyuan Bai */ +/** + * \file + * \ingroup mtp + * Declaration of classes ns3::MultithreadedSimulatorImpl + */ + #ifndef MULTITHREADED_SIMULATOR_IMPL_H #define MULTITHREADED_SIMULATOR_IMPL_H @@ -31,6 +37,10 @@ namespace ns3 { +/** + * @brief + * Implementation of the multithreaded simulator + */ class MultithreadedSimulatorImpl : public SimulatorImpl { public: @@ -66,6 +76,17 @@ class MultithreadedSimulatorImpl : public SimulatorImpl // Inherited from Object virtual void DoDispose(); + /** + * @brief Automatically divides the to-be-simulated topology + * + * This method is called at the beginning of MultithreadedSimulatorImpl::Run. + * It will set each node a systemId. Then it creates logical processes according + * to the number of partitions, and transfer old events to newly created logical + * processes. + * + * If manual partition is enabled by calling MtpInterface::Enable with two parameters, + * this method will not be called. + */ void Partition(); bool m_partition;