F5/unison
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tcp: Documentation and minor fixes for BBR

Browse Source
This commit is contained in:
Mohit P. Tahiliani
2021-05-26 00:19:23 +05:30
committed by Tom Henderson
parent b9e35ffb02
commit 490a1b83e4
7 changed files with 196 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
examples/tcp/tcp-bbr-example.cc
View File

@@ -31,7 +31,8 @@
//
// This program runs by default for 100 seconds and creates a new directory
// called 'bbr-results' in the ns-3 root directory. The program creates one
// sub-directory in 'bbr-results' directory and three .plotme files.
// sub-directory called 'pcap' in 'bbr-results' directory (if pcap generation
// is enabled) and three .dat files.
//
// (1) 'pcap' sub-directory contains six PCAP files:
// * bbr-0-0.pcap for the interface on Sender
@@ -40,9 +41,17 @@
// * bbr-2-1.pcap for the second interface on R1
// * bbr-3-0.pcap for the first interface on R2
// * bbr-3-1.pcap for the second interface on R2
// (2) cwnd.plotme contains congestion window trace for the sender node
// (3) throughput.plotme contains sender side throughput trace
// (4) queueSize.plotme contains queue length trace from the bottleneck link
// (2) cwnd.dat file contains congestion window trace for the sender node
// (3) throughput.dat file contains sender side throughput trace
// (4) queueSize.dat file contains queue length trace from the bottleneck link
//
// BBR algorithm enters PROBE_RTT phase in every 10 seconds. The congestion
// window is fixed to 4 segments in this phase with a goal to achieve a better
// estimate of minimum RTT (because queue at the bottleneck link tends to drain
// when the congestion window is reduced to 4 segments).
//
// The congestion window and queue occupancy traces output by this program show
// periodic drops every 10 seconds when BBR algorithm is in PROBE_RTT phase.
#include "ns3/core-module.h"
#include "ns3/network-module.h"
@@ -54,13 +63,9 @@
using namespace ns3;
Ptr<UniformRandomVariable> uv = CreateObject<UniformRandomVariable> ();
double stopTime = 100;
std::string dir;
std::string str;
uint32_t prev = 0;
double prevtime = 0;
Time prevTime = Seconds (0);
// Calculate throughput
static void
@@ -68,10 +73,10 @@ TraceThroughput (Ptr<FlowMonitor> monitor)
{
FlowMonitor::FlowStatsContainer stats = monitor->GetFlowStats ();
auto itr = stats.begin ();
double curtime = Simulator::Now ().GetSeconds ();
std::ofstream thr (dir + "/throughput.plotme", std::ios::out | std::ios::app);
thr << curtime << " " << 8 * (itr->second.txBytes - prev) / (1000 * 1000 * (curtime - prevtime)) << std::endl;
prevtime = curtime;
Time curTime = Now ();
std::ofstream thr (dir + "/throughput.dat", std::ios::out | std::ios::app);
thr << curTime << " " << 8 * (itr->second.txBytes - prev) / (1000 * 1000 * (curTime.GetSeconds () - prevTime.GetSeconds ())) << std::endl;
prevTime = curTime;
prev = itr->second.txBytes;
Simulator::Schedule (Seconds (0.2), &TraceThroughput, monitor);
}
@@ -81,7 +86,7 @@ void CheckQueueSize (Ptr<QueueDisc> qd)
{
uint32_t qsize = qd->GetCurrentSize ().GetValue ();
Simulator::Schedule (Seconds (0.2), &CheckQueueSize, qd);
std::ofstream q (dir + "/queueSize.plotme", std::ios::out | std::ios::app);
std::ofstream q (dir + "/queueSize.dat", std::ios::out | std::ios::app);
q << Simulator::Now ().GetSeconds () << " " << qsize << std::endl;
q.close ();
}
@@ -95,7 +100,7 @@ static void CwndTracer (Ptr<OutputStreamWrapper> stream, uint32_t oldval, uint32
void TraceCwnd (uint32_t nodeId, uint32_t socketId)
{
AsciiTraceHelper ascii;
Ptr<OutputStreamWrapper> stream = ascii.CreateFileStream (dir + "/cwnd.plotme");
Ptr<OutputStreamWrapper> stream = ascii.CreateFileStream (dir + "/cwnd.dat");
Config::ConnectWithoutContext ("/NodeList/" + std::to_string (nodeId) + "/$ns3::TcpL4Protocol/SocketList/" + std::to_string (socketId) + "/CongestionWindow", MakeBoundCallback (&CwndTracer, stream));
}
@@ -110,21 +115,20 @@ int main (int argc, char *argv [])
strftime (buffer, sizeof (buffer), "%d-%m-%Y-%I-%M-%S", timeinfo);
std::string currentTime (buffer);
uint32_t stream = 1;
std::string tcpTypeId = "TcpBbr";
std::string queueDisc = "FifoQueueDisc";
uint32_t delAckCount = 2;
bool bql = true;
bool enablePcap = false;
Time stopTime = Seconds (100);
CommandLine cmd (__FILE__);
cmd.AddValue ("stream", "Seed value for random variable", stream);
cmd.AddValue ("tcpTypeId", "Transport protocol to use: TcpNewReno, TcpBbr", tcpTypeId);
cmd.AddValue ("delAckCount", "Delayed ACK count", delAckCount);
cmd.AddValue ("enablePcap", "Enable/Disable pcap file generation", enablePcap);
cmd.AddValue ("stopTime", "Stop time for applications / simulation time will be stopTime + 1", stopTime);
cmd.Parse (argc, argv);
uv->SetStream (stream);
queueDisc = std::string ("ns3::") + queueDisc;
Config::SetDefault ("ns3::TcpL4Protocol::SocketType", StringValue ("ns3::" + tcpTypeId));
@@ -198,19 +202,18 @@ int main (int argc, char *argv [])
ApplicationContainer sourceApps = source.Install (sender.Get (0));
sourceApps.Start (Seconds (0.0));
Simulator::Schedule (Seconds (0.2), &TraceCwnd, 0, 0);
sourceApps.Stop (Seconds (stopTime));
sourceApps.Stop (stopTime);
// Install application on the receiver
PacketSinkHelper sink ("ns3::TcpSocketFactory", InetSocketAddress (Ipv4Address::GetAny (), port));
ApplicationContainer sinkApps = sink.Install (receiver.Get (0));
sinkApps.Start (Seconds (0.0));
sinkApps.Stop (Seconds (stopTime));
sinkApps.Stop (stopTime);
// Create a new directory to store the output of the program
dir = "bbr-results/" + currentTime + "/";
std::string dirToSave = "mkdir -p " + dir;
system (dirToSave.c_str ());
system ((dirToSave + "/pcap/").c_str ());
// The plotting scripts are provided in the following repository, if needed:
// https://github.com/mohittahiliani/BBR-Validation/
@@ -219,7 +222,7 @@ int main (int argc, char *argv [])
// from the link given above and place it in the ns-3 root directory.
// Uncomment the following three lines to generate plots for Congestion
// Window, sender side throughput and queue occupancy on the bottleneck link.
//
//
// system (("cp -R PlotScripts/gnuplotScriptCwnd " + dir).c_str ());
// system (("cp -R PlotScripts/gnuplotScriptThroughput " + dir).c_str ());
// system (("cp -R PlotScripts/gnuplotScriptQueueSize " + dir).c_str ());
@@ -230,15 +233,19 @@ int main (int argc, char *argv [])
qd = tch.Install (routers.Get (0)->GetDevice (1));
Simulator::ScheduleNow (&CheckQueueSize, qd.Get (0));
// Enable PCAP traces
bottleneckLink.EnablePcapAll (dir + "/pcap/bbr", true);
// Generate PCAP traces if it is enabled
if (enablePcap)
{
system ((dirToSave + "/pcap/").c_str ());
bottleneckLink.EnablePcapAll (dir + "/pcap/bbr", true);
}
// Check for dropped packets using Flow Monitor
FlowMonitorHelper flowmon;
Ptr<FlowMonitor> monitor = flowmon.InstallAll ();
Simulator::Schedule (Seconds (0 + 0.000001), &TraceThroughput, monitor);
Simulator::Stop (Seconds (stopTime));
Simulator::Stop (stopTime + TimeStep (1));
Simulator::Run ();
Simulator::Destroy ();

BIN
src/internet/doc/figures/ns3-bbr-vs-linux-bbr.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 38 KiB

115
src/internet/doc/tcp.rst
View File

@@ -51,10 +51,11 @@ connection setup and close logic. Several congestion control algorithms
are supported, with CUBIC the default, and NewReno, Westwood, Hybla, HighSpeed,
Vegas, Scalable, Veno, Binary Increase Congestion Control (BIC), Yet Another
HighSpeed TCP (YeAH), Illinois, H-TCP, Low Extra Delay Background Transport
(LEDBAT), TCP Low Priority (TCP-LP) and and Data Center TCP (DCTCP) also supported. The model also supports
Selective Acknowledgements (SACK), Proportional Rate Reduction (PRR) and
Explicit Congestion Notification (ECN). Multipath-TCP is not yet supported in
the |ns3| releases.
(LEDBAT), TCP Low Priority (TCP-LP), Data Center TCP (DCTCP) and Bottleneck
Bandwidth and RTT (BBR) also supported. The model also supports Selective
Acknowledgements (SACK), Proportional Rate Reduction (PRR) and Explicit
Congestion Notification (ECN). Multipath-TCP is not yet supported in the |ns3|
releases.
Model history
+++++++++++++
@@ -1040,6 +1041,85 @@ environment. Some differences were noted:
More information about DCTCP is available in the RFC 8257:
https://tools.ietf.org/html/rfc8257
BBR
^^^
BBR (class :cpp:class:`TcpBbr`) is a congestion control algorithm that
regulates the sending rate by deriving an estimate of the bottleneck's
available bandwidth and RTT of the path. It seeks to operate at an optimal
point where sender experiences maximum delivery rate with minimum RTT. It
creates a network model comprising maximum delivery rate with minimum RTT
observed so far, and then estimates BDP (maximum bandwidth * minimum RTT)
to control the maximum amount of inflight data. BBR controls congestion by
limiting the rate at which packets are sent. It caps the cwnd to one BDP
and paces out packets at a rate which is adjusted based on the latest estimate
of delivery rate. BBR algorithm is agnostic to packet losses and ECN marks.
pacing_gain controls the rate of sending data and cwnd_gain controls the amount
of data to send.
The following is a high level overview of BBR congestion control algorithm:
On receiving an ACK:
rtt = now - packet.sent_time
update_minimum_rtt (rtt)
delivery_rate = estimate_delivery_rate (packet)
update_maximum_bandwidth (delivery_rate)
After transmitting a data packet:
bdp = max_bandwidth * min_rtt
if (cwnd * bdp < inflight)
return
if (now > nextSendTime)
{
transmit (packet)
nextSendTime = now + packet.size / (pacing_gain * max_bandwidth)
}
else
return
Schedule (nextSendTime, Send)
To enable BBR on all TCP sockets, the following configuration can be used:
::
Config::SetDefault ("ns3::TcpL4Protocol::SocketType", TypeIdValue (TcpBbr::GetTypeId ()));
To enable BBR on a chosen TCP socket, the following configuration can be used
(note that an appropriate Node ID must be used instead of 1):
::
Config::Set ("$ns3::NodeListPriv/NodeList/1/$ns3::TcpL4Protocol/SocketType", TypeIdValue (TcpBbr::GetTypeId ()));
The ns-3 implementation of BBR is based on its Linux implementation. Linux 5.4
kernel implementation has been used to validate the behavior of ns-3
implementation of BBR (See below section on Validation).
In addition, the following unit tests have been written to validate the
implementation of BBR in ns-3:
* BBR should enable (if not already done) TCP pacing feature.
* Test to validate the values of pacing_gain and cwnd_gain in different phases
of BBR.
An example program, examples/tcp/tcp-bbr-example.cc, is provided to experiment
with BBR for one long running flow. This example uses a simple topology
consisting of one sender, one receiver and two routers to examine congestion
window, throughput and queue control. A program similar to this has been run
using the Network Stack Tester (NeST) using BBR from Linux kernel 5.4, and the
results were compared against ns-3 results.
More information about BBR is available in the following Internet Draft:
https://tools.ietf.org/html/draft-cardwell-iccrg-bbr-congestion-control-00
More information about Delivery Rate Estimation is in the following draft:
https://tools.ietf.org/html/draft-cheng-iccrg-delivery-rate-estimation-00
For an academic peer-reviewed paper on the BBR implementation in ns-3,
please refer to:
* Vivek Jain, Viyom Mittal and Mohit P. Tahiliani. "Design and Implementation of TCP BBR in ns-3." In Proceedings of the 10th Workshop on ns-3, pp. 16-22. 2018. (https://dl.acm.org/doi/abs/10.1145/3199902.3199911)
Support for Explicit Congestion Notification (ECN)
++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -1312,6 +1392,7 @@ section below on :ref:`Writing-tcp-tests`.
* **tcp-ledbat-test:** Unit tests on the LEDBAT congestion control
* **tcp-lp-test:** Unit tests on the TCP-LP congestion control
* **tcp-dctcp-test:** Unit tests on the DCTCP congestion control
* **tcp-bbr-test:** Unit tests on the BBR congestion control
* **tcp-option:** Unit tests on TCP options
* **tcp-pkts-acked-test:** Unit test the number of time that PktsAcked is called
* **tcp-rto-test:** Unit test behavior after a RTO occurs
@@ -1439,6 +1520,32 @@ a similar scenario but with ECN enabled throughout.
TCP ECN operation is tested in the ARED and RED tests that are documented in the traffic-control
module documentation.
Like DCTCP and TCP CUBIC, the ns-3 implementation of TCP BBR was validated
against the BBR implementation of Linux kernel 5.4 using Network Stack Tester
(NeST). NeST is a python package which allows the users to emulate kernel space
protocols using Linux network namespaces. Figure :ref:`fig-ns3-bbr-vs-linux-bbr`
compares the congestion window evolution between ns-3 and Linux for a single
flow operating over a 10 Mbps link with 10 ms base RTT and FIFO queue
discipline.
.. _fig-ns3-bbr-vs-linux-bbr:
.. figure:: figures/ns3-bbr-vs-linux-bbr.*
:scale: 80 %
:align: center
Congestion window evolution: ns-3 BBR vs. Linux BBR (using NeST)
It can be observed that the congestion window traces for ns-3 BBR closely
overlap with Linux BBR. The periodic drops in congestion window every 10
seconds depict the PROBE_RTT phase of the BBR algorithm. In this phase, BBR
algorithm keeps the congestion window fixed to 4 segments.
The example program, examples/tcp-bbr-example.cc has been used to obtain the
congestion window curve shown in Figure :ref:`fig-ns3-bbr-vs-linux-bbr`. The
detailed instructions to reproduce ns-3 plot and NeST plot can be found at:
https://github.com/mohittahiliani/BBR-Validation
Writing a new congestion control algorithm
++++++++++++++++++++++++++++++++++++++++++

16
src/internet/model/tcp-bbr.cc
View File

@@ -39,6 +39,11 @@ TcpBbr::GetTypeId (void)
.SetParent<TcpCongestionOps> ()
.AddConstructor<TcpBbr> ()
.SetGroupName ("Internet")
.AddAttribute ("Stream",
"Random number stream (default is set to 4 to align with Linux results)",
UintegerValue (4),
MakeUintegerAccessor (&TcpBbr::SetStream),
MakeUintegerChecker<uint32_t> ())
.AddAttribute ("HighGain",
"Value of high gain",
DoubleValue (2.89),
@@ -50,12 +55,12 @@ TcpBbr::GetTypeId (void)
MakeUintegerAccessor (&TcpBbr::m_bandwidthWindowLength),
MakeUintegerChecker<uint32_t> ())
.AddAttribute ("RttWindowLength",
"Length of bandwidth windowed filter",
"Length of RTT windowed filter",
TimeValue (Seconds (10)),
MakeTimeAccessor (&TcpBbr::m_rtPropFilterLen),
MakeTimeChecker ())
.AddAttribute ("ProbeRttDuration",
"Length of bandwidth windowed filter",
"Time to be spent in PROBE_RTT phase",
TimeValue (MilliSeconds (200)),
MakeTimeAccessor (&TcpBbr::m_probeRttDuration),
MakeTimeChecker ())
@@ -109,6 +114,7 @@ TcpBbr::TcpBbr (const TcpBbr &sock)
m_rtPropFilterLen (sock.m_rtPropFilterLen),
m_rtPropStamp (sock.m_rtPropStamp),
m_isInitialized (sock.m_isInitialized),
m_uv (sock.m_uv),
m_delivered (sock.m_delivered),
m_appLimited (sock.m_appLimited),
m_txItemDelivered (sock.m_txItemDelivered),
@@ -122,15 +128,13 @@ TcpBbr::TcpBbr (const TcpBbr &sock)
m_hasSeenRtt (sock.m_hasSeenRtt)
{
NS_LOG_FUNCTION (this);
m_uv = CreateObject<UniformRandomVariable> ();
}
int64_t
TcpBbr::AssignStreams (int64_t stream)
void
TcpBbr::SetStream (uint32_t stream)
{
NS_LOG_FUNCTION (this << stream);
m_uv->SetStream (stream);
return 1;
}
void

12
src/internet/model/tcp-bbr.h
View File

@@ -80,13 +80,11 @@ public:
/**
* Assign a fixed random variable stream number to the random variables
* used by this model. Return the number of streams (possibly zero) that
* have been assigned.
* used by this model.
*
* \param stream first stream index to use
* \return the number of stream indices assigned by this model
*/
virtual int64_t AssignStreams (int64_t stream);
virtual void SetStream (uint32_t stream);
virtual std::string GetName () const;
virtual bool HasCongControl () const;
@@ -353,7 +351,7 @@ private:
Time m_probeRtPropStamp {Seconds (0)}; //!< The wall clock time at which the current BBR.RTProp sample was obtained.
Time m_probeRttDoneStamp {Seconds (0)}; //!< Time to exit from BBR_PROBE_RTT state
bool m_probeRttRoundDone {false}; //!< True when it is time to exit BBR_PROBE_RTT
bool m_packetConservation {false}; //!<
bool m_packetConservation {false}; //!< Enable/Disable packet conservation mode
uint32_t m_priorCwnd {0}; //!< The last-known good congestion window
bool m_idleRestart {false}; //!< When restarting from idle, set it true
uint32_t m_targetCWnd {0}; //!< Target value for congestion window, adapted to the estimated BDP
@@ -370,9 +368,9 @@ private:
Ptr<UniformRandomVariable> m_uv {nullptr}; //!< Uniform Random Variable
uint64_t m_delivered {0}; //!< The total amount of data in bytes delivered so far
uint32_t m_appLimited {0}; //!< The index of the last transmitted packet marked as application-limited
uint32_t m_txItemDelivered {0}; //!<
uint32_t m_txItemDelivered {0}; //!< The number of bytes already delivered at the time of new packet transmission
uint32_t m_extraAckedGain {1}; //!< Gain factor for adding extra ack to cwnd
uint32_t m_extraAcked[2] {0, 0}; //!< Maximum excess data acked in epoch
uint32_t m_extraAcked [2] {0, 0}; //!< Maximum excess data acked in epoch
uint32_t m_extraAckedWinRtt {0}; //!< Age of extra acked in rtt
uint32_t m_extraAckedWinRttLength {5}; //!< Window length of extra acked window
uint32_t m_ackEpochAckedResetThresh {1 << 17}; //!< Max allowed val for m_ackEpochAcked, after which sampling epoch is reset

2
src/internet/model/tcp-socket-state.h
View File

@@ -207,7 +207,7 @@ public:
EcnCodePoint_t m_ectCodePoint {Ect0}; //!< ECT code point to use
uint32_t m_lastAckedSackedBytes {0}; //!< Last acked and sacked recorded upon receiving last acknowledgment
uint32_t m_lastAckedSackedBytes {0}; //!< The number of bytes acked and sacked as indicated by the current ACK received. This is similar to acked_sacked variable in Linux
/**
* \brief Get cwnd in segments rather than bytes

68
src/internet/test/tcp-bbr-test.cc
View File

@@ -48,8 +48,7 @@ private:
TcpBbrPacingEnableTest::TcpBbrPacingEnableTest (bool pacing, const std::string &name)
: TestCase (name),
m_pacing (pacing)
{
}
{}
void
TcpBbrPacingEnableTest::DoRun ()
@@ -93,8 +92,7 @@ TcpBbrCheckGainValuesTest::TcpBbrCheckGainValuesTest (TcpBbr::BbrMode_t state,
: TestCase (name),
m_mode (state),
m_highGain (highGain)
{
}
{}
void
TcpBbrCheckGainValuesTest::DoRun ()
@@ -113,34 +111,40 @@ TcpBbrCheckGainValuesTest::ExecuteTest ()
TcpBbr::BbrMode_t desiredMode;
switch (m_mode)
{
case TcpBbr::BBR_STARTUP:
cong->EnterStartup ();
desiredPacingGain = m_highGain;
desiredCwndGain = m_highGain;
actualPacingGain = cong->GetPacingGain ();
actualCwndGain = cong->GetCwndGain ();
desiredMode = TcpBbr::BBR_STARTUP;
break;
case TcpBbr::BBR_DRAIN:
cong->EnterDrain ();
desiredPacingGain = 1 / m_highGain;
desiredCwndGain = m_highGain;
desiredMode = TcpBbr::BBR_DRAIN;
break;
case TcpBbr::BBR_PROBE_BW:
cong->EnterProbeBW ();
desiredPacingGain = 1;
desiredCwndGain = 2;
desiredMode = TcpBbr::BBR_PROBE_BW;
break;
case TcpBbr::BBR_PROBE_RTT:
cong->EnterProbeRTT ();
desiredPacingGain = 1;
desiredCwndGain = 1;
desiredMode = TcpBbr::BBR_PROBE_RTT;
break;
default:
NS_ASSERT (false);
case TcpBbr::BBR_STARTUP:
cong->EnterStartup ();
desiredPacingGain = m_highGain;
desiredCwndGain = m_highGain;
actualPacingGain = cong->GetPacingGain ();
actualCwndGain = cong->GetCwndGain ();
desiredMode = TcpBbr::BBR_STARTUP;
break;
case TcpBbr::BBR_DRAIN:
cong->EnterDrain ();
desiredPacingGain = 1 / m_highGain;
desiredCwndGain = m_highGain;
desiredMode = TcpBbr::BBR_DRAIN;
break;
case TcpBbr::BBR_PROBE_BW:
cong->EnterProbeBW ();
// The value of desiredPacingGain is sensitive to the setting of random
// variable stream. The value of 1.25 has been used in this test with a
// stream value of 4 (default for TCP BBR). Note that if the stream value
// is changed, this test might fail because when BBR enters the PROBE_BW
// phase, the value of actualPacingGain is chosen randomly from 1.25,
// 0.75, 1, 1, 1, 1, 1, 1.
desiredPacingGain = 1.25;
desiredCwndGain = 2;
desiredMode = TcpBbr::BBR_PROBE_BW;
break;
case TcpBbr::BBR_PROBE_RTT:
cong->EnterProbeRTT ();
desiredPacingGain = 1;
desiredCwndGain = 1;
desiredMode = TcpBbr::BBR_PROBE_RTT;
break;
default:
NS_ASSERT (false);
}
actualPacingGain = cong->GetPacingGain ();