From 3911b5f19f329817cc2e050a32ea426e2dbfd2b1 Mon Sep 17 00:00:00 2001 From: Vivek Jain Date: Mon, 18 Jun 2018 22:47:55 +0200 Subject: [PATCH] tcp: Added RateOps class and Linux implementation to calculate the rate sample (incorporating comments from Natale P.) --- src/internet/doc/tcp.rst | 29 ++ src/internet/model/tcp-rate-ops.cc | 292 +++++++++++++ src/internet/model/tcp-rate-ops.h | 260 +++++++++++ src/internet/test/tcp-general-test.cc | 4 + src/internet/test/tcp-general-test.h | 19 + src/internet/test/tcp-rate-ops-test.cc | 579 +++++++++++++++++++++++++ src/internet/wscript | 3 + 7 files changed, 1186 insertions(+) create mode 100644 src/internet/model/tcp-rate-ops.cc create mode 100644 src/internet/model/tcp-rate-ops.h create mode 100644 src/internet/test/tcp-rate-ops-test.cc diff --git a/src/internet/doc/tcp.rst b/src/internet/doc/tcp.rst index d6d42bb5c..4d276533c 100644 --- a/src/internet/doc/tcp.rst +++ b/src/internet/doc/tcp.rst @@ -1146,6 +1146,35 @@ ExitRecovery is called just prior to exiting recovery phase in order to perform required congestion window ajustments. UpdateBytesSent is used to keep track of bytes sent and is called whenever a data packet is sent during recovery phase. +Delivery Rate Estimation +++++++++++++++++++++++++ +Current TCP implementation measures the approximate value of the delivery rate of +inflight data based on Delivery Rate Estimation. + +As high level idea, keep in mind that the algorithm keeps track of 2 variables: + +1. `delivered`: Total amount of data delivered so far. + +2. `deliveredStamp`: Last time `delivered` was updated. + +When a packet is transmitted, the value of `delivered (d0)` and `deliveredStamp (t0)` +is stored in its respective TcpTxItem. + +When an acknowledgement comes for this packet, the value of `delivered` and `deliveredStamp` +is updated to `d1` and `t1` in the same TcpTxItem. + +After processing the acknowledgement, the rate sample is calculated and then passed +to a congestion avoidance algorithm: + +.. math:: delivery_rate = (d1 - d0)/(t1 - t0) + + +The implementation to estimate delivery rate is a joint work between TcpTxBuffer and TcpRateOps. +For more information, please take a look at their doxygen documentation. + +The implementation follows the Internet draft (Delivery Rate Estimation): +https://tools.ietf.org/html/draft-cheng-iccrg-delivery-rate-estimation-00 + Current limitations +++++++++++++++++++ diff --git a/src/internet/model/tcp-rate-ops.cc b/src/internet/model/tcp-rate-ops.cc new file mode 100644 index 000000000..1b030f154 --- /dev/null +++ b/src/internet/model/tcp-rate-ops.cc @@ -0,0 +1,292 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2018 Natale Patriciello + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +#include "tcp-rate-ops.h" +#include "ns3/log.h" +#include "ns3/simulator.h" + +namespace ns3 { + +NS_LOG_COMPONENT_DEFINE ("TcpRateOps"); +NS_OBJECT_ENSURE_REGISTERED (TcpRateOps); + +TypeId +TcpRateOps::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::TcpRateOps") + .SetParent () + .SetGroupName ("Internet") + ; + return tid; +} + +NS_OBJECT_ENSURE_REGISTERED (TcpRateLinux); + +TypeId +TcpRateLinux::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::TcpRateLinux") + .SetParent () + .SetGroupName ("Internet") + .AddTraceSource ("TcpRateUpdated", + "Tcp rate information has been updated", + MakeTraceSourceAccessor (&TcpRateLinux::m_rateTrace), + "ns3::TcpRateLinux::TcpRateUpdated") + .AddTraceSource ("TcpRateSampleUpdated", + "Tcp rate sample has been updated", + MakeTraceSourceAccessor (&TcpRateLinux::m_rateSampleTrace), + "ns3::TcpRateLinux::TcpRateSampleUpdated") + ; + return tid; +} + +const TcpRateOps::TcpRateSample & +TcpRateLinux::GenerateSample (uint32_t delivered, uint32_t lost, bool is_sack_reneg, + uint32_t priorInFlight, const Time &minRtt) +{ + NS_LOG_FUNCTION (this << delivered << lost << is_sack_reneg); + + /* Clear app limited if bubble is acked and gone. */ + if (m_rate.m_appLimited != 0 && m_rate.m_delivered > m_rate.m_appLimited) + { + NS_LOG_INFO ("Updating Rate m_appLimited to zero"); + m_rate.m_appLimited = 0; + } + + NS_LOG_INFO ("Updating RateSample m_ackedSacked=" << delivered << + ", m_bytesLoss=" << lost << " and m_priorInFlight" << priorInFlight); + m_rateSample.m_ackedSacked = delivered; /* freshly ACKed or SACKed */ + m_rateSample.m_bytesLoss = lost; /* freshly marked lost */ + m_rateSample.m_priorInFlight = priorInFlight; + + /* Return an invalid sample if no timing information is available or + * in recovery from loss with SACK reneging. Rate samples taken during + * a SACK reneging event may overestimate bw by including packets that + * were SACKed before the reneg. + */ + if (m_rateSample.m_priorTime == Seconds (0) || is_sack_reneg) + { + NS_LOG_INFO ("PriorTime is zero, invalidating sample"); + m_rateSample.m_delivered = -1; + m_rateSample.m_interval = Seconds (0); + m_rateSampleTrace (m_rateSample); + return m_rateSample; + } + + // LINUX: + // /* Model sending data and receiving ACKs as separate pipeline phases + // * for a window. Usually the ACK phase is longer, but with ACK + // * compression the send phase can be longer. To be safe we use the + // * longer phase. + // */ + // auto snd_us = m_rateSample.m_interval; /* send phase */ + // auto ack_us = Simulator::Now () - m_rateSample.m_prior_mstamp; + // m_rateSample.m_interval = std::max (snd_us, ack_us); + + m_rateSample.m_interval = std::max (m_rateSample.m_sendElapsed, m_rateSample.m_ackElapsed); + m_rateSample.m_delivered = m_rate.m_delivered - m_rateSample.m_priorDelivered; + NS_LOG_INFO ("Updating sample interval=" << m_rateSample.m_interval << " and delivered data" << m_rateSample.m_delivered); + + /* Normally we expect m_interval >= minRtt. + * Note that rate may still be over-estimated when a spuriously + * retransmistted skb was first (s)acked because "interval_us" + * is under-estimated (up to an RTT). However continuously + * measuring the delivery rate during loss recovery is crucial + * for connections suffer heavy or prolonged losses. + */ + if (m_rateSample.m_interval < minRtt) + { + NS_LOG_INFO ("Sampling interval is invalid"); + m_rateSample.m_interval = Seconds (0); + m_rateSample.m_priorTime = Seconds (0); // To make rate sample invalid + m_rateSampleTrace (m_rateSample); + return m_rateSample; + } + + /* Record the last non-app-limited or the highest app-limited bw */ + if (!m_rateSample.m_isAppLimited || + (m_rateSample.m_delivered * m_rate.m_rateInterval >= + m_rate.m_rateDelivered * m_rateSample.m_interval)) + { + m_rate.m_rateDelivered = m_rateSample.m_delivered; + m_rate.m_rateInterval = m_rateSample.m_interval; + m_rate.m_rateAppLimited = m_rateSample.m_isAppLimited; + m_rateSample.m_deliveryRate = DataRate (m_rateSample.m_delivered * 8.0 / m_rateSample.m_interval.GetSeconds ()); + NS_LOG_INFO ("Updating delivery rate=" << m_rateSample.m_deliveryRate); + } + + m_rateSampleTrace (m_rateSample); + return m_rateSample; +} + +void +TcpRateLinux::CalculateAppLimited (uint32_t cWnd, uint32_t in_flight, + uint32_t segmentSize, const SequenceNumber32 &tailSeq, + const SequenceNumber32 &nextTx, const uint32_t lostOut, + const uint32_t retransOut) +{ + NS_LOG_FUNCTION (this); + + /* Missing checks from Linux: + * - Nothing in sending host's qdisc queues or NIC tx queue. NOT IMPLEMENTED + */ + if (tailSeq - nextTx < static_cast (segmentSize) && // We have less than one packet to send. + in_flight < cWnd && // We are not limited by CWND. + lostOut <= retransOut) // All lost packets have been retransmitted. + { + m_rate.m_appLimited = std::max (m_rate.m_delivered + in_flight, 1UL); + m_rateTrace (m_rate); + } + + // m_appLimited will be reset once in GenerateSample, if it has to be. + // else + // { + // m_rate.m_appLimited = 0U; + // } +} + +void +TcpRateLinux::SkbDelivered (TcpTxItem * skb) +{ + NS_LOG_FUNCTION (this << skb); + + TcpTxItem::RateInformation & skbInfo = skb->GetRateInformation (); + + if (skbInfo.m_deliveredTime == Time::Max ()) + { + return; + } + + m_rate.m_delivered += skb->GetSeqSize (); + m_rate.m_deliveredTime = Simulator::Now (); + + if (m_rateSample.m_priorDelivered == 0 || + skbInfo.m_delivered > m_rateSample.m_priorDelivered) + { + m_rateSample.m_priorDelivered = skbInfo.m_delivered; + m_rateSample.m_priorTime = skbInfo.m_deliveredTime; + m_rateSample.m_isAppLimited = skbInfo.m_isAppLimited; + m_rateSample.m_sendElapsed = skb->GetLastSent () - skbInfo.m_firstSent; + m_rateSample.m_ackElapsed = Simulator::Now () - skbInfo.m_deliveredTime; + + m_rateSampleTrace (m_rateSample); + + m_rate.m_firstSentTime = skb->GetLastSent (); + } + + /* Mark off the skb delivered once it's taken into account to avoid being + * used again when it's cumulatively acked, in case it was SACKed. + */ + skbInfo.m_deliveredTime = Time::Max (); + m_rate.m_txItemDelivered = skbInfo.m_delivered; + m_rateTrace (m_rate); +} + +void +TcpRateLinux::SkbSent (TcpTxItem *skb, bool isStartOfTransmission) +{ + NS_LOG_FUNCTION (this << skb << isStartOfTransmission); + + TcpTxItem::RateInformation & skbInfo = skb->GetRateInformation (); + + /* In general we need to start delivery rate samples from the + * time we received the most recent ACK, to ensure we include + * the full time the network needs to deliver all in-flight + * packets. If there are no packets in flight yet, then we + * know that any ACKs after now indicate that the network was + * able to deliver those packets completely in the sampling + * interval between now and the next ACK. + * + * Note that we use the entire window size instead of bytes_in_flight + * because the latter is a guess based on RTO and loss-marking + * heuristics. We don't want spurious RTOs or loss markings to cause + * a spuriously small time interval, causing a spuriously high + * bandwidth estimate. + */ + if (isStartOfTransmission) + { + NS_LOG_INFO ("Starting of a transmission at time " << Simulator::Now ().GetSeconds ()); + m_rate.m_firstSentTime = Simulator::Now (); + m_rate.m_deliveredTime = Simulator::Now (); + m_rateTrace (m_rate); + } + + skbInfo.m_firstSent = m_rate.m_firstSentTime; + skbInfo.m_deliveredTime = m_rate.m_deliveredTime; + skbInfo.m_isAppLimited = (m_rate.m_appLimited != 0); + skbInfo.m_delivered = m_rate.m_delivered; +} + +std::ostream & +operator<< (std::ostream & os, TcpRateLinux::TcpRateConnection const & rate) +{ + os << "m_delivered = " << rate.m_delivered << std::endl; + os << "m_deliveredTime = " << rate.m_deliveredTime << std::endl; + os << "m_firstSentTime = " << rate.m_firstSentTime << std::endl; + os << "m_appLimited = " << rate.m_appLimited << std::endl; + os << "m_rateDelivered = " << rate.m_rateDelivered << std::endl; + os << "m_rateInterval = " << rate.m_rateInterval << std::endl; + os << "m_rateAppLimited = " << rate.m_rateAppLimited << std::endl; + os << "m_txItemDelivered = " << rate.m_txItemDelivered << std::endl; + return os; +} + +std::ostream & +operator<< (std::ostream & os, TcpRateLinux::TcpRateSample const & sample) +{ + os << "m_deliveryRate = " << sample.m_deliveryRate << std::endl; + os << " m_isAppLimited = " << sample.m_isAppLimited << std::endl; + os << " m_interval = " << sample.m_interval << std::endl; + os << " m_delivered = " << sample.m_delivered << std::endl; + os << " m_priorDelivered = " << sample.m_priorDelivered << std::endl; + os << " m_priorTime = " << sample.m_priorTime << std::endl; + os << " m_sendElapsed = " << sample.m_sendElapsed << std::endl; + os << " m_ackElapsed = " << sample.m_ackElapsed << std::endl; + os << " m_bytesLoss = " << sample.m_bytesLoss << std::endl; + os << " m_priorInFlight= " << sample.m_priorInFlight << std::endl; + os << " m_ackedSacked = " << sample.m_ackedSacked << std::endl; + return os; +} + +bool +operator== (TcpRateLinux::TcpRateSample const & lhs, TcpRateLinux::TcpRateSample const & rhs) +{ + return (lhs.m_deliveryRate == rhs.m_deliveryRate && + lhs.m_isAppLimited == rhs.m_isAppLimited && + lhs.m_interval == rhs.m_interval && + lhs.m_delivered == rhs.m_delivered && + lhs.m_priorDelivered == rhs.m_priorDelivered && + lhs.m_priorTime == rhs.m_priorTime && + lhs.m_sendElapsed == rhs.m_sendElapsed && + lhs.m_ackElapsed == rhs.m_ackElapsed + ); +} + +bool +operator== (TcpRateLinux::TcpRateConnection const & lhs, + TcpRateLinux::TcpRateConnection const & rhs) +{ + return (lhs.m_delivered == rhs.m_delivered && + lhs.m_deliveredTime == rhs.m_deliveredTime && + lhs.m_firstSentTime == rhs.m_firstSentTime && + lhs.m_appLimited == rhs.m_appLimited + ); +} + + +} // namespace ns3 diff --git a/src/internet/model/tcp-rate-ops.h b/src/internet/model/tcp-rate-ops.h new file mode 100644 index 000000000..494c4b188 --- /dev/null +++ b/src/internet/model/tcp-rate-ops.h @@ -0,0 +1,260 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2018 Natale Patriciello + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +#pragma once + +#include "ns3/object.h" +#include "ns3/tcp-tx-item.h" +#include "ns3/traced-callback.h" +#include "ns3/data-rate.h" +#include "ns3/traced-value.h" + +namespace ns3 { + +/** + * \brief Interface for all operations that involve a Rate monitoring for TCP. + * + * The interface is meant to take information to generate rate information + * valid for congestion avoidance algorithm such as BBR. + * + * Please take a look to the TcpRateLinux class for an example. + */ +class TcpRateOps : public Object +{ +public: + struct TcpRateSample; + struct TcpRateConnection; + + /** + * Get the type ID. + * \brief Get the type ID. + * \return the object TypeId + */ + static TypeId GetTypeId (void); + /** + * \brief Put the rate information inside the sent skb + * + * Snapshot the current delivery information in the skb, to generate + * a rate sample later when the skb is (s)acked in SkbDelivered (). + * + * \param skb The SKB sent + * \param isStartOfTransmission true if this is a start of transmission + * (i.e., in_flight == 0) + */ + virtual void SkbSent (TcpTxItem *skb, bool isStartOfTransmission) = 0; + + /** + * \brief Update the Rate information after an item is received + * + * When an skb is sacked or acked, we fill in the rate sample with the (prior) + * delivery information when the skb was last transmitted. + * + * If an ACK (s)acks multiple skbs (e.g., stretched-acks), this function is + * called multiple times. We favor the information from the most recently + * sent skb, i.e., the skb with the highest prior_delivered count. + * + * \param skb The SKB delivered ((s)ACKed) + */ + virtual void SkbDelivered (TcpTxItem * skb) = 0; + + /** + * \brief If a gap is detected between sends, it means we are app-limited. + * \return TODO What the Linux kernel is setting in tp->app_limited? + * https://elixir.bootlin.com/linux/latest/source/net/ipv4/tcp_rate.c#L177 + * + * \param cWnd Congestion Window + * \param in_flight In Flight size (in bytes) + * \param segmentSize Segment size + * \param tailSeq Tail Sequence + * \param nextTx NextTx + * \param lostOut Number of lost bytes + * \param retransOut Number of retransmitted bytes + */ + virtual void CalculateAppLimited (uint32_t cWnd, uint32_t in_flight, + uint32_t segmentSize, const SequenceNumber32 &tailSeq, + const SequenceNumber32 &nextTx, const uint32_t lostOut, + const uint32_t retransOut) = 0; + + /** + * + * \brief Generate a TcpRateSample to feed a congestion avoidance algorithm. + * + * This function will be called after an ACK (or a SACK) is received. The + * (S)ACK carries some implicit information, such as how many segments have been + * lost or delivered. These values will be this function input. + * + * \param delivered number of delivered segments (e.g., receiving a cumulative + * ACK means having more than 1 segment delivered) relative to the most recent + * (S)ACK received + * \param lost number of segments that we detected as lost after the reception + * of the most recent (S)ACK + * \param is_sack_reneg Is SACK reneged? + * \param minRtt Minimum RTT so far + * \return The TcpRateSample that will be used for CA + */ + virtual const TcpRateSample & GenerateSample (uint32_t delivered, uint32_t lost, + bool is_sack_reneg, uint32_t priorInFlight, + const Time &minRtt) = 0; + + /** + * \return The information about the rate connection + * + */ + virtual const TcpRateConnection & GetConnectionRate () = 0; + + /** + * \brief Rate Sample structure + * + * A rate sample measures the number of (original/retransmitted) data + * packets delivered "delivered" over an interval of time "interval_us". + * The tcp_rate code fills in the rate sample, and congestion + * control modules that define a cong_control function to run at the end + * of ACK processing can optionally chose to consult this sample when + * setting cwnd and pacing rate. + * A sample is invalid if "delivered" or "interval_us" is negative. + */ + struct TcpRateSample + { + DataRate m_deliveryRate {DataRate ("0bps")};//!< The delivery rate sample + bool m_isAppLimited {false}; //!< Indicates whether the rate sample is application-limited + Time m_interval {Seconds (0.0)}; //!< The length of the sampling interval + int32_t m_delivered {0}; //!< The amount of data marked as delivered over the sampling interval + uint32_t m_priorDelivered {0}; //!< The delivered count of the most recent packet delivered + Time m_priorTime {Seconds (0.0)}; //!< The delivered time of the most recent packet delivered + Time m_sendElapsed {Seconds (0.0)}; //!< Send time interval calculated from the most recent packet delivered + Time m_ackElapsed {Seconds (0.0)}; //!< ACK time interval calculated from the most recent packet delivered + uint32_t m_bytesLoss {0}; //!< The amount of data marked as lost from the most recent ack received + uint32_t m_priorInFlight {0}; //!< The value if bytes in flight prior to last received ack + uint32_t m_ackedSacked {0}; //!< The amount of data acked and sacked in the last received ack + /** + * \brief Is the sample valid? + * \return true if the sample is valid, false otherwise. + */ + bool IsValid () const + { + return (m_priorTime != Seconds (0.0) || m_interval != Seconds (0.0)); + } + }; + + /** + * \brief Information about the connection rate + * + * In this struct, the values are for the entire connection, and not just + * for an interval of time + */ + struct TcpRateConnection + { + uint64_t m_delivered {0}; //!< The total amount of data in bytes delivered so far + Time m_deliveredTime {Seconds (0)}; //!< Simulator time when m_delivered was last updated + Time m_firstSentTime {Seconds (0)}; //!< The send time of the packet that was most recently marked as delivered + uint32_t m_appLimited {0}; //!< The index of the last transmitted packet marked as application-limited + uint32_t m_txItemDelivered {0}; //!< The value of delivered when the acked item was sent + int32_t m_rateDelivered {0}; //!< The amount of data delivered considered to calculate delivery rate. + Time m_rateInterval {Seconds (0)}; //!< The value of interval considered to calculate delivery rate. + bool m_rateAppLimited {false}; //!< Was sample was taken when data is app limited? + }; +}; + +/** + * \brief Linux management and generation of Rate information for TCP + * + * This class is inspired by what Linux is performing in tcp_rate.c + */ +class TcpRateLinux : public TcpRateOps +{ +public: + /** + * Get the type ID. + * \brief Get the type ID. + * \return the object TypeId + */ + static TypeId GetTypeId (void); + virtual ~TcpRateLinux () override {} + + virtual void SkbSent (TcpTxItem *skb, bool isStartOfTransmission) override; + virtual void SkbDelivered (TcpTxItem * skb) override; + virtual void CalculateAppLimited (uint32_t cWnd, uint32_t in_flight, + uint32_t segmentSize, const SequenceNumber32 &tailSeq, + const SequenceNumber32 &nextTx, const uint32_t lostOut, + const uint32_t retransOut) override; + virtual const TcpRateSample & GenerateSample (uint32_t delivered, uint32_t lost, + bool is_sack_reneg, uint32_t priorInFlight, + const Time &minRtt) override; + virtual const TcpRateConnection & GetConnectionRate () override { return m_rate; } + + /** + * TracedCallback signature for tcp rate update events. + * + * The callback will be fired each time the rate is updated. + * + * \param [in] rate The rate information. + */ + typedef void (* TcpRateUpdated)(const TcpRateConnection &rate); + + /** + * TracedCallback signature for tcp rate sample update events. + * + * The callback will be fired each time the rate sample is updated. + * + * \param [in] sample The rate sample that will be passed to congestion control + * algorithms. + */ + typedef void (* TcpRateSampleUpdated)(const TcpRateSample &sample); + +private: + // Rate sample related variables + TcpRateConnection m_rate; //!< Rate information + TcpRateSample m_rateSample; //!< Rate sample (continuosly updated) + + TracedCallback m_rateTrace; //!< Rate trace + TracedCallback m_rateSampleTrace; //!< Rate Sample trace +}; + +/** + * \brief Output operator. + * \param os The output stream. + * \param sample the TcpRateLinux::TcpRateSample to print. + * \returns The output stream. + */ +std::ostream & operator<< (std::ostream & os, TcpRateLinux::TcpRateSample const & sample); + +/** + * \brief Output operator. + * \param os The output stream. + * \param rate the TcpRateLinux::TcpRateConnection to print. + * \returns The output stream. + */ +std::ostream & operator<< (std::ostream & os, TcpRateLinux::TcpRateConnection const & rate); + +/** + * Comparison operator + * \param lhs left operand + * \param rhs right operand + * \return true if the operands are equal + */ +bool operator== (TcpRateLinux::TcpRateSample const & lhs, TcpRateLinux::TcpRateSample const & rhs); + +/** + * Comparison operator + * \param lhs left operand + * \param rhs right operand + * \return true if the operands are equal + */ +bool operator== (TcpRateLinux::TcpRateConnection const & lhs, TcpRateLinux::TcpRateConnection const & rhs); + +} //namespace ns3 diff --git a/src/internet/test/tcp-general-test.cc b/src/internet/test/tcp-general-test.cc index 8c06a4d82..b7f1ec355 100644 --- a/src/internet/test/tcp-general-test.cc +++ b/src/internet/test/tcp-general-test.cc @@ -224,6 +224,10 @@ TcpGeneralTest::DoRun (void) MakeCallback (&TcpGeneralTest::NextTxSeqTrace, this)); m_senderSocket->TraceConnectWithoutContext ("HighestSequence", MakeCallback (&TcpGeneralTest::HighestTxSeqTrace, this)); + m_senderSocket->m_rateOps->TraceConnectWithoutContext ("TcpRateUpdated", + MakeCallback (&TcpGeneralTest::RateUpdatedTrace, this)); + m_senderSocket->m_rateOps->TraceConnectWithoutContext ("TcpRateSampleUpdated", + MakeCallback (&TcpGeneralTest::RateSampleUpdatedTrace, this)); m_remoteAddr = InetSocketAddress (serverAddress, 4477); diff --git a/src/internet/test/tcp-general-test.h b/src/internet/test/tcp-general-test.h index 31c205254..5940b3cf3 100644 --- a/src/internet/test/tcp-general-test.h +++ b/src/internet/test/tcp-general-test.h @@ -24,6 +24,7 @@ #include "ns3/tcp-socket-base.h" #include "ns3/tcp-congestion-ops.h" #include "ns3/tcp-recovery-ops.h" +#include "ns3/tcp-rate-ops.h" #include "ns3/test.h" namespace ns3 { @@ -786,6 +787,24 @@ protected: NS_UNUSED (newValue); } + /** + * \brief Track the rate value of TcpRateLinux. + * \param rate updated value of TcpRate. + */ + virtual void RateUpdatedTrace (const TcpRateLinux::TcpRateConnection &rate) + { + NS_UNUSED (rate); + } + + /** + * \brief Track the rate sample value of TcpRateLinux. + * \param sample updated value of TcpRateSample. + */ + virtual void RateSampleUpdatedTrace (const TcpRateLinux::TcpRateSample &sample) + { + NS_UNUSED (sample); + } + /** * \brief Socket closed normally * \param who the socket closed (SENDER or RECEIVER) diff --git a/src/internet/test/tcp-rate-ops-test.cc b/src/internet/test/tcp-rate-ops-test.cc new file mode 100644 index 000000000..8e02cb8ae --- /dev/null +++ b/src/internet/test/tcp-rate-ops-test.cc @@ -0,0 +1,579 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2018 NITK Surathkal + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Authors: Vivek Jain + * Viyom Mittal + * Mohit P. Tahiliani + */ + +#include "ns3/test.h" +#include "ns3/tcp-tx-item.h" +#include "ns3/log.h" +#include "ns3/tcp-rate-ops.h" +#include "tcp-general-test.h" +#include "ns3/config.h" +#include "tcp-error-model.h" +#include "ns3/tcp-tx-buffer.h" + +using namespace ns3; + +NS_LOG_COMPONENT_DEFINE ("TcpRateOpsTestSuite"); + +class MimicCongControl; + +/** + * \ingroup internet-tests + * \ingroup tests + * + * \brief The TcpRateLinux Basic Test + */ +class TcpRateLinuxBasicTest : public TestCase +{ +public: + TcpRateLinuxBasicTest (uint32_t cWnd, SequenceNumber32 tailSeq, SequenceNumber32 nextTx, + uint32_t lostOut, uint32_t retransOut, uint32_t testCase, + std::string testName); + +private: + virtual void DoRun (void); + + void SendSkb (TcpTxItem * skb); + void SkbDelivered (TcpTxItem * skb); + + TcpRateLinux m_rateOps; + uint32_t m_cWnd; + uint32_t m_inFlight; + uint32_t m_segmentSize; + uint32_t m_delivered; + Time m_deliveredTime; + SequenceNumber32 m_tailSeq; + SequenceNumber32 m_nextTx; + uint32_t m_lostOut; + uint32_t m_retransOut; + uint32_t m_testCase; + std::vector m_skbs; +}; + +TcpRateLinuxBasicTest::TcpRateLinuxBasicTest (uint32_t cWnd, SequenceNumber32 tailSeq, + SequenceNumber32 nextTx, uint32_t lostOut, + uint32_t retransOut, uint32_t testCase, std::string testName) + : TestCase (testName), + m_cWnd (cWnd), + m_inFlight (0), + m_segmentSize (1), + m_delivered (0), + m_deliveredTime (Seconds (0)), + m_tailSeq (tailSeq), + m_nextTx (nextTx), + m_lostOut (lostOut), + m_retransOut (retransOut), + m_testCase (testCase) +{ +} + +void +TcpRateLinuxBasicTest::DoRun () +{ + for (uint8_t i = 0; i < 100; ++i) + { + m_skbs.push_back (new TcpTxItem ()); + Simulator::Schedule (Time (Seconds (i * 0.01)), &TcpRateLinuxBasicTest::SendSkb, this, m_skbs [i]); + } + + for (uint8_t i = 0; i < 100; ++i) + { + Simulator::Schedule (Time (Seconds ((i + 1) * 0.1)), &TcpRateLinuxBasicTest::SkbDelivered, this, m_skbs [i]); + } + + Simulator::Run (); + Simulator::Destroy (); + + for (uint8_t i = 0; i < 100; ++i) + { + delete m_skbs[i]; + } +} + +void +TcpRateLinuxBasicTest::SendSkb (TcpTxItem * skb) +{ + bool isStartOfTransmission = m_inFlight == 0; + m_rateOps.CalculateAppLimited (m_cWnd, m_inFlight, m_segmentSize, m_tailSeq, m_nextTx, 0, 0); + m_rateOps.SkbSent (skb, isStartOfTransmission); + m_inFlight += skb->GetSeqSize (); + + NS_TEST_ASSERT_MSG_EQ (skb->GetRateInformation ().m_delivered, m_delivered, "SKB should have delivered equal to current value of total delivered"); + + if (isStartOfTransmission) + { + NS_TEST_ASSERT_MSG_EQ (skb->GetRateInformation ().m_deliveredTime, Simulator::Now (), "SKB should have updated the delivered time to current value"); + } + else + { + NS_TEST_ASSERT_MSG_EQ (skb->GetRateInformation ().m_deliveredTime, m_deliveredTime, "SKB should have updated the delivered time to current value"); + } +} + +void +TcpRateLinuxBasicTest::SkbDelivered (TcpTxItem * skb) +{ + m_rateOps.SkbDelivered (skb); + m_inFlight -= skb->GetSeqSize (); + m_delivered += skb->GetSeqSize (); + m_deliveredTime = Simulator::Now (); + + NS_TEST_ASSERT_MSG_EQ (skb->GetRateInformation ().m_deliveredTime, Time::Max (), "SKB should have delivered time as Time::Max ()"); + + if (m_testCase == 1) + { + NS_TEST_ASSERT_MSG_EQ (skb->GetRateInformation ().m_isAppLimited, false, "Socket should not be applimited"); + } + else if (m_testCase == 2) + { + NS_TEST_ASSERT_MSG_EQ (skb->GetRateInformation ().m_isAppLimited, true, "Socket should be applimited"); + } +} + +/** + * \ingroup internet-test + * \ingroup tests + * + * \brief Behaves as NewReno except HasCongControl returns true + */ +class MimicCongControl : public TcpNewReno +{ +public: + /** + * \brief Get the type ID. + * \return the object TypeId + */ + static TypeId GetTypeId (void); + + MimicCongControl () + { + } + + virtual bool HasCongControl () const + { + return true; + } + +}; + +TypeId +MimicCongControl::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::MimicCongControl") + .SetParent () + .AddConstructor () + .SetGroupName ("Internet") + ; + return tid; +} + +/** + * \ingroup internet-tests + * \ingroup tests + * + * \brief The TcpRateLinux Test uses sender-receiver model to test its functionality. + * This test case uses the bytes inflight trace to check whether rate sample + * correctly sets the value of m_deliveredTime and m_firstSentTime. This is + * done using rate trace. Further, Using Rx trace, m_isDupAck is maintained to + * track duplicate acknowledgments. This, in turn, is used to see whether rate + * sample is updated properly (in case of SACK) or not (in case of non SACK). + */ +class TcpRateLinuxWithSocketsTest : public TcpGeneralTest +{ +public: + /** + * \brief Constructor. + * \param desc Description. + * \param sackEnabled To use SACK or not + * \param toDrop Packets to drop. + */ + TcpRateLinuxWithSocketsTest (const std::string &desc, bool sackEnabled, + std::vector &toDrop); + +protected: + /** + * \brief Create and install the socket to install on the sender + * \param node sender node pointer + * \return the socket to be installed in the sender + */ + virtual Ptr CreateSenderSocket (Ptr node); + + /** + * \brief Create a receiver error model. + * \returns The receiver error model. + */ + virtual Ptr CreateReceiverErrorModel (); + + /** + * \brief Receive a packet. + * \param p The packet. + * \param h The TCP header. + * \param who Who the socket belongs to (sender or receiver). + */ + virtual void Rx (const Ptr p, const TcpHeader&h, SocketWho who); + + /** + * \brief Track the bytes in flight. + * \param oldValue previous value. + * \param newValue actual value. + */ + virtual void BytesInFlightTrace (uint32_t oldValue, uint32_t newValue); + + /** + * \brief Called when a packet is dropped. + * \param ipH The IP header. + * \param tcpH The TCP header. + * \param p The packet. + */ + void PktDropped (const Ipv4Header &ipH, const TcpHeader& tcpH, Ptr p); + + /** + * \brief Configure the test. + */ + void ConfigureEnvironment (); + + /** + * \brief Do the final checks. + */ + void FinalChecks (); + + /** + * \brief Track the rate value of TcpRateLinux. + * \param rate updated value of TcpRate. + */ + virtual void RateUpdatedTrace (const TcpRateLinux::TcpRateConnection &rate); + + /** + * \brief Track the rate sample value of TcpRateLinux. + * \param sample updated value of TcpRateSample. + */ + virtual void RateSampleUpdatedTrace (const TcpRateLinux::TcpRateSample &sample); + +private: + Ptr m_congCtl; //!< Dummy congestion control. + bool m_sackEnabled; //!< Sack Variable + std::vector m_toDrop; //!< List of SequenceNumber to drop + uint32_t m_bytesInFlight; //!< Bytes inflight + SequenceNumber32 m_lastAckRecv {SequenceNumber32 (1)}; //!< Last ACK received. + bool m_isDupAck; //!< Whether ACK is DupAck + TcpRateLinux::TcpRateConnection m_prevRate; //!< Previous rate + TcpRateLinux::TcpRateSample m_prevRateSample; //!< Previous rate sample +}; + +TcpRateLinuxWithSocketsTest::TcpRateLinuxWithSocketsTest (const std::string &desc, bool sackEnabled, + std::vector &toDrop) + : TcpGeneralTest (desc), + m_sackEnabled (sackEnabled), + m_toDrop (toDrop) +{ +} + +Ptr +TcpRateLinuxWithSocketsTest::CreateSenderSocket (Ptr node) +{ + Ptr s = TcpGeneralTest::CreateSenderSocket (node); + m_congCtl = CreateObject (); + s->SetCongestionControlAlgorithm (m_congCtl); + return s; +} + +void +TcpRateLinuxWithSocketsTest::ConfigureEnvironment () +{ + TcpGeneralTest::ConfigureEnvironment (); + SetAppPktCount (300); + SetPropagationDelay (MilliSeconds (50)); + SetTransmitStart (Seconds (2.0)); + + Config::SetDefault ("ns3::TcpSocketBase::Sack", BooleanValue (m_sackEnabled)); +} + +Ptr +TcpRateLinuxWithSocketsTest::CreateReceiverErrorModel () +{ + Ptr m_errorModel = CreateObject (); + for (std::vector::iterator it = m_toDrop.begin (); it != m_toDrop.end (); ++it) + { + m_errorModel->AddSeqToKill (SequenceNumber32 (*it)); + } + + m_errorModel->SetDropCallback (MakeCallback (&TcpRateLinuxWithSocketsTest::PktDropped, this)); + + return m_errorModel; +} + +void +TcpRateLinuxWithSocketsTest::PktDropped (const Ipv4Header &ipH, const TcpHeader &tcpH, + Ptr p) +{ + NS_LOG_DEBUG ("Drop seq= " << tcpH.GetSequenceNumber () << " size " << p->GetSize ()); +} + +void +TcpRateLinuxWithSocketsTest::Rx (const Ptr p, const TcpHeader &h, SocketWho who) +{ + if (who == SENDER) + { + if (h.GetAckNumber () == m_lastAckRecv + && m_lastAckRecv != SequenceNumber32 (1) + && (h.GetFlags () & TcpHeader::FIN) == 0) + { + m_isDupAck = true; + } + else + { + m_isDupAck = false; + m_lastAckRecv = h.GetAckNumber (); + } + } +} + +void +TcpRateLinuxWithSocketsTest::BytesInFlightTrace (uint32_t oldValue, uint32_t newValue) +{ + NS_UNUSED (oldValue); + m_bytesInFlight = newValue; +} + +void +TcpRateLinuxWithSocketsTest::RateUpdatedTrace (const TcpRateLinux::TcpRateConnection &rate) +{ + NS_LOG_DEBUG ("Rate updated " << rate); + if (m_bytesInFlight == 0) + { + NS_TEST_ASSERT_MSG_EQ (rate.m_firstSentTime, Simulator::Now (), "FirstSentTime should be current time when bytes inflight is zero"); + NS_TEST_ASSERT_MSG_EQ (rate.m_deliveredTime, Simulator::Now (), "Delivered time should be current time when bytes inflight is zero"); + } + NS_TEST_ASSERT_MSG_GT_OR_EQ (rate.m_delivered, m_prevRate.m_delivered, "Total delivered should not be lesser than previous values"); + m_prevRate = rate; +} + +void +TcpRateLinuxWithSocketsTest::RateSampleUpdatedTrace (const TcpRateLinux::TcpRateSample &sample) +{ + NS_LOG_DEBUG ("Rate sample updated " << sample); + if (m_isDupAck) + { + if (!m_sackEnabled) + { + NS_TEST_ASSERT_MSG_EQ (m_prevRateSample, sample, "RateSample should not update due to DupAcks"); + } + else + { + if (sample.m_ackedSacked == 0) + { + NS_TEST_ASSERT_MSG_EQ (m_prevRateSample, sample, "RateSample should not update as nothing is acked or sacked"); + } + } + } + m_prevRateSample = sample; +} + +void +TcpRateLinuxWithSocketsTest::FinalChecks () +{ +} + +/** + * \ingroup internet-tests + * \ingroup tests + * + * \brief The TcpRateLinuxWithBufferTest tests rate sample functionality with arbitary SACK scenario. + * Check the value of delivered against a home-made guess + */ +class TcpRateLinuxWithBufferTest : public TestCase +{ +public: + /** + * \brief Constructor. + * \param segmentSize Segment size to use. + * \param desc Description. + */ + TcpRateLinuxWithBufferTest (uint32_t segmentSize, std::string desc); + +private: + virtual void DoRun (void); + virtual void DoTeardown (void); + + /** + * \brief Track the rate value of TcpRateLinux. + * \param rate updated value of TcpRate. + */ + virtual void RateUpdatedTrace (const TcpRateLinux::TcpRateConnection &rate); + + /** + * \brief Track the rate sample value of TcpRateLinux. + * \param sample updated value of TcpRateSample. + */ + virtual void RateSampleUpdatedTrace (const TcpRateLinux::TcpRateSample &sample); + + /** \brief Test with acks without drop */ + void TestWithStraightAcks (); + + /** \brief Test with arbitary SACK scenario */ + void TestWithSackBlocks (); + + uint32_t m_expectedDelivered {0}; //!< Amount of expected delivered data + uint32_t m_expectedAckedSacked {0}; //!< Amount of expected acked sacked data + uint32_t m_segmentSize; //!< Segment size + TcpTxBuffer m_txBuf; //!< Tcp Tx buffer + Ptr m_rateOps; //!< Rate operations +}; + +TcpRateLinuxWithBufferTest::TcpRateLinuxWithBufferTest (uint32_t segmentSize, + std::string testString) + : TestCase (testString), + m_segmentSize (segmentSize) +{ + m_rateOps = CreateObject (); + m_rateOps->TraceConnectWithoutContext ("TcpRateUpdated", + MakeCallback (&TcpRateLinuxWithBufferTest::RateUpdatedTrace, this)); + m_rateOps->TraceConnectWithoutContext ("TcpRateSampleUpdated", + MakeCallback (&TcpRateLinuxWithBufferTest::RateSampleUpdatedTrace, this)); +} + +void +TcpRateLinuxWithBufferTest::DoRun () +{ + Simulator::Schedule (Seconds (0.0), &TcpRateLinuxWithBufferTest::TestWithSackBlocks, this); + Simulator::Run (); + Simulator::Destroy (); +} + +void +TcpRateLinuxWithBufferTest::RateUpdatedTrace (const TcpRateLinux::TcpRateConnection &rate) +{ + NS_LOG_DEBUG ("Rate updated " << rate); + NS_TEST_ASSERT_MSG_EQ (rate.m_delivered, m_expectedDelivered, "Delivered data is not equal to expected delivered"); +} + +void +TcpRateLinuxWithBufferTest::RateSampleUpdatedTrace (const TcpRateLinux::TcpRateSample &sample) +{ + NS_LOG_DEBUG ("Rate sample updated " << sample); + NS_TEST_ASSERT_MSG_EQ (sample.m_ackedSacked, m_expectedAckedSacked, "AckedSacked bytes is not equal to expected AckedSacked bytes"); +} + +void +TcpRateLinuxWithBufferTest::TestWithSackBlocks () +{ + SequenceNumber32 head (1); + m_txBuf.SetHeadSequence (head); + SequenceNumber32 ret; + Ptr sack = CreateObject (); + m_txBuf.SetSegmentSize (m_segmentSize); + m_txBuf.SetDupAckThresh (3); + + m_txBuf.Add(Create (10 * m_segmentSize)); + + // Send 10 Segments + for (uint8_t i = 0; i < 10; ++i) + { + bool isStartOfTransmission = m_txBuf.BytesInFlight () == 0; + TcpTxItem *outItem = m_txBuf.CopyFromSequence (m_segmentSize, SequenceNumber32((i * m_segmentSize) + 1)); + m_rateOps->SkbSent (outItem, isStartOfTransmission); + } + + uint32_t priorInFlight = m_txBuf.BytesInFlight (); + // ACK 2 Segments + for (uint8_t i = 1; i <= 2; ++i) + { + priorInFlight = m_txBuf.BytesInFlight (); + m_expectedDelivered += m_segmentSize; + m_txBuf.DiscardUpTo (SequenceNumber32 (m_segmentSize * i + 1), MakeCallback (&TcpRateOps::SkbDelivered, m_rateOps)); + m_expectedAckedSacked = m_segmentSize; + m_rateOps->GenerateSample (m_segmentSize, 0, false, priorInFlight, Seconds (0)); + } + + priorInFlight = m_txBuf.BytesInFlight (); + sack->AddSackBlock (TcpOptionSack::SackBlock (SequenceNumber32 (m_segmentSize * 4 + 1), SequenceNumber32 (m_segmentSize * 5 + 1))); + m_expectedDelivered += m_segmentSize; + m_txBuf.Update(sack->GetSackList(), MakeCallback (&TcpRateOps::SkbDelivered, m_rateOps)); + m_expectedAckedSacked = m_segmentSize; + m_rateOps->GenerateSample (m_segmentSize, 0, false, priorInFlight, Seconds (0)); + + priorInFlight = m_txBuf.BytesInFlight (); + sack->AddSackBlock (TcpOptionSack::SackBlock (SequenceNumber32 (m_segmentSize * 3 + 1), SequenceNumber32 (m_segmentSize * 4 + 1))); + m_expectedDelivered += m_segmentSize; + m_txBuf.Update(sack->GetSackList(), MakeCallback (&TcpRateOps::SkbDelivered, m_rateOps)); + m_rateOps->GenerateSample (m_segmentSize, 0, false, priorInFlight, Seconds (0)); + + priorInFlight = m_txBuf.BytesInFlight (); + // Actual delivered should be increased by one segment even multiple blocks are acked. + m_expectedDelivered += m_segmentSize; + m_txBuf.DiscardUpTo (SequenceNumber32 (m_segmentSize * 5 + 1), MakeCallback (&TcpRateOps::SkbDelivered, m_rateOps)); + m_rateOps->GenerateSample (m_segmentSize, 0, false, priorInFlight, Seconds (0)); + + + priorInFlight = m_txBuf.BytesInFlight (); + // ACK rest of the segments + for (uint8_t i = 6; i <= 10; ++i) + { + m_expectedDelivered += m_segmentSize; + m_txBuf.DiscardUpTo (SequenceNumber32 (m_segmentSize * i + 1), MakeCallback (&TcpRateOps::SkbDelivered, m_rateOps)); + } + m_expectedAckedSacked = 5 * m_segmentSize; + TcpRateOps::TcpRateSample rateSample = m_rateOps->GenerateSample (5 * m_segmentSize, 0, false, priorInFlight, Seconds (0)); +} + +void +TcpRateLinuxWithBufferTest::DoTeardown () +{ +} + + +/** + * \ingroup internet-test + * \ingroup tests + * + * \brief the TestSuite for the TcpRateLinux test case + */ +class TcpRateOpsTestSuite : public TestSuite +{ +public: + TcpRateOpsTestSuite () + : TestSuite ("tcp-rate-ops", UNIT) + { + AddTestCase (new TcpRateLinuxBasicTest (1000, SequenceNumber32 (20), SequenceNumber32 (11), 1, 0, 0, "Testing SkbDelivered and SkbSent"), TestCase::QUICK); + AddTestCase (new TcpRateLinuxBasicTest (1000, SequenceNumber32 (11), SequenceNumber32 (11), 2, 0, 0, "Testing SkbDelivered and SkbSent with app limited data"), TestCase::QUICK); + + std::vector toDrop; + toDrop.push_back (4001); + AddTestCase (new TcpRateLinuxWithSocketsTest ("Checking Rate sample value without SACK, one drop", false, toDrop), + TestCase::QUICK); + + AddTestCase (new TcpRateLinuxWithSocketsTest ("Checking Rate sample value with SACK, one drop", true, toDrop), + TestCase::QUICK); + toDrop.push_back (4001); + AddTestCase (new TcpRateLinuxWithSocketsTest ("Checking Rate sample value without SACK, two drop", false, toDrop), + TestCase::QUICK); + + AddTestCase (new TcpRateLinuxWithSocketsTest ("Checking Rate sample value with SACK, two drop", true, toDrop), + TestCase::QUICK); + + AddTestCase (new TcpRateLinuxWithBufferTest (1000, "Checking rate sample values with arbitary SACK Block"), TestCase::QUICK); + + AddTestCase (new TcpRateLinuxWithBufferTest (500, "Checking rate sample values with arbitary SACK Block"), TestCase::QUICK); + + } +}; + +static TcpRateOpsTestSuite g_TcpRateOpsTestSuite; //!< Static variable for test initialization diff --git a/src/internet/wscript b/src/internet/wscript index 3245b6f49..ed19722e5 100644 --- a/src/internet/wscript +++ b/src/internet/wscript @@ -163,6 +163,7 @@ def build(bld): 'model/tcp-rx-buffer.cc', 'model/tcp-tx-buffer.cc', 'model/tcp-tx-item.cc', + 'model/tcp-rate-ops.cc', 'model/tcp-option.cc', 'model/tcp-option-rfc793.cc', 'model/tcp-option-winscale.cc', @@ -292,6 +293,7 @@ def build(bld): 'test/tcp-rx-buffer-test.cc', 'test/tcp-endpoint-bug2211.cc', 'test/tcp-datasentcb-test.cc', + 'test/tcp-rate-ops-test.cc', 'test/ipv4-rip-test.cc', 'test/tcp-close-test.cc', 'test/icmp-test.cc', @@ -404,6 +406,7 @@ def build(bld): 'model/tcp-socket-state.h', 'model/tcp-tx-buffer.h', 'model/tcp-tx-item.h', + 'model/tcp-rate-ops.h', 'model/tcp-rx-buffer.h', 'model/tcp-recovery-ops.h', 'model/tcp-prr-recovery.h',