From ff8083ca2e03115bc18f56faf8ff39a78db4da14 Mon Sep 17 00:00:00 2001 From: Pasquale Imputato Date: Tue, 19 May 2020 13:28:06 +0200 Subject: [PATCH] fd-net-device: add support for emulation in netmap mode --- doc/models/Makefile | 1 + src/fd-net-device/doc/netmap-net-device.rst | 79 +++ src/fd-net-device/examples/fd-emu-onoff.cc | 73 ++- src/fd-net-device/examples/fd-emu-ping.cc | 68 ++- src/fd-net-device/examples/fd-emu-send.cc | 230 +++++++++ src/fd-net-device/examples/fd-emu-tc.cc | 313 ++++++++++++ src/fd-net-device/examples/wscript | 13 +- .../helper/emu-fd-net-device-helper.cc | 19 + .../helper/emu-fd-net-device-helper.h | 8 + .../helper/fd-net-device-helper.cc | 6 + .../helper/netmap-device-creator.cc | 95 ++++ .../helper/netmap-net-device-helper.cc | 477 ++++++++++++++++++ .../helper/netmap-net-device-helper.h | 100 ++++ src/fd-net-device/model/fd-net-device.cc | 54 +- src/fd-net-device/model/fd-net-device.h | 28 +- src/fd-net-device/model/netmap-net-device.cc | 396 +++++++++++++++ src/fd-net-device/model/netmap-net-device.h | 219 ++++++++ src/fd-net-device/wscript | 39 +- .../utils/net-device-queue-interface.cc | 30 +- .../utils/net-device-queue-interface.h | 26 +- 20 files changed, 2221 insertions(+), 53 deletions(-) create mode 100644 src/fd-net-device/doc/netmap-net-device.rst create mode 100644 src/fd-net-device/examples/fd-emu-send.cc create mode 100644 src/fd-net-device/examples/fd-emu-tc.cc create mode 100644 src/fd-net-device/helper/netmap-device-creator.cc create mode 100644 src/fd-net-device/helper/netmap-net-device-helper.cc create mode 100644 src/fd-net-device/helper/netmap-net-device-helper.h create mode 100644 src/fd-net-device/model/netmap-net-device.cc create mode 100644 src/fd-net-device/model/netmap-net-device.h diff --git a/doc/models/Makefile b/doc/models/Makefile index cf72df707..c846969bf 100644 --- a/doc/models/Makefile +++ b/doc/models/Makefile @@ -38,6 +38,7 @@ SOURCES = \ $(SRC)/mpi/doc/distributed.rst \ $(SRC)/energy/doc/energy.rst \ $(SRC)/fd-net-device/doc/fd-net-device.rst \ + $(SRC)/fd-net-device/doc/netmap-net-device.rst \ $(SRC)/tap-bridge/doc/tap.rst \ $(SRC)/mesh/doc/source/mesh.rst \ $(SRC)/mesh/doc/source/mesh-user.rst \ diff --git a/src/fd-net-device/doc/netmap-net-device.rst b/src/fd-net-device/doc/netmap-net-device.rst new file mode 100644 index 000000000..f5d64f51b --- /dev/null +++ b/src/fd-net-device/doc/netmap-net-device.rst @@ -0,0 +1,79 @@ +Netmap NetDevice +------------------------- +.. include:: replace.txt +.. highlight:: cpp + +.. heading hierarchy: + ------------- Chapter + ************* Section (#.#) + ============= Subsection (#.#.#) + ############# Paragraph (no number) + +The ``fd-net-device`` module provides the ``NetmapNetDevice`` class, a class derived +from the ``FdNetDevice`` which is able to read and write traffic using a netmap file descriptor. +This netmap file descriptor must be associated to a real ethernet device in the host machine. +The ``NetmapNetDeviceHelper`` class supports the configuration of a ``NetmapNetDevice``. + +netmap is a fast packets processing that bypasses the host networking stack and gains +direct access to network device. + +Model Description +***************** + + + +Design +====== + + + +Scope and Limitations +===================== +The main scope of ``NetmapNetDevice`` is to support the flow-control between the physical device and +the upper layer and using at best the computational resources to process packets. + + +Usage +***** + + + +Helpers +======= + + + +Attributes +========== + + + +Output +====== + + + +Examples +======== + +Several examples are provided: + +* ``fd-emu-onoff.cc``: This example is aimed at measuring the throughput of the + NetmapNetDevice when using the NetmapNetDeviceHelper to attach the simulated + device to a real device in the host machine. This is achieved by saturating + the channel with TCP or UDP traffic. +* ``fd-emu-ping.cc``: This example uses the NetmapNetDevice to send ICMP + traffic over a real device. +* ``fd-emu-tc.cc``: This example configures a router on a machine with two interfaces + in emulated mode through netmap. The aim is to explore different qdiscs behaviours on the backlog of a + device emulated bottleneck side. +* ``fd-emu-send.cc``: This example builds a node with a device in emulation mode through netmap. + The aim is to measure the maximum tx rate in pps achievable with NetmapNetDevice on a specific machine. + +Note that all the examples run in emulation mode through netmap (with NetmapNetDevice) and raw socket (with FdNetDevice). + +Acknowledgments +*************** +The NetmapNetDevice has been presented and evaluated in + +* Pasquale Imputato, Stefano Avallone, Enhancing the fidelity of network emulation through direct access to device buffers, Journal of Network and Computer Applications, Volume 130, 2019, Pages 63-75, (http://www.sciencedirect.com/science/article/pii/S1084804519300220) diff --git a/src/fd-net-device/examples/fd-emu-onoff.cc b/src/fd-net-device/examples/fd-emu-onoff.cc index dcd4fc87e..d09c66170 100644 --- a/src/fd-net-device/examples/fd-emu-onoff.cc +++ b/src/fd-net-device/examples/fd-emu-onoff.cc @@ -1,6 +1,7 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* * Copyright (c) 2012 University of Washington, 2012 INRIA + * 2017 Università' degli Studi di Napoli Federico II * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -16,6 +17,7 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * * Author: Alina Quereilhac + * Extended by: Pasquale Imputato * */ @@ -28,10 +30,11 @@ // | +----------------+ | | +----------------+ | // | | ns-3 IPv4 | | | | ns-3 IPv4 | | // | +----------------+ | | +----------------+ | -// | | FdNetDevice | | | | FdNetDevice | | +// | | FdNetDevice or | | | | FdNetDevice or | | +// | | NetmapNetDevice| | | | NetmapNetDevice| | // | | 10.1.1.1 | | | | 10.1.1.2 | | // | +----------------+ | | +----------------+ | -// | | raw socket | | | | raw socket | | +// | | fd | | | | fd | | // | +----------------+ | | +----------------+ | // | | eth0 | | | | eth0 | | // +-------+------+-------+ +--------+------+-------+ @@ -41,9 +44,9 @@ // | | // +----------------------------+ // -// This example is aimed at measuring the throughput of the FdNetDevice -// when using the EmuFdNetDeviceHelper. This is achieved by saturating -// the channel with TCP traffic. Then the throughput can be obtained from +// This example is aimed at measuring the throughput of the FdNetDevice (NetmapNetDevice) +// when using the EmuFdNetDeviceHelper (NetmapNetDeviceHelper). This is achieved by saturating +// the channel with TCP or UDP traffic. Then the throughput can be obtained from // the generated .pcap files. // // To run this example you will need two hosts (client & server). @@ -59,13 +62,20 @@ // // both machines: $ sudo ip link set eth0 promisc on // -// 4 - Give root suid to the raw socket creator binary. +// 3' - If you run emulation in netmap mode, you need before to load the netmap.ko module. +// The user is in charge to configure and build netmap separately. +// +// 4 - Give root suid to the raw or netmap socket creator binary. // If the --enable-sudo option was used to configure ns-3 with waf, then the following // step will not be necessary. // // both hosts: $ sudo chown root.root build/src/fd-net-device/ns3-dev-raw-sock-creator // both hosts: $ sudo chmod 4755 build/src/fd-net-device/ns3-dev-raw-sock-creator // +// or (if you run emulation in netmap mode): +// both hosts: $ sudo chown root.root build/src/fd-net-device/ns3-dev-netmap-device-creator +// both hosts: $ sudo chmod 4755 build/src/fd-net-device/ns3-dev-netmap-device-creator +// // 5 - Run the server side: // // server host: $ ./waf --run="fd-emu-onoff --serverMode=1" @@ -95,8 +105,8 @@ int main (int argc, char *argv[]) { uint16_t sinkPort = 8000; - uint32_t packetSize = 10000; // bytes - std::string dataRate ("1000Mb/s"); + uint32_t packetSize = 1400; // bytes + std::string dataRate("1000Mb/s"); bool serverMode = false; std::string deviceName ("eth0"); @@ -105,6 +115,13 @@ main (int argc, char *argv[]) std::string netmask ("255.255.255.0"); std::string macClient ("00:00:00:00:00:01"); std::string macServer ("00:00:00:00:00:02"); + std::string transportProt = "Tcp"; + std::string socketType; +#ifdef HAVE_PACKET_H + std::string emuMode ("raw"); +#else // HAVE_NETMAP_USER_H is true (otherwise this example is not compiled) + std::string emuMode ("netmap"); +#endif CommandLine cmd (__FILE__); cmd.AddValue ("deviceName", "Device name", deviceName); @@ -115,8 +132,19 @@ main (int argc, char *argv[]) cmd.AddValue ("mac-client", "Mac Address for Server Client : 00:00:00:00:00:01", macClient); cmd.AddValue ("mac-server", "Mac Address for Server Default : 00:00:00:00:00:02", macServer); cmd.AddValue ("data-rate", "Data rate defaults to 1000Mb/s", dataRate); + cmd.AddValue ("transportProt", "Transport protocol to use: Tcp, Udp", transportProt); + cmd.AddValue ("emuMode", "Emulation mode in {raw, netmap}", emuMode); cmd.Parse (argc, argv); + if (transportProt.compare ("Tcp") == 0) + { + socketType = "ns3::TcpSocketFactory"; + } + else + { + socketType = "ns3::UdpSocketFactory"; + } + Ipv4Address remoteIp; Ipv4Address localIp; Mac48AddressValue localMac; @@ -144,9 +172,31 @@ main (int argc, char *argv[]) Ptr node = CreateObject (); NS_LOG_INFO ("Create Device"); - EmuFdNetDeviceHelper emu; - emu.SetDeviceName (deviceName); - NetDeviceContainer devices = emu.Install (node); + FdNetDeviceHelper* helper = nullptr; + +#ifdef HAVE_PACKET_H + if (emuMode == "raw") + { + EmuFdNetDeviceHelper* raw = new EmuFdNetDeviceHelper; + raw->SetDeviceName (deviceName); + helper = raw; + } +#endif +#ifdef HAVE_NETMAP_USER_H + if (emuMode == "netmap") + { + NetmapNetDeviceHelper* netmap = new NetmapNetDeviceHelper; + netmap->SetDeviceName (deviceName); + helper = netmap; + } +#endif + + if (helper == nullptr) + { + NS_ABORT_MSG (emuMode << " not supported."); + } + + NetDeviceContainer devices = helper->Install (node); Ptr device = devices.Get (0); device->SetAttribute ("Address", localMac); @@ -193,6 +243,7 @@ main (int argc, char *argv[]) Simulator::Stop (Seconds (61.0)); Simulator::Run (); Simulator::Destroy (); + delete helper; return 0; } diff --git a/src/fd-net-device/examples/fd-emu-ping.cc b/src/fd-net-device/examples/fd-emu-ping.cc index a47eaef86..7714c7cb4 100644 --- a/src/fd-net-device/examples/fd-emu-ping.cc +++ b/src/fd-net-device/examples/fd-emu-ping.cc @@ -1,6 +1,7 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2012 University of Washington, 2012 INRIA + * Copyright (c) 2012 University of Washington, 2012 INRIA + * 2017 Università' degli Studi di Napoli Federico II * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -29,7 +30,8 @@ // | +----------------+ | // | | ns-3 IPv4 | | // | +----------------+ | -// | | FdNetDevice | | +// | | FdNetDevice or | | +// | | NetmapNetDevice| | // |--+----------------+--+ // | | eth0 | | // | +------+ | @@ -46,6 +48,8 @@ // device name in as a command-line argument // 2) The host device must be set to promiscuous mode // (e.g. "sudo ifconfig eth0 promisc") +// 2') If you run emulation in netmap mode, you need before to load the netmap.ko module. +// The user is in charge to configure and build netmap separately. // 3) Be aware that ns-3 will generate a fake mac address, and that in // some enterprise networks, this may be considered bad form to be // sending packets out of your device with "unauthorized" mac addresses @@ -59,12 +63,16 @@ // 'netstat -rn' command and find the IP address of the default gateway // on your host. Search for "Ipv4Address gateway" and replace the string // "1.2.3.4" string with the gateway IP address. -/// 6) Give root suid to the raw socket creator binary. +/// 6) Give root suid to the raw or netmap socket creator binary. // If the --enable-sudo option was used to configure ns-3 with waf, then the following // step will not be necessary. // // $ sudo chown root.root build/src/fd-net-device/ns3-dev-raw-sock-creator // $ sudo chmod 4755 build/src/fd-net-device/ns3-dev-raw-sock-creator +// +// or (if you run emulation in netmap mode): +// $ sudo chown root.root build/src/fd-net-device/ns3-dev-netmap-device-creator +// $ sudo chmod 4755 build/src/fd-net-device/ns3-dev-netmap-device-creator // #include "ns3/abort.h" @@ -92,7 +100,14 @@ main (int argc, char *argv[]) NS_LOG_INFO ("Ping Emulation Example"); std::string deviceName ("eth0"); - std::string remote ("173.194.34.51"); // example.com + std::string remote ("8.8.8.8"); + std::string localAddress ("1.2.3.4"); + std::string localGateway ("1.2.3.4"); +#ifdef HAVE_PACKET_H + std::string emuMode ("raw"); +#else // HAVE_NETMAP_USER_H is true (otherwise this example is not compiled) + std::string emuMode ("netmap"); +#endif // // Allow the user to override any of the defaults at run-time, via @@ -101,10 +116,13 @@ main (int argc, char *argv[]) CommandLine cmd (__FILE__); cmd.AddValue ("deviceName", "Device name", deviceName); cmd.AddValue ("remote", "Remote IP address (dotted decimal only please)", remote); + cmd.AddValue ("localIp", "Local IP address (dotted decimal only please)", localAddress); + cmd.AddValue ("gateway", "Gateway address (dotted decimal only please)", localGateway); + cmd.AddValue ("emuMode", "Emulation mode in {raw, netmap}", emuMode); cmd.Parse (argc, argv); Ipv4Address remoteIp (remote.c_str ()); - Ipv4Address localIp ("1.2.3.4"); + Ipv4Address localIp (localAddress.c_str ()); NS_ABORT_MSG_IF (localIp == "1.2.3.4", "You must change the local IP address before running this example"); Ipv4Mask localMask ("255.255.255.0"); @@ -149,9 +167,32 @@ main (int argc, char *argv[]) // OUI flying around. Be aware. // NS_LOG_INFO ("Create Device"); - EmuFdNetDeviceHelper emu; - emu.SetDeviceName (deviceName); - NetDeviceContainer devices = emu.Install (node); + + FdNetDeviceHelper* helper = nullptr; + +#ifdef HAVE_PACKET_H + if (emuMode == "raw") + { + EmuFdNetDeviceHelper* raw = new EmuFdNetDeviceHelper; + raw->SetDeviceName (deviceName); + helper = raw; + } +#endif +#ifdef HAVE_NETMAP_USER_H + if (emuMode == "netmap") + { + NetmapNetDeviceHelper* netmap = new NetmapNetDeviceHelper; + netmap->SetDeviceName (deviceName); + helper = netmap; + } +#endif + + if (helper == nullptr) + { + NS_ABORT_MSG (emuMode << " not supported."); + } + + NetDeviceContainer devices = helper->Install (node); Ptr device = devices.Get (0); device->SetAttribute ("Address", Mac48AddressValue (Mac48Address::Allocate ())); @@ -188,7 +229,7 @@ main (int argc, char *argv[]) // the default gateway on your host and add it below, replacing the // "1.2.3.4" string. // - Ipv4Address gateway ("1.2.3.4"); + Ipv4Address gateway (localGateway.c_str ()); NS_ABORT_MSG_IF (gateway == "1.2.3.4", "You must change the gateway IP address before running this example"); Ipv4StaticRoutingHelper ipv4RoutingHelper; @@ -207,7 +248,7 @@ main (int argc, char *argv[]) app->SetAttribute ("Verbose", BooleanValue (true) ); node->AddApplication (app); app->SetStartTime (Seconds (1.0)); - app->SetStopTime (Seconds (21.0)); + app->SetStopTime (Seconds (22.0)); // // Give the application a name. This makes life much easier when constructing @@ -223,14 +264,15 @@ main (int argc, char *argv[]) // // Enable a promiscuous pcap trace to see what is coming and going on our device. // - emu.EnablePcap ("emu-ping", device, true); + helper->EnablePcap (emuMode + "-emu-ping", device, true); // // Now, do the actual emulation. // - NS_LOG_INFO ("Run Emulation."); - Simulator::Stop (Seconds (22.0)); + NS_LOG_INFO ("Run Emulation in " << emuMode << " mode."); + Simulator::Stop (Seconds (23.0)); Simulator::Run (); Simulator::Destroy (); + delete helper; NS_LOG_INFO ("Done."); } diff --git a/src/fd-net-device/examples/fd-emu-send.cc b/src/fd-net-device/examples/fd-emu-send.cc new file mode 100644 index 000000000..e90ffbcf3 --- /dev/null +++ b/src/fd-net-device/examples/fd-emu-send.cc @@ -0,0 +1,230 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2017 Universita' degli Studi di Napoli Federico II + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Pasquale Imputato + */ + +/* + * This example builds a node with a device in emulation mode in {raw, netmap}. + * The aim is to measure the maximum tx rate in pps achievable with NetmapNetDevice and FdNetDevice on a specific machine. + * The emulated device must be connected and in promiscuous mode. + * + * If you run emulation in netmap mode, you need before to load the netmap.ko module. + * The user is in charge to configure and build netmap separately. + * + */ + +#include "ns3/abort.h" +#include "ns3/core-module.h" +#include "ns3/internet-module.h" +#include "ns3/network-module.h" +#include "ns3/fd-net-device-module.h" +#include "ns3/internet-apps-module.h" +#include "ns3/ipv4-static-routing-helper.h" +#include "ns3/ipv4-list-routing-helper.h" +#include "ns3/applications-module.h" +#include "ns3/traffic-control-module.h" + +#include +#include + +using namespace ns3; + +NS_LOG_COMPONENT_DEFINE ("NetmapEmulationSendExample"); + +// this function sends a number packets by means the SendFrom method or +// the Write method (depending from the level value) of a FdNetDevice or of a NetmapNetDevice +// (depending from the emulation mode value). + +static void +Send (Ptr dev, int level, std::string emuMode) +{ + + Ptr device = DynamicCast (dev); + + int packets = 10000000; + + Mac48Address sender = Mac48Address("00:00:00:aa:00:01"); + Mac48Address receiver = Mac48Address("ff:ff:ff:ff:ff:ff"); + + int packetsSize = 64; + Ptr packet = Create (packetsSize); + EthernetHeader header; + + ssize_t len = (size_t) packet->GetSize (); + uint8_t *buffer = (uint8_t*)malloc (len); + packet->CopyData (buffer, len); + + int sent = 0; + int failed = 0; + + Ptr ndq = nullptr; + if (emuMode == "netmap") + { + Ptr ndqi = dev->GetObject (); + ndq = ndqi->GetTxQueue (0); + } + + + std::cout << ((level == 0) ? "Writing" : "Sending") << std::endl; + + // period to print the stats + std::chrono::milliseconds period (1000); + + auto t1 = std::chrono::high_resolution_clock::now(); + + while (packets > 0) + { + // in case of netmap emulated device we check for + // available slot in the netmap transmission ring + if (ndq) + { + while (ndq->IsStopped ()) + { + usleep (10); + } + } + + if (level == 1) + { + if (device->SendFrom (packet, sender, receiver, 0) == false) + { + failed++; + } + sent++; + packet->RemoveHeader (header); + } + + if (level == 0) + { + if (device->Write (buffer, len) != len) + { + failed++; + } + sent++; + } + + auto t2 = std::chrono::high_resolution_clock::now(); + + if (t2 - t1 >= period) + { + + // print stats + std::chrono::duration dur = (t2 - t1); // in ms + double estimatedThr = ((sent - failed) * packetsSize * 8) / 1000000; // in Mbps + + std::cout << sent << " packets sent in "<< dur.count () << " ms, failed " << failed <<" (" << estimatedThr << " Mbps estimanted throughput)" << std::endl ; + + sent = 0; + failed = 0; + t1 = std::chrono::high_resolution_clock::now(); + + } + + packets--; + + } + +} + +int +main (int argc, char *argv[]) +{ + NS_LOG_INFO ("Emu Send Example"); + + std::string deviceName ("eno1"); + int level = 0; + std::string localAddress ("1.2.3.4"); + std::string localGateway ("1.2.3.4"); + +#ifdef HAVE_PACKET_H + std::string emuMode ("raw"); +#else // HAVE_NETMAP_USER_H is true (otherwise this example is not compiled) + std::string emuMode ("netmap"); +#endif + + CommandLine cmd; + cmd.AddValue ("deviceName", "Device name", deviceName); + cmd.AddValue ("level", "Enable send (1) or write (0) level test", level); + cmd.AddValue ("localIp", "Local IP address (dotted decimal only please)", localAddress); + cmd.AddValue ("gateway", "Gateway address (dotted decimal only please)", localGateway); + cmd.AddValue ("emuMode", "Emulation mode in {raw, netmap}", emuMode); + + cmd.Parse (argc, argv); + + Ipv4Address localIp (localAddress.c_str ()); + NS_ABORT_MSG_IF (localIp == "1.2.3.4", "You must change the local IP address before running this example"); + + Ipv4Mask localMask (localGateway.c_str ()); + NS_ABORT_MSG_IF (localMask == "1.2.3.4", "You must change the local mask before running this example"); + + GlobalValue::Bind ("SimulatorImplementationType", StringValue ("ns3::RealtimeSimulatorImpl")); + + GlobalValue::Bind ("ChecksumEnabled", BooleanValue (true)); + + NS_LOG_INFO ("Create Node"); + Ptr node = CreateObject (); + + NS_LOG_INFO ("Create Device"); + + FdNetDeviceHelper* helper = nullptr; + +#ifdef HAVE_PACKET_H + if (emuMode == "raw") + { + EmuFdNetDeviceHelper* raw = new EmuFdNetDeviceHelper; + raw->SetDeviceName (deviceName); + helper = raw; + } +#endif +#ifdef HAVE_NETMAP_USER_H + if (emuMode == "netmap") + { + NetmapNetDeviceHelper* netmap = new NetmapNetDeviceHelper; + netmap->SetDeviceName (deviceName); + helper = netmap; + } +#endif + + if (helper == nullptr) + { + NS_ABORT_MSG (emuMode << " not supported."); + } + + NetDeviceContainer devices = helper->Install (node); + Ptr device = devices.Get (0); + device->SetAttribute ("Address", Mac48AddressValue (Mac48Address::Allocate ())); + + + // NS_LOG_INFO ("Add Internet Stack"); + // InternetStackHelper internetStackHelper; + // internetStackHelper.Install (node); + + + // TrafficControlHelper tch; + // tch.SetRootQueueDisc ("ns3::PfifoFastQueueDisc"); + // tch.Install (devices); + + Simulator::Schedule (Seconds (3), &Send, device, level, emuMode); + + NS_LOG_INFO ("Run Emulation."); + Simulator::Stop (Seconds (6.0)); + Simulator::Run (); + Simulator::Destroy (); + delete helper; + NS_LOG_INFO ("Done."); +} diff --git a/src/fd-net-device/examples/fd-emu-tc.cc b/src/fd-net-device/examples/fd-emu-tc.cc new file mode 100644 index 000000000..ae1559b80 --- /dev/null +++ b/src/fd-net-device/examples/fd-emu-tc.cc @@ -0,0 +1,313 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2017 Universita' degli Studi di Napoli Federico II + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Pasquale Imputato + */ + +/* + * node + * -------------------------------------------------- + * | | + * | pfifo_fast queueDiscType [pfifo_fast]| + * | bql [false] | + * | interface 0 interface 1 | + * | | | | + * -------------------------------------------------- + * | | + * 1 Gbps access incoming link | | 100 Mbps bottleneck outgoing link + * ----------------------------------- ----------------------------------- + * + * This example builds a node with two interfaces in emulation mode in {raw, netmap}. + * The aim is to explore different qdiscs behaviours on the backlog of a device emulated bottleneck side. + * + * If you run emulation in netmap mode, you need before to load the netmap.ko module. + * The user is in charge to configure and build netmap separately. + * + */ + +#include "ns3/abort.h" +#include "ns3/core-module.h" +#include "ns3/internet-module.h" +#include "ns3/network-module.h" +#include "ns3/fd-net-device-module.h" +#include "ns3/internet-apps-module.h" +#include "ns3/ipv4-list-routing-helper.h" +#include "ns3/traffic-control-module.h" + +using namespace ns3; + +NS_LOG_COMPONENT_DEFINE ("TrafficControlEmu"); + +void +TcPacketsInQueue (Ptr q, Ptr stream) +{ + Simulator::Schedule (Seconds (0.001), &TcPacketsInQueue, q, stream); + *stream->GetStream () << Simulator::Now ().GetSeconds () << " backlog " << q->GetNPackets () << "p " << q->GetNBytes () << "b "<< " dropped " + << q->GetStats ().nTotalDroppedPackets << "p " << q->GetStats ().nTotalDroppedBytes << "b " << std::endl; +} + +#ifdef HAVE_NETMAP_USER_H +void +Inflight (Ptr dev, Ptr stream) +{ + Simulator::Schedule (Seconds (0.001), &Inflight, dev, stream); + *stream->GetStream () << dev->GetBytesInNetmapTxRing () << std::endl; +} +#endif + +int +main (int argc, char *argv[]) +{ + NS_LOG_INFO ("Traffic Control Emulation Example"); + + std::string deviceName0 ("enx503f56005a2a"); + std::string deviceName1 ("eno1"); + std::string ip0 ("10.0.1.2"); + std::string ip1 ("10.0.2.1"); + std::string mask0 ("255.255.255.0"); + std::string mask1 ("255.255.255.0"); + std::string m0 ("00:00:00:aa:00:01"); + std::string m1 ("00:00:00:aa:00:02"); + std::string queueDiscType = "PfifoFast"; + bool bql = false; + + uint32_t index = 0; + bool writer = true; +#ifdef HAVE_PACKET_H + std::string emuMode ("raw"); +#else // HAVE_NETMAP_USER_H is true (otherwise this example is not compiled) + std::string emuMode ("netmap"); +#endif + + CommandLine cmd; + cmd.AddValue ("deviceName0", "Device name", deviceName0); + cmd.AddValue ("deviceName1", "Device name", deviceName1); + cmd.AddValue ("ip0", "Local IP address", ip0); + cmd.AddValue ("ip1", "Local IP address", ip1); + cmd.AddValue ("mask0", "Local mask", mask0); + cmd.AddValue ("mask1", "Local mask", mask1); + cmd.AddValue ("m0", "Mac address", m0); + cmd.AddValue ("m1", "Mac address", m1); + cmd.AddValue ("writer", "Enable write stats", writer); + cmd.AddValue ("queueDiscType", "Bottleneck queue disc type in {PfifoFast, ARED, CoDel, FqCoDel, PIE}", queueDiscType); + cmd.AddValue ("bql", "Enable byte queue limits on bottleneck netdevice", bql); + cmd.AddValue ("index", "Experiment index", index); + cmd.AddValue ("emuMode", "Emulation mode in {raw, netmap}", emuMode); + cmd.Parse (argc, argv); + + Ipv4Address localIp0 (ip0.c_str ()); + Ipv4Address localIp1 (ip1.c_str ()); + Ipv4Mask netmask0 (mask0.c_str ()); + Ipv4Mask netmask1 (mask1.c_str ()); + Mac48AddressValue mac0 (m0.c_str ()); + Mac48AddressValue mac1 (m1.c_str ()); + + // + // Since we are using a real piece of hardware we need to use the realtime + // simulator. + // + GlobalValue::Bind ("SimulatorImplementationType", StringValue ("ns3::RealtimeSimulatorImpl")); + + // + // Since we are going to be talking to real-world machines, we need to enable + // calculation of checksums in our protocols. + // + GlobalValue::Bind ("ChecksumEnabled", BooleanValue (true)); + + // + // In such a simple topology, the use of the helper API can be a hindrance + // so we drop down into the low level API and do it manually. + // + // First we need a single node. + // + NS_LOG_INFO ("Create Node"); + Ptr node = CreateObject (); + + // + // Create an emu device, allocate a MAC address and point the device to the + // Linux device name. The device needs a transmit queueing discipline so + // create a droptail queue and give it to the device. Finally, "install" + // the device into the node. + // + // Do understand that the ns-3 allocated MAC address will be sent out over + // your network since the emu net device will spoof it. By default, this + // address will have an Organizationally Unique Identifier (OUI) of zero. + // The Internet Assigned Number Authority IANA + // + // http://www.iana.org/assignments/ethernet-numberslocalIp + // + // reports that this OUI is unassigned, and so should not conflict with + // real hardware on your net. It may raise all kinds of red flags in a + // real environment to have packets from a device with an obviously bogus + // OUI flying around. Be aware. + // + + NS_LOG_INFO ("Create Devices"); + + FdNetDeviceHelper *helper0 = nullptr; + FdNetDeviceHelper *helper1 = nullptr; + +#ifdef HAVE_PACKET_H + if (emuMode == "raw") + { + EmuFdNetDeviceHelper *raw0 = new EmuFdNetDeviceHelper; + raw0->SetDeviceName (deviceName0); + helper0 = raw0; + + EmuFdNetDeviceHelper *raw1 = new EmuFdNetDeviceHelper; + raw1->SetDeviceName (deviceName1); + helper1 = raw1; + } +#endif +#ifdef HAVE_NETMAP_USER_H + if (emuMode == "netmap") + { + NetmapNetDeviceHelper *netmap0 = new NetmapNetDeviceHelper; + netmap0->SetDeviceName (deviceName0); + helper0 = netmap0; + + NetmapNetDeviceHelper *netmap1 = new NetmapNetDeviceHelper; + netmap1->SetDeviceName (deviceName1); + helper1 = netmap1; + } +#endif + + if ((helper0 == nullptr) || (helper1 == nullptr)) + { + NS_ABORT_MSG (emuMode << " not supported."); + } + + NetDeviceContainer devices0 = helper0->Install (node); + NetDeviceContainer devices1 = helper1->Install (node); + + Ptr device0 = devices0.Get (0); + device0->SetAttribute ("Address", mac0); + + Ptr device1 = devices1.Get (0); + device1->SetAttribute ("Address", mac1); + + // + // Add a default internet stack to the node. This gets us the ns-3 versions + // of ARP, IPv4, ICMP, UDP and TCP. + // + NS_LOG_INFO ("Add Internet Stack"); + InternetStackHelper internetStackHelper; + internetStackHelper.Install (node); + + Ptr ipv4 = node->GetObject (); + + NS_LOG_INFO ("Create IPv4 Interface 0"); + uint32_t interface0 = ipv4->AddInterface (device0); + Ipv4InterfaceAddress address0 = Ipv4InterfaceAddress (localIp0, netmask0); + ipv4->AddAddress (interface0, address0); + ipv4->SetMetric (interface0, 0); + ipv4->SetForwarding (interface0, true); + ipv4->SetUp (interface0); + + NS_LOG_INFO ("Create IPv4 Interface 1"); + uint32_t interface1 = ipv4->AddInterface (device1); + Ipv4InterfaceAddress address1 = Ipv4InterfaceAddress (localIp1, netmask1); + ipv4->AddAddress (interface1, address1); + ipv4->SetMetric (interface1, 0); + ipv4->SetForwarding (interface1, true); + ipv4->SetUp (interface1); + + Ipv4GlobalRoutingHelper::PopulateRoutingTables (); + + // Traffic control configurations + // Access link side + TrafficControlHelper tch0; + tch0.SetRootQueueDisc ("ns3::PfifoFastQueueDisc", "MaxSize", StringValue ("1000p")); + tch0.Install (devices0); + + // Bottleneck link side + TrafficControlHelper tch1; + + if (queueDiscType.compare ("PfifoFast") == 0) + { + tch1.SetRootQueueDisc ("ns3::PfifoFastQueueDisc", "MaxSize", StringValue ("1000p")); + } + else if (queueDiscType.compare ("ARED") == 0) + { + tch1.SetRootQueueDisc ("ns3::RedQueueDisc"); + Config::SetDefault ("ns3::RedQueueDisc::ARED", BooleanValue (true)); + Config::SetDefault ("ns3::RedQueueDisc::MaxSize", + QueueSizeValue (QueueSize (QueueSizeUnit::BYTES, 1000000))); + Config::SetDefault ("ns3::RedQueueDisc::MeanPktSize", UintegerValue (1000)); + Config::SetDefault ("ns3::RedQueueDisc::LinkBandwidth", StringValue ("100Mbps")); + Config::SetDefault ("ns3::RedQueueDisc::MinTh", DoubleValue (83333)); + Config::SetDefault ("ns3::RedQueueDisc::MinTh", DoubleValue (250000)); + } + else if (queueDiscType.compare ("CoDel") == 0) + { + tch1.SetRootQueueDisc ("ns3::CoDelQueueDisc"); + } + else if (queueDiscType.compare ("FqCoDel") == 0) + { + tch1.SetRootQueueDisc ("ns3::FqCoDelQueueDisc"); + } + else if (queueDiscType.compare ("PIE") == 0) + { + tch1.SetRootQueueDisc ("ns3::PieQueueDisc"); + Config::SetDefault ("ns3::PieQueueDisc::MaxSize", + QueueSizeValue (QueueSize (QueueSizeUnit::PACKETS, 1000))); + Config::SetDefault ("ns3::PieQueueDisc::QueueDelayReference", TimeValue (Seconds (0.02))); + Config::SetDefault ("ns3::PieQueueDisc::Tupdate", TimeValue (Seconds (0.032))); + Config::SetDefault ("ns3::PieQueueDisc::A", DoubleValue (2)); + Config::SetDefault ("ns3::PieQueueDisc::B", DoubleValue (20)); + } + else + { + NS_ABORT_MSG ("--queueDiscType not valid"); + } + + if (bql) + { + tch1.SetQueueLimits ("ns3::DynamicQueueLimits"); + } + + QueueDiscContainer qdiscs = tch1.Install (devices1); + Ptr q = qdiscs.Get (0); + + if (writer) + { + AsciiTraceHelper ascii; + Ptr stream = ascii.CreateFileStream ("exp-" + std::to_string (index) + "-router-tc-stats.txt"); + Simulator::Schedule (Seconds (0.001), &TcPacketsInQueue, q, stream); + +#ifdef HAVE_NETMAP_USER_H + if (emuMode.compare ("netmap") == 0) + { + Ptr dev = StaticCast (device1); + Ptr stream2 = ascii.CreateFileStream ("exp-" + std::to_string (index) + "-router-inflight.txt"); + Simulator::Schedule (Seconds (0.001), &Inflight, dev, stream2); + } +#endif + + Ipv4GlobalRoutingHelper g; + Ptr routingStream = Create ("router-routes.txt", std::ios::out); + g.PrintRoutingTableAllAt (Seconds (3), routingStream); + } + + NS_LOG_INFO ("Run Emulation."); + Simulator::Stop (Seconds (50.0)); + Simulator::Run (); + Simulator::Destroy (); + delete helper0; + delete helper1; + NS_LOG_INFO ("Done."); +} diff --git a/src/fd-net-device/examples/wscript b/src/fd-net-device/examples/wscript index f606c1429..7439199d0 100644 --- a/src/fd-net-device/examples/wscript +++ b/src/fd-net-device/examples/wscript @@ -16,13 +16,19 @@ def build(bld): obj = bld.create_ns3_program('realtime-fd2fd-onoff', ['fd-net-device', 'internet', 'applications']) obj.source = 'realtime-fd2fd-onoff.cc' - if bld.env['ENABLE_TAP']: + if bld.env['ENABLE_EMU' or 'ENABLE_NETMAP_EMU']: obj = bld.create_ns3_program('fd-emu-ping', ['fd-net-device', 'internet', 'internet-apps']) obj.source = 'fd-emu-ping.cc' - obj = bld.create_ns3_program('fd-emu-udp-echo', ['fd-net-device', 'internet', 'applications']) - obj.source = 'fd-emu-udp-echo.cc' obj = bld.create_ns3_program('fd-emu-onoff', ['fd-net-device', 'internet', 'applications']) obj.source = 'fd-emu-onoff.cc' + obj = bld.create_ns3_program('fd-emu-tc', ['fd-net-device', 'internet', 'internet-apps', 'applications', 'traffic-control']) + obj.source = 'fd-emu-tc.cc' + obj = bld.create_ns3_program('fd-emu-send', ['fd-net-device', 'internet', 'internet-apps', 'applications', 'traffic-control']) + obj.source = 'fd-emu-send.cc' + + if bld.env['ENABLE_EMU']: + obj = bld.create_ns3_program('fd-emu-udp-echo', ['fd-net-device', 'internet', 'applications']) + obj.source = 'fd-emu-udp-echo.cc' if bld.env['ENABLE_DPDKNETDEV']: obj = bld.create_ns3_program('fd-dpdk-emu-ping', ['fd-net-device', 'internet', 'internet-apps', 'applications']) @@ -39,4 +45,3 @@ def build(bld): if bld.env['ENABLE_PLANETLAB']: obj = bld.create_ns3_program('fd-planetlab-ping', ['fd-net-device', 'internet', 'internet-apps']) obj.source = 'fd-planetlab-ping.cc' - diff --git a/src/fd-net-device/helper/emu-fd-net-device-helper.cc b/src/fd-net-device/helper/emu-fd-net-device-helper.cc index 9c8dbf3e2..ee256b020 100644 --- a/src/fd-net-device/helper/emu-fd-net-device-helper.cc +++ b/src/fd-net-device/helper/emu-fd-net-device-helper.cc @@ -67,6 +67,7 @@ EmuFdNetDeviceHelper::EmuFdNetDeviceHelper () { m_deviceName = "undefined"; m_dpdkMode = false; + m_hostQdiscBypass = false; } void @@ -92,6 +93,12 @@ EmuFdNetDeviceHelper::SetDpdkMode (int argc, char **argv) #endif } +void +EmuFdNetDeviceHelper::HostQdiscBypass (bool hostQdiscBypass) +{ + m_hostQdiscBypass = hostQdiscBypass; +} + std::string EmuFdNetDeviceHelper::GetDeviceName (void) { @@ -172,6 +179,18 @@ EmuFdNetDeviceHelper::SetFileDescriptor (Ptr device) const NS_FATAL_ERROR ("EmuFdNetDeviceHelper::SetFileDescriptor (): Can't get interface flags"); } + if (m_hostQdiscBypass) + { + static const int32_t sock_qdisc_bypass = 1; + int32_t sock_qdisc_ret = setsockopt (fd, SOL_PACKET, PACKET_QDISC_BYPASS, &sock_qdisc_bypass, + sizeof (sock_qdisc_bypass)); + + if (sock_qdisc_ret == -1) + { + NS_LOG_ERROR ("Cannot use the qdisc bypass option"); + } + } + // // This device only works if the underlying interface is up in promiscuous // mode. We could have turned it on in the socket creator, but the situation diff --git a/src/fd-net-device/helper/emu-fd-net-device-helper.h b/src/fd-net-device/helper/emu-fd-net-device-helper.h index 67d9eb923..00b6591f4 100644 --- a/src/fd-net-device/helper/emu-fd-net-device-helper.h +++ b/src/fd-net-device/helper/emu-fd-net-device-helper.h @@ -70,6 +70,12 @@ public: */ void SetDpdkMode (int argc, char **argv); + /** + * \brief Request host qdisc bypass + * \param hostQdiscBypass to enable host qdisc bypass + */ + void HostQdiscBypass (bool hostQdiscBypass); + protected: /** * This method creates an ns3::FdNetDevice attached to a physical network @@ -111,6 +117,8 @@ protected: * Dpdk EAL arguements list. */ char **m_ealArgv; + + bool m_hostQdiscBypass; }; } // namespace ns3 diff --git a/src/fd-net-device/helper/fd-net-device-helper.cc b/src/fd-net-device/helper/fd-net-device-helper.cc index f71857ea1..198a7af73 100644 --- a/src/fd-net-device/helper/fd-net-device-helper.cc +++ b/src/fd-net-device/helper/fd-net-device-helper.cc @@ -42,6 +42,12 @@ FdNetDeviceHelper::FdNetDeviceHelper () m_deviceFactory.SetTypeId ("ns3::FdNetDevice"); } +void +FdNetDeviceHelper::SetTypeId (std::string type) +{ + m_deviceFactory.SetTypeId (type); +} + void FdNetDeviceHelper::SetAttribute (std::string n1, const AttributeValue &v1) { diff --git a/src/fd-net-device/helper/netmap-device-creator.cc b/src/fd-net-device/helper/netmap-device-creator.cc new file mode 100644 index 000000000..8f99bfe54 --- /dev/null +++ b/src/fd-net-device/helper/netmap-device-creator.cc @@ -0,0 +1,95 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) University of Washington + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "creator-utils.h" + +#define EMU_MAGIC 65867 + +using namespace ns3; + +int +main (int argc, char *argv[]) +{ + int c; + char *path = NULL; + + opterr = 0; + + while ((c = getopt (argc, argv, "vp:")) != -1) + { + switch (c) + { + case 'v': + gVerbose = true; + break; + case 'p': + path = optarg; + break; + } + } + + // + // This program is spawned by an emu net device running in a simulation. It + // wants to create a raw socket as described below. We are going to do the + // work here since we're running suid root. Once we create the raw socket, + // we have to send it back to the emu net device. We do that over a Unix + // (local interprocess) socket. The emu net device created a socket to + // listen for our response on, and it is expected to have encoded the address + // information as a string and to have passed that string as an argument to + // us. We see it here as the "path" string. We can't do anything useful + // unless we have that string. + // + ABORT_IF (path == NULL, "path is a required argument", 0); + LOG ("Provided path is \"" << path << "\""); + // + // The whole reason for all of the hoops we went through to call out to this + // program will pay off here. We created this program to run as suid root + // in order to keep the main simulation program from having to be run with + // root privileges. We need root privileges to be able to open a raw socket + // though. So all of these hoops are to allow us to execute the following + // single line of code: + // + LOG ("Creating netmap fd"); + int sock = open ("/dev/netmap", O_RDWR); + ABORT_IF (sock == -1, "CreateSocket(): Unable to open netmap fd", 1); + + // + // Send the socket back to the emu net device so it can go about its business + // + SendSocket (path, sock, EMU_MAGIC); + + return 0; +} + diff --git a/src/fd-net-device/helper/netmap-net-device-helper.cc b/src/fd-net-device/helper/netmap-net-device-helper.cc new file mode 100644 index 000000000..62a63126e --- /dev/null +++ b/src/fd-net-device/helper/netmap-net-device-helper.cc @@ -0,0 +1,477 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2017 Universita' degli Studi di Napoli Federico II + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Pasquale Imputato + */ + +#include "netmap-net-device-helper.h" +#include "encode-decode.h" + +#include "ns3/abort.h" +#include "ns3/config.h" +#include "ns3/fd-net-device.h" +#include "ns3/log.h" +#include "ns3/names.h" +#include "ns3/object-factory.h" +#include "ns3/packet.h" +#include "ns3/simulator.h" +#include "ns3/trace-helper.h" +#include "ns3/netmap-net-device.h" +#include "ns3/uinteger.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace ns3 { + +NS_LOG_COMPONENT_DEFINE ("NetmapNetDeviceHelper"); + +#define EMU_MAGIC 65867 + +NetmapNetDeviceHelper::NetmapNetDeviceHelper () +{ + m_deviceName = "undefined"; + SetTypeId ("ns3::NetmapNetDevice"); +} + +std::string +NetmapNetDeviceHelper::GetDeviceName (void) +{ + return m_deviceName; +} + +void +NetmapNetDeviceHelper::SetDeviceName (std::string deviceName) +{ + m_deviceName = deviceName; +} + +Ptr +NetmapNetDeviceHelper::InstallPriv (Ptr node) const +{ + Ptr d = FdNetDeviceHelper::InstallPriv (node); + Ptr device = d->GetObject (); + + SetDeviceAttributes (device); + + int fd = CreateFileDescriptor (); + Ptr netmapDevice = DynamicCast (device); + SwitchInNetmapMode (fd, netmapDevice); + + // Aggregate NetDeviceQueueInterface object + Ptr ndqi = CreateObjectWithAttributes ( + "TxQueuesType", TypeIdValue (NetDeviceQueueLock::GetTypeId ()), + "NTxQueues", UintegerValue (1)); + + device->AggregateObject (ndqi); + netmapDevice->SetNetDeviceQueue (ndqi->GetTxQueue (0)); + + return device; +} + +void +NetmapNetDeviceHelper::SetDeviceAttributes (Ptr device) const +{ + + if (m_deviceName == "undefined") + { + NS_FATAL_ERROR ("NetmapNetDeviceHelper::SetFileDescriptor (): m_deviceName is not set"); + } + + // + // Call out to a separate process running as suid root in order to get a raw + // socket. We do this to avoid having the entire simulation running as root. + // + int fd = socket (PF_INET, SOCK_DGRAM, 0); + + // + // Figure out which interface index corresponds to the device name in the corresponding attribute. + // + struct ifreq ifr; + bzero (&ifr, sizeof(ifr)); + strncpy ((char *)ifr.ifr_name, m_deviceName.c_str (), IFNAMSIZ - 1); + + NS_LOG_LOGIC ("Getting interface index"); + int32_t rc = ioctl (fd, SIOCGIFINDEX, &ifr); + if (rc == -1) + { + NS_FATAL_ERROR ("NetmapNetDeviceHelper::SetFileDescriptor (): Can't get interface index"); + } + + rc = ioctl (fd, SIOCGIFFLAGS, &ifr); + if (rc == -1) + { + NS_FATAL_ERROR ("NetmapNetDeviceHelper::SetFileDescriptor (): Can't get interface flags"); + } + + // + // This device only works if the underlying interface is up in promiscuous + // mode. We could have turned it on in the socket creator, but the situation + // is that we expect these devices to be used in conjunction with virtual + // machines with connected host-only (simulated) networks, or in a testbed. + // There is a lot of setup and configuration happening outside of this one + // issue, and we expect that configuration to include choosing a valid + // interface (e.g, "ath1"), ensuring that the device supports promiscuous + // mode, and placing it in promiscuous mode. We just make sure of the + // end result. + // + if ((ifr.ifr_flags & IFF_PROMISC) == 0) + { + NS_FATAL_ERROR ("NetmapNetDeviceHelper::SetFileDescriptor (): " + << m_deviceName.c_str () + << " is not in promiscuous mode. Please config the interface in promiscuous " + "mode before to run the simulation."); + } + + if ((ifr.ifr_flags & IFF_BROADCAST) != IFF_BROADCAST) + { + // We default m_isBroadcast to true but turn it off here if not + // supported, because in the common case, overlying IP code will + // assert during configuration time if this is false, before this + // method has a chance to set it during runtime + device->SetIsBroadcast (false); + } + + if ((ifr.ifr_flags & IFF_MULTICAST) == IFF_MULTICAST) + { + // This one is OK to enable at runtime + device->SetIsMulticast (true); + } + + // Set the MTU of the device to the mtu of the associated network interface +// struct ifreq ifr2; +// +// bzero (&ifr2, sizeof (ifr2)); +// strcpy (ifr2.ifr_name, m_deviceName.c_str ()); + +// int32_t mtufd = socket (PF_INET, SOCK_DGRAM, IPPROTO_IP); + + rc = ioctl (fd, SIOCGIFMTU, &ifr); + if (rc == -1) + { + NS_FATAL_ERROR ("FdNetDevice::SetFileDescriptor (): Can't ioctl SIOCGIFMTU"); + } + + NS_LOG_DEBUG ("Device MTU " << ifr.ifr_mtu); + device->SetMtu (ifr.ifr_mtu); + + close (fd); + +} + +int +NetmapNetDeviceHelper::CreateFileDescriptor (void) const +{ + NS_LOG_FUNCTION (this); + + // + // We want to create a raw socket for our net device. Unfortunately for us + // you have to have root privileges to do that. Instead of running the + // entire simulation as root, we decided to make a small program who's whole + // reason for being is to run as suid root and create a raw socket. We're + // going to fork and exec that program soon, but we need to have a socket + // to talk to it with. So we create a local interprocess (Unix) socket + // for that purpose. + // + int sock = socket (PF_UNIX, SOCK_DGRAM, 0); + if (sock == -1) + { + NS_FATAL_ERROR ("NetmapNetDeviceHelper::CreateFileDescriptor(): Unix socket creation error, errno = " << strerror (errno)); + } + + // + // Bind to that socket and let the kernel allocate an endpoint + // + struct sockaddr_un un; + memset (&un, 0, sizeof (un)); + un.sun_family = AF_UNIX; + int status = bind (sock, (struct sockaddr*)&un, sizeof (sa_family_t)); + if (status == -1) + { + NS_FATAL_ERROR ("NetmapNetDeviceHelper::CreateFileDescriptor(): Could not bind(): errno = " << strerror (errno)); + } + + NS_LOG_INFO ("Created Unix socket"); + NS_LOG_INFO ("sun_family = " << un.sun_family); + NS_LOG_INFO ("sun_path = " << un.sun_path); + + // + // We have a socket here, but we want to get it there -- to the program we're + // going to exec. What we'll do is to do a getsockname and then encode the + // resulting address information as a string, and then send the string to the + // program as an argument. So we need to get the sock name. + // + socklen_t len = sizeof (un); + status = getsockname (sock, (struct sockaddr*)&un, &len); + if (status == -1) + { + NS_FATAL_ERROR ("NetmapNetDeviceHelper::CreateFileDescriptor(): Could not getsockname(): errno = " << strerror (errno)); + } + + // + // Now encode that socket name (family and path) as a string of hex digits + // + std::string path = BufferToString ((uint8_t *)&un, len); + NS_LOG_INFO ("Encoded Unix socket as \"" << path << "\""); + // + // Fork and exec the process to create our socket. If we're us (the parent) + // we wait for the child (the socket creator) to complete and read the + // socket it created using the ancillary data mechanism. + // + // Tom Goff reports the possiblility of a deadlock when trying to acquire the + // python GIL here. He says that this might be due to trying to access Python + // objects after fork() without calling PyOS_AfterFork() to properly reset + // Python state (including the GIL). There is no code to cause the problem + // here in emu, but this was visible in similar code in tap-bridge. + // + pid_t pid = ::fork (); + if (pid == 0) + { + NS_LOG_DEBUG ("Child process"); + + // + // build a command line argument from the encoded endpoint string that + // the socket creation process will use to figure out how to respond to + // the (now) parent process. + // + std::ostringstream oss; + + oss << "-p" << path; + + + + NS_LOG_INFO ("Parameters set to \"" << oss.str () << "\""); + + // + // Execute the socket creation process image. + // + status = ::execlp (NETMAP_DEV_CREATOR, + NETMAP_DEV_CREATOR, // argv[0] (filename) + oss.str ().c_str (), // argv[1] (-pcmsg_level == SOL_SOCKET + && cmsg->cmsg_type == SCM_RIGHTS) + { + // + // This is the type of message we want. Check to see if the magic + // number is correct and then pull out the socket we care about if + // it matches + // + if (magic == EMU_MAGIC) + { + NS_LOG_INFO ("Got SCM_RIGHTS with correct magic " << magic); + int *rawSocket = (int*)CMSG_DATA (cmsg); + NS_LOG_INFO ("Got the socket from the socket creator = " << *rawSocket); + return *rawSocket; + } + else + { + NS_LOG_INFO ("Got SCM_RIGHTS, but with bad magic " << magic); + } + } + } + NS_FATAL_ERROR ("Did not get the raw socket from the socket creator"); + } +} + +void +NetmapNetDeviceHelper::SwitchInNetmapMode (int fd, Ptr device) const +{ + NS_LOG_FUNCTION (this << fd << device); + NS_ASSERT (device); + + if (m_deviceName == "undefined") + { + NS_FATAL_ERROR ("NetmapNetDevice: m_deviceName is not set"); + } + + if (fd == -1) + { + NS_FATAL_ERROR ("NetmapNetDevice: fd is not set"); + } + + struct nmreq nmr; + memset (&nmr, 0, sizeof (nmr)); + + nmr.nr_version = NETMAP_API; + + // setting the interface name in the netmap request + strncpy (nmr.nr_name, m_deviceName.c_str (), m_deviceName.length ()); + + // switch the interface in netmap mode + int code = ioctl (fd, NIOCREGIF, &nmr); + if (code == -1) + { + NS_FATAL_ERROR ("Switching failed"); + } + + // memory mapping + uint8_t *memory = (uint8_t *) mmap (0, nmr.nr_memsize, PROT_WRITE | PROT_READ, + MAP_SHARED, fd, 0); + + if (memory == MAP_FAILED) + { + NS_FATAL_ERROR ("Memory mapping failed"); + } + + // getting the base struct of the interface in netmap mode + struct netmap_if *nifp = NETMAP_IF (memory, nmr.nr_offset); + + if (!nifp) + { + NS_FATAL_ERROR ("Failed getting the base struct of the interface in netmap mode"); + } + + device->SetNetmapInterfaceRepresentation (nifp); + device->SetTxRingsInfo (nifp->ni_tx_rings, nmr.nr_tx_slots); + device->SetRxRingsInfo (nifp->ni_rx_rings, nmr.nr_rx_slots); + + device->SetFileDescriptor (fd); + +} + +} // namespace ns3 diff --git a/src/fd-net-device/helper/netmap-net-device-helper.h b/src/fd-net-device/helper/netmap-net-device-helper.h new file mode 100644 index 000000000..ba5a8eac8 --- /dev/null +++ b/src/fd-net-device/helper/netmap-net-device-helper.h @@ -0,0 +1,100 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2017 Universita' degli Studi di Napoli Federico II + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Pasquale Imputato + */ + +#ifndef NETMAP_NET_DEVICE_HELPER_H +#define NETMAP_NET_DEVICE_HELPER_H + +#include + +#include "ns3/attribute.h" +#include "ns3/fd-net-device.h" +#include "ns3/fd-net-device-helper.h" +#include "ns3/object-factory.h" +#include "ns3/net-device-container.h" +#include "ns3/node-container.h" +#include "ns3/netmap-net-device.h" + +namespace ns3 { + +class NetmapNetDevice; + +/** + * \ingroup fd-net-device + * \brief build a set of FdNetDevice objects attached to a physical network + * interface + * + */ +class NetmapNetDeviceHelper : public FdNetDeviceHelper +{ +public: + + NetmapNetDeviceHelper (); + virtual ~NetmapNetDeviceHelper () + { + } + + /** + * \brief Get the device name of this device. + * \returns The device name of this device. + */ + std::string GetDeviceName (void); + + /** + * \brief Set the device name of this device. + * \param deviceName The device name of this device. + */ + void SetDeviceName (std::string deviceName); + +protected: + + /** + * \brief This method creates an ns3::FdNetDevice attached to a physical network + * interface + * \param node The node to install the device in + * \returns A container holding the added net device. + */ + Ptr InstallPriv (Ptr node) const; + + /** + * \brief Sets device flags and MTU. + * \param device the FdNetDevice + */ + virtual void SetDeviceAttributes (Ptr device) const; + + /** + * \brief Call out to a separate process running as suid root in order to get a raw + * socket. We do this to avoid having the entire simulation running as root. + * \return the rawSocket number + */ + virtual int CreateFileDescriptor (void) const; + + /** + * \brief Switch the fd in netmap mode. + * \param fd the file descriptor + * \param device the NetmapNetDevice + */ + void SwitchInNetmapMode (int fd, Ptr device) const; + + std::string m_deviceName; //!< The unix/linux name of the underlying device (e.g., eth0) +}; + +} // namespace ns3 + +#endif /* NETMAP_NET_DEVICE_HELPER_H */ diff --git a/src/fd-net-device/model/fd-net-device.cc b/src/fd-net-device/model/fd-net-device.cc index 98acf9342..40c926f7f 100644 --- a/src/fd-net-device/model/fd-net-device.cc +++ b/src/fd-net-device/model/fd-net-device.cc @@ -247,14 +247,37 @@ FdNetDevice::StartDevice (void) // m_nodeId = GetNode ()->GetId (); - m_fdReader = Create (); - // 22 bytes covers 14 bytes Ethernet header with possible 8 bytes LLC/SNAP - m_fdReader->SetBufferSize (m_mtu + 22); + m_fdReader = DoCreateFdReader (); m_fdReader->Start (m_fd, MakeCallback (&FdNetDevice::ReceiveCallback, this)); + DoFinishStartingDevice (); + NotifyLinkUp (); } +Ptr +FdNetDevice::DoCreateFdReader (void) +{ + NS_LOG_FUNCTION (this); + + Ptr fdReader = Create (); + // 22 bytes covers 14 bytes Ethernet header with possible 8 bytes LLC/SNAP + fdReader->SetBufferSize (m_mtu + 22); + return fdReader; +} + +void +FdNetDevice::DoFinishStartingDevice (void) +{ + NS_LOG_FUNCTION (this); +} + +void +FdNetDevice::DoFinishStoppingDevice (void) +{ + NS_LOG_FUNCTION (this); +} + void FdNetDevice::StopDevice (void) { @@ -272,17 +295,7 @@ FdNetDevice::StopDevice (void) m_fd = -1; } - { - CriticalSection cs (m_pendingReadMutex); - - while (!m_pendingQueue.empty ()) - { - std::pair next = m_pendingQueue.front (); - m_pendingQueue.pop (); - - FreeBuffer (next.first); - } - } + DoFinishStoppingDevice (); } void @@ -605,7 +618,8 @@ FdNetDevice::SendFrom (Ptr packet, const Address& src, const Address& de AddPIHeader (buffer, len); } - ssize_t written = Write(buffer, len); + ssize_t written = Write (buffer, len); + free (buffer); if (written == -1 || (size_t) written != len) { @@ -621,9 +635,7 @@ FdNetDevice::Write (uint8_t *buffer, size_t length) { NS_LOG_FUNCTION (this << buffer << length); - uint32_t ret = write (m_fd, buffer, length); - FreeBuffer (buffer); - return ret; + return write (m_fd, buffer, length); } void @@ -635,6 +647,12 @@ FdNetDevice::SetFileDescriptor (int fd) } } +int +FdNetDevice::GetFileDescriptor (void) const +{ + return m_fd; +} + void FdNetDevice::SetAddress (Address address) { diff --git a/src/fd-net-device/model/fd-net-device.h b/src/fd-net-device/model/fd-net-device.h index cee8a6044..6b9e9920b 100644 --- a/src/fd-net-device/model/fd-net-device.h +++ b/src/fd-net-device/model/fd-net-device.h @@ -212,6 +212,11 @@ protected: * Callback to invoke when a new frame is received */ void ReceiveCallback (uint8_t *buf, ssize_t len); + /** + * Get the associated file descriptor. + * \return the associated file descriptor + */ + int GetFileDescriptor (void) const; private: /** @@ -232,6 +237,27 @@ private: */ void StopDevice (void); + /** + * Create the FdReader object + * \return the created FdReader object + */ + virtual Ptr DoCreateFdReader (void); + + /** + * Complete additional actions, if any, to spin up down the device + */ + virtual void DoFinishStartingDevice (void); + + /** + * Complete additional actions, if any, to tear down the device + */ + virtual void DoFinishStoppingDevice (void); + + /** + * Callback to invoke when a new frame is received + */ + void ReceiveCallback (uint8_t *buf, ssize_t len); + /** * Forward the frame to the appropriate callback for processing */ @@ -279,7 +305,7 @@ private: /** * Reader for the file descriptor. */ - Ptr m_fdReader; + Ptr m_fdReader; /** * The net device mac address. diff --git a/src/fd-net-device/model/netmap-net-device.cc b/src/fd-net-device/model/netmap-net-device.cc new file mode 100644 index 000000000..82133748e --- /dev/null +++ b/src/fd-net-device/model/netmap-net-device.cc @@ -0,0 +1,396 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2017 Universita' degli Studi di Napoli Federico II + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Pasquale Imputato + */ + +#include "netmap-net-device.h" +#include "ns3/system-thread.h" +#include +#include + +namespace ns3 { + +NS_LOG_COMPONENT_DEFINE ("NetmapNetDevice"); + +TypeId +NetDeviceQueueLock::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::NetDeviceQueueLock") + .SetParent () + .SetGroupName ("Network") + .AddConstructor (); + return tid; +} + +NetDeviceQueueLock::NetDeviceQueueLock () +{ +} + +NetDeviceQueueLock::~NetDeviceQueueLock () +{ +} + +bool +NetDeviceQueueLock::IsStopped (void) const +{ + m_mutex.lock (); + bool stopped = NetDeviceQueue::IsStopped (); + m_mutex.unlock (); + return stopped; +} + +void +NetDeviceQueueLock::Start (void) +{ + m_mutex.lock (); + NetDeviceQueue::Start (); + m_mutex.unlock (); +} + +void +NetDeviceQueueLock::Stop (void) +{ + m_mutex.lock (); + NetDeviceQueue::Stop (); + m_mutex.unlock (); +} + +void +NetDeviceQueueLock::Wake (void) +{ + m_mutex.lock (); + NetDeviceQueue::Wake (); + m_mutex.unlock (); +} + +void +NetDeviceQueueLock::NotifyQueuedBytes (uint32_t bytes) +{ + m_mutex.lock (); + NetDeviceQueue::NotifyQueuedBytes (bytes); + m_mutex.unlock (); +} + +void +NetDeviceQueueLock::NotifyTransmittedBytes (uint32_t bytes) +{ + m_mutex.lock (); + NetDeviceQueue::NotifyTransmittedBytes (bytes); + m_mutex.unlock (); +} + +NetmapNetDeviceFdReader::NetmapNetDeviceFdReader () + : m_bufferSize (65536), // Defaults to maximum TCP window size + m_nifp (nullptr) +{ +} + +void +NetmapNetDeviceFdReader::SetBufferSize (uint32_t bufferSize) +{ + NS_LOG_FUNCTION (this << bufferSize); + m_bufferSize = bufferSize; +} + +void +NetmapNetDeviceFdReader::SetNetmapIfp (struct netmap_if *nifp) +{ + NS_LOG_FUNCTION (this << nifp); + m_nifp = nifp; +} + +FdReader::Data +NetmapNetDeviceFdReader::DoRead (void) +{ + NS_LOG_FUNCTION (this); + + uint8_t *buf = (uint8_t *) malloc (m_bufferSize); + NS_ABORT_MSG_IF (buf == 0, "malloc() failed"); + + NS_LOG_LOGIC ("Calling read on fd " << m_fd); + + struct netmap_ring *rxring; + uint16_t len = 0; + uint32_t rxRingIndex = 0; + + // we have a packet in one of the receiver rings + // we check for the first non empty receiver ring + while (rxRingIndex < m_nifp->ni_rx_rings) + { + rxring = NETMAP_RXRING (m_nifp, rxRingIndex); + + if (!nm_ring_empty (rxring)) + { + uint32_t i = rxring->cur; + uint8_t *buffer = (uint8_t *) NETMAP_BUF (rxring, rxring->slot[i].buf_idx); + len = rxring->slot[i].len; + NS_LOG_DEBUG ("Received a packet of " << len << " bytes"); + + // copy buffer in the destination memory area + memcpy (buf, buffer, len); + + // advance the netmap pointers and sync the fd + rxring->head = rxring->cur = nm_ring_next (rxring, i); + + ioctl (m_fd, NIOCRXSYNC, NULL); + + break; + } + + rxRingIndex++; + } + + if (len <= 0) + { + free (buf); + buf = 0; + len = 0; + } + NS_LOG_LOGIC ("Read " << len << " bytes on fd " << m_fd); + return FdReader::Data (buf, len); +} + +NS_OBJECT_ENSURE_REGISTERED (NetmapNetDevice); + +TypeId +NetmapNetDevice::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::NetmapNetDevice") + .SetParent () + .SetGroupName ("FdNetDevice") + .AddConstructor (); + return tid; +} + +NetmapNetDevice::NetmapNetDevice () +{ + NS_LOG_FUNCTION (this); + m_nifp = nullptr; + m_nTxRings = 0; + m_nTxRingsSlots = 0; + m_nRxRings = 0; + m_nRxRingsSlots = 0; + m_queue = nullptr; + m_totalQueuedBytes = 0; + m_syncAndNotifyQueueThread = nullptr; + m_syncAndNotifyQueueThreadRun = false; +} + +NetmapNetDevice::~NetmapNetDevice () +{ + NS_LOG_FUNCTION (this); + m_nifp = nullptr; + m_queue = nullptr; +} + +Ptr +NetmapNetDevice::DoCreateFdReader (void) +{ + NS_LOG_FUNCTION (this); + + Ptr fdReader = Create (); + // 22 bytes covers 14 bytes Ethernet header with possible 8 bytes LLC/SNAP + fdReader->SetBufferSize (GetMtu () + 22); + fdReader->SetNetmapIfp (m_nifp); + return fdReader; +} + +void +NetmapNetDevice::DoFinishStartingDevice (void) +{ + NS_LOG_FUNCTION (this); + + m_syncAndNotifyQueueThreadRun = true; + m_syncAndNotifyQueueThread = Create (MakeCallback (&NetmapNetDevice::SyncAndNotifyQueue, this)); + m_syncAndNotifyQueueThread->Start (); +} + + +void +NetmapNetDevice::DoFinishStoppingDevice (void) +{ + NS_LOG_FUNCTION (this); + + m_queue->Stop (); + + m_syncAndNotifyQueueThreadRun = false; + m_syncAndNotifyQueueThread->Join (); + m_syncAndNotifyQueueThread = nullptr; +} + +uint32_t +NetmapNetDevice::GetBytesInNetmapTxRing () +{ + NS_LOG_FUNCTION (this); + + struct netmap_ring *txring; + txring = NETMAP_TXRING (m_nifp, 0); + + int tail = txring->tail; + + // the netmap ring has one slot reserved + int inQueue = (m_nTxRingsSlots - 1) - nm_ring_space (txring); + + uint32_t bytesInQueue = 0; + + for (int i = 1; i < inQueue; i++) + { + bytesInQueue += txring->slot[tail].len; + tail++; + tail = tail % m_nTxRingsSlots; + } + + return bytesInQueue; +} + +void +NetmapNetDevice::SetNetDeviceQueue (Ptr queue) +{ + NS_LOG_FUNCTION (this); + + m_queue = queue; +} + +void +NetmapNetDevice::SetNetmapInterfaceRepresentation (struct netmap_if *nifp) +{ + NS_LOG_FUNCTION (this << nifp); + + m_nifp = nifp; +} + +void +NetmapNetDevice::SetTxRingsInfo (uint32_t nTxRings, uint32_t nTxRingsSlots) +{ + NS_LOG_FUNCTION (this << nTxRings << nTxRingsSlots); + + m_nTxRings = nTxRings; + m_nTxRingsSlots = nTxRingsSlots; +} + +void +NetmapNetDevice::SetRxRingsInfo (uint32_t nRxRings, uint32_t nRxRingsSlots) +{ + NS_LOG_FUNCTION (this << nRxRings << nRxRingsSlots); + + m_nRxRings = nRxRings; + m_nRxRingsSlots = nRxRingsSlots; +} + +int +NetmapNetDevice::GetSpaceInNetmapTxRing () const +{ + NS_LOG_FUNCTION (this); + + struct netmap_ring *txring; + txring = NETMAP_TXRING (m_nifp, 0); + + return nm_ring_space (txring); +} + +// This function runs in a separate thread. +void +NetmapNetDevice::SyncAndNotifyQueue () +{ + NS_LOG_FUNCTION (this); + + struct netmap_ring *txring = NETMAP_TXRING (m_nifp, 0); + + uint32_t prevTotalTransmittedBytes = 0; + + while (m_syncAndNotifyQueueThreadRun == true) + { + // we sync the netmap ring periodically. + // the traffic control layer can write packets during the period between two syncs. + ioctl (GetFileDescriptor (), NIOCTXSYNC, NULL); + + // we need of a nearly periodic notification to queue limits of the transmitted bytes. + uint32_t totalTransmittedBytes = m_totalQueuedBytes - GetBytesInNetmapTxRing (); + uint32_t deltaBytes = totalTransmittedBytes - prevTotalTransmittedBytes; + NS_LOG_DEBUG (deltaBytes << " delta transmitted bytes"); + prevTotalTransmittedBytes = totalTransmittedBytes; + if (m_queue) + { + m_queue->NotifyTransmittedBytes (deltaBytes); + + if (GetSpaceInNetmapTxRing () >= 32) // WAKE_THRESHOLD + { + if (m_queue->IsStopped ()) + { + m_queue->Wake (); + } + } + } + + // we use a period to sync, check and notify of 200 us; it is a value close to the interrupt coalescence + // period of a real device + usleep (200); + + NS_LOG_DEBUG ("Space in the netmap ring of " << nm_ring_space (txring) << " packets"); + } + + ioctl (GetFileDescriptor (), NIOCTXSYNC, NULL); + +} + +ssize_t +NetmapNetDevice::Write (uint8_t *buffer, size_t length) +{ + NS_LOG_FUNCTION (this << buffer << length); + + struct netmap_ring *txring; + + // we use one ring also in case of multiqueue device to perform an accurate flow control on that ring + txring = NETMAP_TXRING (m_nifp, 0); + + uint16_t ret = -1; + + if (m_queue->IsStopped ()) + { + // the device queue is stopped and we cannot write other packets + return ret; + } + + if (!nm_ring_empty (txring)) + { + + uint32_t i = txring->cur; + uint8_t *buf = (uint8_t *) NETMAP_BUF (txring, txring->slot[i].buf_idx); + + memcpy (buf, buffer, length); + txring->slot[i].len = length; + + txring->head = txring->cur = nm_ring_next (txring, i); + + ret = length; + + // we update the total transmitted bytes counter and notify queue limits of the queued bytes + m_totalQueuedBytes += length; + m_queue->NotifyQueuedBytes (length); + + // if there is no room for other packets then stop the queue. + if (nm_ring_space (txring) == 0) + { + m_queue->Stop (); + } + } + + return ret; +} + +} // namespace ns3 diff --git a/src/fd-net-device/model/netmap-net-device.h b/src/fd-net-device/model/netmap-net-device.h new file mode 100644 index 000000000..29d0e3812 --- /dev/null +++ b/src/fd-net-device/model/netmap-net-device.h @@ -0,0 +1,219 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2017 Universita' degli Studi di Napoli Federico II + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Pasquale Imputato + */ + +#ifndef NETMAP_NET_DEVICE_H +#define NETMAP_NET_DEVICE_H + +#include "ns3/net-device-queue-interface.h" +#include +#include "fd-net-device.h" +#include +#include + +namespace ns3 { + +class SystemThread; + +/** + * \ingroup fd-net-device + * + * \brief Network device transmission queue with lock + * + * This class stores information about a single transmission queue + * of a network device that is exposed to queue discs. This class extends + * the NetDeviceQueue base class by introducing a lock for methods which + * require mutual exclusion on data access in emulation. + */ +class NetDeviceQueueLock : public NetDeviceQueue +{ +public: + /** + * \brief Get the type ID. + * \return the object TypeId + */ + static TypeId GetTypeId (void); + + NetDeviceQueueLock (); + virtual ~NetDeviceQueueLock (); + + /** + * Called by the device to start this device transmission queue. + * This is the analogous to the netif_tx_start_queue function of the Linux kernel. + */ + virtual void Start (void); + + /** + * Called by the device to stop this device transmission queue. + * This is the analogous to the netif_tx_stop_queue function of the Linux kernel. + */ + virtual void Stop (void); + + /** + * Called by the device to wake the queue disc associated with this + * device transmission queue. This is done by invoking the wake callback. + * This is the analogous to the netif_tx_wake_queue function of the Linux kernel. + */ + virtual void Wake (void); + + /** + * \brief Get the status of the device transmission queue. + * \return true if the device transmission queue is stopped. + * + * Called by queue discs to enquire about the status of a given transmission queue. + * This is the analogous to the netif_xmit_stopped function of the Linux kernel. + */ + virtual bool IsStopped (void) const; + + /** + * \brief Called by the netdevice to report the number of bytes queued to the device queue + * \param bytes number of bytes queued to the device queue + */ + virtual void NotifyQueuedBytes (uint32_t bytes); + + /** + * \brief Called by the netdevice to report the number of bytes it is going to transmit + * \param bytes number of bytes the device is going to transmit + */ + virtual void NotifyTransmittedBytes (uint32_t bytes); + +private: + mutable std::mutex m_mutex; //!< Mutex to serialize the operations performed on the queue +}; + +/** + * \ingroup fd-net-device + * + * \brief This class performs the actual data reading from the netmap ring. + */ +class NetmapNetDeviceFdReader : public FdReader +{ +public: + NetmapNetDeviceFdReader (); + + /** + * \brief Set size of the read buffer. + * \param bufferSize the size of the read buffer + */ + void SetBufferSize (uint32_t bufferSize); + + /** + * \brief Set netmap interface representation. + * \param nifp the netmap interface representation + */ + void SetNetmapIfp (struct netmap_if *nifp); + +private: + FdReader::Data DoRead (void); + + uint32_t m_bufferSize; //!< size of the read buffer + struct netmap_if *m_nifp; //!< Netmap interface representation +}; + +/** + * \ingroup fd-net-device + * + * \brief a NetDevice to read/write network traffic from/into a netmap file descriptor. + * + * A NetmapNetDevice object will read and write packets from/to a netmap file descriptor. + * + */ +class NetmapNetDevice : public FdNetDevice +{ +public: + /** + * \brief Get the type ID. + * \return the object TypeId + */ + static TypeId GetTypeId (void); + + NetmapNetDevice (); + virtual ~NetmapNetDevice (); + + /** + * \brief Get the number of bytes currently in the netmap transmission ring. + * \return the number of bytes in the netmap transmission ring. + */ + uint32_t GetBytesInNetmapTxRing (); + + /** + * \brief Get the number of slots currently available in the netmap transmission ring. + * \return the number of slots currently available in the netmap transmission ring. + */ + int GetSpaceInNetmapTxRing () const; + + /** + * \brief Set the NetDeviceQueue + * \param queue the NetDeviceQueue + */ + void SetNetDeviceQueue (Ptr queue); + + /** + * \brief Set the netmap interface representation + * \param nifp the pointer to netmap interface representation + */ + void SetNetmapInterfaceRepresentation (struct netmap_if *nifp); + + /** + * \brief Set the netmap transmission rings info + * \param nTxRings the number of transmission rings + * \param nTxRingsSlots the number of slots for each transmission ring + */ + void SetTxRingsInfo (uint32_t nTxRings, uint32_t nTxRingsSlots); + + /** + * \brief Set the netmap receiver rings info + * \param nTxRings the number of receiver rings + * \param nTxRingsSlots the number of slots for each receiver ring + */ + void SetRxRingsInfo (uint32_t nRxRings, uint32_t nRxRingsSlots); + + /** + * \brief The function Writes a packet into the netmap transmission ring. + * \param buffer the pointer to packet + * \param lenght the packet lenght + * \return the number of written bytes + */ + virtual ssize_t Write (uint8_t *buffer, size_t length); + +private: + Ptr DoCreateFdReader (void); + void DoFinishStartingDevice (void); + void DoFinishStoppingDevice (void); + + /** + * \brief This function syncs netmap ring and notifies netdevice queue. + * This function runs in a separate thread. + */ + virtual void SyncAndNotifyQueue (); + + struct netmap_if *m_nifp; //!< Netmap interface representation + uint32_t m_nTxRings; //!< Number of transmission rings + uint32_t m_nTxRingsSlots; //!< Number of slots in the transmission rings + uint32_t m_nRxRings; //!< Number of receiver rings + uint32_t m_nRxRingsSlots; //!< Number of slots in the receiver rings + Ptr m_queue; //!< NetDevice queue + uint32_t m_totalQueuedBytes; //!< Total queued bytes + Ptr m_syncAndNotifyQueueThread; //!< Thread used to perform the flow control + std::atomic m_syncAndNotifyQueueThreadRun; //!< Running flag of the flow control thread +}; + +} // namespace ns3 + +#endif /* NETMAP_NET_DEVICE_H */ diff --git a/src/fd-net-device/wscript b/src/fd-net-device/wscript index 1ff6ee8f9..b273662e7 100644 --- a/src/fd-net-device/wscript +++ b/src/fd-net-device/wscript @@ -93,7 +93,7 @@ def configure(conf): conf.env['MODULES_NOT_BUILT'].append('fd-net-device') # Next, check for whether specialized FdNetDevice features are enabled - # such as tap device support, raw socket support, and planetlab + # such as tap device support, raw socket support, netmap support, and planetlab if conf.env['ENABLE_FDNETDEV']: conf.env['ENABLE_TAP'] = conf.check_nonfatal( @@ -127,6 +127,22 @@ def configure(conf): False, "needs netpacket/packet.h") + # Enable use of netmap EMU support + conf.env['ENABLE_NETMAP_EMU'] = conf.check_nonfatal( + header_name='net/netmap_user.h', + define_name='HAVE_NETMAP_USER_H') and have_sysioctl and have_netif + + if conf.env['ENABLE_NETMAP_EMU']: + conf.report_optional_feature("NetmapFdNetDevice", + "Netmap emulation FdNetDevice", + True, + "Netmap emulation support enabled") + else: + conf.report_optional_feature("NetmapFdNetDevice", + "Netmap emulation FdNetDevice", + False, + "needs net/netmap_user.h") + # Enable use of PlanetLab TAP helper # TODO: How to validate (sysname, nodename, release, version, machine) = os.uname() @@ -224,6 +240,27 @@ def build(bld): module.env.append_value("DEFINES", "RAW_SOCK_CREATOR=\"%s\"" % (creator.target,)) + if bld.env['ENABLE_NETMAP_EMU']: + module.source.extend([ + 'model/netmap-net-device.cc', + 'helper/netmap-net-device-helper.cc', + ]) + + headers.source.extend([ + 'model/netmap-net-device.h', + 'helper/netmap-net-device-helper.h', + ]) + + creator = bld.create_suid_program('netmap-device-creator') + creator.source = [ + 'helper/netmap-device-creator.cc', + 'helper/encode-decode.cc', + 'helper/creator-utils.cc', + ] + + module.env.append_value("DEFINES", + "NETMAP_DEV_CREATOR=\"%s\"" % (creator.target,)) + if bld.env['ENABLE_PLANETLAB']: module.source.extend([ 'helper/planetlab-fd-net-device-helper.cc', diff --git a/src/network/utils/net-device-queue-interface.cc b/src/network/utils/net-device-queue-interface.cc index 3e3850ff3..2b8b54211 100644 --- a/src/network/utils/net-device-queue-interface.cc +++ b/src/network/utils/net-device-queue-interface.cc @@ -29,6 +29,17 @@ namespace ns3 { NS_LOG_COMPONENT_DEFINE ("NetDeviceQueueInterface"); +TypeId +NetDeviceQueue::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::NetDeviceQueue") + .SetParent () + .SetGroupName("Network") + .AddConstructor () + ; + return tid; +} + NetDeviceQueue::NetDeviceQueue () : m_stoppedByDevice (false), m_stoppedByQueueLimits (false), @@ -170,6 +181,12 @@ NetDeviceQueueInterface::GetTypeId (void) .SetParent () .SetGroupName("Network") .AddConstructor () + .AddAttribute ("TxQueuesType", + "The type of transmission queues to be used", + TypeId::ATTR_CONSTRUCT, + TypeIdValue (NetDeviceQueue::GetTypeId ()), + MakeTypeIdAccessor (&NetDeviceQueueInterface::SetTxQueuesType), + MakeTypeIdChecker ()) .AddAttribute ("NTxQueues", "The number of device transmission queues", TypeId::ATTR_GET | TypeId::ATTR_CONSTRUCT, UintegerValue (1), @@ -228,6 +245,17 @@ NetDeviceQueueInterface::NotifyNewAggregate (void) Object::NotifyNewAggregate (); } +void +NetDeviceQueueInterface::SetTxQueuesType (TypeId type) +{ + NS_LOG_FUNCTION (this << type); + + NS_ABORT_MSG_IF (!m_txQueuesVector.empty (), "Cannot call SetTxQueuesType after creating device queues"); + + m_txQueues = ObjectFactory (); + m_txQueues.SetTypeId (type); +} + void NetDeviceQueueInterface::SetNTxQueues (std::size_t numTxQueues) { @@ -239,7 +267,7 @@ NetDeviceQueueInterface::SetNTxQueues (std::size_t numTxQueues) // create the netdevice queues for (std::size_t i = 0; i < numTxQueues; i++) { - m_txQueuesVector.push_back (Create ()); + m_txQueuesVector.push_back (m_txQueues.Create ()->GetObject ()); } } diff --git a/src/network/utils/net-device-queue-interface.h b/src/network/utils/net-device-queue-interface.h index 033c5b369..d33b4f64f 100644 --- a/src/network/utils/net-device-queue-interface.h +++ b/src/network/utils/net-device-queue-interface.h @@ -27,6 +27,7 @@ #include "ns3/ptr.h" #include "ns3/log.h" #include "ns3/net-device.h" +#include "ns3/object-factory.h" namespace ns3 { @@ -52,9 +53,15 @@ class QueueItem; * * This class roughly models the struct netdev_queue of Linux. */ -class NetDeviceQueue : public SimpleRefCount +class NetDeviceQueue : public Object { public: + /** + * \brief Get the type ID. + * \return the object TypeId + */ + static TypeId GetTypeId (void); + NetDeviceQueue (); virtual ~NetDeviceQueue(); @@ -84,7 +91,7 @@ public: * Called by queue discs to enquire about the status of a given transmission queue. * This is the analogous to the netif_xmit_stopped function of the Linux kernel. */ - bool IsStopped (void) const; + virtual bool IsStopped (void) const; /** * \brief Notify this NetDeviceQueue that the NetDeviceQueueInterface was @@ -116,13 +123,13 @@ public: * \brief Called by the netdevice to report the number of bytes queued to the device queue * \param bytes number of bytes queued to the device queue */ - void NotifyQueuedBytes (uint32_t bytes); + virtual void NotifyQueuedBytes (uint32_t bytes); /** * \brief Called by the netdevice to report the number of bytes it is going to transmit * \param bytes number of bytes the device is going to transmit */ - void NotifyTransmittedBytes (uint32_t bytes); + virtual void NotifyTransmittedBytes (uint32_t bytes); /** * \brief Reset queue limits state @@ -251,6 +258,16 @@ public: */ std::size_t GetNTxQueues (void) const; + /** + * \brief Set the type of device transmission queues to create. + * \param type type of device transmission queues to create. + * + * This method is called when the TxQueuesType attribute is set to create + * the corresponding type of device transmission queues. It cannot be + * called again afterwards. + */ + void SetTxQueuesType (TypeId type); + /** * \brief Set the number of device transmission queues to create. * \param numTxQueues number of device transmission queues to create. @@ -293,6 +310,7 @@ protected: virtual void NotifyNewAggregate (void); private: + ObjectFactory m_txQueues; //!< Device transmission queues TypeId std::vector< Ptr > m_txQueuesVector; //!< Device transmission queues SelectQueueCallback m_selectQueueCallback; //!< Select queue callback };