diff --git a/doc/manual/source/how-to-write-tests.rst b/doc/manual/source/how-to-write-tests.rst index 2f93b5713..374c47831 100644 --- a/doc/manual/source/how-to-write-tests.rst +++ b/doc/manual/source/how-to-write-tests.rst @@ -114,14 +114,99 @@ use the ``ASSERT`` variants. How to add an example program to the test suite *********************************************** -One can "smoke test" that examples compile and run successfully +There are two methods for adding an example program to the the test +suite. Normally an example is added using only one of these methods +to avoid running the example twice. + +First, you can "smoke test" that examples compile and run successfully to completion (without memory leaks) using the ``examples-to-run.py`` script located in your module's test directory. Briefly, by including an instance of this file in your test directory, you can cause the -test runner to execute the examples listed. It is usually best to make -sure that you select examples that have reasonably short run times so as -to not bog down the tests. See the example in ``src/lte/test/`` -directory. +test runner to execute the examples listed. It is usually best to +make sure that you select examples that have reasonably short run +times so as to not bog down the tests. See the example in +``src/lte/test/`` directory. The exit status of the example will be +checked when run and a non-zero exit status can be used to indicate +that the example has failed. This is the easiest way to add an example +to the test suite but has limited checks. + +The second method you can use to add an example to the test suite is +more complicated but enables checking of the example output +(``std::out`` and ``std::err``). This approach uses the test suite +framework with a specialized ``TestSuite`` or ``TestCase`` class +designed to run an example and compare the output with a specified +known "good" reference file. To use an example program as a test you +need to create a test suite file and add it to the appropriate list in +your module wscript file. The "good" output reference file needs to be +generated for detecting regressions. + +If you are thinking about using this class, strongly consider using a +standard test instead. The TestSuite class has better checking using +the ``NS_TEST_*`` macros and in almost all cases is the better approach. +If your test can be done with a TestSuite class you will be asked by +the reviewers to rewrite the test when you do a pull request. + +Let's assume your module is called ``mymodule``, and the example +program is ``mymodule/examples/mod-example.cc``. First you should +create a test file ``mymodule/test/mymodule-examples-test-suite.cc`` +which looks like this: + +:: + + #include "ns3/example-as-test.h" + static ns3::ExampleAsTestSuite g_modExampleOne ("mymodule-example-mod-example-one", "mod-example", NS_TEST_SOURCEDIR, "--arg-one"); + static ns3::ExampleAsTestSuite g_modExampleTwo ("mymodule-example-mod-example-two", "mod-example", NS_TEST_SOURCEDIR, "--arg-two"); + +The arguments to the constructor are the name of the test suite, the +example to run, the directory that contains the "good" reference file +(the macro ``NS_TEST_SOURCEDIR`` is normally the correct directory), +and command line arguments for the example. In the preceding code the +same example is run twice with different arguments. + +You then need to add that newly created test suite file to the list of +test sources in ``mymodule/wscript``. + +Since you modified a wscript file you need to reconfigure and rebuild +everything. + +You just added new tests so you will need to generate the "good" +output reference files that will be used to verify the example: + +.. sourcecode :: bash + + ./test.py --suite="mymodule-example-*" --update + +This will run all tests starting with "mymodule-example-" and save new +"good" reference files. Updating the reference files should be done +when you create the test and whenever output changes. When updating +the reference output you should inspect it to ensure that it is valid. +The reference files should be committed with the new test. + +This completes the process of adding a new example. + +You can now run the test with the standard ``test.py`` script. For +example to run the suites you just added: + +.. sourcecode:: bash + + ./test.py --suite="mymodule-example-*" + +This will run all ``mymodule-example-...`` tests and report whether they +produce output matching the reference files. + +You can also add multiple examples as test cases to a ``TestSuite`` +using ``ExampleAsTestCase``. See +``src/core/test/examples-as-tests-test-suite.cc`` for examples of +setting examples as tests. + +When setting up an example for use by this class you should be very +careful about what output the example generates. For example, writing +output which includes simulation time (especially high resolution +time) makes the test sensitive to potentially minor changes in event +times. This makes the reference output hard to verify and hard to +keep up-to-date. Output as little as needed for the example and +include only behavioral state that is important for determining if the +example has run correctly. Testing for boolean outcomes **************************** diff --git a/doc/manual/source/test-framework.rst b/doc/manual/source/test-framework.rst index 5f62e00a5..d2dc67f3d 100644 --- a/doc/manual/source/test-framework.rst +++ b/doc/manual/source/test-framework.rst @@ -500,10 +500,11 @@ implementation. Examples ++++++++ -The examples are tested by the framework to make sure they built and will -run. Nothing is checked, and currently the pcap files are just written off -into /tmp to be discarded. If the examples run (don't crash) they pass this -smoke test. +The examples are tested by the framework to make sure they built and +will run. Limited checking is done on examples; currently the pcap +files are just written off into /tmp to be discarded. If the example +runs (don't crash) and the exit status is zero, the example will pass +the smoke test. Performance Tests +++++++++++++++++ diff --git a/src/core/examples/hash-example.cc b/src/core/examples/hash-example.cc index 71a24425a..d0e4ccb9d 100644 --- a/src/core/examples/hash-example.cc +++ b/src/core/examples/hash-example.cc @@ -148,7 +148,7 @@ public: * Add a string to the Collider. * * \param [in] phrase The string to add. - * \return true If this was a new string. + * \return \c true If this was a new string. */ bool Add (const std::string phrase) { @@ -455,7 +455,7 @@ public: * CommandLine callback function to add a file argument to the list. * * \param [in] file The word file to add. - * \return true Tf the file is new to the list. + * \return \c true If the file is new to the list. */ bool Add (const std::string file) { @@ -467,6 +467,12 @@ public: return true; } + /** \return The default dictionary path. */ + static std::string GetDefault (void) + { + return "/usr/share/dict/words"; + } + /** * Add phrases from the files into the dict. * @@ -476,7 +482,7 @@ public: { if (m_files.size () == 0) { - Add ("/usr/share/dict/web2"); + Add (GetDefault ()); } std::cout << "Hashing the dictionar" @@ -541,7 +547,9 @@ main (int argc, char *argv[]) cmd.Usage ("Find hash collisions in the dictionary."); cmd.AddValue ("dict", "Dictionary file to hash", MakeCallback (&DictFiles::Add, - &files)); + &files), + DictFiles::GetDefault ()); + cmd.AddValue ("time", "Run timing test", timing); cmd.Parse (argc, argv); diff --git a/src/core/examples/system-path-examples.cc b/src/core/examples/system-path-examples.cc new file mode 100644 index 000000000..1fcb75f25 --- /dev/null +++ b/src/core/examples/system-path-examples.cc @@ -0,0 +1,86 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2020 Lawrence Livermore National Laboratory + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Peter D. Barnes, Jr. + */ + +#include +#include +#include + +#include "ns3/core-module.h" + +/** + * \file + * \ingroup core-examples + * \ingroup systempath + * Example program illustrating use of ns3::SystemPath + */ + +using namespace ns3; +using namespace ns3::SystemPath; + +int main (int argc, char *argv[]) +{ + std::string path = "/usr/share/dict/"; + + CommandLine cmd; + cmd.Usage ("SystemPath examples.\n"); + + cmd.AddValue ("path", "Path to demonstrate SystemPath functions.", path); + cmd.Parse (argc, argv); + + // Show initial values: + std::cout << std::endl; + std::cout << cmd.GetName () << ":" << std::endl; + + std::cout << "FindSelfDirectory: " << FindSelfDirectory () << std::endl; + + std::cout << "Demonstration path: " << path << std::endl; + std::cout << "Exists? " + << (Exists (path) ? "yes" : "no") << std::endl; + + auto foo = Append (path, "foo"); + std::cout << "Append 'foo': " << foo << std::endl; + std::cout << "Exists? " + << (Exists (foo) ? "yes" : "no") << std::endl; + + std::cout << "Split path:\n"; + auto items = Split (path); + for (auto item : items) + { + std::cout << " '" << item << "'\n"; + } + std::cout << std::endl; + + std::cout << "Successive Joins: \n"; + for (auto it = items.begin (); it != items.end (); ++it) + { + auto partial = Join (items.begin (), it); + std::cout << " '" << partial << "'\n"; + } + + std::cout << "Files in the directory: \n"; + auto files = ReadFiles (path); + for (auto item : files) + { + std::cout << " '" << item << "'\n"; + } + + + return 0; +} diff --git a/src/core/examples/wscript b/src/core/examples/wscript index 6a34452ef..d96c43a9b 100644 --- a/src/core/examples/wscript +++ b/src/core/examples/wscript @@ -46,6 +46,10 @@ def build(bld): obj = bld.create_ns3_program('empirical-random-variable-example', ['core', 'flow-monitor']) obj.source = 'empirical-random-variable-example.cc' + + obj = bld.create_ns3_program('system-path-examples', + ['core']) + obj.source = 'system-path-examples.cc' if bld.env['ENABLE_THREADING'] and bld.env["ENABLE_REAL_TIME"]: obj = bld.create_ns3_program('main-test-sync', ['network']) diff --git a/src/network/utils/ascii-file.cc b/src/core/model/ascii-file.cc similarity index 94% rename from src/network/utils/ascii-file.cc rename to src/core/model/ascii-file.cc index aeeb0cfa0..ed230f600 100644 --- a/src/network/utils/ascii-file.cc +++ b/src/core/model/ascii-file.cc @@ -14,20 +14,21 @@ * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * + * * Author: Mitch Watrous (watrous@u.washington.edu) * * This file is based on pcap-file.cc by Craig Dowell (craigdo@ee.washington.edu) */ +#include "ascii-file.h" +#include "assert.h" +#include "fatal-error.h" +#include "fatal-impl.h" #include #include -#include "ns3/assert.h" -#include "ns3/fatal-error.h" -#include "ns3/fatal-impl.h" -#include "ascii-file.h" + // -// This file is used as part of the ns-3 test framework, so please refrain from +// This file is used as part of the ns-3 test framework, so please refrain from // adding any ns-3 specific constructs such as Packet to this file. // namespace ns3 { @@ -44,12 +45,12 @@ AsciiFile::~AsciiFile () Close (); } -bool +bool AsciiFile::Fail (void) const { return m_file.fail (); } -bool +bool AsciiFile::Eof (void) const { return m_file.eof (); @@ -81,7 +82,7 @@ AsciiFile::Read (std::string& line) bool AsciiFile::Diff (std::string const & f1, - std::string const & f2, + std::string const & f2, uint64_t & lineNumber) { AsciiFile ascii1, ascii2; diff --git a/src/network/utils/ascii-file.h b/src/core/model/ascii-file.h similarity index 97% rename from src/network/utils/ascii-file.h rename to src/core/model/ascii-file.h index 3115138cb..0492263da 100644 --- a/src/network/utils/ascii-file.h +++ b/src/core/model/ascii-file.h @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * + * * Author: Mitch Watrous (watrous@u.washington.edu) * * This file is based on pcap-file.h by Craig Dowell (craigdo@ee.washington.edu) @@ -65,23 +65,23 @@ public: /** * \brief Read next line from file - * + * * \param line [out] line from file - * + * */ void Read (std::string& line); /** * \brief Compare two ASCII files line-by-line - * + * * \return true if files are different, false otherwise - * + * * \param f1 First ASCII file name * \param f2 Second ASCII file name * \param lineNumber [out] Line number of first different line. */ static bool Diff (std::string const & f1, - std::string const & f2, + std::string const & f2, uint64_t & lineNumber); private: diff --git a/src/network/utils/ascii-test.h b/src/core/model/ascii-test.h similarity index 71% rename from src/network/utils/ascii-test.h rename to src/core/model/ascii-test.h index 2439b40cf..82f28221d 100644 --- a/src/network/utils/ascii-test.h +++ b/src/core/model/ascii-test.h @@ -35,15 +35,15 @@ * \param expectedFilename The name of the reference file to read in * including its path */ -#define NS_ASCII_TEST_EXPECT_EQ(gotFilename, expectedFilename) \ +#define NS_ASCII_TEST_EXPECT_EQ(gotFilename, expectedFilename) \ do { \ - uint64_t line(0); \ - bool diff = AsciiFile::Diff (gotFilename, expectedFilename, line); \ - NS_TEST_EXPECT_MSG_EQ (diff, false, \ - "ASCII traces " << gotFilename << \ - " and " << expectedFilename << \ - " differ starting from line " << line); \ - } while (false) + uint64_t line (0); \ + bool diff = AsciiFile::Diff (gotFilename, expectedFilename, line); \ + NS_TEST_EXPECT_MSG_EQ (diff, false, \ + "ASCII traces " << gotFilename << \ + " and " << expectedFilename << \ + " differ starting from line " << line); \ + } while (false) #endif /* ASCII_TEST_H */ diff --git a/src/core/model/command-line.cc b/src/core/model/command-line.cc index eb2debf8e..140d33492 100644 --- a/src/core/model/command-line.cc +++ b/src/core/model/command-line.cc @@ -638,6 +638,18 @@ CommandLine::HandleArgument (const std::string &name, const std::string &value) } } +bool +CommandLine::CallbackItem::HasDefault (void) const +{ + return m_default != ""; +} + +std::string +CommandLine::CallbackItem::GetDefault (void) const +{ + return m_default; +} + bool CommandLine::CallbackItem::Parse (const std::string value) { @@ -649,13 +661,16 @@ CommandLine::CallbackItem::Parse (const std::string value) void CommandLine::AddValue (const std::string &name, const std::string &help, - ns3::Callback callback) + ns3::Callback callback, + std::string defaultValue /* = "" */) + { NS_LOG_FUNCTION (this << &name << &help << &callback); CallbackItem *item = new CallbackItem (); item->m_name = name; item->m_help = help; item->m_callback = callback; + item->m_default = defaultValue; m_options.push_back (item); } diff --git a/src/core/model/command-line.h b/src/core/model/command-line.h index ad195918c..2095f7e44 100644 --- a/src/core/model/command-line.h +++ b/src/core/model/command-line.h @@ -286,13 +286,16 @@ public: * \param [in] help The help text used by \c --help * \param [in] callback A Callback function that will be invoked to parse and * store the value. + * \param [in] defaultValue Optional default value for argument. * * The callback should have the signature * CommandLine::Callback */ void AddValue (const std::string &name, const std::string &help, - ns3::Callback callback); + ns3::Callback callback, + const std::string defaultValue = ""); + /** * Add a program argument as a shorthand for an Attribute. @@ -437,7 +440,8 @@ private: }; // class UserItem /** - * Extension of Item for strings. + * \ingroup commandline + * \brief Extension of Item for strings. */ class StringItem : public Item { @@ -457,6 +461,10 @@ private: class CallbackItem : public Item { public: + // Inherited + bool HasDefault (void) const; + std::string GetDefault (void) const; + /** * Parse from a string. * @@ -465,6 +473,7 @@ private: */ virtual bool Parse (const std::string value); ns3::Callback m_callback; /**< The Callback */ + std::string m_default; /**< The default value, as a string, if it exists. */ }; // class CallbackItem diff --git a/src/core/model/example-as-test.cc b/src/core/model/example-as-test.cc new file mode 100644 index 000000000..835f8f8c8 --- /dev/null +++ b/src/core/model/example-as-test.cc @@ -0,0 +1,143 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2020 Lawrence Livermore National Laboratory + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Peter D. Barnes, Jr. + */ + +#include "example-as-test.h" +#include "ascii-test.h" +#include "log.h" +#include "unused.h" +#include "assert.h" + +#include +#include +#include // itoa(), system () + +/** + * \file + * \ingroup testing + * Implementation of classes ns3::ExampleAsTestSuite and ns3::ExampleTestCase. + */ + +namespace ns3 { + +NS_LOG_COMPONENT_DEFINE ("ExampleAsTestCase"); + +// Running tests as examples currently requires bash shell; uses Unix +// piping that does not work on Windows. +#if defined(NS3_ENABLE_EXAMPLES) && !defined (__win32__) + +ExampleAsTestCase::ExampleAsTestCase (const std::string name, + const std::string program, + const std::string dataDir, + const std::string args /* = "" */) + : TestCase (name), + m_program (program), + m_dataDir (dataDir), + m_args (args) +{ + NS_LOG_FUNCTION (this << name << program << dataDir << args); +} + +ExampleAsTestCase::~ExampleAsTestCase (void) +{ + NS_LOG_FUNCTION_NOARGS (); +} + +std::string +ExampleAsTestCase::GetCommandTemplate (void) const +{ + NS_LOG_FUNCTION_NOARGS (); + std::string command ("%s "); + command += m_args; + return command; +} + +std::string +ExampleAsTestCase::GetPostProcessingCommand (void) const +{ + NS_LOG_FUNCTION_NOARGS (); + std::string command (""); + return command; +} + +void +ExampleAsTestCase::DoRun (void) +{ + NS_LOG_FUNCTION_NOARGS (); + // Set up the output file names + SetDataDir (m_dataDir); + std::string refFile = CreateDataDirFilename (GetName () + ".reflog"); + std::string testFile = CreateTempDirFilename (GetName () + ".reflog"); + + std::stringstream ss; + + // Use bash as shell to allow use of PIPESTATUS + ss << "bash -c './waf --run-no-build " << m_program + << " --command-template=\"" << GetCommandTemplate () << "\"" + + // redirect std::clog, std::cerr to std::cout + << " 2>&1 " + + // Suppress the waf lines from output; waf output contains directory paths which will + // obviously differ during a test run + << " | grep -v 'Waf:' " + << GetPostProcessingCommand () + << " > " << testFile + + // Get the status of waf + << "; exit ${PIPESTATUS[0]}'"; + + int status = std::system (ss.str ().c_str ()); + + std::cout << "command: " << ss.str () << "\n" + << "status: " << status << "\n" + << "refFile: " << refFile << "\n" + << "testFile: " << testFile << "\n" + << std::endl; + std::cout << "testFile contents:" << std::endl; + + std::ifstream logF (testFile); + std::string line; + while (getline (logF, line)) + { + std::cout << line << "\n"; + } + logF.close (); + + // Make sure the example didn't outright crash + NS_TEST_ASSERT_MSG_EQ (status, 0, "example " + m_program + " failed"); + + // Compare the testFile to the reference file + NS_ASCII_TEST_EXPECT_EQ (testFile, refFile); +} + +ExampleAsTestSuite::ExampleAsTestSuite (const std::string name, + const std::string program, + const std::string dataDir, + const std::string args /* = "" */, + const TestDuration duration /* =QUICK */) + : TestSuite (name, EXAMPLE) +{ + NS_LOG_FUNCTION (this << name << program << dataDir << args << duration); + AddTestCase (new ExampleAsTestCase (name, program, dataDir, args), duration); +} + +#endif // NS3_ENABLE_EXAMPLES && !defined (__win32__) + +} // namespace ns3 diff --git a/src/core/model/example-as-test.h b/src/core/model/example-as-test.h new file mode 100644 index 000000000..2c2de0266 --- /dev/null +++ b/src/core/model/example-as-test.h @@ -0,0 +1,208 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2020 Lawrence Livermore National Laboratory + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Peter D. Barnes, Jr. + */ + +#ifndef NS3_EXAMPLE_AS_TEST_SUITE_H +#define NS3_EXAMPLE_AS_TEST_SUITE_H + +#include "ns3/test.h" + +#include + +/** + * \file + * \ingroup testing + * Enable examples to be run as meaningful tests. + * Declaration of classes ns3::ExampleAsTestSuite and ns3::ExampleAsTestCase. + */ + +namespace ns3 { + + +/** + * \ingroup testing + * Execute an example program as a test, by comparing the output + * to a reference file. + * + * User can subclass and override the GetCommandTemplate and + * GetPostProcessingCommand methods if more complex + * example invocation patterns are required. + * + * \see examples-as-tests-test-suite.cc + */ +class ExampleAsTestCase : public TestCase +{ +public: + /** + * Constructor. + * \param [in] name The test case name, typically the program name + * and summary of the arguments, such as `my-example-foo` + * \param [in] program The actual example program names, such as `my-example` + * \param [in] dataDir The location of the reference file. + * This is normally provided by the symbol + * `NS_TEST_SOURCEDIR` in the `module-examples-test-suite.cc` + * file. + * The reference file should be named after + * the test case name, + * for example `my-example-foo.log`. If you use + * the `--update` argument to `test.py` or + * `test-runner` the reference file will be created + * with the correct name. + * \param [in] args Any additional arguments to the program. + */ + ExampleAsTestCase (const std::string name, + const std::string program, + const std::string dataDir, + const std::string args = ""); + + /** Destructor. */ + virtual ~ExampleAsTestCase (void); + + /** + * Customization point for more complicated patterns + * to invoke the example program. + * + * \returns The string to be given to the `waf --command-template=` argument. + */ + virtual std::string GetCommandTemplate (void) const; + + /** + * Customization point for tests requiring post-processing of stdout. + * + * For example to sort return "| sort" + * + * Default is "", no processing step. + * + * \returns The string of post-processing commands + */ + virtual std::string GetPostProcessingCommand (void) const; + + // Inherited + virtual void DoRun (void); + +protected: + std::string m_program; /**< The program to run. */ + std::string m_dataDir; /**< The source directory for the test. */ + std::string m_args; /**< Any additional arguments to the program. */ + +}; // class ExampleAsTestCase + +/** + * \ingroup testing + * Execute an example program as a test suite. + * + * You can use this TestSuite to add an example to the test suite with + * checking of the example output (`std::out` and `std::err`). This + * is an alternative to adding an example using the + * `examples-to-run.py` file. The key difference between the two + * methods is what criteria is used to for success. Examples added to + * `examples-to-run.py` will be run and the exit status checked + * (non-zero indicates failure). ExampleAsTestSuite adds checking of + * output against a specified known "good" reference file. + * + * \warning If you are thinking about using this class, strongly + * consider using a standard test instead. The TestSuite class has + * better checking using the NS_TEST_* macros and in almost all cases + * is the better approach. If your test can be done with a TestSuite + * class you will be asked by the reviewers to rewrite the test when + * you do a pull request. + * + * \par Test Addition + * + * To use an example program as a test you need to create a test suite + * file and add it to the appropriate list in your module wscript + * file. The "good" output reference file needs to be generated for + * detecting regressions. + * + * Let's assume your module is called `mymodule`, and the example + * program is `mymodule/examples/mod-example.cc`. First you should + * create a test file `mymodule/test/mymodule-examples-test-suite.cc` + * which looks like this: + * + * \code{.cpp} + * #include "ns3/example-as-test.h" + * static ns3::ExampleAsTestSuite g_modExampleOne ("mymodule-example-mod-example-one", "mod-example", NS_TEST_SOURCEDIR, "--arg-one"); + * static ns3::ExampleAsTestSuite g_modExampleTwo ("mymodule-example-mod-example-two", "mod-example", NS_TEST_SOURCEDIR, "--arg-two"); + * \endcode + * + * The arguments to the constructor is the name of the test suite, the + * example to run, the directory that contains the "good" reference file + * (the macro `NS_TEST_SOURCEDIR` is normally the correct directory), + * and command line arguments for the example. In the preceding code + * the same example is run twice with different arguments. + * + * You then need to add that newly created test suite file to the list + * of test sources in `mymodule/wscript`. + * + * Since you modified a wscript file you need to reconfigure and + * rebuild everything. + * + * You just added new tests so you will need to generate the "good" + * output reference files that will be used to verify the example: + * + * `./test.py --suite="mymodule-example-*" --update` + * + * This will run all tests starting with "mymodule-example-" and save + * new "good" reference files. Updating the reference file should be + * done when you create the test and whenever output changes. When + * updating the reference output you should inspect it to ensure that + * it is valid. The reference files should be committed with the new + * test. + * + * \par Test Verification + * + * You can run the test with the standard `test.py` script. For + * example to run the suites you just added: + * + * `./test.py --suite="mymodule-example-*"` + * + * This will run all `mymodule-example-...` tests and report whether they + * produce output matching the reference files. + * + * \par Writing good examples for testing + * + * When setting up an example for use by this class you should be very + * careful about what output the example generates. For example, + * writing output which includes simulation time (especially high + * resolution time) makes the test sensitive to potentially minor + * changes in event times. This makes the reference output hard to + * verify and hard to keep up-to-date. Output as little as needed for + * the example and include only behavioral state that is important for + * determining if the example has run correctly. + * + */ +class ExampleAsTestSuite : public TestSuite +{ +public: + /** + * \copydoc ExampleAsTestCase::ExampleAsTestCase + * \param [in] duration Amount of time this test takes to execute + * (defaults to QUICK). + */ + ExampleAsTestSuite (const std::string name, + const std::string program, + const std::string dataDir, + const std::string args = "", + const TestDuration duration = QUICK); +}; // class ExampleAsTestSuite + + +} // namespace ns3 + +#endif /* NS3_EXAMPLE_TEST_SUITE_H */ diff --git a/src/core/model/system-path.cc b/src/core/model/system-path.cc index f3fa4da31..d24bcc7de 100644 --- a/src/core/model/system-path.cc +++ b/src/core/model/system-path.cc @@ -23,10 +23,11 @@ #include "log.h" #include "ns3/core-config.h" +#include #include // getenv #include #include // strlen - +#include #if defined (HAVE_DIRENT_H) && defined (HAVE_SYS_TYPES_H) /** Do we have an \c opendir function? */ @@ -74,6 +75,57 @@ namespace ns3 { NS_LOG_COMPONENT_DEFINE ("SystemPath"); +// unnamed namespace for internal linkage +namespace { +/** + * \ingroup systempath + * Get the list of files located in a file system directory with error. + * + * \param [in] path A path which identifies a directory + * \return Tuple with a list of the filenames which are located in the input directory or error flag \c true if directory doesn't exist. + */ +std::tuple, bool> ReadFilesNoThrow (std::string path) +{ + NS_LOG_FUNCTION (path); + std::list files; + +#if defined HAVE_OPENDIR + DIR *dp = opendir (path.c_str ()); + if (dp == NULL) + { + return std::make_tuple (files, true); + } + struct dirent *de = readdir (dp); + while (de != 0) + { + files.push_back (de->d_name); + de = readdir (dp); + } + closedir (dp); +#elif defined (HAVE_FIND_FIRST_FILE) + /** \todo untested */ + HANDLE hFind; + WIN32_FIND_DATA fileData; + + hFind = FindFirstFile (path.c_str (), &FindFileData); + if (hFind == INVALID_HANDLE_VALUE) + { + return std::make_tuple (files, true); + } + do + { + files.push_back (fileData.cFileName); + } + while (FindNextFile (hFind, &fileData)); + FindClose (hFind); +#else +#error "No support for reading a directory on this platform" +#endif + return std::make_tuple (files, false); +} + +} // unnamed namespace + namespace SystemPath { /** @@ -224,11 +276,16 @@ std::list Split (std::string path) std::string Join (std::list::const_iterator begin, std::list::const_iterator end) { - NS_LOG_FUNCTION (&begin << &end); + NS_LOG_FUNCTION (*begin << *end); std::string retval = ""; for (std::list::const_iterator i = begin; i != end; i++) { - if (i == begin) + if (*i == "") + { + // skip empty strings in the path list + continue; + } + else if (i == begin) { retval = *i; } @@ -243,39 +300,13 @@ std::string Join (std::list::const_iterator begin, std::list ReadFiles (std::string path) { NS_LOG_FUNCTION (path); + bool err; std::list files; -#if defined HAVE_OPENDIR - DIR *dp = opendir (path.c_str ()); - if (dp == NULL) + std::tie (files, err) = ReadFilesNoThrow (path); + if (err) { NS_FATAL_ERROR ("Could not open directory=" << path); } - struct dirent *de = readdir (dp); - while (de != 0) - { - files.push_back (de->d_name); - de = readdir (dp); - } - closedir (dp); -#elif defined (HAVE_FIND_FIRST_FILE) - /** \todo untested */ - HANDLE hFind; - WIN32_FIND_DATA fileData; - - hFind = FindFirstFile (path.c_str (), &FindFileData); - if (hFind == INVALID_HANDLE_VALUE) - { - NS_FATAL_ERROR ("Could not open directory=" << path); - } - do - { - files.push_back (fileData.cFileName); - } - while (FindNextFile (hFind, &fileData)); - FindClose (hFind); -#else -#error "No support for reading a directory on this platform" -#endif return files; } @@ -358,6 +389,52 @@ MakeDirectories (std::string path) } } +bool +Exists (const std::string path) +{ + NS_LOG_FUNCTION (path); + + bool err; + auto dirpath = Dirname (path); + std::list files; + tie (files, err) = ReadFilesNoThrow (dirpath); + if (err) + { + // Directory doesn't exist + NS_LOG_LOGIC ("directory doesn't exist: " << dirpath); + return false; + } + NS_LOG_LOGIC ("directory exists: " << dirpath); + + // Check if the file itself exists + auto tokens = Split (path); + std::string file = tokens.back (); + + if (file == "") + { + // Last component was a directory, not a file name + // We already checked that the directory exists, + // so return true + NS_LOG_LOGIC ("directory path exists: " << path); + return true; + } + + files = ReadFiles (dirpath); + + auto it = std::find (files.begin (), files.end (), file); + if (it == files.end ()) + { + // File itself doesn't exist + NS_LOG_LOGIC ("file itself doesn't exist: " << file); + return false; + } + + NS_LOG_LOGIC ("file itself exists: " << file); + return true; + +} // Exists() + + } // namespace SystemPath } // namespace ns3 diff --git a/src/core/model/system-path.h b/src/core/model/system-path.h index 2e3a9e1eb..e065adbb4 100644 --- a/src/core/model/system-path.h +++ b/src/core/model/system-path.h @@ -138,6 +138,15 @@ std::string MakeTemporaryDirectoryName (void); */ void MakeDirectories (std::string path); +/** + * \ingroup systempath + * Check if a path exists. + * Path can be a file or directory. + * \param [in] path The path to check. + * \returns \c true if the \pname{path} exists. + */ +bool Exists (const std::string path); + } // namespace SystemPath diff --git a/src/core/test/core-example-command-line.reflog b/src/core/test/core-example-command-line.reflog new file mode 100644 index 000000000..7b1e8a962 --- /dev/null +++ b/src/core/test/core-example-command-line.reflog @@ -0,0 +1,20 @@ + +command-line-example +Initial values: +intArg: 1 +boolArg: false +strArg: "strArg default" +anti: "false" +cbArg: "cbArg default" +nonOpt1: 1 +nonOpt2: 1 + +Final values: +intArg: 1 +boolArg: false +strArg: "strArg default" +anti: "false" +cbArg: "cbArg default" +nonOpt1: 1 +nonOpt2: 1 +Number of extra non-option arguments:0 diff --git a/src/core/test/core-example-sample-random-variable.reflog b/src/core/test/core-example-sample-random-variable.reflog new file mode 100644 index 000000000..855176f1b --- /dev/null +++ b/src/core/test/core-example-sample-random-variable.reflog @@ -0,0 +1 @@ +0.816532 diff --git a/src/core/test/core-example-simulator.reflog b/src/core/test/core-example-simulator.reflog new file mode 100644 index 000000000..5a9a72a64 --- /dev/null +++ b/src/core/test/core-example-simulator.reflog @@ -0,0 +1,3 @@ +ExampleFunction received event at 10s +RandomFunction received event at 18.1653s +Member method received event at 20s started at 10s diff --git a/src/core/test/examples-as-tests-test-suite.cc b/src/core/test/examples-as-tests-test-suite.cc new file mode 100644 index 000000000..8752f4523 --- /dev/null +++ b/src/core/test/examples-as-tests-test-suite.cc @@ -0,0 +1,88 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2020 Lawrence Livermore National Laboratory + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Peter D. Barnes, Jr. + */ + +#include "ns3/example-as-test.h" +#include "ns3/system-path.h" + +#include + +using namespace ns3; + +/** + * \file + * \ingroup core-tests + * \ingroup examples-as-tests + * Examples-as-tests test suite + */ + +/** + * \ingroup core-tests + * \defgroup examples-as-tests Examples as tests test suite + * + * Runs several examples as tests in order to test ExampleAsTestSuite and ExampleAsTestCase. + */ +namespace ns3 { + +namespace tests { +/** + * \ingroup examples-as-tests + * Run examples as tests, checking stdout for regressions. + */ +class ExamplesAsTestsTestSuite : public TestSuite +{ +public: + ExamplesAsTestsTestSuite (); +}; + + +ExamplesAsTestsTestSuite::ExamplesAsTestsTestSuite () + : TestSuite ("examples-as-tests-test-suite", UNIT) +{ + AddTestCase (new ExampleAsTestCase ("core-example-simulator", "sample-simulator", NS_TEST_SOURCEDIR)); + + AddTestCase (new ExampleAsTestCase ("core-example-sample-random-variable", "sample-random-variable", NS_TEST_SOURCEDIR)); + + AddTestCase (new ExampleAsTestCase ("core-example-command-line", "command-line-example", NS_TEST_SOURCEDIR)); +} + +/** + * \ingroup examples-tests + * ExampleAsTestsTestSuite instance variable. + * Tests multiple examples in a single TestSuite using AddTestCase to add the examples to the suite. + */ +static ExamplesAsTestsTestSuite g_examplesAsTestsTestSuite; + +/** + * \ingroup examples-tests + * ExampleTestSuite instance variables. + * + * Tests ExampleTestSuite which runs a single example as test suite as specified in constructor arguments. + */ + +static ExampleAsTestSuite g_exampleCommandLineTest ("core-example-command-line", "command-line-example", NS_TEST_SOURCEDIR); + +} // namespace tests + +} // namespace ns3 + + + + + diff --git a/src/core/wscript b/src/core/wscript index cc5476b55..d40ea512d 100644 --- a/src/core/wscript +++ b/src/core/wscript @@ -209,11 +209,15 @@ def build(bld): 'model/hash-fnv.cc', 'model/hash.cc', 'model/des-metrics.cc', + 'model/ascii-file.cc', 'model/node-printer.cc', 'model/time-printer.cc', 'model/show-progress.cc', ] + if (bld.env['ENABLE_EXAMPLES']): + core.source.append('model/example-as-test.cc') + core_test = bld.create_ns3_module_test_library('core') core_test.source = [ 'test/attribute-test-suite.cc', @@ -240,6 +244,9 @@ def build(bld): 'test/type-id-test-suite.cc', ] + if (bld.env['ENABLE_EXAMPLES']): + core_test.source.append('test/examples-as-tests-test-suite.cc') + headers = bld(features='ns3header') headers.module = 'core' headers.source = [ @@ -326,11 +333,16 @@ def build(bld): 'model/non-copyable.h', 'model/build-profile.h', 'model/des-metrics.h', + 'model/ascii-file.h', + 'model/ascii-test.h', 'model/node-printer.h', 'model/time-printer.h', 'model/show-progress.h', ] + if (bld.env['ENABLE_EXAMPLES']): + headers.source.append('model/example-as-test.h') + if sys.platform == 'win32': core.source.extend([ 'model/win32-system-wall-clock-ms.cc', diff --git a/src/network/wscript b/src/network/wscript index f71be7631..be77c6975 100644 --- a/src/network/wscript +++ b/src/network/wscript @@ -24,7 +24,6 @@ def build(bld): 'model/tag-buffer.cc', 'model/trailer.cc', 'utils/address-utils.cc', - 'utils/ascii-file.cc', 'utils/crc32.cc', 'utils/data-rate.cc', 'utils/drop-tail-queue.cc', @@ -111,8 +110,6 @@ def build(bld): 'model/tag-buffer.h', 'model/trailer.h', 'utils/address-utils.h', - 'utils/ascii-file.h', - 'utils/ascii-test.h', 'utils/crc32.h', 'utils/data-rate.h', 'utils/drop-tail-queue.h', diff --git a/wscript b/wscript index ed18619cb..85d70b667 100644 --- a/wscript +++ b/wscript @@ -769,6 +769,9 @@ def register_ns3_script(bld, name, dependencies=('core',)): def add_examples_programs(bld): env = bld.env if env['ENABLE_EXAMPLES']: + # Add a define, so this is testable from code + env.append_value('DEFINES', 'NS3_ENABLE_EXAMPLES') + try: for dir in os.listdir('examples'): if dir.startswith('.') or dir == 'CVS':