diff --git a/src/core/model/assert.h b/src/core/model/assert.h index b7132e85b..7a819c27f 100644 --- a/src/core/model/assert.h +++ b/src/core/model/assert.h @@ -68,7 +68,7 @@ { \ if (!(condition)) \ { \ - std::cerr << "assert failed. cond=\"" << #condition << "\", "; \ + std::cerr << "NS_ASSERT failed, cond=\"" << #condition << "\", "; \ NS_FATAL_ERROR_NO_MSG(); \ } \ } while (false) @@ -88,7 +88,7 @@ { \ if (!(condition)) \ { \ - std::cerr << "assert failed. cond=\"" << #condition << "\", "; \ + std::cerr << "NS_ASSERT failed, cond=\"" << #condition << "\", "; \ NS_FATAL_ERROR(message); \ } \ } while (false) diff --git a/src/core/model/example-as-test.cc b/src/core/model/example-as-test.cc index 3f1695776..3ff6e203b 100644 --- a/src/core/model/example-as-test.cc +++ b/src/core/model/example-as-test.cc @@ -22,6 +22,7 @@ #include "ascii-test.h" #include "assert.h" #include "environment-variable.h" +#include "fatal-error.h" #include "log.h" #include // itoa(), system () @@ -46,11 +47,13 @@ NS_LOG_COMPONENT_DEFINE("ExampleAsTestCase"); ExampleAsTestCase::ExampleAsTestCase(const std::string name, const std::string program, const std::string dataDir, - const std::string args /* = "" */) + const std::string args /* = "" */, + const bool shouldNotErr /* = true */) : TestCase(name), m_program(program), m_dataDir(dataDir), - m_args(args) + m_args(args), + m_shouldNotErr(shouldNotErr) { NS_LOG_FUNCTION(this << name << program << dataDir << args); } @@ -87,6 +90,14 @@ ExampleAsTestCase::DoRun() std::string testFile = CreateTempDirFilename(GetName() + ".reflog"); std::string post = GetPostProcessingCommand(); + if (!m_shouldNotErr) + { + // Strip any system- or compiler-dependent messages + // resulting from invoking NS_FATAL..., which in turn + // calls std::terminate + post += " | sed '1,/" + std::string(NS_FATAL_MSG) + "/!d' "; + } + std::stringstream ss; ss << "python3 ./ns3 run " << m_program << " --no-build --command-template=\"" @@ -104,25 +115,30 @@ ExampleAsTestCase::DoRun() int status = std::system(ss.str().c_str()); - std::cout << "command: " << ss.str() << "\n" - << "status: " << status << "\n" - << "refFile: " << refFile << "\n" - << "testFile: " << testFile << "\n" - << std::endl; - std::cout << "testFile contents:" << std::endl; + std::cout << "\n" + << GetName() << ":\n" + << " command: " << ss.str() << "\n" + << " status: " << status << "\n" + << " refFile: " << refFile << "\n" + << " testFile: " << testFile << "\n" + << " testFile contents:" << std::endl; std::ifstream logF(testFile); std::string line; while (getline(logF, line)) { - std::cout << line << "\n"; + std::cout << "--- " << line << "\n"; } logF.close(); - // Make sure the example didn't outright crash - NS_TEST_ASSERT_MSG_EQ(status, 0, "example " + m_program + " failed"); + if (m_shouldNotErr) + { + // Make sure the example didn't outright crash + NS_TEST_ASSERT_MSG_EQ(status, 0, "example " + m_program + " failed"); + } - // Check that we're not just introspecting the command-line + // If we're just introspecting the command-line + // we've run the example and we're done auto [found, intro] = EnvironmentVariable::Get("NS_COMMANDLINE_INTROSPECTION"); if (found) { @@ -137,11 +153,12 @@ ExampleAsTestSuite::ExampleAsTestSuite(const std::string name, const std::string program, const std::string dataDir, const std::string args /* = "" */, - const TestDuration duration /* =QUICK */) + const TestDuration duration /* =QUICK */, + const bool shouldNotErr /* = true */) : TestSuite(name, EXAMPLE) { - NS_LOG_FUNCTION(this << name << program << dataDir << args << duration); - AddTestCase(new ExampleAsTestCase(name, program, dataDir, args), duration); + NS_LOG_FUNCTION(this << name << program << dataDir << args << duration << shouldNotErr); + AddTestCase(new ExampleAsTestCase(name, program, dataDir, args, shouldNotErr), duration); } #endif // NS3_ENABLE_EXAMPLES diff --git a/src/core/model/example-as-test.h b/src/core/model/example-as-test.h index 08724dcbe..4301817eb 100644 --- a/src/core/model/example-as-test.h +++ b/src/core/model/example-as-test.h @@ -39,8 +39,8 @@ namespace ns3 * Execute an example program as a test, by comparing the output * to a reference file. * - * User can subclass and override the GetCommandTemplate and - * GetPostProcessingCommand methods if more complex + * User can subclass and override the GetCommandTemplate() and + * GetPostProcessingCommand() methods if more complex * example invocation patterns are required. * * \see examples-as-tests-test-suite.cc @@ -64,11 +64,17 @@ class ExampleAsTestCase : public TestCase * `test-runner` the reference file will be created * with the correct name. * \param [in] args Any additional arguments to the program. + * \param [in] shouldNotErr Whether an error return status should be + * considered a test failure. This is useful when testing + * error detection which might return a non-zero status. + * The output (on `std::cout` and `std::cerr`) will + * be compared to the reference logs as normal. */ ExampleAsTestCase(const std::string name, const std::string program, const std::string dataDir, - const std::string args = ""); + const std::string args = "", + const bool shouldNotErr = true); /** Destructor. */ ~ExampleAsTestCase() override; @@ -84,9 +90,15 @@ class ExampleAsTestCase : public TestCase /** * Customization point for tests requiring post-processing of stdout. * - * For example to sort return "| sort" + * For example to sort return `"| sort"` * - * Default is "", no processing step. + * One common case is to mask memory addresses, which can change + * when things are built on different platforms, recompiled locally, + * or even from run to run. A simple post-processing filter could be + * + * `"| sed -E 's/0x[0-9a-fA-F]{8,}/0x-address/g'"` + * + * Default is `""`, no additional processing. * * \returns The string of post-processing commands */ @@ -99,6 +111,7 @@ class ExampleAsTestCase : public TestCase std::string m_program; /**< The program to run. */ std::string m_dataDir; /**< The source directory for the test. */ std::string m_args; /**< Any additional arguments to the program. */ + bool m_shouldNotErr; /**< Whether error return status is a test failure. */ }; // class ExampleAsTestCase @@ -135,12 +148,13 @@ class ExampleAsTestCase : public TestCase * which looks like this: * * \code{.cpp} - * #include "ns3/example-as-test.h" - * static ns3::ExampleAsTestSuite g_modExampleOne ("mymodule-example-mod-example-one", - * "mod-example", NS_TEST_SOURCEDIR, "--arg-one"); static ns3::ExampleAsTestSuite g_modExampleTwo - * ("mymodule-example-mod-example-two", "mod-example", NS_TEST_SOURCEDIR, "--arg-two"); \endcode + * #include "ns3/example-as-test.h" + * static ns3::ExampleAsTestSuite g_modExampleOne("mymodule-example-mod-example-one", + * "mod-example", NS_TEST_SOURCEDIR, "--arg-one"); + * static ns3::ExampleAsTestSuite g_modExampleTwo("mymodule-example-mod-example-two", + * "mod-example", NS_TEST_SOURCEDIR, "--arg-two"); \endcode * - * The arguments to the constructor is the name of the test suite, the + * The arguments to the constructor are the name of the test suite, the * example to run, the directory that contains the "good" reference file * (the macro `NS_TEST_SOURCEDIR` is normally the correct directory), * and command line arguments for the example. In the preceding code @@ -204,7 +218,8 @@ class ExampleAsTestSuite : public TestSuite const std::string program, const std::string dataDir, const std::string args = "", - const TestDuration duration = QUICK); + const TestDuration duration = QUICK, + const bool shouldNotErr = true); }; // class ExampleAsTestSuite } // namespace ns3 diff --git a/src/core/model/fatal-error.h b/src/core/model/fatal-error.h index 0539a26b0..67c199505 100644 --- a/src/core/model/fatal-error.h +++ b/src/core/model/fatal-error.h @@ -26,6 +26,7 @@ #include #include #include +#include /** * \file @@ -52,6 +53,21 @@ * on the attempt to execute the flush() function. */ +namespace ns3 +{ + +/** + * \ingroup fatal + * + * \brief Output string marking imminent invocation of std::terminate. + * + * This is useful to know when capturing output and you want to strip + * system and compiler-dependent messages generated by std::terminate. + */ +constexpr std::string_view NS_FATAL_MSG{"NS_FATAL, terminating"}; + +} // namespace ns3 + /** * \ingroup fatal * @@ -77,7 +93,10 @@ std::cerr << "file=" << __FILE__ << ", line=" << __LINE__ << std::endl; \ ::ns3::FatalImpl::FlushStreams(); \ if (fatal) \ + { \ + std::cerr << ns3::NS_FATAL_MSG << std::endl; \ std::terminate(); \ + } \ } while (false) /** diff --git a/src/mpi/test/mpi-test-suite.cc b/src/mpi/test/mpi-test-suite.cc index 92ece7c0a..0c461f7aa 100644 --- a/src/mpi/test/mpi-test-suite.cc +++ b/src/mpi/test/mpi-test-suite.cc @@ -43,7 +43,8 @@ class MpiTestCase : public ExampleAsTestCase const std::string program, const std::string dataDir, const int ranks, - const std::string args = ""); + const std::string args = "", + const bool shouldNotErr = true); /** Destructor */ ~MpiTestCase() override @@ -75,8 +76,9 @@ MpiTestCase::MpiTestCase(const std::string name, const std::string program, const std::string dataDir, const int ranks, - const std::string args /* = "" */) - : ExampleAsTestCase(name, program, dataDir, args), + const std::string args /* = "" */, + const bool shouldNotErr /* = true */) + : ExampleAsTestCase(name, program, dataDir, args, shouldNotErr), m_ranks(ranks) { } @@ -114,10 +116,11 @@ class MpiTestSuite : public TestSuite const std::string dataDir, const int ranks, const std::string args = "", - const TestDuration duration = QUICK) + const TestDuration duration = QUICK, + const bool shouldNotErr = true) : TestSuite(name, EXAMPLE) { - AddTestCase(new MpiTestCase(name, program, dataDir, ranks, args), duration); + AddTestCase(new MpiTestCase(name, program, dataDir, ranks, args, shouldNotErr), duration); } }; // class MpiTestSuite