F5/unison
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

core: allow examples to return an error in examples-as-tests

Browse Source
This commit is contained in:
Peter D. Barnes, Jr
2023-01-20 14:14:43 -08:00
committed by Peter Barnes
parent 63912dfc4a
commit 34b6c4f097
5 changed files with 87 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/core/model/assert.h
View File

@@ -68,7 +68,7 @@
{ \
if (!(condition)) \
{ \
std::cerr << "assert failed. cond=\"" << #condition << "\", "; \
std::cerr << "NS_ASSERT failed, cond=\"" << #condition << "\", "; \
NS_FATAL_ERROR_NO_MSG(); \
} \
} while (false)
@@ -88,7 +88,7 @@
{ \
if (!(condition)) \
{ \
std::cerr << "assert failed. cond=\"" << #condition << "\", "; \
std::cerr << "NS_ASSERT failed, cond=\"" << #condition << "\", "; \
NS_FATAL_ERROR(message); \
} \
} while (false)

47
src/core/model/example-as-test.cc
View File

@@ -22,6 +22,7 @@
#include "ascii-test.h"
#include "assert.h"
#include "environment-variable.h"
#include "fatal-error.h"
#include "log.h"
#include <cstdlib> // itoa(), system ()
@@ -46,11 +47,13 @@ NS_LOG_COMPONENT_DEFINE("ExampleAsTestCase");
ExampleAsTestCase::ExampleAsTestCase(const std::string name,
const std::string program,
const std::string dataDir,
const std::string args /* = "" */)
const std::string args /* = "" */,
const bool shouldNotErr /* = true */)
: TestCase(name),
m_program(program),
m_dataDir(dataDir),
m_args(args)
m_args(args),
m_shouldNotErr(shouldNotErr)
{
NS_LOG_FUNCTION(this << name << program << dataDir << args);
}
@@ -87,6 +90,14 @@ ExampleAsTestCase::DoRun()
std::string testFile = CreateTempDirFilename(GetName() + ".reflog");
std::string post = GetPostProcessingCommand();
if (!m_shouldNotErr)
{
// Strip any system- or compiler-dependent messages
// resulting from invoking NS_FATAL..., which in turn
// calls std::terminate
post += " | sed '1,/" + std::string(NS_FATAL_MSG) + "/!d' ";
}
std::stringstream ss;
ss << "python3 ./ns3 run " << m_program << " --no-build --command-template=\""
@@ -104,25 +115,30 @@ ExampleAsTestCase::DoRun()
int status = std::system(ss.str().c_str());
std::cout << "command: " << ss.str() << "\n"
<< "status: " << status << "\n"
<< "refFile: " << refFile << "\n"
<< "testFile: " << testFile << "\n"
<< std::endl;
std::cout << "testFile contents:" << std::endl;
std::cout << "\n"
<< GetName() << ":\n"
<< " command: " << ss.str() << "\n"
<< " status: " << status << "\n"
<< " refFile: " << refFile << "\n"
<< " testFile: " << testFile << "\n"
<< " testFile contents:" << std::endl;
std::ifstream logF(testFile);
std::string line;
while (getline(logF, line))
{
std::cout << line << "\n";
std::cout << "--- " << line << "\n";
}
logF.close();
// Make sure the example didn't outright crash
NS_TEST_ASSERT_MSG_EQ(status, 0, "example " + m_program + " failed");
if (m_shouldNotErr)
{
// Make sure the example didn't outright crash
NS_TEST_ASSERT_MSG_EQ(status, 0, "example " + m_program + " failed");
}
// Check that we're not just introspecting the command-line
// If we're just introspecting the command-line
// we've run the example and we're done
auto [found, intro] = EnvironmentVariable::Get("NS_COMMANDLINE_INTROSPECTION");
if (found)
{
@@ -137,11 +153,12 @@ ExampleAsTestSuite::ExampleAsTestSuite(const std::string name,
const std::string program,
const std::string dataDir,
const std::string args /* = "" */,
const TestDuration duration /* =QUICK */)
const TestDuration duration /* =QUICK */,
const bool shouldNotErr /* = true */)
: TestSuite(name, EXAMPLE)
{
NS_LOG_FUNCTION(this << name << program << dataDir << args << duration);
AddTestCase(new ExampleAsTestCase(name, program, dataDir, args), duration);
NS_LOG_FUNCTION(this << name << program << dataDir << args << duration << shouldNotErr);
AddTestCase(new ExampleAsTestCase(name, program, dataDir, args, shouldNotErr), duration);
}
#endif // NS3_ENABLE_EXAMPLES

37
src/core/model/example-as-test.h
View File

@@ -39,8 +39,8 @@ namespace ns3
* Execute an example program as a test, by comparing the output
* to a reference file.
*
* User can subclass and override the GetCommandTemplate and
* GetPostProcessingCommand methods if more complex
* User can subclass and override the GetCommandTemplate() and
* GetPostProcessingCommand() methods if more complex
* example invocation patterns are required.
*
* \see examples-as-tests-test-suite.cc
@@ -64,11 +64,17 @@ class ExampleAsTestCase : public TestCase
* `test-runner` the reference file will be created
* with the correct name.
* \param [in] args Any additional arguments to the program.
* \param [in] shouldNotErr Whether an error return status should be
* considered a test failure. This is useful when testing
* error detection which might return a non-zero status.
* The output (on `std::cout` and `std::cerr`) will
* be compared to the reference logs as normal.
*/
ExampleAsTestCase(const std::string name,
const std::string program,
const std::string dataDir,
const std::string args = "");
const std::string args = "",
const bool shouldNotErr = true);
/** Destructor. */
~ExampleAsTestCase() override;
@@ -84,9 +90,15 @@ class ExampleAsTestCase : public TestCase
/**
* Customization point for tests requiring post-processing of stdout.
*
* For example to sort return "| sort"
* For example to sort return `"| sort"`
*
* Default is "", no processing step.
* One common case is to mask memory addresses, which can change
* when things are built on different platforms, recompiled locally,
* or even from run to run. A simple post-processing filter could be
*
* `"| sed -E 's/0x[0-9a-fA-F]{8,}/0x-address/g'"`
*
* Default is `""`, no additional processing.
*
* \returns The string of post-processing commands
*/
@@ -99,6 +111,7 @@ class ExampleAsTestCase : public TestCase
std::string m_program; /**< The program to run. */
std::string m_dataDir; /**< The source directory for the test. */
std::string m_args; /**< Any additional arguments to the program. */
bool m_shouldNotErr; /**< Whether error return status is a test failure. */
}; // class ExampleAsTestCase
@@ -135,12 +148,13 @@ class ExampleAsTestCase : public TestCase
* which looks like this:
*
* \code{.cpp}
* #include "ns3/example-as-test.h"
* static ns3::ExampleAsTestSuite g_modExampleOne ("mymodule-example-mod-example-one",
* "mod-example", NS_TEST_SOURCEDIR, "--arg-one"); static ns3::ExampleAsTestSuite g_modExampleTwo
* ("mymodule-example-mod-example-two", "mod-example", NS_TEST_SOURCEDIR, "--arg-two"); \endcode
* #include "ns3/example-as-test.h"
* static ns3::ExampleAsTestSuite g_modExampleOne("mymodule-example-mod-example-one",
* "mod-example", NS_TEST_SOURCEDIR, "--arg-one");
* static ns3::ExampleAsTestSuite g_modExampleTwo("mymodule-example-mod-example-two",
* "mod-example", NS_TEST_SOURCEDIR, "--arg-two"); \endcode
*
* The arguments to the constructor is the name of the test suite, the
* The arguments to the constructor are the name of the test suite, the
* example to run, the directory that contains the "good" reference file
* (the macro `NS_TEST_SOURCEDIR` is normally the correct directory),
* and command line arguments for the example. In the preceding code
@@ -204,7 +218,8 @@ class ExampleAsTestSuite : public TestSuite
const std::string program,
const std::string dataDir,
const std::string args = "",
const TestDuration duration = QUICK);
const TestDuration duration = QUICK,
const bool shouldNotErr = true);
}; // class ExampleAsTestSuite
} // namespace ns3

19
src/core/model/fatal-error.h
View File

@@ -26,6 +26,7 @@
#include <cstdlib>
#include <exception>
#include <iostream>
#include <string_view>
/**
* \file
@@ -52,6 +53,21 @@
* on the attempt to execute the flush() function.
*/
namespace ns3
{
/**
* \ingroup fatal
*
* \brief Output string marking imminent invocation of std::terminate.
*
* This is useful to know when capturing output and you want to strip
* system and compiler-dependent messages generated by std::terminate.
*/
constexpr std::string_view NS_FATAL_MSG{"NS_FATAL, terminating"};
} // namespace ns3
/**
* \ingroup fatal
*
@@ -77,7 +93,10 @@
std::cerr << "file=" << __FILE__ << ", line=" << __LINE__ << std::endl; \
::ns3::FatalImpl::FlushStreams(); \
if (fatal) \
{ \
std::cerr << ns3::NS_FATAL_MSG << std::endl; \
std::terminate(); \
} \
} while (false)
/**

13
src/mpi/test/mpi-test-suite.cc
View File

@@ -43,7 +43,8 @@ class MpiTestCase : public ExampleAsTestCase
const std::string program,
const std::string dataDir,
const int ranks,
const std::string args = "");
const std::string args = "",
const bool shouldNotErr = true);
/** Destructor */
~MpiTestCase() override
@@ -75,8 +76,9 @@ MpiTestCase::MpiTestCase(const std::string name,
const std::string program,
const std::string dataDir,
const int ranks,
const std::string args /* = "" */)
: ExampleAsTestCase(name, program, dataDir, args),
const std::string args /* = "" */,
const bool shouldNotErr /* = true */)
: ExampleAsTestCase(name, program, dataDir, args, shouldNotErr),
m_ranks(ranks)
{
}
@@ -114,10 +116,11 @@ class MpiTestSuite : public TestSuite
const std::string dataDir,
const int ranks,
const std::string args = "",
const TestDuration duration = QUICK)
const TestDuration duration = QUICK,
const bool shouldNotErr = true)
: TestSuite(name, EXAMPLE)
{
AddTestCase(new MpiTestCase(name, program, dataDir, ranks, args), duration);
AddTestCase(new MpiTestCase(name, program, dataDir, ranks, args, shouldNotErr), duration);
}
}; // class MpiTestSuite