F5/unison
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

core, test: Add --only-fullness option to execute only test cases with the specified duration

Browse Source 
Allows for tiered execution of progressively slower/faster tests
./test.py -of QUICK
./test.py -of EXTENSIVE
./test.py -of TAKES_FOREVER
This commit is contained in:
Gabriel Ferreira
2025-09-14 15:29:11 +02:00
parent 56dee9c8a7
commit dddc5740ee
2 changed files with 99 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
src/core/model/test.cc
View File

@@ -224,12 +224,12 @@ class TestRunnerImpl : public Singleton<TestRunnerImpl>
*
* @param [in] testName Include a specific test by name.
* @param [in] testType Include all tests of give type.
* @param [in] maximumTestDuration Restrict to tests shorter than this.
* @param [in] filteredTestDuration Restrict tests that match these durations.
* @returns The list of tests matching the filter constraints.
*/
std::list<TestCase*> FilterTests(std::string testName,
TestSuite::Type testType,
TestCase::Duration maximumTestDuration);
std::vector<TestCase::Duration> filteredTestDuration);
/** Container type for the test. */
typedef std::vector<TestSuite*> TestSuiteVector;
@@ -752,33 +752,39 @@ TestRunnerImpl::PrintHelp(const char* program_name) const
<< "Usage: " << program_name << " [OPTIONS]" << std::endl
<< std::endl
<< "Options: " << std::endl
<< " --help : print these options" << std::endl
<< " --print-test-name-list : print the list of names of tests available" << std::endl
<< " --list : an alias for --print-test-name-list" << std::endl
<< " --print-test-types : print the type of tests along with their names" << std::endl
<< " --print-test-type-list : print the list of types of tests available" << std::endl
<< " --print-temp-dir : print name of temporary directory before running "
<< " --help : print these options" << std::endl
<< " --print-test-name-list : print the list of names of tests available" << std::endl
<< " --list : an alias for --print-test-name-list" << std::endl
<< " --print-test-types : print the type of tests along with their names"
<< std::endl
<< " the tests" << std::endl
<< " --test-type=TYPE : process only tests of type TYPE" << std::endl
<< " --test-name=NAME : process only test whose name matches NAME" << std::endl
<< " --suite=NAME : an alias (here for compatibility reasons only) " << std::endl
<< " for --test-name=NAME" << std::endl
<< " --assert-on-failure : when a test fails, crash immediately (useful" << std::endl
<< " when running under a debugger" << std::endl
<< " --stop-on-failure : when a test fails, stop immediately" << std::endl
<< " --fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
<< " EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE " << std::endl
<< " includes QUICK and TAKES_FOREVER includes " << std::endl
<< " QUICK and EXTENSIVE (only QUICK tests are " << std::endl
<< " run by default)" << std::endl
<< " --verbose : print details of test execution" << std::endl
<< " --xml : format test run output as xml" << std::endl
<< " --tempdir=DIR : set temp dir for tests to store output files" << std::endl
<< " --datadir=DIR : set data dir for tests to read reference files" << std::endl
<< " --out=FILE : send test result to FILE instead of standard output"
<< " --print-test-type-list : print the list of types of tests available" << std::endl
<< " --print-temp-dir : print name of temporary directory before running "
<< std::endl
<< " --append=FILE : append test result to FILE instead of standard output"
<< " the tests" << std::endl
<< " --test-type=TYPE : process only tests of type TYPE" << std::endl
<< " --test-name=NAME : process only test whose name matches NAME" << std::endl
<< " --suite=NAME : an alias (here for compatibility reasons only) "
<< std::endl
<< " for --test-name=NAME" << std::endl
<< " --assert-on-failure : when a test fails, crash immediately (useful" << std::endl
<< " when running under a debugger" << std::endl
<< " --stop-on-failure : when a test fails, stop immediately" << std::endl
<< " --fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
<< " EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE " << std::endl
<< " includes QUICK and TAKES_FOREVER includes " << std::endl
<< " QUICK and EXTENSIVE (only QUICK tests are " << std::endl
<< " run by default)" << std::endl
<< " --only-fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
<< " EXTENSIVE, TAKES_FOREVER (only tests marked " << std::endl
<< " with fullness will be executed)" << std::endl
<< " --verbose : print details of test execution" << std::endl
<< " --xml : format test run output as xml" << std::endl
<< " --tempdir=DIR : set temp dir for tests to store output files" << std::endl
<< " --datadir=DIR : set data dir for tests to read reference files"
<< std::endl
<< " --out=FILE : send test result to FILE instead of standard output"
<< std::endl
<< " --append=FILE : append test result to FILE instead of standard output"
<< std::endl;
}
@@ -827,7 +833,7 @@ TestRunnerImpl::PrintTestTypeList() const
std::list<TestCase*>
TestRunnerImpl::FilterTests(std::string testName,
TestSuite::Type testType,
TestCase::Duration maximumTestDuration)
std::vector<TestCase::Duration> filteredTestDuration)
{
NS_LOG_FUNCTION(this << testName << testType);
std::list<TestCase*> tests;
@@ -852,7 +858,10 @@ TestRunnerImpl::FilterTests(std::string testName,
// If this test case takes longer than the maximum test
// duration that should be run, then don't run it.
if (testCase->m_duration > maximumTestDuration)
auto it = std::find(filteredTestDuration.begin(),
filteredTestDuration.end(),
testCase->m_duration);
if (it == filteredTestDuration.end())
{
// Free this test case's memory.
delete *j;
@@ -882,13 +891,14 @@ TestRunnerImpl::Run(int argc, char* argv[])
std::string testTypeString = "";
std::string out = "";
std::string fullness = "";
std::string onlyFullness = "";
bool xml = false;
bool append = false;
bool printTempDir = false;
bool printTestTypeList = false;
bool printTestNameList = false;
bool printTestTypeAndName = false;
TestCase::Duration maximumTestDuration = TestCase::Duration::QUICK;
std::vector<TestCase::Duration> filteredTestDuration{TestCase::Duration::QUICK};
char* progname = argv[0];
char** argi = argv;
@@ -962,15 +972,17 @@ TestRunnerImpl::Run(int argc, char* argv[])
// Set the maximum test length allowed.
if (fullness == "QUICK")
{
maximumTestDuration = TestCase::Duration::QUICK;
filteredTestDuration = {TestCase::Duration::QUICK};
}
else if (fullness == "EXTENSIVE")
{
maximumTestDuration = TestCase::Duration::EXTENSIVE;
filteredTestDuration = {TestCase::Duration::QUICK, TestCase::Duration::EXTENSIVE};
}
else if (fullness == "TAKES_FOREVER")
{
maximumTestDuration = TestCase::Duration::TAKES_FOREVER;
filteredTestDuration = {TestCase::Duration::QUICK,
TestCase::Duration::EXTENSIVE,
TestCase::Duration::TAKES_FOREVER};
}
else
{
@@ -979,6 +991,30 @@ TestRunnerImpl::Run(int argc, char* argv[])
return 3;
}
}
else if (arg.find("--only-fullness=") != std::string::npos)
{
onlyFullness = arg.substr(arg.find_first_of('=') + 1);
// Set the maximum test length allowed.
if (onlyFullness == "QUICK")
{
filteredTestDuration = {TestCase::Duration::QUICK};
}
else if (onlyFullness == "EXTENSIVE")
{
filteredTestDuration = {TestCase::Duration::EXTENSIVE};
}
else if (onlyFullness == "TAKES_FOREVER")
{
filteredTestDuration = {TestCase::Duration::TAKES_FOREVER};
}
else
{
// Wrong fullness option
PrintHelp(progname);
return 4;
}
}
else
{
// Print the help if arg == "--help" or arg is an un-recognized command-line argument
@@ -1015,7 +1051,15 @@ TestRunnerImpl::Run(int argc, char* argv[])
return 1;
}
std::list<TestCase*> tests = FilterTests(testName, testType, maximumTestDuration);
if (!fullness.empty() && !onlyFullness.empty())
{
std::cout << "Invalid simultaneous use of '--fullness' and '--only-fullness'. "
<< "Use only one of them." << std::endl;
PrintHelp(progname);
return 5;
}
std::list<TestCase*> tests = FilterTests(testName, testType, filteredTestDuration);
if (m_tempDir.empty())
{

27
test.py
View File

@@ -1657,11 +1657,15 @@ def run_tests():
multiple = ""
else:
multiple = " --stop-on-failure"
if len(args.fullness):
fullness = args.fullness.upper()
fullness = " --fullness=%s" % fullness
else:
fullness = " --fullness=QUICK"
if args.fullness:
if len(args.fullness):
fullness = args.fullness.upper()
fullness = " --fullness=%s" % fullness
else:
fullness = " --fullness=QUICK"
elif args.only_fullness:
fullness = " --only-fullness=%s" % args.only_fullness.upper()
path_cmd = os.path.join(
"utils", test_runner_name + " --test-name=%s%s%s" % (test, multiple, fullness)
@@ -2292,8 +2296,9 @@ def main(argv):
default=False,
help="If examples use reference data files, get them to re-generate them",
)
fullness_group = parser.add_mutually_exclusive_group(required=False)
parser.add_argument(
fullness_group.add_argument(
"-f",
"--fullness",
action="store",
@@ -2303,6 +2308,16 @@ def main(argv):
help="choose the duration of tests to run: QUICK, EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE includes QUICK and TAKES_FOREVER includes QUICK and EXTENSIVE (only QUICK tests are run by default)",
)
fullness_group.add_argument(
"-of",
"--only-fullness",
action="store",
type=str,
default=None,
choices=["QUICK", "EXTENSIVE", "TAKES_FOREVER"],
help="choose the duration of tests to run: QUICK, EXTENSIVE, or TAKES_FOREVER (only tests marked with fullness will be executed)",
)
parser.add_argument(
"-g",
"--grind",