F5/unison
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.

Browse Source
This commit is contained in:
Gustavo J. A. M. Carneiro
2009-01-24 12:25:52 +00:00
parent 55930dc4fe
commit b2c212c239
24 changed files with 204 additions and 355 deletions
Download Patch File Download Diff File Expand all files Collapse all files

306
regression.py
View File

@@ -9,6 +9,7 @@ import urllib
import Build
import Options
import Utils
import Task
# local modules
import wutils
@@ -21,106 +22,6 @@ def dev_null():
return open("/dev/null", "w")
### Regression testing
class Regression(object):
def __init__(self, testdir, reference_traces):
self.testdir = testdir
self.reference_traces = reference_traces
self.env = Build.bld.env
def run_test(self, verbose, generate, testName, arguments=[], pyscript=None, refTestName=None):
"""
@param verbose: enable verbose execution
@param generate: generate new traces instead of comparing with the reference
@param testName: name of the test
@arguments: list of extra parameters to pass to the program to be tested
@pyscript: if not None, the test is written in Python and this
parameter contains the path to the python script, relative to
the project root dir
@param refTestName: if not None, this is the name of the directory under refDirName
that contains the reference traces. Otherwise "refDirname/testName + .ref" is used.
"""
if not isinstance(arguments, list):
raise TypeError
if refTestName is None:
refTestDirName = os.path.join(self.reference_traces, (testName + ".ref"))
else:
refTestDirName = os.path.join(self.reference_traces, refTestName)
if not os.path.exists(self.reference_traces):
print"No reference trace repository"
return 1
if generate:
if not os.path.exists(refTestDirName):
print "creating new " + refTestDirName
os.mkdir(refTestDirName)
if pyscript is None:
tmpl = "%s"
for arg in arguments:
tmpl = tmpl + " " + arg
wutils.run_program(testName, tmpl, cwd=refTestDirName)
else:
argv = [self.env['PYTHON'], os.path.join(Options.cwd_launch, *os.path.split(pyscript))] + arguments
wutils.run_argv(argv, cwd=refTestDirName)
print "Remember to commit " + refTestDirName
return 0
else:
if not os.path.exists(refTestDirName):
print "Cannot locate reference traces in " + refTestDirName
return 1
if refTestName is None:
traceDirName = testName + ".ref"
else:
traceDirName = refTestName
traceDirName = os.path.join('regression', 'traces', traceDirName)
try:
shutil.rmtree(traceDirName)
except OSError:
pass
os.mkdir(traceDirName)
if pyscript is None:
wutils.run_program(testName,
command_template=wutils.get_command_template(*arguments),
cwd=traceDirName)
else:
argv = [self.env['PYTHON'], os.path.join('..', '..', '..', *os.path.split(pyscript))] + arguments
wutils.run_argv(argv, cwd=traceDirName)
if verbose:
#diffCmd = "diff traces " + refTestDirName + " | head"
diffCmd = subprocess.Popen([self.env['DIFF'], traceDirName, refTestDirName],
stdout=subprocess.PIPE)
headCmd = subprocess.Popen("head", stdin=diffCmd.stdout)
rc2 = headCmd.wait()
diffCmd.stdout.close()
rc1 = diffCmd.wait()
rc = rc1 or rc2
else:
rc = subprocess.Popen([self.env['DIFF'], traceDirName, refTestDirName], stdout=dev_null()).wait()
if rc:
print "----------"
print "Traces differ in test: test-" + testName
print "Reference traces in directory: regression/" + refTestDirName
print "Traces in directory: traces"
print "Rerun regression test as: " + \
"\"./waf --regression --regression-tests=test-" + testName + "\""
print "Then do \"diff -u regression/" + refTestDirName + " regression/" + traceDirName +\
"\" for details"
print "----------"
return rc
def _find_tests(testdir):
"""Return a list of test modules in the test directory
@@ -136,7 +37,161 @@ def _find_tests(testdir):
tests.sort()
return tests
def run_regression(reference_traces):
class regression_test_task(Task.TaskBase):
after = 'cc cxx cc_link cxx_link'
color = 'BLUE'
def __init__(self, env, test_name, test_scripts_dir, build_traces_dir, reference_traces):
super(regression_test_task, self).__init__()
self.env = env
self.test_name = test_name
self.test_scripts_dir = test_scripts_dir
self.build_traces_dir = build_traces_dir
self.reference_traces_dir = reference_traces
def __str__(self):
return 'regression-test (%s)\n' % self.test_name
def run(self):
"""Run a single test"""
sys.path.insert(0, self.test_scripts_dir)
try:
mod = __import__(self.test_name, globals(), locals(), [])
finally:
sys.path.remove(self.test_scripts_dir)
assert self.test_name.startswith('test-')
short_name = self.test_name[len('test-'):]
trace_dir_name = getattr(mod, "trace_dir_name", None)
if trace_dir_name is None:
trace_dir_name = "%s.ref" % short_name
trace_output_path = os.path.join(self.build_traces_dir, trace_dir_name)
reference_traces_path = os.path.join(self.reference_traces_dir, trace_dir_name)
if hasattr(mod, 'get_arguments'):
arguments = mod.get_arguments(self.env, '..')
else:
arguments = getattr(mod, "arguments", [])
pyscript = getattr(mod, "pyscript", None)
if pyscript:
is_pyscript = True
program = pyscript
else:
is_pyscript = False
program = getattr(mod, "program", short_name)
if hasattr(mod, 'may_run'):
reason_cannot_run = mod.may_run(self.env)
else:
reason_cannot_run = None
if reason_cannot_run:
print "SKIP %s (%s)" % (self.test_name, reason_cannot_run)
self.result = None
return 0
if Options.options.regression_generate:
# clean the target dir
shutil.rmtree(reference_traces_path, ignore_errors=True)
os.makedirs(reference_traces_path)
result = self.run_reference_generate(reference_traces_path, program, arguments, is_pyscript)
if result == 0:
print "GENERATE " + self.test_name
else:
print "GENERATE FAIL " + self.test_name
else:
# clean the target dir
shutil.rmtree(trace_output_path, ignore_errors=True)
os.makedirs(trace_output_path)
# run it
result = self.run_reference_test(reference_traces_path, trace_output_path, program, arguments, is_pyscript)
if result == 0:
print "PASS " + self.test_name
else:
print "FAIL " + self.test_name
self.result = result
return 0
def run_reference_test(self, reference_traces_path, trace_output_path, program, arguments, is_pyscript):
if not os.path.exists(reference_traces_path):
print "Cannot locate reference traces in " + reference_traces_path
return 1
if is_pyscript:
script = os.path.abspath(os.path.join('..', *os.path.split(program)))
argv = [self.env['PYTHON'], script] + arguments
wutils.run_argv(argv, cwd=trace_output_path)
else:
wutils.run_program(program,
command_template=wutils.get_command_template(*arguments),
cwd=trace_output_path)
if Options.options.verbose:
#diffCmd = "diff traces " + refTestDirName + " | head"
diffCmd = subprocess.Popen([self.env['DIFF'], trace_output_path, reference_traces_path],
stdout=subprocess.PIPE)
headCmd = subprocess.Popen("head", stdin=diffCmd.stdout)
rc2 = headCmd.wait()
diffCmd.stdout.close()
rc1 = diffCmd.wait()
rc = rc1 or rc2
else:
rc = subprocess.Popen([self.env['DIFF'], trace_output_path, reference_traces_path], stdout=dev_null()).wait()
if rc:
print "----------"
print "Traces differ in test: ", self.test_name
print "Reference traces in directory: " + reference_traces_path
print "Traces in directory: " + trace_output_path
print "Run the following command for details:"
print "\tdiff -u %s %s" % (reference_traces_path, trace_output_path)
print "----------"
return rc
def run_reference_generate(self, trace_output_path, program, arguments, is_pyscript):
if is_pyscript:
script = os.path.abspath(os.path.join('..', *os.path.split(program)))
argv = [self.env['PYTHON'], script] + arguments
retval = wutils.run_argv(argv, cwd=trace_output_path)
else:
retval = wutils.run_program(program,
command_template=wutils.get_command_template(*arguments),
cwd=trace_output_path)
return retval
class regression_test_collector_task(Task.TaskBase):
after = 'regression_test_task'
color = 'BLUE'
def __init__(self, test_tasks):
super(regression_test_collector_task, self).__init__()
self.test_tasks = test_tasks
def __str__(self):
return 'regression-test-collector\n'
def run(self):
failed_tests = [test for test in self.test_tasks if test.result is not None and test.result != 0]
skipped_tests = [test for test in self.test_tasks if test.result is None]
print "Regression testing summary:"
if skipped_tests:
print "SKIP: %i of %i tests have been skipped (%s)" % (
len(skipped_tests), len(self.test_tasks),
', '.join([test.test_name for test in skipped_tests]))
if failed_tests:
print "FAIL: %i of %i tests have failed (%s)" % (
len(failed_tests), len(self.test_tasks),
', '.join([test.test_name for test in failed_tests]))
return 1
else:
print "PASS: %i of %i tests passed" % (len(self.test_tasks) - len(skipped_tests),
len(self.test_tasks))
return 0
def run_regression(bld, reference_traces):
"""Execute regression tests. Called with cwd set to the 'regression' subdir of ns-3.
@param reference_traces: reference traces directory.
@@ -148,57 +203,18 @@ def run_regression(reference_traces):
print "Tests directory does not exist"
sys.exit(3)
sys.path.append(testdir)
sys.modules['tracediff'] = Regression(testdir, reference_traces)
if Options.options.regression_tests:
tests = Options.options.regression_tests.split(',')
else:
tests = _find_tests(testdir)
print "========== Running Regression Tests =========="
env = Build.bld.env
if not os.path.exists(reference_traces):
print "Reference traces directory (%s) does not exist" % reference_traces
return 3
bad = []
test_scripts_dir = bld.path.find_dir('regression/tests').abspath()
build_traces_dir = bld.path.find_or_declare('regression/traces').abspath(bld.env)
tasks = []
for test in tests:
try:
result = _run_regression_test(test)
if result == 0:
if Options.options.regression_generate:
print "GENERATE " + test
else:
print "PASS " + test
else:
bad.append(test)
print "FAIL " + test
except NotImplementedError:
print "SKIP " + test
return (len(bad) > 0)
def _run_regression_test(test):
"""Run a single test.
Arguments:
test -- the name of the test
"""
traces_dir = os.path.join("regression", "traces")
if os.path.exists(traces_dir):
files = os.listdir(traces_dir)
for file in files:
if file == '.' or file == '..':
continue
shutil.rmtree(os.path.join("traces", file), ignore_errors=True)
else:
os.mkdir(traces_dir)
mod = __import__(test, globals(), locals(), [])
return mod.run(verbose=(Options.options.verbose > 0),
generate=Options.options.regression_generate)
tasks.append(regression_test_task(bld.env, test, test_scripts_dir, build_traces_dir, reference_traces))
regression_test_collector_task(tasks)

18
regression/tests/test-csma-bridge.py
View File

@@ -2,15 +2,13 @@
"""Generic trace-comparison-type regression test."""
import os
import sys
import tracediff
import os.path
def run(verbose, generate):
"""Execute a test."""
if tracediff.env['ENABLE_PYTHON_BINDINGS']:
return tracediff.run_test(verbose, generate,
"csma-bridge", pyscript=os.path.join('examples', 'csma-bridge.py'))
def may_run(env):
"""Returns 0 when it can run, return non-zero or string (reason) when it cannot run"""
if env['ENABLE_PYTHON_BINDINGS']:
return 0
else:
print >> sys.stderr, "Skipping csma-bridge: Python bindings not available."
raise NotImplementedError
return "Python bindings not available."
pyscript = os.path.join('examples', 'csma-bridge.py')

8
regression/tests/test-csma-broadcast.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "csma-broadcast")

9
regression/tests/test-csma-multicast.py
View File

@@ -1,12 +1,3 @@
#! /usr/bin/env python
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "csma-multicast")

8
regression/tests/test-csma-one-subnet.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "csma-one-subnet")

9
regression/tests/test-csma-packet-socket.py
View File

@@ -2,12 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate,
"csma-packet-socket")

8
regression/tests/test-csma-ping.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "csma-ping")

8
regression/tests/test-csma-raw-ip-socket.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "csma-raw-ip-socket")

8
regression/tests/test-csma-star.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "csma-star")

8
regression/tests/test-dynamic-global-routing.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "dynamic-global-routing")

8
regression/tests/test-global-routing-slash32.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "global-routing-slash32")

15
regression/tests/test-ns2-mob.py
View File

@@ -2,11 +2,12 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
import os.path
def run(verbose, generate):
"""Execute a test."""
arguments = ["../../../samples/ns2-mob.tr", "out.tr"]
return tracediff.run_test(verbose, generate, "main-ns2-mob", arguments=arguments)
program = "main-ns2-mob"
def get_arguments(env, top_dir):
ns2_tracefile = os.path.abspath(os.path.join(top_dir, "samples", "ns2-mob.tr"))
return [ns2_tracefile, "out.tr"]
trace_dir_name = "main-ns2-mob.ref"

8
regression/tests/test-realtime-udp-echo.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "realtime-udp-echo")

8
regression/tests/test-second.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "second")

9
regression/tests/test-simple-error-model.py
View File

@@ -2,12 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate,
"simple-error-model")

9
regression/tests/test-simple-global-routing.py
View File

@@ -2,12 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate,
"simple-global-routing")

9
regression/tests/test-simple-point-to-point-olsr.py
View File

@@ -2,12 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate,
"simple-point-to-point-olsr")

8
regression/tests/test-static-routing-slash32.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "static-routing-slash32")

9
regression/tests/test-tcp-large-transfer.py
View File

@@ -2,12 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate,
"tcp-large-transfer")

41
regression/tests/test-tcp-nsc-lfn.py
View File

@@ -2,32 +2,25 @@
"""Trace-comparison-type regression test for the Network Simulation Cradle."""
import os
import shutil
import sys
import tracediff
import platform
def run(verbose, generate):
"""Run a Network Simulation Cradle test involving two TCP streams."""
if not tracediff.env['ENABLE_NSC']:
print >>sys.stderr, "Skipping tcp-nsc-lfn: NSC not available."
raise NotImplementedError
testName = "tcp-nsc-lfn"
arguments = ["--ns3::OnOffApplication::DataRate=40000", "--runtime=20"]
platform_bits = platform.architecture()[0]
if platform_bits == "64bit":
traceDirName = testName + "_64bit.ref"
elif platform_bits == "32bit":
traceDirName = testName + "_32bit.ref"
def may_run(env):
if not env['NSC_ENABLED']:
return "NSC not available"
else:
# Something unexpected. How should we signal an error here? Rasing a
# string might not be the best idea?
raise "Unknown architecture, not 64 or 32 bit?"
return 0
platform_bits = platform.architecture()[0]
if platform_bits == "64bit":
trace_dir_name = "tcp-nsc-lfn_64bit.ref"
elif platform_bits == "32bit":
trace_dir_name = "tcp-nsc-lfn_32bit.ref"
else:
raise AssertionError("Unknown architecture, not 64 or 32 bit?")
del platform_bits
arguments = ["--ns3::OnOffApplication::DataRate=40000", "--runtime=20"]
return tracediff.run_test(verbose, generate,
testName, arguments=arguments, refTestName=traceDirName)

8
regression/tests/test-third.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "third")

8
regression/tests/test-udp-echo.py
View File

@@ -2,11 +2,3 @@
"""Generic trace-comparison-type regression test."""
import os
import shutil
import tracediff
def run(verbose, generate):
"""Execute a test."""
#print tracediff.env
return tracediff.run_test(verbose, generate, "udp-echo")

8
regression/tests/test-wifi-wired-bridging.py
View File

@@ -2,11 +2,5 @@
"""Compare that Wifi-Wired Bridging generates correct traces."""
import os
import shutil
import tracediff
arguments = ["--SendIp=0"]
def run(verbose, generate):
"""Execute a test."""
return tracediff.run_test(verbose, generate, "wifi-wired-bridging", ["--SendIp=0"])

21
wscript
View File

@@ -415,6 +415,15 @@ def build(bld):
if not Options.options.compile_targets:
Options.options.compile_targets = os.path.basename(program_name)
if Options.options.regression or Options.options.regression_generate:
if not env['DIFF']:
raise Utils.WafError("Cannot run regression tests: the 'diff' program is not installed.")
regression_traces = env['REGRESSION_TRACES']
if not regression_traces:
raise Utils.WafError("Cannot run regression tests: reference traces directory not given"
" (--with-regression-traces configure option)")
regression.run_regression(bld, regression_traces)
def get_command_template(*arguments):
@@ -435,18 +444,6 @@ def shutdown():
if Options.commands['check']:
_run_waf_check()
if Options.options.regression or Options.options.regression_generate:
if not env['DIFF']:
raise Utils.WafError("Cannot run regression tests: the 'diff' program is not installed.")
regression_traces = env['REGRESSION_TRACES']
if not regression_traces:
raise Utils.WafError("Cannot run regression tests: reference traces directory not given"
" (--with-regression-traces configure option)")
retval = regression.run_regression(regression_traces)
if retval:
sys.exit(retval)
if Options.options.lcov_report:
lcov_report()