Integrate regression testing into the main waf script.
This commit is contained in:
Gustavo J. A. M. Carneiro
@@ -1,175 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
# regression.py adapted from python language regression scripts.
|
||||
|
||||
"""Regression test.
|
||||
|
||||
This will find all modules whose name is "test-*" in the tests
|
||||
directory, and run them.
|
||||
|
||||
Command line options:
|
||||
|
||||
-v: verbose -- run tests in verbose mode (print diff output)
|
||||
-g: generate -- write the output file for a test instead of comparing it
|
||||
|
||||
If non-option arguments are present, they are names for tests to run.
|
||||
If no test names are given, all tests are run.
|
||||
|
||||
"""
|
||||
|
||||
import getopt
|
||||
import sys
|
||||
import os
|
||||
import urllib
|
||||
import subprocess
|
||||
|
||||
verbose = 0
|
||||
generate = 0
|
||||
|
||||
#
|
||||
# The directory in which the tarball of the reference traces lives. This is
|
||||
# used if Mercurial is not on the system.
|
||||
#
|
||||
refUrl = "http://www.nsnam.org/releases/"
|
||||
|
||||
#
|
||||
# The name of the tarball to find the reference traces in if there is no
|
||||
# mercurial on the system. It is expected to be created using tar -cjf and
|
||||
# will be extracted using tar -xjf
|
||||
#
|
||||
refTarName = "ns-3.0.12-ref-traces.tar.bz2"
|
||||
|
||||
#
|
||||
# The path to the Mercurial repository used to find the reference traces if
|
||||
# we find "hg" on the system. We expect that the repository will be named
|
||||
# identically to refDirName below
|
||||
#
|
||||
refRepo = "http://code.nsnam.org/"
|
||||
|
||||
#
|
||||
# The local directory name into which the reference traces will go in either
|
||||
# case (net or hg).
|
||||
#
|
||||
refDirName = "ns-3-dev-ref-traces"
|
||||
|
||||
def main(tests = None, testdir = None):
|
||||
"""Execute regression tests.
|
||||
|
||||
Arguments:
|
||||
tests -- a list of strings containing test names (optional)
|
||||
testdir -- the directory in which to look for tests (optional)
|
||||
"""
|
||||
|
||||
global verbose
|
||||
global generate
|
||||
global refUrl
|
||||
global refTarName
|
||||
global refRepo
|
||||
global refRepoName
|
||||
global refDirName
|
||||
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'vg')
|
||||
except getopt.error, msg:
|
||||
print msg
|
||||
print __doc__
|
||||
return 2
|
||||
|
||||
for o, a in opts:
|
||||
if o == '-v': verbose = 1
|
||||
if o == '-g': generate = 1
|
||||
|
||||
if not testdir:
|
||||
testdir = os.path.join(os.curdir, "tests")
|
||||
|
||||
if not os.path.exists(testdir):
|
||||
print "Tests directory does not exist"
|
||||
return 3
|
||||
|
||||
sys.path.append(testdir)
|
||||
|
||||
for i in range(len(args)):
|
||||
if args[i][-3:] == '.py':
|
||||
args[i] = args[i][:-3]
|
||||
|
||||
if not tests:
|
||||
tests = args
|
||||
|
||||
if not tests:
|
||||
tests = findtests(testdir)
|
||||
if not generate:
|
||||
print "========== Running Unit Tests =========="
|
||||
os.system("./waf check")
|
||||
|
||||
print "========== Running Regression Tests =========="
|
||||
|
||||
if os.system("hg version > /dev/null 2>&1") == 0:
|
||||
print "Synchronizing reference traces using Mercurial."
|
||||
if not os.path.exists(refDirName):
|
||||
os.system("hg clone " + refRepo + refDirName + " > /dev/null 2>&1")
|
||||
else:
|
||||
os.chdir(refDirName)
|
||||
os.system("hg pull " + refRepo + refDirName + " > /dev/null 2>&1")
|
||||
os.chdir("..")
|
||||
else:
|
||||
print "Synchronizing reference traces from web."
|
||||
urllib.urlretrieve(refUrl + refTarName, refTarName)
|
||||
os.system("tar -xjf " + refTarName)
|
||||
|
||||
print "Done."
|
||||
|
||||
if not os.path.exists(refDirName):
|
||||
print "Reference traces directory does not exist"
|
||||
return 3
|
||||
|
||||
bad = []
|
||||
|
||||
|
||||
for test in tests:
|
||||
result = run_test(test)
|
||||
if result == 0:
|
||||
if generate:
|
||||
print "GENERATE " + test
|
||||
else:
|
||||
print "PASS " + test
|
||||
else:
|
||||
bad.append(test)
|
||||
print "FAIL " + test
|
||||
|
||||
return len(bad) > 0
|
||||
|
||||
def findtests(testdir):
|
||||
"""Return a list of test modules in the test directory
|
||||
|
||||
Arguments:
|
||||
testdir -- the directory to look in for tests
|
||||
"""
|
||||
names = os.listdir(testdir)
|
||||
tests = []
|
||||
for name in names:
|
||||
if name[:5] == "test-" and name[-3:] == ".py":
|
||||
testname = name[:-3]
|
||||
tests.append(testname)
|
||||
tests.sort()
|
||||
return tests
|
||||
|
||||
def run_test(test):
|
||||
"""Run a single test.
|
||||
|
||||
Arguments:
|
||||
test -- the name of the test
|
||||
"""
|
||||
if os.path.exists("traces"):
|
||||
files = os.listdir("traces")
|
||||
for file in files:
|
||||
if file == '.' or file == '..':
|
||||
continue
|
||||
path = "traces" + os.sep + file
|
||||
os.remove(path)
|
||||
else:
|
||||
os.mkdir("traces")
|
||||
|
||||
mod = __import__(test, globals(), locals(), [])
|
||||
return mod.run(verbose, generate, refDirName)
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,57 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
|
||||
"""Generic trace-comparison-type regression test."""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
|
||||
def run_test(verbose, generate, refDirName, testName):
|
||||
"""Execute a test."""
|
||||
|
||||
refTestDirName = refDirName + "/" + testName + ".ref"
|
||||
|
||||
if not os.path.exists(refDirName):
|
||||
print"No reference trace repository"
|
||||
return 1
|
||||
|
||||
if generate:
|
||||
if not os.path.exists(refTestDirName):
|
||||
print "creating new " + refTestDirName
|
||||
os.mkdir(refTestDirName)
|
||||
|
||||
os.system("./waf --cwd regression/" + refTestDirName +
|
||||
" --run " + testName + " > /dev/null 2>&1")
|
||||
|
||||
print "Remember to commit " + refTestDirName
|
||||
return 0
|
||||
else:
|
||||
if not os.path.exists(refTestDirName):
|
||||
print "Cannot locate reference traces"
|
||||
return 1
|
||||
|
||||
shutil.rmtree("traces");
|
||||
os.mkdir("traces")
|
||||
|
||||
os.system("./waf --cwd regression/traces --run " +
|
||||
testName + " > /dev/null 2>&1")
|
||||
|
||||
if verbose:
|
||||
diffCmd = "diff traces " + refTestDirName + " | head"
|
||||
else:
|
||||
diffCmd = "diff traces " + refTestDirName + \
|
||||
" > /dev/null 2>&1"
|
||||
|
||||
rc = os.system(diffCmd)
|
||||
if rc:
|
||||
print "----------"
|
||||
print "Traces differ in test: test-" + testName
|
||||
print "Reference traces in directory: " + refTestDirName
|
||||
print "Traces in directory: traces"
|
||||
print "Rerun regression test as: " + \
|
||||
"\"python regression.py test-" + testName + "\""
|
||||
print "Then do \"diff -u traces " + refTestDirName + \
|
||||
"\" for details"
|
||||
print "----------"
|
||||
return rc
|
||||
|
||||
200
wscript
200
wscript
@@ -12,6 +12,7 @@ import pproc as subprocess
|
||||
|
||||
Params.g_autoconfig = 1
|
||||
|
||||
|
||||
# the following two variables are used by the target "waf dist"
|
||||
VERSION = file("VERSION").read().strip()
|
||||
APPNAME = 'ns'
|
||||
@@ -20,6 +21,33 @@ APPNAME = 'ns'
|
||||
srcdir = '.'
|
||||
blddir = 'build'
|
||||
|
||||
#
|
||||
# The directory in which the tarball of the reference traces lives. This is
|
||||
# used if Mercurial is not on the system.
|
||||
#
|
||||
REGRESSION_TRACES_URL = "http://www.nsnam.org/releases/"
|
||||
|
||||
#
|
||||
# The name of the tarball to find the reference traces in if there is no
|
||||
# mercurial on the system. It is expected to be created using tar -cjf and
|
||||
# will be extracted using tar -xjf
|
||||
#
|
||||
REGRESSION_TRACES_TAR_NAME = "ns-3.0.12-ref-traces.tar.bz2"
|
||||
|
||||
#
|
||||
# The path to the Mercurial repository used to find the reference traces if
|
||||
# we find "hg" on the system. We expect that the repository will be named
|
||||
# identically to refDirName below
|
||||
#
|
||||
REGRESSION_TRACES_REPO = "http://code.nsnam.org/"
|
||||
|
||||
#
|
||||
# The local directory name (relative to the 'regression' dir) into
|
||||
# which the reference traces will go in either case (net or hg).
|
||||
#
|
||||
REGRESSION_TRACES_DIR_NAME = "ns-3-dev-ref-traces"
|
||||
|
||||
|
||||
def dist_hook():
|
||||
shutil.rmtree("doc/html", True)
|
||||
shutil.rmtree("doc/latex", True)
|
||||
@@ -95,6 +123,17 @@ def set_options(opt):
|
||||
action="store_true", default=False,
|
||||
dest='shell')
|
||||
|
||||
opt.add_option('--regression',
|
||||
help=("Enable regression testing; only used for the 'check' target"),
|
||||
default=False, dest='regression', action="store_true")
|
||||
opt.add_option('--regression-generate',
|
||||
help=("Generate new regression test traces; only used for the 'check' target"),
|
||||
default=False, dest='regression_generate', action="store_true")
|
||||
opt.add_option('--regression-tests',
|
||||
help=('For regression testing, only run/generate the indicated regression tests, '
|
||||
'specified as a comma separated list of test names'),
|
||||
dest='regression_tests', type="string")
|
||||
|
||||
# options provided in a script in a subdirectory named "src"
|
||||
opt.sub_options('src')
|
||||
|
||||
@@ -256,7 +295,7 @@ def get_command_template():
|
||||
if Params.g_options.valgrind:
|
||||
if Params.g_options.command_template:
|
||||
Params.fatal("Options --command-template and --valgrind are conflicting")
|
||||
return "valgrind %s"
|
||||
return "valgrind --leak-check=full %s"
|
||||
else:
|
||||
return (Params.g_options.command_template or '%s')
|
||||
|
||||
@@ -273,6 +312,14 @@ def shutdown():
|
||||
if Params.g_commands['check']:
|
||||
_run_waf_check()
|
||||
|
||||
if Params.g_options.regression or Params.g_options.regression_generate:
|
||||
_dir = os.getcwd()
|
||||
os.chdir("regression")
|
||||
try:
|
||||
run_regression()
|
||||
finally:
|
||||
os.chdir(_dir)
|
||||
|
||||
if Params.g_options.lcov_report:
|
||||
lcov_report()
|
||||
|
||||
@@ -588,3 +635,154 @@ def DistDir(appname, version):
|
||||
return TMPFOLDER
|
||||
|
||||
Scripting.DistDir = DistDir
|
||||
|
||||
|
||||
|
||||
|
||||
### Regression testing
|
||||
class Regression(object):
|
||||
def __init__(self, testdir):
|
||||
self.testdir = testdir
|
||||
|
||||
def run_test(self, verbose, generate, refDirName, testName):
|
||||
refTestDirName = os.path.join(refDirName, (testName + ".ref"))
|
||||
|
||||
if not os.path.exists(refDirName):
|
||||
print"No reference trace repository"
|
||||
return 1
|
||||
|
||||
if generate:
|
||||
if not os.path.exists(refTestDirName):
|
||||
print "creating new " + refTestDirName
|
||||
os.mkdir(refTestDirName)
|
||||
|
||||
#os.system("./waf --cwd regression/" + refTestDirName +
|
||||
# " --run " + testName + " > /dev/null 2>&1")
|
||||
Params.g_options.cwd_launch = refTestDirName
|
||||
run_program(testName)
|
||||
|
||||
print "Remember to commit " + refTestDirName
|
||||
return 0
|
||||
else:
|
||||
if not os.path.exists(refTestDirName):
|
||||
print "Cannot locate reference traces"
|
||||
return 1
|
||||
|
||||
shutil.rmtree("traces");
|
||||
os.mkdir("traces")
|
||||
|
||||
#os.system("./waf --cwd regression/traces --run " +
|
||||
# testName + " > /dev/null 2>&1")
|
||||
Params.g_options.cwd_launch = "traces"
|
||||
run_program(testName, command_template=get_command_template())
|
||||
|
||||
if verbose:
|
||||
diffCmd = "diff traces " + refTestDirName + " | head"
|
||||
else:
|
||||
diffCmd = "diff traces " + refTestDirName + \
|
||||
" > /dev/null 2>&1"
|
||||
|
||||
rc = os.system(diffCmd)
|
||||
if rc:
|
||||
print "----------"
|
||||
print "Traces differ in test: test-" + testName
|
||||
print "Reference traces in directory: " + refTestDirName
|
||||
print "Traces in directory: traces"
|
||||
print "Rerun regression test as: " + \
|
||||
"\"python regression.py test-" + testName + "\""
|
||||
print "Then do \"diff -u traces " + refTestDirName + \
|
||||
"\" for details"
|
||||
print "----------"
|
||||
return rc
|
||||
|
||||
def _find_tests(testdir):
|
||||
"""Return a list of test modules in the test directory
|
||||
|
||||
Arguments:
|
||||
testdir -- the directory to look in for tests
|
||||
"""
|
||||
names = os.listdir(testdir)
|
||||
tests = []
|
||||
for name in names:
|
||||
if name[:5] == "test-" and name[-3:] == ".py":
|
||||
testname = name[:-3]
|
||||
tests.append(testname)
|
||||
tests.sort()
|
||||
return tests
|
||||
|
||||
def run_regression():
|
||||
"""Execute regression tests."""
|
||||
|
||||
testdir = "tests"
|
||||
if not os.path.exists(testdir):
|
||||
print "Tests directory does not exist"
|
||||
sys.exit(3)
|
||||
|
||||
sys.path.append(testdir)
|
||||
sys.modules['tracediff'] = Regression(testdir)
|
||||
|
||||
if Params.g_options.regression_tests:
|
||||
tests = Params.g_options.regression_tests.split(',')
|
||||
else:
|
||||
tests = _find_tests(testdir)
|
||||
|
||||
print "========== Running Regression Tests =========="
|
||||
|
||||
if os.system("hg version > /dev/null 2>&1") == 0:
|
||||
print "Synchronizing reference traces using Mercurial."
|
||||
if not os.path.exists(REGRESSION_TRACES_DIR_NAME):
|
||||
os.system("hg clone " + REGRESSION_TRACES_REPO + REGRESSION_TRACES_DIR_NAME + " > /dev/null 2>&1")
|
||||
else:
|
||||
_dir = os.getcwd()
|
||||
os.chdir(REGRESSION_TRACES_DIR_NAME)
|
||||
try:
|
||||
os.system("hg pull " + REGRESSION_TRACES_REPO + REGRESSION_TRACES_DIR_NAME + " > /dev/null 2>&1")
|
||||
finally:
|
||||
os.chdir("..")
|
||||
else:
|
||||
print "Synchronizing reference traces from web."
|
||||
urllib.urlretrieve(REGRESSION_TRACES_URL + REGRESSION_TRACES_TAR_NAME, REGRESSION_TRACES_TAR_NAME)
|
||||
os.system("tar -xjf %s" % (REGRESSION_TRACES_TAR_NAME,))
|
||||
|
||||
print "Done."
|
||||
|
||||
if not os.path.exists(REGRESSION_TRACES_DIR_NAME):
|
||||
print "Reference traces directory does not exist"
|
||||
return 3
|
||||
|
||||
bad = []
|
||||
|
||||
for test in tests:
|
||||
result = _run_regression_test(test)
|
||||
if result == 0:
|
||||
if Params.g_options.regression_generate:
|
||||
print "GENERATE " + test
|
||||
else:
|
||||
print "PASS " + test
|
||||
else:
|
||||
bad.append(test)
|
||||
print "FAIL " + test
|
||||
|
||||
return len(bad) > 0
|
||||
|
||||
|
||||
def _run_regression_test(test):
|
||||
"""Run a single test.
|
||||
|
||||
Arguments:
|
||||
test -- the name of the test
|
||||
"""
|
||||
if os.path.exists("traces"):
|
||||
files = os.listdir("traces")
|
||||
for file in files:
|
||||
if file == '.' or file == '..':
|
||||
continue
|
||||
path = "traces" + os.sep + file
|
||||
os.remove(path)
|
||||
else:
|
||||
os.mkdir("traces")
|
||||
|
||||
mod = __import__(test, globals(), locals(), [])
|
||||
return mod.run(verbose=(Params.g_options.verbose > 0),
|
||||
generate=Params.g_options.regression_generate,
|
||||
refDirName=REGRESSION_TRACES_DIR_NAME)
|
||||
|
||||
Reference in New Issue
Block a user