add test programs to test waf and test.py
This commit is contained in:
Siddharth Santurkar
125
utils/tests/TestBase.py
Normal file
125
utils/tests/TestBase.py
Normal file
@@ -0,0 +1,125 @@
|
||||
#! /usr/bin/env python
|
||||
## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
|
||||
#
|
||||
# Copyright (c) 2014 Siddharth Santurkar
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation;
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
#
|
||||
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
import subprocess
|
||||
import argparse
|
||||
import os
|
||||
|
||||
def print_case_in_file(case_string, out):
|
||||
for i in range(100):
|
||||
print("-", end = '', file = out)
|
||||
print(file=out)
|
||||
print("running test case " + case_string, end='\n\n', file = out)
|
||||
out.flush()
|
||||
|
||||
def print_failed_cases(failed_cases):
|
||||
print("\nFailed Cases:")
|
||||
for case in failed_cases:
|
||||
print(case)
|
||||
|
||||
def print_cmds(cmds):
|
||||
print('Commands to be executed:')
|
||||
for cmd in cmds:
|
||||
print(cmd.replace(sys.executable, ''))
|
||||
|
||||
def set_workdir():
|
||||
dir_files = [ f for f in os.listdir('.') if os.path.exists(f) ]
|
||||
if not 'VERSION' in dir_files and not 'waf' in dir_files:
|
||||
if os.path.split(os.path.abspath('.'))[1] == 'tests' and os.path.split(os.path.abspath(os.pardir))[1] == 'utils':
|
||||
os.chdir('../../')
|
||||
else:
|
||||
print('Error: Invalid working directory')
|
||||
sys.exit(1)
|
||||
|
||||
class TestBaseClass:
|
||||
"""
|
||||
Generic class for testing tools based on provided commands and test cases.
|
||||
"""
|
||||
|
||||
def __init__(self, argv, desc, mode):
|
||||
"""
|
||||
Provide input argument list, description and mode of the suite being executed.
|
||||
"""
|
||||
self.my_env = os.environ
|
||||
set_workdir()
|
||||
self.my_env['LD_LIBRARY_PATH'] = os.getcwd() + "/build"
|
||||
self.mode = mode
|
||||
self.outfile = 'test-port-'+self.mode+'.out'
|
||||
self.options = self.parseargs(argv , desc)
|
||||
|
||||
def parseargs(self, argv, desc):
|
||||
"""
|
||||
Parses the commandline arguments
|
||||
"""
|
||||
parser = argparse.ArgumentParser(description = desc)
|
||||
parser.add_argument('-f', '--file', action='store', dest='out_file', default = self.outfile,
|
||||
metavar="FILE",
|
||||
help='File to be used for storing the command specific output (Default: '+self.outfile+')')
|
||||
parser.add_argument('-c', action='store_true', dest='cmds', default=False,
|
||||
help='List out all the commands being tested')
|
||||
parser.add_argument('-m', action='store_true', dest='mute', default=False,
|
||||
help='Sends only stderr output to FILE')
|
||||
parser.add_argument('-x', '--customcmd', action='store', dest='custcmd', default = None,
|
||||
help='Enter a comma-separated list of commands to override the existing ones. NOT APPLICABLE FOR TEST-PY SUITE.')
|
||||
return parser.parse_args(argv)
|
||||
|
||||
def override_cmds(self):
|
||||
"""
|
||||
Can be used by importing suite to handle custom commands
|
||||
"""
|
||||
return self.options.custcmd
|
||||
|
||||
def runtests(self, cmds):
|
||||
"""
|
||||
Execute the tests.
|
||||
"""
|
||||
if self.options.cmds:
|
||||
print_cmds(cmds)
|
||||
return
|
||||
|
||||
final_return = 0
|
||||
total_tests = len(cmds)
|
||||
passed = 0
|
||||
progress = 0.0
|
||||
failed_cases = []
|
||||
with open(self.options.out_file, 'w') as out:
|
||||
outstream = out
|
||||
with open(os.devnull, 'w') as sink:
|
||||
if self.options.mute:
|
||||
outstream = sink
|
||||
for cmd in cmds:
|
||||
case_string = cmd.replace(sys.executable, '')
|
||||
print("running test case: " + case_string)
|
||||
print_case_in_file(case_string, out)
|
||||
progress += 1
|
||||
ret = subprocess.call(cmd, shell=True, env=self.my_env, stdout=outstream, stderr=out)
|
||||
if not ret:
|
||||
passed += 1
|
||||
else:
|
||||
final_return = 1
|
||||
failed_cases.append(case_string)
|
||||
print("[ %s out of %s ] test cases passed; Progress = %.2f%% \n" % (passed, total_tests, progress*100/total_tests))
|
||||
if final_return != 0:
|
||||
print_failed_cases(failed_cases)
|
||||
else:
|
||||
print("\nAll cases passed")
|
||||
print ("Detailed output available in " + self.options.out_file, end='\n\n')
|
||||
return final_return
|
||||
121
utils/tests/test-test.py
Normal file
121
utils/tests/test-test.py
Normal file
@@ -0,0 +1,121 @@
|
||||
#! /usr/bin/env python
|
||||
## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
|
||||
#
|
||||
# Copyright (c) 2014 Siddharth Santurkar
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation;
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
#
|
||||
|
||||
# NOTE: Run this script with the Python3 interpreter if the python3 compatibility
|
||||
# of the ns-3 unit test runner needs to be tested.
|
||||
|
||||
# The following options of test.py are being tested for poratability by this script.
|
||||
# To see the options supported by this script, run with the -h option on the command line
|
||||
#
|
||||
# -h, --help show this help message and exit
|
||||
# -b BUILDPATH, --buildpath=BUILDPATH
|
||||
# specify the path where ns-3 was built (defaults to the
|
||||
# build directory for the current variant)
|
||||
# -c KIND, --constrain=KIND
|
||||
# constrain the test-runner by kind of test
|
||||
# -d, --duration print the duration of each test suite and example
|
||||
# -e EXAMPLE, --example=EXAMPLE
|
||||
# specify a single example to run (no relative path is
|
||||
# needed)
|
||||
# -u, --update-data If examples use reference data files, get them to re-
|
||||
# generate them
|
||||
# -f FULLNESS, --fullness=FULLNESS
|
||||
# choose the duration of tests to run: QUICK, EXTENSIVE,
|
||||
# or TAKES_FOREVER, where EXTENSIVE includes QUICK and
|
||||
# TAKES_FOREVER includes QUICK and EXTENSIVE (only QUICK
|
||||
# tests are run by default)
|
||||
# -g, --grind run the test suites and examples using valgrind
|
||||
# -k, --kinds print the kinds of tests available
|
||||
# -l, --list print the list of known tests
|
||||
# -m, --multiple report multiple failures from test suites and test
|
||||
# cases
|
||||
# -n, --nowaf do not run waf before starting testing
|
||||
# -p PYEXAMPLE, --pyexample=PYEXAMPLE
|
||||
# specify a single python example to run (with relative
|
||||
# path)
|
||||
# -r, --retain retain all temporary files (which are normally
|
||||
# deleted)
|
||||
# -s TEST-SUITE, --suite=TEST-SUITE
|
||||
# specify a single test suite to run
|
||||
# -t TEXT-FILE, --text=TEXT-FILE
|
||||
# write detailed test results into TEXT-FILE.txt
|
||||
# -v, --verbose print progress and informational messages
|
||||
# -w HTML-FILE, --web=HTML-FILE, --html=HTML-FILE
|
||||
# write detailed test results into HTML-FILE.html
|
||||
# -x XML-FILE, --xml=XML-FILE
|
||||
# write detailed test results into XML-FILE.xml
|
||||
|
||||
|
||||
|
||||
from __future__ import print_function
|
||||
from TestBase import TestBaseClass
|
||||
import sys
|
||||
|
||||
def main(argv):
|
||||
"""
|
||||
Prepares test cases and executes
|
||||
"""
|
||||
test_cases = [
|
||||
'',
|
||||
'-h',
|
||||
'--help',
|
||||
'-b build/',
|
||||
'--buildpath=build/',
|
||||
'-c performance',
|
||||
'--constrain=performance',
|
||||
'-d',
|
||||
'--duration',
|
||||
'-e socket-options-ipv6',
|
||||
'--example=socket-options-ipv6',
|
||||
'-u',
|
||||
'--update-data',
|
||||
'-f EXTENSIVE --fullness=EXTENSIVE'
|
||||
'-g',
|
||||
'--grind',
|
||||
'-l',
|
||||
'--list',
|
||||
'-m',
|
||||
'--multiple',
|
||||
'-n',
|
||||
'--nowaf',
|
||||
'-p first',
|
||||
'--pyexample=first',
|
||||
'-r',
|
||||
'--retain',
|
||||
'-s ns3-tcp-interoperability',
|
||||
'--suite=ns3-tcp-interoperability',
|
||||
'-t t_opt.txt',
|
||||
'--text=t_opt.txt && rm -rf t_opt.txt',
|
||||
'-v',
|
||||
'--verbose',
|
||||
'-w t_opt.html && rm -rf t_opt.html',
|
||||
'--web=t_opt.html && rm -rf t_opt.html',
|
||||
'--html=t_opt.html && rm -rf t_opt.html',
|
||||
'-x t_opt.xml && rm -rf t_opt.xml',
|
||||
'--xml=t_opt.xml && rm -rf t_opt.xml',
|
||||
]
|
||||
|
||||
configure_string = sys.executable + ' waf configure --enable-tests --enable-examples'
|
||||
clean_string = sys.executable + ' waf clean'
|
||||
cmd_execute_list = [ '%s && %s test.py %s && %s' % (configure_string, sys.executable, option, clean_string) for option in test_cases]
|
||||
runner = TestBaseClass(argv[1:], "Test suite for the ns-3 unit test runner" , 'test-py')
|
||||
return runner.runtests(cmd_execute_list)
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
195
utils/tests/test-waf.py
Normal file
195
utils/tests/test-waf.py
Normal file
@@ -0,0 +1,195 @@
|
||||
#! /usr/bin/env python
|
||||
## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
|
||||
#
|
||||
# Copyright (c) 2014 Siddharth Santurkar
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation;
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
#
|
||||
|
||||
# NOTE: Run this script with the Python3 interpreter if the python3 compatibility
|
||||
# of the ns-3 unit test runner needs to be tested.
|
||||
|
||||
# The following options of waf are being tested for poratability by this script.
|
||||
# To see the options supported by this script, run with the -h option on the command line
|
||||
#
|
||||
# build : executes the build (pre: configure, post: clean)
|
||||
# check : run the equivalent of the old ns-3 unit tests using test.py
|
||||
# clean : cleans the project
|
||||
# configure: configures the project (pore: None, post: distclean)
|
||||
# dist : makes a tarball for redistributing the sources (pre:none )
|
||||
# distcheck: checks if the project compiles (tarball from 'dist') (pre: dist, post: rm -rf ns-3*.tar.bz2)
|
||||
# docs : build all the documentation: doxygen, manual, tutorial, models (pre: configure; post: distclean)
|
||||
# doxygen : do a full build, generate the introspected doxygen and then the doxygen
|
||||
# install : installs the targets on the system (pre: configure, post: uninstall )
|
||||
# list : lists the targets to execute (pre: configure)
|
||||
# shell : run a shell with an environment suitably modified to run locally built programs (pre:configure)
|
||||
# sphinx : build the Sphinx documentation: manual, tutorial, models
|
||||
# step : executes tasks in a step-by-step fashion, for debugging (pre: configure)
|
||||
# uninstall: removes the targets installed (pre: install, post uninstall)
|
||||
# *update : updates the plugins from the *waflib/extras* directory
|
||||
|
||||
|
||||
|
||||
|
||||
from __future__ import print_function
|
||||
from TestBase import TestBaseClass
|
||||
import sys
|
||||
|
||||
def replace(pre, post, main_cmd_list):
|
||||
if pre:
|
||||
pre = pre + ' && '
|
||||
else:
|
||||
pre = ''
|
||||
if post:
|
||||
post = ' && ' + post
|
||||
else:
|
||||
post = ''
|
||||
return [ pre + main_cmd + post for main_cmd in main_cmd_list ]
|
||||
|
||||
def main(argv):
|
||||
"""
|
||||
Prepares test cases and executes
|
||||
"""
|
||||
runner = TestBaseClass(argv[1:], "Test suite for the ns-3 Waf build system", 'waf')
|
||||
in_cmds = runner.override_cmds()
|
||||
if in_cmds:
|
||||
cmds = in_cmds.split(',')
|
||||
else:
|
||||
cmds = ['basic', 'build', 'configure', 'step', 'clean', 'dist', 'list']
|
||||
|
||||
config_test_cases = [
|
||||
"--enable-gcov",
|
||||
"--enable-sudo",
|
||||
"--enable-sudo",
|
||||
"--enable-tests",
|
||||
"--disable-tests",
|
||||
"--enable-examples",
|
||||
"--disable-examples",
|
||||
"--doxygen-no-build",
|
||||
"--enable-static",
|
||||
"--enable-mpi",
|
||||
"--enable-rpath",
|
||||
"--enable-modules=build/utils/test-runner.cc.1.o",
|
||||
"--boost-static",
|
||||
"--boost-mt",
|
||||
"--boost-linkage_autodetect",
|
||||
"--boost-python=33",
|
||||
"--disable-gtk",
|
||||
"--int64x64=cairo",
|
||||
"--disable-pthread",
|
||||
"--force-planetlab",
|
||||
"--nopyc",
|
||||
"--nopyo",
|
||||
"--disable-python",
|
||||
"--apiscan=all",
|
||||
"--with-python=/usr/bin/python2.7",
|
||||
"--no32bit-scan",
|
||||
"-o test_out && rm -rf test_out",
|
||||
"--out=test_out && rm -rf test_out",
|
||||
"-t test_top && rm -rf test_top",
|
||||
"--top=test_top && rm -rf test_top",
|
||||
"--download",
|
||||
"--check-c-compiler=gc",
|
||||
"--check-cxx-compiler=g++",
|
||||
]
|
||||
|
||||
basic_test_cases = [
|
||||
"--version",
|
||||
"-h",
|
||||
"--help",
|
||||
]
|
||||
|
||||
build_test_cases = [
|
||||
"-j10",
|
||||
"--jobs=10",
|
||||
"-d optimized",
|
||||
"-d debug",
|
||||
"-d release",
|
||||
"--build-profile optimized",
|
||||
"--build-profile debug",
|
||||
"--build-profile release",
|
||||
"-p",
|
||||
"--progress",
|
||||
]
|
||||
|
||||
step_test_cases = [
|
||||
"--files=\"*/main.c,*/test/main.o\"",
|
||||
]
|
||||
|
||||
install_test_cases = [
|
||||
"-f",
|
||||
"--force",
|
||||
"--prefix=./test-prefix && rm -rf ./test-prefix",
|
||||
"--exec-prefix=.",
|
||||
"--bindir=./test-prefix/bin --sbindir=./test-prefix/sbin --libexecdir=./test-prefix/libexec --sysconfdir=./test-prefix/etc --sharedstatedir=./test-prefix/com --localstatedir=./test-prefix/var --libdir=./test-prefix/lib --includedir=./test-prefix/include --oldincludedir=./test-prefix/usr/include --datarootdir=./test-prefix/share --datadir=./test-prefix/share_root --infodir=./test-prefix/info --localedir=./test-prefix/locale --mandir=./test-prefix/man --docdir=./test-prefix/doc/package --htmldir=./test-prefix/doc --dvidir=./test-prefix/doc --pdfdir=./test-prefix/doc --psdir=./test-prefix/doc && rm -rf ./test-prefix",
|
||||
]
|
||||
|
||||
common_test_cases = [
|
||||
"",
|
||||
"-k",
|
||||
"--keep",
|
||||
"-v",
|
||||
"--verbose",
|
||||
"--nocache",
|
||||
"--zones=task_gen",
|
||||
"--zones=deps",
|
||||
"--zones=tasks",
|
||||
"--no-task-lines",
|
||||
]
|
||||
|
||||
test_case_mappings = {
|
||||
'basic' : basic_test_cases,
|
||||
'configure' : config_test_cases,
|
||||
'build' : build_test_cases,
|
||||
'step' : step_test_cases,
|
||||
'install' : install_test_cases,
|
||||
}
|
||||
|
||||
waf_string = sys.executable + ' waf'
|
||||
cmd_execute_list = []
|
||||
for cmd in cmds:
|
||||
if cmd == 'basic':
|
||||
cmd_list = []
|
||||
else:
|
||||
cmd_list = ['%s %s %s' % (waf_string, cmd, option) for option in common_test_cases ]
|
||||
if cmd in test_case_mappings:
|
||||
cmd_list += ['%s %s %s' % (waf_string, cmd, option) for option in test_case_mappings[cmd] ]
|
||||
if cmd == 'basic':
|
||||
cmd_list.append('%s configure && %s build && %s --run scratch/myfirst' % tuple([waf_string]*3))
|
||||
cmd_list.append('%s configure && %s build && %s --pyrun scratch/myfirst.py' % tuple([waf_string]*3))
|
||||
if cmd == 'build':
|
||||
cmd_list = replace(waf_string+' configure', waf_string+' clean', cmd_list)
|
||||
cmd_list.append('%s configure --enable-gcov && %s build --lcov-report && %s clean' % tuple([waf_string]*3))
|
||||
elif cmd == 'configure':
|
||||
cmd_list = replace(None, waf_string+' distclean', cmd_list)
|
||||
elif cmd == 'distcheck':
|
||||
cmd_list = replace(waf_string+' dist', 'rm -rf ns-3*.tar.bz2', cmd_list)
|
||||
elif cmd == 'docs':
|
||||
cmd_list = replace(waf_string+' configure', waf_string+' distclean', cmd_list)
|
||||
elif cmd == 'install':
|
||||
cmd_list = replace(waf_string+' configure', waf_string+' uninstall', cmd_list)
|
||||
elif cmd == 'list':
|
||||
cmd_list = replace(waf_string+' configure', waf_string +' distclean', cmd_list)
|
||||
elif cmd == 'shell':
|
||||
cmd_list = replace(waf_string+' configure', waf_string+' distclean', cmd_list)
|
||||
elif cmd == 'step':
|
||||
cmd_list = replace(waf_string+' configure', waf_string+' distclean', cmd_list)
|
||||
elif cmd == 'uninstall':
|
||||
cmd_list = replace(waf_string+' install', None, cmd_list)
|
||||
cmd_execute_list += cmd_list
|
||||
|
||||
return runner.runtests(cmd_execute_list)
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
Reference in New Issue
Block a user