diff --git a/utils/tests/TestBase.py b/utils/tests/TestBase.py new file mode 100644 index 000000000..9c829c564 --- /dev/null +++ b/utils/tests/TestBase.py @@ -0,0 +1,125 @@ +#! /usr/bin/env python +## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*- +# +# Copyright (c) 2014 Siddharth Santurkar +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation; +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +from __future__ import print_function +import sys +import subprocess +import argparse +import os + +def print_case_in_file(case_string, out): + for i in range(100): + print("-", end = '', file = out) + print(file=out) + print("running test case " + case_string, end='\n\n', file = out) + out.flush() + +def print_failed_cases(failed_cases): + print("\nFailed Cases:") + for case in failed_cases: + print(case) + +def print_cmds(cmds): + print('Commands to be executed:') + for cmd in cmds: + print(cmd.replace(sys.executable, '')) + +def set_workdir(): + dir_files = [ f for f in os.listdir('.') if os.path.exists(f) ] + if not 'VERSION' in dir_files and not 'waf' in dir_files: + if os.path.split(os.path.abspath('.'))[1] == 'tests' and os.path.split(os.path.abspath(os.pardir))[1] == 'utils': + os.chdir('../../') + else: + print('Error: Invalid working directory') + sys.exit(1) + +class TestBaseClass: + """ + Generic class for testing tools based on provided commands and test cases. + """ + + def __init__(self, argv, desc, mode): + """ + Provide input argument list, description and mode of the suite being executed. + """ + self.my_env = os.environ + set_workdir() + self.my_env['LD_LIBRARY_PATH'] = os.getcwd() + "/build" + self.mode = mode + self.outfile = 'test-port-'+self.mode+'.out' + self.options = self.parseargs(argv , desc) + + def parseargs(self, argv, desc): + """ + Parses the commandline arguments + """ + parser = argparse.ArgumentParser(description = desc) + parser.add_argument('-f', '--file', action='store', dest='out_file', default = self.outfile, + metavar="FILE", + help='File to be used for storing the command specific output (Default: '+self.outfile+')') + parser.add_argument('-c', action='store_true', dest='cmds', default=False, + help='List out all the commands being tested') + parser.add_argument('-m', action='store_true', dest='mute', default=False, + help='Sends only stderr output to FILE') + parser.add_argument('-x', '--customcmd', action='store', dest='custcmd', default = None, + help='Enter a comma-separated list of commands to override the existing ones. NOT APPLICABLE FOR TEST-PY SUITE.') + return parser.parse_args(argv) + + def override_cmds(self): + """ + Can be used by importing suite to handle custom commands + """ + return self.options.custcmd + + def runtests(self, cmds): + """ + Execute the tests. + """ + if self.options.cmds: + print_cmds(cmds) + return + + final_return = 0 + total_tests = len(cmds) + passed = 0 + progress = 0.0 + failed_cases = [] + with open(self.options.out_file, 'w') as out: + outstream = out + with open(os.devnull, 'w') as sink: + if self.options.mute: + outstream = sink + for cmd in cmds: + case_string = cmd.replace(sys.executable, '') + print("running test case: " + case_string) + print_case_in_file(case_string, out) + progress += 1 + ret = subprocess.call(cmd, shell=True, env=self.my_env, stdout=outstream, stderr=out) + if not ret: + passed += 1 + else: + final_return = 1 + failed_cases.append(case_string) + print("[ %s out of %s ] test cases passed; Progress = %.2f%% \n" % (passed, total_tests, progress*100/total_tests)) + if final_return != 0: + print_failed_cases(failed_cases) + else: + print("\nAll cases passed") + print ("Detailed output available in " + self.options.out_file, end='\n\n') + return final_return diff --git a/utils/tests/test-test.py b/utils/tests/test-test.py new file mode 100644 index 000000000..c7f668df6 --- /dev/null +++ b/utils/tests/test-test.py @@ -0,0 +1,121 @@ +#! /usr/bin/env python +## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*- +# +# Copyright (c) 2014 Siddharth Santurkar +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation; +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +# NOTE: Run this script with the Python3 interpreter if the python3 compatibility +# of the ns-3 unit test runner needs to be tested. + +# The following options of test.py are being tested for poratability by this script. +# To see the options supported by this script, run with the -h option on the command line +# +# -h, --help show this help message and exit +# -b BUILDPATH, --buildpath=BUILDPATH +# specify the path where ns-3 was built (defaults to the +# build directory for the current variant) +# -c KIND, --constrain=KIND +# constrain the test-runner by kind of test +# -d, --duration print the duration of each test suite and example +# -e EXAMPLE, --example=EXAMPLE +# specify a single example to run (no relative path is +# needed) +# -u, --update-data If examples use reference data files, get them to re- +# generate them +# -f FULLNESS, --fullness=FULLNESS +# choose the duration of tests to run: QUICK, EXTENSIVE, +# or TAKES_FOREVER, where EXTENSIVE includes QUICK and +# TAKES_FOREVER includes QUICK and EXTENSIVE (only QUICK +# tests are run by default) +# -g, --grind run the test suites and examples using valgrind +# -k, --kinds print the kinds of tests available +# -l, --list print the list of known tests +# -m, --multiple report multiple failures from test suites and test +# cases +# -n, --nowaf do not run waf before starting testing +# -p PYEXAMPLE, --pyexample=PYEXAMPLE +# specify a single python example to run (with relative +# path) +# -r, --retain retain all temporary files (which are normally +# deleted) +# -s TEST-SUITE, --suite=TEST-SUITE +# specify a single test suite to run +# -t TEXT-FILE, --text=TEXT-FILE +# write detailed test results into TEXT-FILE.txt +# -v, --verbose print progress and informational messages +# -w HTML-FILE, --web=HTML-FILE, --html=HTML-FILE +# write detailed test results into HTML-FILE.html +# -x XML-FILE, --xml=XML-FILE +# write detailed test results into XML-FILE.xml + + + +from __future__ import print_function +from TestBase import TestBaseClass +import sys + +def main(argv): + """ + Prepares test cases and executes + """ + test_cases = [ + '', + '-h', + '--help', + '-b build/', + '--buildpath=build/', + '-c performance', + '--constrain=performance', + '-d', + '--duration', + '-e socket-options-ipv6', + '--example=socket-options-ipv6', + '-u', + '--update-data', + '-f EXTENSIVE --fullness=EXTENSIVE' + '-g', + '--grind', + '-l', + '--list', + '-m', + '--multiple', + '-n', + '--nowaf', + '-p first', + '--pyexample=first', + '-r', + '--retain', + '-s ns3-tcp-interoperability', + '--suite=ns3-tcp-interoperability', + '-t t_opt.txt', + '--text=t_opt.txt && rm -rf t_opt.txt', + '-v', + '--verbose', + '-w t_opt.html && rm -rf t_opt.html', + '--web=t_opt.html && rm -rf t_opt.html', + '--html=t_opt.html && rm -rf t_opt.html', + '-x t_opt.xml && rm -rf t_opt.xml', + '--xml=t_opt.xml && rm -rf t_opt.xml', + ] + + configure_string = sys.executable + ' waf configure --enable-tests --enable-examples' + clean_string = sys.executable + ' waf clean' + cmd_execute_list = [ '%s && %s test.py %s && %s' % (configure_string, sys.executable, option, clean_string) for option in test_cases] + runner = TestBaseClass(argv[1:], "Test suite for the ns-3 unit test runner" , 'test-py') + return runner.runtests(cmd_execute_list) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) \ No newline at end of file diff --git a/utils/tests/test-waf.py b/utils/tests/test-waf.py new file mode 100644 index 000000000..55f38312c --- /dev/null +++ b/utils/tests/test-waf.py @@ -0,0 +1,195 @@ +#! /usr/bin/env python +## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*- +# +# Copyright (c) 2014 Siddharth Santurkar +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation; +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +# NOTE: Run this script with the Python3 interpreter if the python3 compatibility +# of the ns-3 unit test runner needs to be tested. + +# The following options of waf are being tested for poratability by this script. +# To see the options supported by this script, run with the -h option on the command line +# +# build : executes the build (pre: configure, post: clean) +# check : run the equivalent of the old ns-3 unit tests using test.py +# clean : cleans the project +# configure: configures the project (pore: None, post: distclean) +# dist : makes a tarball for redistributing the sources (pre:none ) +# distcheck: checks if the project compiles (tarball from 'dist') (pre: dist, post: rm -rf ns-3*.tar.bz2) +# docs : build all the documentation: doxygen, manual, tutorial, models (pre: configure; post: distclean) +# doxygen : do a full build, generate the introspected doxygen and then the doxygen +# install : installs the targets on the system (pre: configure, post: uninstall ) +# list : lists the targets to execute (pre: configure) +# shell : run a shell with an environment suitably modified to run locally built programs (pre:configure) +# sphinx : build the Sphinx documentation: manual, tutorial, models +# step : executes tasks in a step-by-step fashion, for debugging (pre: configure) +# uninstall: removes the targets installed (pre: install, post uninstall) +# *update : updates the plugins from the *waflib/extras* directory + + + + +from __future__ import print_function +from TestBase import TestBaseClass +import sys + +def replace(pre, post, main_cmd_list): + if pre: + pre = pre + ' && ' + else: + pre = '' + if post: + post = ' && ' + post + else: + post = '' + return [ pre + main_cmd + post for main_cmd in main_cmd_list ] + +def main(argv): + """ + Prepares test cases and executes + """ + runner = TestBaseClass(argv[1:], "Test suite for the ns-3 Waf build system", 'waf') + in_cmds = runner.override_cmds() + if in_cmds: + cmds = in_cmds.split(',') + else: + cmds = ['basic', 'build', 'configure', 'step', 'clean', 'dist', 'list'] + + config_test_cases = [ + "--enable-gcov", + "--enable-sudo", + "--enable-sudo", + "--enable-tests", + "--disable-tests", + "--enable-examples", + "--disable-examples", + "--doxygen-no-build", + "--enable-static", + "--enable-mpi", + "--enable-rpath", + "--enable-modules=build/utils/test-runner.cc.1.o", + "--boost-static", + "--boost-mt", + "--boost-linkage_autodetect", + "--boost-python=33", + "--disable-gtk", + "--int64x64=cairo", + "--disable-pthread", + "--force-planetlab", + "--nopyc", + "--nopyo", + "--disable-python", + "--apiscan=all", + "--with-python=/usr/bin/python2.7", + "--no32bit-scan", + "-o test_out && rm -rf test_out", + "--out=test_out && rm -rf test_out", + "-t test_top && rm -rf test_top", + "--top=test_top && rm -rf test_top", + "--download", + "--check-c-compiler=gc", + "--check-cxx-compiler=g++", + ] + + basic_test_cases = [ + "--version", + "-h", + "--help", + ] + + build_test_cases = [ + "-j10", + "--jobs=10", + "-d optimized", + "-d debug", + "-d release", + "--build-profile optimized", + "--build-profile debug", + "--build-profile release", + "-p", + "--progress", + ] + + step_test_cases = [ + "--files=\"*/main.c,*/test/main.o\"", + ] + + install_test_cases = [ + "-f", + "--force", + "--prefix=./test-prefix && rm -rf ./test-prefix", + "--exec-prefix=.", + "--bindir=./test-prefix/bin --sbindir=./test-prefix/sbin --libexecdir=./test-prefix/libexec --sysconfdir=./test-prefix/etc --sharedstatedir=./test-prefix/com --localstatedir=./test-prefix/var --libdir=./test-prefix/lib --includedir=./test-prefix/include --oldincludedir=./test-prefix/usr/include --datarootdir=./test-prefix/share --datadir=./test-prefix/share_root --infodir=./test-prefix/info --localedir=./test-prefix/locale --mandir=./test-prefix/man --docdir=./test-prefix/doc/package --htmldir=./test-prefix/doc --dvidir=./test-prefix/doc --pdfdir=./test-prefix/doc --psdir=./test-prefix/doc && rm -rf ./test-prefix", + ] + + common_test_cases = [ + "", + "-k", + "--keep", + "-v", + "--verbose", + "--nocache", + "--zones=task_gen", + "--zones=deps", + "--zones=tasks", + "--no-task-lines", + ] + + test_case_mappings = { + 'basic' : basic_test_cases, + 'configure' : config_test_cases, + 'build' : build_test_cases, + 'step' : step_test_cases, + 'install' : install_test_cases, + } + + waf_string = sys.executable + ' waf' + cmd_execute_list = [] + for cmd in cmds: + if cmd == 'basic': + cmd_list = [] + else: + cmd_list = ['%s %s %s' % (waf_string, cmd, option) for option in common_test_cases ] + if cmd in test_case_mappings: + cmd_list += ['%s %s %s' % (waf_string, cmd, option) for option in test_case_mappings[cmd] ] + if cmd == 'basic': + cmd_list.append('%s configure && %s build && %s --run scratch/myfirst' % tuple([waf_string]*3)) + cmd_list.append('%s configure && %s build && %s --pyrun scratch/myfirst.py' % tuple([waf_string]*3)) + if cmd == 'build': + cmd_list = replace(waf_string+' configure', waf_string+' clean', cmd_list) + cmd_list.append('%s configure --enable-gcov && %s build --lcov-report && %s clean' % tuple([waf_string]*3)) + elif cmd == 'configure': + cmd_list = replace(None, waf_string+' distclean', cmd_list) + elif cmd == 'distcheck': + cmd_list = replace(waf_string+' dist', 'rm -rf ns-3*.tar.bz2', cmd_list) + elif cmd == 'docs': + cmd_list = replace(waf_string+' configure', waf_string+' distclean', cmd_list) + elif cmd == 'install': + cmd_list = replace(waf_string+' configure', waf_string+' uninstall', cmd_list) + elif cmd == 'list': + cmd_list = replace(waf_string+' configure', waf_string +' distclean', cmd_list) + elif cmd == 'shell': + cmd_list = replace(waf_string+' configure', waf_string+' distclean', cmd_list) + elif cmd == 'step': + cmd_list = replace(waf_string+' configure', waf_string+' distclean', cmd_list) + elif cmd == 'uninstall': + cmd_list = replace(waf_string+' install', None, cmd_list) + cmd_execute_list += cmd_list + + return runner.runtests(cmd_execute_list) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) \ No newline at end of file