diff --git a/waf b/waf index ca4d42c18..65ab0a8e3 100755 Binary files a/waf and b/waf differ diff --git a/waf-tools/cflags.py b/waf-tools/cflags.py new file mode 100644 index 000000000..1738c88f9 --- /dev/null +++ b/waf-tools/cflags.py @@ -0,0 +1,192 @@ +import Logs +import Options +import Utils + + +class CompilerTraits(object): + def get_warnings_flags(self, level): + """get_warnings_flags(level) -> list of cflags""" + raise NotImplementedError + + def get_optimization_flags(self, level): + """get_optimization_flags(level) -> list of cflags""" + raise NotImplementedError + + def get_debug_flags(self, level): + """get_debug_flags(level) -> (list of cflags, list of cppdefines)""" + raise NotImplementedError + + +class GccTraits(CompilerTraits): + def __init__(self): + super(GccTraits, self).__init__() + # cumulative list of warnings per level + self.warnings_flags = [['-Wall'], ['-Werror'], ['-Wextra']] + + def get_warnings_flags(self, level): + warnings = [] + for l in range(level): + if l < len(self.warnings_flags): + warnings.extend(self.warnings_flags[l]) + else: + break + return warnings + + def get_optimization_flags(self, level): + if level == 0: + return ['-O0'] + elif level == 1: + return ['-O'] + elif level == 2: + return ['-O2'] + elif level == 3: + return ['-O3'] + + def get_debug_flags(self, level): + if level == 0: + return (['-g0'], ['NDEBUG']) + elif level == 1: + return (['-g'], []) + elif level >= 2: + return (['-ggdb', '-g3'], ['_DEBUG']) + + +class IccTraits(CompilerTraits): + def __init__(self): + super(IccTraits, self).__init__() + # cumulative list of warnings per level + # icc is _very_ verbose with -Wall, -Werror is barely achievable + self.warnings_flags = [[], [], ['-Wall']] + + def get_warnings_flags(self, level): + warnings = [] + for l in range(level): + if l < len(self.warnings_flags): + warnings.extend(self.warnings_flags[l]) + else: + break + return warnings + + def get_optimization_flags(self, level): + if level == 0: + return ['-O0'] + elif level == 1: + return ['-O'] + elif level == 2: + return ['-O2'] + elif level == 3: + return ['-O3'] + + def get_debug_flags(self, level): + if level == 0: + return (['-g0'], ['NDEBUG']) + elif level == 1: + return (['-g'], []) + elif level >= 2: + return (['-ggdb', '-g3'], ['_DEBUG']) + + + +class MsvcTraits(CompilerTraits): + def __init__(self): + super(MsvcTraits, self).__init__() + # cumulative list of warnings per level + self.warnings_flags = [['/W2'], ['/WX'], ['/Wall']] + + def get_warnings_flags(self, level): + warnings = [] + for l in range(level): + if l < len(self.warnings_flags): + warnings.extend(self.warnings_flags[l]) + else: + break + return warnings + + def get_optimization_flags(self, level): + if level == 0: + return ['/Od'] + elif level == 1: + return [] + elif level == 2: + return ['/O2'] + elif level == 3: + return ['/Ox'] + + def get_debug_flags(self, level): + if level == 0: + return ([], ['NDEBUG']) + elif level == 1: + return (['/ZI', '/RTC1'], []) + elif level >= 2: + return (['/ZI', '/RTC1'], ['_DEBUG']) + + + +gcc = GccTraits() +icc = IccTraits() +msvc = MsvcTraits() + +# how to map env['COMPILER_CC'] or env['COMPILER_CXX'] into a traits object +compiler_mapping = { + 'gcc': gcc, + 'g++': gcc, + 'msvc': msvc, + 'icc': icc, + 'icpc': icc, +} + +profiles = { + # profile name: [optimization_level, warnings_level, debug_level] + 'default': [2, 1, 1], + 'debug': [0, 2, 3], + 'release': [3, 1, 0], + } + +default_profile = 'default' + +def set_options(opt): + assert default_profile in profiles + opt.add_option('-d', '--build-profile', + action='store', + default=default_profile, + help=("Specify the build profile. " + "Build profiles control the default compilation flags" + " used for C/C++ programs, if CCFLAGS/CXXFLAGS are not" + " set set in the environment. [Allowed Values: %s]" + % ", ".join([repr(p) for p in profiles.keys()])), + choices=profiles.keys(), + dest='build_profile') + +def detect(conf): + cc = conf.env['COMPILER_CC'] or None + cxx = conf.env['COMPILER_CXX'] or None + if not (cc or cxx): + raise Utils.WafError("neither COMPILER_CC nor COMPILER_CXX are defined; " + "maybe the compiler_cc or compiler_cxx tool has not been configured yet?") + + try: + compiler = compiler_mapping[cc] + except KeyError: + try: + compiler = compiler_mapping[cxx] + except KeyError: + Logs.warn("No compiler flags support for compiler %r or %r" + % (cc, cxx)) + return + + opt_level, warn_level, dbg_level = profiles[Options.options.build_profile] + + optimizations = compiler.get_optimization_flags(opt_level) + debug, debug_defs = compiler.get_debug_flags(dbg_level) + warnings = compiler.get_warnings_flags(warn_level) + + if cc and not conf.env['CCFLAGS']: + conf.env.append_value('CCFLAGS', optimizations) + conf.env.append_value('CCFLAGS', debug) + conf.env.append_value('CCFLAGS', warnings) + conf.env.append_value('CCDEFINES', debug_defs) + if cxx and not conf.env['CXXFLAGS']: + conf.env.append_value('CXXFLAGS', optimizations) + conf.env.append_value('CXXFLAGS', debug) + conf.env.append_value('CXXFLAGS', warnings) + conf.env.append_value('CXXDEFINES', debug_defs) diff --git a/waf-tools/command.py b/waf-tools/command.py new file mode 100644 index 000000000..5034acf3c --- /dev/null +++ b/waf-tools/command.py @@ -0,0 +1,134 @@ +from TaskGen import feature, taskgen, before, task_gen +import Node, Task, Utils, Build, pproc, Constants +import Options + +import shellcmd +shellcmd.subprocess = pproc # the WAF version of the subprocess module is supposedly less buggy + +from Logs import debug, error +shellcmd.debug = debug + +import Task + +import re + + +arg_rx = re.compile(r"(?P\$\$)|(?P\$\{(?P\w+)(?P.*?)\})", re.M) + +class command_task(Task.Task): + color = "BLUE" + def __init__(self, env, generator): + Task.Task.__init__(self, env, normal=1, generator=generator) + + def __str__(self): + "string to display to the user" + env = self.env + src_str = ' '.join([a.nice_path(env) for a in self.inputs]) + tgt_str = ' '.join([a.nice_path(env) for a in self.outputs]) + if self.outputs: + sep = ' -> ' + else: + sep = '' + + pipeline = shellcmd.Pipeline() + pipeline.parse(self.generator.command) + cmd = pipeline.get_abbreviated_command() + + return 'command (%s): %s%s%s\n' % (cmd, src_str, sep, tgt_str) + + def _subst_arg(self, arg, direction, namespace): + """ + @param arg: the command argument (or stdin/stdout/stderr) to substitute + @param direction: direction of the argument: 'in', 'out', or None + """ + def repl(match): + if match.group('dollar'): + return "$" + elif match.group('subst'): + var = match.group('var') + code = match.group('code') + result = eval(var+code, namespace) + if isinstance(result, Node.Node): + if var == 'TGT': + return result.bldpath(self.env) + elif var == 'SRC': + return result.srcpath(self.env) + else: + raise ValueError("Bad subst variable %r" % var) + elif result is self.inputs: + if len(self.inputs) == 1: + return result[0].srcpath(self.env) + else: + raise ValueError("${SRC} requested but have multiple sources; which one?") + elif result is self.outputs: + if len(self.outputs) == 1: + return result[0].bldpath(self.env) + else: + raise ValueError("${TGT} requested but have multiple targets; which one?") + else: + return result + return None + + return arg_rx.sub(repl, arg) + + def run(self): + pipeline = shellcmd.Pipeline() + pipeline.parse(self.generator.command) + namespace = self.env.get_merged_dict() + if self.generator.variables is not None: + namespace.update(self.generator.variables) + namespace.update(env=self.env, SRC=self.inputs, TGT=self.outputs) + for cmd in pipeline.pipeline: + if isinstance(cmd, shellcmd.Command): + if isinstance(cmd.stdin, basestring): + cmd.stdin = self._subst_arg(cmd.stdin, 'in', namespace) + if isinstance(cmd.stdout, basestring): + cmd.stdout = self._subst_arg(cmd.stdout, 'out', namespace) + if isinstance(cmd.stderr, basestring): + cmd.stderr = self._subst_arg(cmd.stderr, 'out', namespace) + for argI in xrange(len(cmd.argv)): + cmd.argv[argI] = self._subst_arg(cmd.argv[argI], None, namespace) + if cmd.env_vars is not None: + env_vars = dict() + for name, value in cmd.env_vars.iteritems(): + env_vars[name] = self._subst_arg(value, None, namespace) + cmd.env_vars = env_vars + elif isinstance(cmd, shellcmd.Chdir): + cmd.dir = self._subst_arg(cmd.dir, None, namespace) + + return pipeline.run(verbose=(Options.options.verbose > 0)) + +@taskgen +@feature('command') +def init_command(self): + Utils.def_attrs(self, + # other variables that can be used in the command: ${VARIABLE} + variables = None) + + + +@taskgen +@feature('command') +@before('apply_core') +def apply_command(self): + self.meths.remove('apply_core') + # create the task + task = self.create_task('command') + setattr(task, "dep_vars", getattr(self, "dep_vars", None)) + # process the sources + inputs = [] + for src in self.to_list(self.source): + node = self.path.find_resource(src) + if node is None: + raise Utils.WafError("source %s not found" % src) + inputs.append(node) + task.set_inputs(inputs) + task.set_outputs([self.path.find_or_declare(tgt) for tgt in self.to_list(self.target)]) + #Task.file_deps = Task.extract_deps + + + +class command_taskgen(task_gen): + def __init__(self, *k, **kw): + task_gen.__init__(self, *k, **kw) + self.features.append('command') diff --git a/waf-tools/pkgconfig.py b/waf-tools/pkgconfig.py new file mode 100644 index 000000000..95d7cf587 --- /dev/null +++ b/waf-tools/pkgconfig.py @@ -0,0 +1,71 @@ +# -*- mode: python; encoding: utf-8 -*- +# Gustavo Carneiro (gjamc) 2008 + +import Options +import Configure +import pproc as subprocess +import config_c + +def detect(conf): + pkg_config = conf.find_program('pkg-config', var='PKG_CONFIG') + if not pkg_config: return + +@Configure.conf +def pkg_check_modules(conf, uselib_name, expression, mandatory=True): + pkg_config = conf.env['PKG_CONFIG'] + if not pkg_config: + if mandatory: + conf.fatal("pkg-config is not available") + else: + return False + + argv = [pkg_config, '--cflags', '--libs', expression] + cmd = subprocess.Popen(argv, stdout=subprocess.PIPE) + out, dummy = cmd.communicate() + retval = cmd.wait() + + msg_checking = ("pkg-config flags for %s" % (uselib_name,)) + if Options.options.verbose: + if retval == 0: + conf.check_message_custom(msg_checking, + ('(%s)' % expression), out) + else: + conf.check_message(msg_checking, ('(%s)' % expression), False) + else: + conf.check_message(msg_checking, '', (retval == 0), '') + conf.log.write('%r: %r (exit code %i)\n' % (argv, out, retval)) + + if retval == 0: + + config_c.parse_flags(out, uselib_name, conf.env) + conf.env[uselib_name] = True + return True + + else: + + conf.env[uselib_name] = False + if mandatory: + raise Configure.ConfigurationError('pkg-config check failed') + else: + return False + +@Configure.conf +def pkg_check_module_variable(conf, module, variable): + pkg_config = conf.env['PKG_CONFIG'] + if not pkg_config: + conf.fatal("pkg-config is not available") + + argv = [pkg_config, '--variable', variable, module] + cmd = subprocess.Popen(argv, stdout=subprocess.PIPE) + out, dummy = cmd.communicate() + retval = cmd.wait() + out = out.rstrip() # strip the trailing newline + + msg_checking = ("pkg-config variable %r in %s" % (variable, module,)) + conf.check_message_custom(msg_checking, '', out) + conf.log.write('%r: %r (exit code %i)\n' % (argv, out, retval)) + + if retval == 0: + return out + else: + raise Configure.ConfigurationError('pkg-config check failed') diff --git a/waf-tools/shellcmd.py b/waf-tools/shellcmd.py new file mode 100644 index 000000000..776593528 --- /dev/null +++ b/waf-tools/shellcmd.py @@ -0,0 +1,345 @@ +# Copyright (C) 2008 Gustavo J. A. M. Carneiro + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import shlex +import subprocess +import sys +import re +import os + +env_var_rx = re.compile(r"^([a-zA-Z0-9_]+)=(\S+)$") + +def debug(message): + print >> sys.stderr, message + + +if sys.platform == 'win32': + dev_null = open("NUL:", "w") +else: + dev_null = open("/dev/null", "w") + +def _open_out_file(filename): + if filename in ['NUL:', '/dev/null']: + return dev_null + else: + return open(filename, 'wb') + + +class Node(object): + pass + +class Op(Node): + pass + +class Pipe(Op): + pass + +class And(Op): + pass + +class Or(Op): + pass + +class Command(Node): + class PIPE(object): + pass # PIPE is a constant + class STDOUT(object): + pass # PIPE is a constant + + def __init__(self, name): + super(Command, self).__init__() + self.name = name # command name + self.argv = [name] # command argv + self.stdin = None + self.stdout = None + self.stderr = None + self.env_vars = None + + def __repr__(self): + return "Command(%r, argv=%r, stdin=%r, stdout=%r, stderr=%r)" \ + % (self.name, self.argv, self.stdin, self.stdout, self.stderr) + +class Chdir(Node): + def __init__(self): + super(Chdir, self).__init__() + self.dir = None + + def __repr__(self): + return "Chdir(%r)" \ + % (self.dir) + +class Pipeline(object): + def __init__(self): + self.current_command = None + self.pipeline = [] + + def _commit_command(self): + assert self.current_command is not None + self.pipeline.append(self.current_command) + self.current_command = None + + def get_abbreviated_command(self): + l = [] + for node in self.pipeline: + if isinstance(node, Command): + l.append(node.name) + if isinstance(node, Chdir): + l.append('cd %s' % node.dir) + elif isinstance(node, Pipe): + l.append('|') + elif isinstance(node, And): + l.append('&&') + elif isinstance(node, And): + l.append('||') + return ' '.join(l) + + def parse(self, command): + self.current_command = None + self.pipeline = [] + + if isinstance(command, list): + tokens = list(command) + else: + tokens = shlex.split(command) + debug("command: shlex: %r" % (tokens,)) + + BEGIN, COMMAND, CHDIR, STDERR, STDOUT, STDIN = range(6) + state = BEGIN + self.current_command = None + env_vars = dict() + + while tokens: + token = tokens.pop(0) + if state == BEGIN: + env_var_match = env_var_rx.match(token) + if env_var_match is not None: + env_vars[env_var_match.group(1)] = env_var_match.group(2) + else: + assert self.current_command is None + if token == 'cd': + self.current_command = Chdir() + assert not env_vars + state = CHDIR + else: + self.current_command = Command(token) + if env_vars: + self.current_command.env_vars = env_vars + env_vars = dict() + state = COMMAND + elif state == COMMAND: + if token == '>': + state = STDOUT + elif token == '2>': + state = STDERR + elif token == '2>&1': + assert self.current_command.stderr is None + self.current_command.stderr = Command.STDOUT + elif token == '<': + state = STDIN + elif token == '|': + assert self.current_command.stdout is None + self.current_command.stdout = Command.PIPE + self._commit_command() + self.pipeline.append(Pipe()) + state = BEGIN + elif token == '&&': + self._commit_command() + self.pipeline.append(And()) + state = BEGIN + elif token == '||': + self._commit_command() + self.pipeline.append(Or()) + state = BEGIN + else: + self.current_command.argv.append(token) + elif state == CHDIR: + if token == '&&': + self._commit_command() + self.pipeline.append(And()) + state = BEGIN + else: + assert self.current_command.dir is None + self.current_command.dir = token + elif state == STDOUT: + assert self.current_command.stdout is None + self.current_command.stdout = token + state = COMMAND + elif state == STDERR: + assert self.current_command.stderr is None + self.current_command.stderr = token + state = COMMAND + elif state == STDIN: + assert self.current_command.stdin is None + self.current_command.stdin = token + state = COMMAND + self._commit_command() + return self.pipeline + + def _exec_piped_commands(self, commands): + retvals = [] + for cmd in commands: + retvals.append(cmd.wait()) + retval = 0 + for r in retvals: + if r: + retval = retvals[-1] + break + return retval + + def run(self, verbose=False): + pipeline = list(self.pipeline) + files_to_close = [] + piped_commands = [] + piped_commands_display = [] + BEGIN, PIPE = range(2) + state = BEGIN + cwd = '.' + while pipeline: + node = pipeline.pop(0) + + if isinstance(node, Chdir): + next_op = pipeline.pop(0) + assert isinstance(next_op, And) + cwd = os.path.join(cwd, node.dir) + if verbose: + piped_commands_display.append("cd %s &&" % node.dir) + continue + + assert isinstance(node, (Command, Chdir)) + cmd = node + if verbose: + if cmd.env_vars: + env_vars_str = ' '.join(['%s=%s' % (key, val) for key, val in cmd.env_vars.iteritems()]) + piped_commands_display.append("%s %s" % (env_vars_str, ' '.join(cmd.argv))) + else: + piped_commands_display.append(' '.join(cmd.argv)) + + if state == PIPE: + stdin = piped_commands[-1].stdout + elif cmd.stdin is not None: + stdin = open(cmd.stdin, "r") + if verbose: + piped_commands_display.append('< %s' % cmd.stdin) + files_to_close.append(stdin) + else: + stdin = None + + if cmd.stdout is None: + stdout = None + elif cmd.stdout is Command.PIPE: + stdout = subprocess.PIPE + else: + stdout = _open_out_file(cmd.stdout) + files_to_close.append(stdout) + if verbose: + piped_commands_display.append('> %s' % cmd.stdout) + + if cmd.stderr is None: + stderr = None + elif cmd.stderr is Command.PIPE: + stderr = subprocess.PIPE + elif cmd.stderr is Command.STDOUT: + stderr = subprocess.STDOUT + if verbose: + piped_commands_display.append('2>&1') + else: + stderr = _open_out_file(cmd.stderr) + files_to_close.append(stderr) + if verbose: + piped_commands_display.append('2> %s' % cmd.stderr) + + if cmd.env_vars: + env = dict(os.environ) + env.update(cmd.env_vars) + else: + env = None + + if cwd == '.': + proc_cwd = None + else: + proc_cwd = cwd + + debug("command: subprocess.Popen(argv=%r, stdin=%r, stdout=%r, stderr=%r, env_vars=%r, cwd=%r)" + % (cmd.argv, stdin, stdout, stderr, cmd.env_vars, proc_cwd)) + proc = subprocess.Popen(cmd.argv, stdin=stdin, stdout=stdout, stderr=stderr, env=env, cwd=proc_cwd) + del stdin, stdout, stderr + piped_commands.append(proc) + + try: + next_node = pipeline.pop(0) + except IndexError: + try: + retval = self._exec_piped_commands(piped_commands) + if verbose: + print "%s: exit code %i" % (' '.join(piped_commands_display), retval) + finally: + for f in files_to_close: + if f is not dev_null: + f.close() + files_to_close = [] + return retval + else: + + if isinstance(next_node, Pipe): + state = PIPE + piped_commands_display.append('|') + + elif isinstance(next_node, Or): + try: + this_retval = self._exec_piped_commands(piped_commands) + finally: + for f in files_to_close: + if f is not dev_null: + f.close() + files_to_close = [] + if this_retval == 0: + if verbose: + print "%s: exit code %i (|| is short-circuited)" % (' '.join(piped_commands_display), retval) + return this_retval + if verbose: + print "%s: exit code %i (|| proceeds)" % (' '.join(piped_commands_display), retval) + state = BEGIN + piped_commands = [] + piped_commands_display = [] + + elif isinstance(next_node, And): + try: + this_retval = self._exec_piped_commands(piped_commands) + finally: + for f in files_to_close: + if f is not dev_null: + f.close() + files_to_close = [] + if this_retval != 0: + if verbose: + print "%s: exit code %i (&& is short-circuited)" % (' '.join(piped_commands_display), retval) + return this_retval + if verbose: + print "%s: exit code %i (&& proceeds)" % (' '.join(piped_commands_display), retval) + state = BEGIN + piped_commands = [] + piped_commands_display = [] + + + +def _main(): + pipeline = Pipeline() + pipeline.parse('./foo.py 2>&1 < xxx | cat && ls') + print pipeline.run() + +if __name__ == '__main__': + _main() + diff --git a/wscript b/wscript index 1d18b155c..adf3b84cf 100644 --- a/wscript +++ b/wscript @@ -27,6 +27,8 @@ import Build import Configure import Scripting +sys.path.insert(0, os.path.abspath('waf-tools')) + import cflags # override the build profiles from waf cflags.profiles = { # profile name: [optimization_level, warnings_level, debug_level]