#!/usr/bin/env python # Copyright 2015 Ben Vanik. All Rights Reserved. """Main build script and tooling for xenia. Run with --help or no arguments for possible commands. """ __author__ = 'ben.vanik@gmail.com (Ben Vanik)' import argparse import os import re import shutil import string import subprocess import sys self_path = os.path.dirname(os.path.abspath(__file__)) def main(): # Add self to the root search path. sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) # Augment path to include our fancy things. os.environ['PATH'] += os.pathsep + os.pathsep.join([ self_path, os.path.abspath('build_tools/'), ]) # Check git exists. if not has_bin('git'): print('ERROR: git must be installed and on PATH.') sys.exit(1) return # Check python version. if not sys.version_info[:2] == (2, 7): # TODO(benvanik): allow things to work with 3, but warn on clang-format. print('ERROR: Python 2.7 must be installed and on PATH') sys.exit(1) return # Grab Visual Studio version and execute shell to set up environment. if sys.platform == 'win32': vs_version = import_vs_environment() if vs_version != 2015: print('ERROR: Visual Studio 2015 not found!') print('Ensure you have the VS140COMNTOOLS environment variable!') sys.exit(1) return # Setup main argument parser and common arguments. parser = argparse.ArgumentParser(prog='xenia-build') # Grab all commands and populate the argument parser for each. subparsers = parser.add_subparsers(title='subcommands', dest='subcommand') commands = discover_commands(subparsers) # If the user passed no args, die nicely. if len(sys.argv) == 1: parser.print_help() sys.exit(1) return # Gather any arguments that we want to pass to child processes. command_args = sys.argv[1:] pass_args = [] try: pass_index = command_args.index('--') pass_args = command_args[pass_index + 1:] command_args = command_args[:pass_index] except: pass # Parse command name and dispatch. args = vars(parser.parse_args(command_args)) command_name = args['subcommand'] try: command = commands[command_name] return_code = command.execute(args, pass_args, os.getcwd()) except Exception as e: raise return_code = 1 sys.exit(return_code) # TODO(benvanik): move to build_tools utils module. def import_vs_environment(): """Finds the installed Visual Studio version and imports interesting environment variables into os.environ. Returns: A version such as 2015 or None if no VS is found. """ version = 0 tools_path = '' if 'VS140COMNTOOLS' in os.environ: version = 2015 tools_path = os.environ['VS140COMNTOOLS'] if version == 0: return None tools_path = os.path.join(tools_path, '..\\..\\vc\\vcvarsall.bat') args = [tools_path, '&&', 'set'] popen = subprocess.Popen( args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) variables, _ = popen.communicate() envvars_to_save = ( 'devenvdir', 'include', 'lib', 'libpath', 'path', 'pathext', 'systemroot', 'temp', 'tmp', 'windowssdkdir', ) for line in variables.splitlines(): for envvar in envvars_to_save: if re.match(envvar + '=', line.lower()): var, setting = line.split('=', 1) if envvar == 'path': setting = os.path.dirname(sys.executable) + os.pathsep + setting os.environ[var.upper()] = setting break os.environ['VSVERSION'] = str(version) return version def has_bin(bin): """Checks whether the given binary is present. """ for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, bin) if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): return True exe_file = exe_file + '.exe' if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): return True return None def shell_call(command, throw_on_error=True, stdout_path=None): """Executes a shell command. Args: command: Command to execute, as a list of parameters. throw_on_error: Whether to throw an error or return the status code. stdout_path: File path to write stdout output to. Returns: If throw_on_error is False the status code of the call will be returned. """ stdout_file = None if stdout_path: stdout_file = open(stdout_path, 'w') result = 0 try: if throw_on_error: result = 1 subprocess.check_call(command, shell=False, stdout=stdout_file) result = 0 else: result = subprocess.call(command, shell=False, stdout=stdout_file) finally: if stdout_file: stdout_file.close() return result def git_submodule_update(): """Runs a full recursive git submodule init and update. Older versions of git do not support 'update --init --recursive'. We could check and run it on versions that do support it and speed things up a bit. """ if True: shell_call([ 'git', 'submodule', 'update', '--init', '--recursive', ]) else: shell_call([ 'git', 'submodule', 'init', ]) shell_call([ 'git', 'submodule', 'foreach', '--recursive', 'git', 'submodule', 'init', ]) shell_call([ 'git', 'submodule', 'update', '--recursive', ]) def get_clang_format_binary(): """Finds a clang-format binary. Aborts if none is found. Returns: A path to the clang-format executable. """ attempts = [ 'C:\\Program Files (x86)\\LLVM\\bin\\clang-format.exe', 'clang-format-3.8', 'clang-format', ] for binary in attempts: if has_bin(binary): return binary print 'ERROR: clang-format is not on PATH' print 'LLVM is available from http://llvm.org/releases/download.html' print 'At least version 3.8 is required.' print 'See docs/style_guide.md for instructions on how to get it.' sys.exit(1) def run_premake(target_os, action): """Runs premake on the main project with the given format. Args: target_os: target --os to pass to premake. action: action to preform. """ shell_call([ 'python', os.path.join('build_tools', 'premake'), '--file=premake5.lua', '--os=%s' % (target_os), '--cc=clang', '--test-suite-mode=combined', '--verbose', action, ]) def run_premake_clean(): """Runs a premake clean operation. """ if sys.platform == 'darwin': run_premake('macosx', 'clean') elif sys.platform == 'win32': run_premake('windows', 'clean') else: run_premake('linux', 'clean') def run_platform_premake(): """Runs all gyp configurations. """ if sys.platform == 'darwin': run_premake('macosx', 'xcode') elif sys.platform == 'win32': run_premake('windows', 'vs2015') else: run_premake('linux', 'gmake') run_premake('linux', 'codelite') def get_build_bin_path(args): """Returns the path of the bin/ path with build results based on the configuration specified in the parsed arguments. Args: args: Parsed arguments. Returns: A full path for the bin folder. """ if sys.platform == 'darwin': platform = 'macosx' elif sys.platform == 'win32': platform = 'windows' else: platform = 'linux' return os.path.join(self_path, 'build', 'bin', platform, args['config']) def discover_commands(subparsers): """Looks for all commands and returns a dictionary of them. In the future commands could be discovered on disk. Args: subparsers: Argument subparsers parent used to add command parsers. Returns: A dictionary containing name-to-Command mappings. """ commands = { 'setup': SetupCommand(subparsers), 'pull': PullCommand(subparsers), 'premake': PremakeCommand(subparsers), 'build': BuildCommand(subparsers), 'gentests': GenTestsCommand(subparsers), 'test': TestCommand(subparsers), 'clean': CleanCommand(subparsers), 'nuke': NukeCommand(subparsers), 'lint': LintCommand(subparsers), 'format': FormatCommand(subparsers), 'style': StyleCommand(subparsers), } if sys.platform == 'win32': commands['devenv'] = DevenvCommand(subparsers) return commands class Command(object): """Base type for commands. """ def __init__(self, subparsers, name, help_short=None, help_long=None, *args, **kwargs): """Initializes a command. Args: subparsers: Argument subparsers parent used to add command parsers. name: The name of the command exposed to the management script. help_short: Help text printed alongside the command when queried. help_long: Extended help text when viewing command help. """ self.name = name self.help_short = help_short self.help_long = help_long self.parser = subparsers.add_parser(name, help=help_short, description=help_long) self.parser.set_defaults(command_handler=self) def execute(self, args, pass_args, cwd): """Executes the command. Args: args: Arguments hash for the command. pass_args: Arguments list to pass to child commands. cwd: Current working directory. Returns: Return code of the command. """ return 1 class SetupCommand(Command): """'setup' command.""" def __init__(self, subparsers, *args, **kwargs): super(SetupCommand, self).__init__( subparsers, name='setup', help_short='Setup the build environment.', *args, **kwargs) def execute(self, args, pass_args, cwd): print('Setting up the build environment...') print('') # Setup submodules. print('- git submodule init / update...') git_submodule_update() print('') print('- running premake...') run_platform_premake() print('') print('Success!') return 0 class PullCommand(Command): """'pull' command.""" def __init__(self, subparsers, *args, **kwargs): super(PullCommand, self).__init__( subparsers, name='pull', help_short='Pulls the repo and all dependencies and rebases changes.', *args, **kwargs) self.parser.add_argument('--merge', action='store_true', help='Merges on master instead of rebasing.') def execute(self, args, pass_args, cwd): print('Pulling...') print('') print('- switching to master...') shell_call([ 'git', 'checkout', 'master', ]) print('') print('- pulling self...') if args['merge']: shell_call([ 'git', 'pull', ]) else: shell_call([ 'git', 'pull', '--rebase', ]) print('') print('- pulling dependencies...') git_submodule_update() print('') print('- running premake...') run_platform_premake() print('') print('Success!') return 0 class PremakeCommand(Command): """'premake' command.""" def __init__(self, subparsers, *args, **kwargs): super(PremakeCommand, self).__init__( subparsers, name='premake', help_short='Runs premake to update all projects.', *args, **kwargs) def execute(self, args, pass_args, cwd): print('Running premake...') print('') # Update premake. run_platform_premake() print('Success!') return 0 class BaseBuildCommand(Command): """Base command for things that require building.""" def __init__(self, subparsers, *args, **kwargs): super(BaseBuildCommand, self).__init__( subparsers, *args, **kwargs) self.parser.add_argument( '--config', choices=['checked', 'debug', 'release'], default='debug', help='Chooses the build configuration.') self.parser.add_argument( '--target', action='append', default=[], help='Builds only the given target(s).') self.parser.add_argument( '--force', action='store_true', help='Forces a full rebuild.') self.parser.add_argument( '--no-premake', action='store_true', help='Skips running premake before building.') def execute(self, args, pass_args, cwd): if not args['no_premake']: print('- running premake...') run_platform_premake() print('') print('- building (%s):%s...' % ( 'all' if not len(args['target']) else ' '.join(args['target']), args['config'])) if sys.platform == 'win32': result = shell_call([ 'devenv', '/nologo', 'build/xenia.sln', '/rebuild' if args['force'] else '/build', args['config'], ] + [('/project ', target) for target in args['target']] + pass_args, throw_on_error=False) elif sys.platform == 'darwin': # TODO(benvanik): other platforms. print('ERROR: don\'t know how to build on this platform.') else: # TODO(benvanik): allow gcc? os.environ['CXX'] = 'clang++' os.environ['CC'] = 'clang' result = shell_call([ 'make', '-Cbuild/', 'config=%s_linux' % (args['config']), ] + pass_args + args['target'], throw_on_error=False) print('') if result != 0: print('ERROR: build failed with one or more errors.') return result return 0 class BuildCommand(BaseBuildCommand): """'build' command.""" def __init__(self, subparsers, *args, **kwargs): super(BuildCommand, self).__init__( subparsers, name='build', help_short='Builds the project.', *args, **kwargs) def execute(self, args, pass_args, cwd): print('Building %s...' % (args['config'])) print('') result = super(BuildCommand, self).execute(args, pass_args, cwd) if not result: print('Success!') return result class TestCommand(BaseBuildCommand): """'test' command.""" def __init__(self, subparsers, *args, **kwargs): super(TestCommand, self).__init__( subparsers, name='test', help_short='Runs automated tests that have been built with `xb build`.', help_long=''' To pass arguments to the test executables separate them with `--`. For example, you can run only the instr_foo.s tests with: $ xb test -- instr_foo ''', *args, **kwargs) self.parser.add_argument( '--no-build', action='store_true', help='Don\'t build before running tests.') self.parser.add_argument( '--continue', action='store_true', help='Don\'t stop when a test errors, but continue running all.') def execute(self, args, pass_args, cwd): print('Testing...') print('') # The test executables that will be built and run. test_targets = args['target'] or [ 'xenia-cpu-frontend-tests', ] args['target'] = test_targets # Build all targets (if desired). if not args['no_build']: result = super(TestCommand, self).execute(args, [], cwd) if result: print('Failed to build, aborting test run.') return result # Ensure all targets exist before we run. test_executables = [ os.path.join(get_build_bin_path(args), test_target) for test_target in test_targets] for test_executable in test_executables: if not has_bin(test_executable): print('ERROR: Unable to find %s - build it.' % (test_executable)) return 1 # Run tests. any_failed = False for test_executable in test_executables: print('- %s' % (test_executable)) result = shell_call([ test_executable, ] + pass_args, throw_on_error=False) if result: any_failed = True if args['continue']: print('ERROR: test failed but continuing due to --continue.') else: print('ERROR: test failed, aborting, use --continue to keep going.') return result if any_failed: print('ERROR: one or more tests failed.') result = 1 return result class GenTestsCommand(Command): """'gentests' command.""" def __init__(self, subparsers, *args, **kwargs): super(GenTestsCommand, self).__init__( subparsers, name='gentests', help_short='Generates test binaries.', help_long=''' Generates test binaries (under src/xenia/cpu/frontend/testing/bin/). Run after modifying test .s files. ''', *args, **kwargs) def execute(self, args, pass_args, cwd): print('Generating test binaries...') print('') binutils_path = os.path.join('third_party', 'binutils-ppc-cygwin') ppc_as = os.path.join(binutils_path, 'powerpc-none-elf-as') ppc_ld = os.path.join(binutils_path, 'powerpc-none-elf-ld') ppc_objdump = os.path.join(binutils_path, 'powerpc-none-elf-objdump') ppc_nm = os.path.join(binutils_path, 'powerpc-none-elf-nm') test_src = os.path.join('src', 'xenia', 'cpu', 'frontend', 'testing') test_bin = os.path.join(test_src, 'bin') # Ensure the test output path exists. if not os.path.exists(test_bin): os.mkdir(test_bin) src_files = [os.path.join(root, name) for root, dirs, files in os.walk('src') for name in files if name.endswith(('.s'))] def make_unix_path(p): """Forces a unix path separator style, as required by binutils. """ return string.replace(p, os.sep, '/') any_errors = False for src_file in src_files: print('- %s' % (src_file)) src_name = os.path.splitext(os.path.basename(src_file))[0] obj_file = os.path.join(test_bin, src_name) + '.o' shell_call([ ppc_as, '-a32', '-be', '-mregnames', '-mpower7', '-maltivec', '-mvsx', '-mvmx128', '-R', '-o%s' % (make_unix_path(obj_file)), make_unix_path(src_file), ]) dis_file = os.path.join(test_bin, src_name) + '.dis' shell_call([ ppc_objdump, '--adjust-vma=0x100000', '-Mpower7', '-Mvmx128', '-D', '-EB', make_unix_path(obj_file), ], stdout_path=dis_file) # Eat the first 4 lines to kill the file path that'll differ across machines. with open(dis_file) as f: dis_file_lines = f.readlines() with open(dis_file, 'w') as f: f.writelines(dis_file_lines[4:]) shell_call([ ppc_ld, '-A powerpc:common32', '-melf32ppc', '-EB', '-nostdlib', '--oformat=binary', '-Ttext=0x80000000', '-e0x80000000', '-o%s' % (make_unix_path(os.path.join(test_bin, src_name) + '.bin')), make_unix_path(obj_file), ]) shell_call([ ppc_nm, '--numeric-sort', make_unix_path(obj_file), ], stdout_path=os.path.join(test_bin, src_name) + '.map') if any_errors: print('ERROR: failed to build one or more tests.') return 1 return 0 class CleanCommand(Command): """'clean' command.""" def __init__(self, subparsers, *args, **kwargs): super(CleanCommand, self).__init__( subparsers, name='clean', help_short='Removes intermediate files and build outputs.', *args, **kwargs) def execute(self, args, pass_args, cwd): print('Cleaning build artifacts...') print('') print('- premake clean...') run_premake_clean() print('') print('Success!') return 0 class NukeCommand(Command): """'nuke' command.""" def __init__(self, subparsers, *args, **kwargs): super(NukeCommand, self).__init__( subparsers, name='nuke', help_short='Removes all build/ output.', *args, **kwargs) def execute(self, args, pass_args, cwd): print('Cleaning build artifacts...') print('') print('- removing build/...') if os.path.isdir('build/'): shutil.rmtree('build/') print('') print('- git reset to master...') shell_call([ 'git', 'reset', '--hard', 'master', ]) print('') print('- running premake...') run_platform_premake() print('') print('Success!') return 0 def find_xenia_source_files(): """Gets all xenia source files in the project. Returns: A list of file paths. """ return [os.path.join(root, name) for root, dirs, files in os.walk('src') for name in files if name.endswith(('.cc', '.c', '.h', '.inl'))] def find_elemental_source_files(): """Gets all elemental-forms source files in the project. Returns: A list of file paths. """ return [os.path.join(root, name) for root, dirs, files in os.walk('third_party/elemental-forms/src/') for name in files if name.endswith(('.cc', '.c', '.h', '.inl'))] def find_all_source_files(): """Gets all interesting source files in the project. Returns: A list of file paths. """ return find_xenia_source_files() + find_elemental_source_files() class LintCommand(Command): """'lint' command.""" def __init__(self, subparsers, *args, **kwargs): super(LintCommand, self).__init__( subparsers, name='lint', help_short='Checks for lint errors with clang-format.', *args, **kwargs) self.parser.add_argument( '--all', action='store_true', help='Lint all files, not just those changed.') self.parser.add_argument( '--origin', action='store_true', help='Lints all files changed relative to origin/master.') def execute(self, args, pass_args, cwd): clang_format_binary = get_clang_format_binary() difftemp = '.difftemp.txt' if args['all']: all_files = find_all_source_files() print('- linting %d files' % (len(all_files))) any_errors = False for file_path in all_files: if os.path.exists(difftemp): os.remove(difftemp) ret = shell_call([ clang_format_binary, '-output-replacements-xml', '-style=file', file_path, ], throw_on_error=False, stdout_path=difftemp) with open(difftemp) as f: had_errors = '