From 8ab71d7e512ce26fe726a2f30e8003b9d13e0d51 Mon Sep 17 00:00:00 2001 From: Ben Vanik Date: Thu, 31 Dec 2015 12:27:26 -0800 Subject: [PATCH] xb gputest and reference repo - woo. --- .gitmodules | 3 + testdata/reference-gpu-traces | 1 + tools/gpu-trace-diff | 40 +++++++++--- xenia-build | 116 ++++++++++++++++++++++++++++++++-- 4 files changed, 145 insertions(+), 15 deletions(-) create mode 160000 testdata/reference-gpu-traces diff --git a/.gitmodules b/.gitmodules index d633d675b..27f9b5be8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,3 +31,6 @@ [submodule "third_party/premake-export-compile-commands"] path = third_party/premake-export-compile-commands url = https://github.com/xenia-project/premake-export-compile-commands.git +[submodule "testdata/reference-gpu-traces"] + path = testdata/reference-gpu-traces + url = https://github.com/xenia-project/reference-gpu-traces.git diff --git a/testdata/reference-gpu-traces b/testdata/reference-gpu-traces new file mode 160000 index 000000000..c91b83312 --- /dev/null +++ b/testdata/reference-gpu-traces @@ -0,0 +1 @@ +Subproject commit c91b833121060dd8b914a41d5996ace3f68dfcd4 diff --git a/tools/gpu-trace-diff b/tools/gpu-trace-diff index 8fa804a9c..ca834d853 100644 --- a/tools/gpu-trace-diff +++ b/tools/gpu-trace-diff @@ -48,7 +48,10 @@ def main(): parser.add_argument('-t', '--trace_file', action='append') parser.add_argument('-p', '--trace_path') parser.add_argument('-o', '--output_path', default='') + parser.add_argument('-r', '--reference_path', default='') parser.add_argument('-u', '--update_reference_files', action='store_true') + parser.add_argument('-n', '--generate_missing_reference_files', + action='store_true') args = vars(parser.parse_args(sys.argv[1:])) exe_path = args['executable'] @@ -60,8 +63,7 @@ def main(): trace_files = args['trace_file'] or [] if args['trace_path']: for child_path in os.listdir(args['trace_path']): - if (child_path.startswith('gpu_trace_') or - os.path.splitext(child_path)[1] == '.trace'): + if (os.path.splitext(child_path)[1] == '.xenia_gpu_trace'): trace_files.append(os.path.join(args['trace_path'], child_path)) # If the user passed no args, die nicely. @@ -70,7 +72,7 @@ def main(): sys.exit(1) return - output_path = args['output_path'].replace('/', '\\') + output_path = args['output_path'].replace('/', os.pathsep) if not os.path.exists(output_path): os.makedirs(output_path) @@ -78,6 +80,12 @@ def main(): if os.path.exists(html_path): os.remove(html_path) + reference_path = args['reference_path'].replace('/', os.pathsep) + if not os.path.exists(reference_path): + print('Reference path %s not found; forcing to update mode') + os.makedirs(reference_path) + args['update_reference_files'] = True + html_file = None if not args['update_reference_files']: html_file = open(html_path, 'w') @@ -97,14 +105,18 @@ def main(): diff_count = 0 for trace_file in trace_files: - trace_file = trace_file.replace('/', '\\') + trace_file = trace_file.replace('/', os.pathsep) base_path = os.path.dirname(trace_file) file_name = os.path.basename(trace_file) - reference_file_path = os.path.join(base_path, 'reference', - file_name + '.png') + reference_file_path = os.path.join(reference_path, file_name + '.png') output_file_path = os.path.join(output_path, file_name + '.png') diff_file_path = os.path.join(output_path, file_name + '.diff.png') + if (args['generate_missing_reference_files'] and + os.path.exists(reference_file_path)): + # Only process tracess that are missing reference files. + continue + print '--------------------------------------------------------------------' print ' Trace: %s' % (trace_file) print 'Reference: %s' % (reference_file_path) @@ -117,9 +129,9 @@ def main(): # Run the trace dump too to produce a new png. run_args = [ - exe_path.replace('/', '\\'), - '--target_trace_file=%s' % (trace_file.replace('\\', '/')), - '--trace_dump_path=%s' % (output_path.replace('\\', '/')), + exe_path.replace('/', os.pathsep), + '--target_trace_file=%s' % (trace_file.replace(os.pathsep, '/')), + '--trace_dump_path=%s' % (output_path.replace(os.pathsep, '/')), ] tries_remaining = 3 while tries_remaining: @@ -166,6 +178,16 @@ def main(): shutil.copy2(output_file_path, reference_file_path) continue + # If we didn't have a reference file for this and are in gen mode, just copy + # and ignore. + if (not os.path.exists(reference_file_path) and + args['generate_missing_reference_files']): + print 'Adding new reference file...' + if not os.path.exists(os.path.dirname(reference_file_path)): + os.makedirs(os.path.dirname(reference_file_path)) + shutil.copy2(output_file_path, reference_file_path) + continue + # Compare files. print ' New: %s' % (output_file_path) reference_image = Image.open(reference_file_path) diff --git a/xenia-build b/xenia-build index ef2652113..46f784f9b 100755 --- a/xenia-build +++ b/xenia-build @@ -137,15 +137,36 @@ def import_vs_environment(): def has_bin(bin): """Checks whether the given binary is present. + + Args: + bin: binary name (without .exe, etc). + + Returns: + True if the binary exists. """ - for path in os.environ["PATH"].split(os.pathsep): + bin_path = get_bin(bin) + if not bin_path: + return False + return True + + +def get_bin(bin): + """Checks whether the given binary is present and returns the path. + + Args: + bin: binary name (without .exe, etc). + + Returns: + Full path to the binary or None if not found. + """ + for path in os.environ['PATH'].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, bin) if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): - return True + return exe_file exe_file = exe_file + '.exe' if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): - return True + return exe_file return None @@ -377,6 +398,7 @@ def discover_commands(subparsers): 'build': BuildCommand(subparsers), 'gentests': GenTestsCommand(subparsers), 'test': TestCommand(subparsers), + 'gputest': GpuTestCommand(subparsers), 'clean': CleanCommand(subparsers), 'nuke': NukeCommand(subparsers), 'lint': LintCommand(subparsers), @@ -540,7 +562,7 @@ class BaseBuildCommand(Command): '--force', action='store_true', help='Forces a full rebuild.') self.parser.add_argument( - '--no-premake', action='store_true', + '--no_premake', action='store_true', help='Skips running premake before building.') def execute(self, args, pass_args, cwd): @@ -615,7 +637,7 @@ class TestCommand(BaseBuildCommand): ''', *args, **kwargs) self.parser.add_argument( - '--no-build', action='store_true', + '--no_build', action='store_true', help='Don\'t build before running tests.') self.parser.add_argument( '--continue', action='store_true', @@ -640,7 +662,7 @@ class TestCommand(BaseBuildCommand): # Ensure all targets exist before we run. test_executables = [ - os.path.join(get_build_bin_path(args), test_target) + get_bin(os.path.join(get_build_bin_path(args), test_target)) for test_target in test_targets] for test_executable in test_executables: if not has_bin(test_executable): @@ -768,6 +790,88 @@ class GenTestsCommand(Command): return 0 +class GpuTestCommand(BaseBuildCommand): + """'gputest' command.""" + + def __init__(self, subparsers, *args, **kwargs): + super(GpuTestCommand, self).__init__( + subparsers, + name='gputest', + help_short='Runs automated GPU diff tests against reference imagery.', + help_long=''' + To pass arguments to the test executables separate them with `--`. + ''', + *args, **kwargs) + self.parser.add_argument( + '--no_build', action='store_true', + help='Don\'t build before running tests.') + self.parser.add_argument( + '--update_reference_files', action='store_true', + help='Update all reference imagery.') + self.parser.add_argument( + '--generate_missing_reference_files', action='store_true', + help='Create reference files for new traces.') + + def execute(self, args, pass_args, cwd): + print('Testinging...') + print('') + + # The test executables that will be built and run. + test_targets = args['target'] or [ + 'xenia-gpu-gl4-trace-dump', + ] + args['target'] = test_targets + + # Build all targets (if desired). + if not args['no_build']: + result = super(GpuTestCommand, self).execute(args, [], cwd) + if result: + print('Failed to build, aborting test run.') + return result + + # Ensure all targets exist before we run. + test_executables = [ + get_bin(os.path.join(get_build_bin_path(args), test_target)) + for test_target in test_targets] + for test_executable in test_executables: + if not has_bin(test_executable): + print('ERROR: Unable to find %s - build it.' % (test_executable)) + return 1 + + output_path = os.path.join(self_path, 'build', 'gputest') + if os.path.isdir(output_path): + shutil.rmtree(output_path) + os.makedirs(output_path) + print('Running tests and outputting to %s...' % (output_path)) + + reference_trace_root = os.path.join(self_path, 'testdata', + 'reference-gpu-traces') + + # Run tests. + any_failed = False + result = shell_call([ + 'python', + os.path.join(self_path, 'tools', 'gpu-trace-diff'), + '--executable=' + test_executables[0], + '--trace_path=' + os.path.join(reference_trace_root, 'traces'), + '--output_path=' + output_path, + '--reference_path=' + os.path.join(reference_trace_root, 'references'), + ] + (['--generate_missing_reference_files'] + if args['generate_missing_reference_files'] else []) + + (['--update_reference_files'] + if args['update_reference_files'] else []) + + pass_args, + throw_on_error=False) + if result: + any_failed = True + + if any_failed: + print('ERROR: one or more tests failed.') + result = 1 + print('Check %s/results.html for more details.' % (output_path)) + return result + + class CleanCommand(Command): """'clean' command."""