xb gputest and reference repo - woo.

.gitmodules

@@ -31,3 +31,6 @@
 [submodule "third_party/premake-export-compile-commands"]
 	path = third_party/premake-export-compile-commands
 	url = https://github.com/xenia-project/premake-export-compile-commands.git
+[submodule "testdata/reference-gpu-traces"]
+	path = testdata/reference-gpu-traces
+	url = https://github.com/xenia-project/reference-gpu-traces.git

testdata/reference-gpu-traces

@@ -0,0 +1 @@
+Subproject commit c91b833121060dd8b914a41d5996ace3f68dfcd4

tools/gpu-trace-diff

@@ -48,7 +48,10 @@ def main():
   parser.add_argument('-t', '--trace_file', action='append')
   parser.add_argument('-p', '--trace_path')
   parser.add_argument('-o', '--output_path', default='')
+  parser.add_argument('-r', '--reference_path', default='')
   parser.add_argument('-u', '--update_reference_files', action='store_true')
+  parser.add_argument('-n', '--generate_missing_reference_files',
+                      action='store_true')
   args = vars(parser.parse_args(sys.argv[1:]))
 
   exe_path = args['executable']
@@ -60,8 +63,7 @@ def main():
   trace_files = args['trace_file'] or []
   if args['trace_path']:
     for child_path in os.listdir(args['trace_path']):
-      if (child_path.startswith('gpu_trace_') or
-          os.path.splitext(child_path)[1] == '.trace'):
+      if (os.path.splitext(child_path)[1] == '.xenia_gpu_trace'):
         trace_files.append(os.path.join(args['trace_path'], child_path))
 
   # If the user passed no args, die nicely.
@@ -70,7 +72,7 @@ def main():
     sys.exit(1)
     return
 
-  output_path = args['output_path'].replace('/', '\\')
+  output_path = args['output_path'].replace('/', os.pathsep)
   if not os.path.exists(output_path):
     os.makedirs(output_path)
 
@@ -78,6 +80,12 @@ def main():
   if os.path.exists(html_path):
     os.remove(html_path)
 
+  reference_path = args['reference_path'].replace('/', os.pathsep)
+  if not os.path.exists(reference_path):
+    print('Reference path %s not found; forcing to update mode')
+    os.makedirs(reference_path)
+    args['update_reference_files'] = True
+
   html_file = None
   if not args['update_reference_files']:
     html_file = open(html_path, 'w')
@@ -97,14 +105,18 @@ def main():
 
   diff_count = 0
   for trace_file in trace_files:
-    trace_file = trace_file.replace('/', '\\')
+    trace_file = trace_file.replace('/', os.pathsep)
     base_path = os.path.dirname(trace_file)
     file_name = os.path.basename(trace_file)
-    reference_file_path = os.path.join(base_path, 'reference',
-                                       file_name + '.png')
+    reference_file_path = os.path.join(reference_path, file_name + '.png')
     output_file_path = os.path.join(output_path, file_name + '.png')
     diff_file_path = os.path.join(output_path, file_name + '.diff.png')
 
+    if (args['generate_missing_reference_files'] and
+        os.path.exists(reference_file_path)):
+      # Only process tracess that are missing reference files.
+      continue
+
     print '--------------------------------------------------------------------'
     print '    Trace: %s' % (trace_file)
     print 'Reference: %s' % (reference_file_path)
@@ -117,9 +129,9 @@ def main():
 
     # Run the trace dump too to produce a new png.
     run_args = [
-        exe_path.replace('/', '\\'),
-        '--target_trace_file=%s' % (trace_file.replace('\\', '/')),
-        '--trace_dump_path=%s' % (output_path.replace('\\', '/')),
+        exe_path.replace('/', os.pathsep),
+        '--target_trace_file=%s' % (trace_file.replace(os.pathsep, '/')),
+        '--trace_dump_path=%s' % (output_path.replace(os.pathsep, '/')),
         ]
     tries_remaining = 3
     while tries_remaining:
@@ -166,6 +178,16 @@ def main():
       shutil.copy2(output_file_path, reference_file_path)
       continue
 
+    # If we didn't have a reference file for this and are in gen mode, just copy
+    # and ignore.
+    if (not os.path.exists(reference_file_path) and
+        args['generate_missing_reference_files']):
+      print 'Adding new reference file...'
+      if not os.path.exists(os.path.dirname(reference_file_path)):
+        os.makedirs(os.path.dirname(reference_file_path))
+      shutil.copy2(output_file_path, reference_file_path)
+      continue
+
     # Compare files.
     print '      New: %s' % (output_file_path)
     reference_image = Image.open(reference_file_path)

xenia-build

@@ -137,15 +137,36 @@ def import_vs_environment():
 
 def has_bin(bin):
   """Checks whether the given binary is present.
+
+  Args:
+    bin: binary name (without .exe, etc).
+
+  Returns:
+    True if the binary exists.
   """
-  for path in os.environ["PATH"].split(os.pathsep):
+  bin_path = get_bin(bin)
+  if not bin_path:
+    return False
+  return True
+
+
+def get_bin(bin):
+  """Checks whether the given binary is present and returns the path.
+
+  Args:
+    bin: binary name (without .exe, etc).
+
+  Returns:
+    Full path to the binary or None if not found.
+  """
+  for path in os.environ['PATH'].split(os.pathsep):
     path = path.strip('"')
     exe_file = os.path.join(path, bin)
     if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
-      return True
+      return exe_file
     exe_file = exe_file + '.exe'
     if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
-      return True
+      return exe_file
   return None
 
 
@@ -377,6 +398,7 @@ def discover_commands(subparsers):
       'build': BuildCommand(subparsers),
       'gentests': GenTestsCommand(subparsers),
       'test': TestCommand(subparsers),
+      'gputest': GpuTestCommand(subparsers),
       'clean': CleanCommand(subparsers),
       'nuke': NukeCommand(subparsers),
       'lint': LintCommand(subparsers),
@@ -540,7 +562,7 @@ class BaseBuildCommand(Command):
         '--force', action='store_true',
         help='Forces a full rebuild.')
     self.parser.add_argument(
-        '--no-premake', action='store_true',
+        '--no_premake', action='store_true',
         help='Skips running premake before building.')
 
   def execute(self, args, pass_args, cwd):
@@ -615,7 +637,7 @@ class TestCommand(BaseBuildCommand):
         ''',
         *args, **kwargs)
     self.parser.add_argument(
-        '--no-build', action='store_true',
+        '--no_build', action='store_true',
         help='Don\'t build before running tests.')
     self.parser.add_argument(
         '--continue', action='store_true',
@@ -640,7 +662,7 @@ class TestCommand(BaseBuildCommand):
 
     # Ensure all targets exist before we run.
     test_executables = [
-        os.path.join(get_build_bin_path(args), test_target)
+        get_bin(os.path.join(get_build_bin_path(args), test_target))
         for test_target in test_targets]
     for test_executable in test_executables:
       if not has_bin(test_executable):
@@ -768,6 +790,88 @@ class GenTestsCommand(Command):
     return 0
 
 
+class GpuTestCommand(BaseBuildCommand):
+  """'gputest' command."""
+
+  def __init__(self, subparsers, *args, **kwargs):
+    super(GpuTestCommand, self).__init__(
+        subparsers,
+        name='gputest',
+        help_short='Runs automated GPU diff tests against reference imagery.',
+        help_long='''
+        To pass arguments to the test executables separate them with `--`.
+        ''',
+        *args, **kwargs)
+    self.parser.add_argument(
+        '--no_build', action='store_true',
+        help='Don\'t build before running tests.')
+    self.parser.add_argument(
+        '--update_reference_files', action='store_true',
+        help='Update all reference imagery.')
+    self.parser.add_argument(
+        '--generate_missing_reference_files', action='store_true',
+        help='Create reference files for new traces.')
+
+  def execute(self, args, pass_args, cwd):
+    print('Testinging...')
+    print('')
+
+    # The test executables that will be built and run.
+    test_targets = args['target'] or [
+        'xenia-gpu-gl4-trace-dump',
+        ]
+    args['target'] = test_targets
+
+    # Build all targets (if desired).
+    if not args['no_build']:
+      result = super(GpuTestCommand, self).execute(args, [], cwd)
+      if result:
+        print('Failed to build, aborting test run.')
+        return result
+
+    # Ensure all targets exist before we run.
+    test_executables = [
+        get_bin(os.path.join(get_build_bin_path(args), test_target))
+        for test_target in test_targets]
+    for test_executable in test_executables:
+      if not has_bin(test_executable):
+        print('ERROR: Unable to find %s - build it.' % (test_executable))
+        return 1
+
+    output_path = os.path.join(self_path, 'build', 'gputest')
+    if os.path.isdir(output_path):
+      shutil.rmtree(output_path)
+    os.makedirs(output_path)
+    print('Running tests and outputting to %s...' % (output_path))
+
+    reference_trace_root = os.path.join(self_path, 'testdata',
+                                        'reference-gpu-traces')
+
+    # Run tests.
+    any_failed = False
+    result = shell_call([
+        'python',
+        os.path.join(self_path, 'tools', 'gpu-trace-diff'),
+        '--executable=' + test_executables[0],
+        '--trace_path=' + os.path.join(reference_trace_root, 'traces'),
+        '--output_path=' + output_path,
+        '--reference_path=' + os.path.join(reference_trace_root, 'references'),
+        ] + (['--generate_missing_reference_files']
+             if args['generate_missing_reference_files'] else []) +
+            (['--update_reference_files']
+             if args['update_reference_files'] else []) +
+        pass_args,
+        throw_on_error=False)
+    if result:
+      any_failed = True
+
+    if any_failed:
+      print('ERROR: one or more tests failed.')
+      result = 1
+    print('Check %s/results.html for more details.' % (output_path))
+    return result
+
+
 class CleanCommand(Command):
   """'clean' command."""