Python 3 support (#755)

* Python 3 support

* Fix import_vs_environment in xenia-build

* Drop support for Python 2

* Fix CIs and xb.bat

* Require Python3.4+ because of Ubuntu Trusty

* popen.communicate returns bytes instead of string

* Useful info in travis

* universal_newlines should be True

* Changed shebang to python 3

* Python 3 shebang fix

* Clang python 3 shebang fix

.travis.yml

@@ -33,6 +33,7 @@ addons:
       - clang-3.8
       - clang-format-3.8
       - libc++-dev
+      - python3
 
 git:
   # We handle submodules ourselves in xenia-build setup.
@@ -43,6 +44,7 @@ before_script:
   - export CC=clang-3.8
   # Dump useful info.
   - $CXX --version
+  - python3 --version
   # Prepare environment (pull dependencies, build tools).
   - travis_retry ./xenia-build setup
 

third_party/clang-format/clang-format-diff.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 #===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===#
 #

third_party/clang-format/git-clang-format

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 #===- git-clang-format - ClangFormat Git Integration ---------*- python -*--===#
 #

tools/build/premake

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3 
 
 # Copyright 2015 Ben Vanik. All Rights Reserved.
 

tools/diff.py

@@ -1,3 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""python diff.py file1 file2 diff"""
+
 import difflib
 import sys
 

tools/gpu-trace-diff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3 
 
 # Copyright 2015 Ben Vanik. All Rights Reserved.
 

tools/ppc-table-gen

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3 
 
 # Copyright 2015 Ben Vanik & shuffle2. All Rights Reserved.
 

xb.bat

@@ -10,8 +10,8 @@ REM ============================================================================
 CALL :check_python
 IF %_RESULT% NEQ 0 (
   ECHO.
-  ECHO Python 2.7 must be installed and on PATH:
+  ECHO Python 3.4+ must be installed and on PATH:
-  ECHO https://www.python.org/ftp/python/2.7.13/python-2.7.13.msi
+  ECHO https://www.python.org/
   GOTO :exit_error
 )
 
@@ -31,13 +31,19 @@ REM ============================================================================
 :check_python
 SETLOCAL
 SET FOUND_PYTHON_EXE=""
-1>NUL 2>NUL CMD /c where python2
+1>NUL 2>NUL CMD /c where python3
 IF NOT ERRORLEVEL 1 (
   ECHO FOUND PYTHON 2
-  SET FOUND_PYTHON_EXE=python2
+  SET FOUND_PYTHON_EXE=python3
 )
 IF %FOUND_PYTHON_EXE% EQU "" (
-  IF EXIST c:\\python27\\python.exe SET FOUND_PYTHON_EXE=C:\\python27\\python.exe
+  IF EXIST c:\\python34\\python.exe SET FOUND_PYTHON_EXE=C:\\python34\\python.exe
+)
+IF %FOUND_PYTHON_EXE% EQU "" (
+  IF EXIST c:\\python35\\python.exe SET FOUND_PYTHON_EXE=C:\\python35\\python.exe
+)
+IF %FOUND_PYTHON_EXE% EQU "" (
+  IF EXIST c:\\python36\\python.exe SET FOUND_PYTHON_EXE=C:\\python36\\python.exe
 )
 IF %FOUND_PYTHON_EXE% EQU "" (
   1>NUL 2>NUL CMD /c where python
@@ -47,13 +53,13 @@ IF %FOUND_PYTHON_EXE% EQU "" (
 )
 IF %FOUND_PYTHON_EXE% EQU "" (
   ECHO ERROR: no Python executable found on PATH.
-  ECHO Make sure you can run 'python' or 'python2' in a Command Prompt.
+  ECHO Make sure you can run 'python' or 'python3' in a Command Prompt.
   ENDLOCAL & SET _RESULT=1
   GOTO :eof
 )
-CMD /C %FOUND_PYTHON_EXE% -c "import sys; sys.exit(1 if not sys.version_info[:2] == (2, 7) else 0)"
+CMD /C %FOUND_PYTHON_EXE% -c "import sys; sys.exit(1 if not sys.version_info[:2] >= (3, 4) else 0)"
 IF %ERRORLEVEL% NEQ 0 (
-  ECHO ERROR: Python version mismatch - not 2.7
+  ECHO ERROR: Python version mismatch - not 3.4+
   ENDLOCAL & SET _RESULT=1
   GOTO :eof
 )

xenia-build

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2015 Ben Vanik. All Rights Reserved.
 
@@ -6,25 +6,24 @@
 
 Run with --help or no arguments for possible commands.
 """
-
+from __future__ import print_function
-__author__ = 'ben.vanik@gmail.com (Ben Vanik)'
-
 
 import argparse
 import os
 import re
 import shutil
-import string
 import subprocess
 import sys
 
+__author__ = 'ben.vanik@gmail.com (Ben Vanik)'
+
 
 self_path = os.path.dirname(os.path.abspath(__file__))
 
 
 def main():
     # Add self to the root search path.
-  sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+    sys.path.insert(0, self_path)
 
     # Augment path to include our fancy things.
     os.environ['PATH'] += os.pathsep + os.pathsep.join([
@@ -36,24 +35,20 @@ def main():
     if not has_bin('git'):
         print('ERROR: git must be installed and on PATH.')
         sys.exit(1)
-    return
 
     # Check python version.
-  if not sys.version_info[:2] == (2, 7):
+    if not sys.version_info[:2] >= (3, 4):
-    # TODO(benvanik): allow things to work with 3, but warn on clang-format.
+        print('ERROR: Python 3.4+ must be installed and on PATH')
-    print('ERROR: Python 2.7 must be installed and on PATH')
         sys.exit(1)
-    return
 
     # Grab Visual Studio version and execute shell to set up environment.
     if sys.platform == 'win32':
         vs_version = import_vs_environment()
-    if vs_version == None:
+        if vs_version is None:
             print('ERROR: Visual Studio not found!')
             print('Please refer to the building guide:')
             print('https://github.com/benvanik/xenia/blob/master/docs/building.md')
             sys.exit(1)
-      return
 
     # Setup main argument parser and common arguments.
     parser = argparse.ArgumentParser(prog='xenia-build')
@@ -67,7 +62,6 @@ def main():
     if len(sys.argv) == 1:
         parser.print_help()
         sys.exit(1)
-    return
 
     # Gather any arguments that we want to pass to child processes.
     command_args = sys.argv[1:]
@@ -76,7 +70,7 @@ def main():
         pass_index = command_args.index('--')
         pass_args = command_args[pass_index + 1:]
         command_args = command_args[:pass_index]
-  except:
+    except Exception:
         pass
 
     # Parse command name and dispatch.
@@ -85,9 +79,8 @@ def main():
     try:
         command = commands[command_name]
         return_code = command.execute(args, pass_args, os.getcwd())
-  except Exception as e:
+    except Exception:
         raise
-    return_code = 1
     sys.exit(return_code)
 
 
@@ -100,7 +93,7 @@ def import_vs_environment():
     """
     version = 0
 
-  candidate_path = subprocess.check_output('third_party/vswhere/vswhere.exe -version "[15,)" -latest -format value -property installationPath', shell=False)
+    candidate_path = subprocess.check_output('third_party/vswhere/vswhere.exe -version "[15,)" -latest -format value -property installationPath', shell=False, universal_newlines=True)
     candidate_path = candidate_path.strip()
 
     tools_path = ''
@@ -117,7 +110,7 @@ def import_vs_environment():
 
     args = [tools_path, 'x64', '&&', 'set']
     popen = subprocess.Popen(
-      args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
     variables, _ = popen.communicate()
     envvars_to_save = (
         'devenvdir',
@@ -144,36 +137,36 @@ def import_vs_environment():
     return version
 
 
-def has_bin(bin):
+def has_bin(binary):
     """Checks whether the given binary is present.
 
     Args:
-    bin: binary name (without .exe, etc).
+      binary: binary name (without .exe, etc).
 
     Returns:
       True if the binary exists.
     """
-  bin_path = get_bin(bin)
+    bin_path = get_bin(binary)
     if not bin_path:
         return False
     return True
 
 
-def get_bin(bin):
+def get_bin(binary):
     """Checks whether the given binary is present and returns the path.
 
     Args:
-    bin: binary name (without .exe, etc).
+      binary: binary name (without .exe, etc).
 
     Returns:
       Full path to the binary or None if not found.
     """
     for path in os.environ['PATH'].split(os.pathsep):
         path = path.strip('"')
-    exe_file = os.path.join(path, bin)
+        exe_file = os.path.join(path, binary)
         if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
             return exe_file
-    exe_file = exe_file + '.exe'
+        exe_file += '.exe'
         if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
             return exe_file
     return None
@@ -238,7 +231,7 @@ def get_git_head_info():
         ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (stdout, stderr) = p.communicate()
     commit_short = stdout.strip() or 'unknown'
-  return (branch_name, commit, commit_short)
+    return branch_name, commit, commit_short
 
 
 def generate_version_h():
@@ -310,12 +303,13 @@ def get_clang_format_binary():
     for binary in attempts:
         if has_bin(binary):
             return binary
-  print 'ERROR: clang-format is not on PATH'
+    print('ERROR: clang-format is not on PATH')
-  print 'LLVM is available from http://llvm.org/releases/download.html'
+    print('LLVM is available from http://llvm.org/releases/download.html')
-  print 'At least version 3.8 is required.'
+    print('At least version 3.8 is required.')
-  print 'See docs/style_guide.md for instructions on how to get it.'
+    print('See docs/style_guide.md for instructions on how to get it.')
     sys.exit(1)
 
+
 def run_premake(target_os, action):
     """Runs premake on the main project with the given format.
 
@@ -327,7 +321,7 @@ def run_premake(target_os, action):
         'python',
         os.path.join('tools', 'build', 'premake'),
         '--file=premake5.lua',
-      '--os=%s' % (target_os),
+        '--os=%s' % target_os,
         '--cc=clang',
         '--test-suite-mode=combined',
         '--verbose',
@@ -611,6 +605,7 @@ class BaseBuildCommand(Command):
         elif sys.platform == 'darwin':
             # TODO(benvanik): other platforms.
             print('ERROR: don\'t know how to build on this platform.')
+            result = 1
         else:
             # TODO(benvanik): allow gcc?
             if 'CXX' not in os.environ:
@@ -797,10 +792,8 @@ class TestCommand(BaseBuildCommand):
         # Run tests.
         any_failed = False
         for test_executable in test_executables:
-      print('- %s' % (test_executable))
+            print('- %s' % test_executable)
-      result = shell_call([
+            result = shell_call([test_executable] + pass_args,
-          test_executable,
-          ] + pass_args,
                                 throw_on_error=False)
             if result:
                 any_failed = True
@@ -856,11 +849,11 @@ class GenTestsCommand(Command):
         def make_unix_path(p):
             """Forces a unix path separator style, as required by binutils.
             """
-      return string.replace(p, os.sep, '/')
+            return p.replace(os.sep, '/')
 
         any_errors = False
         for src_file in src_files:
-      print('- %s' % (src_file))
+            print('- %s' % src_file)
             src_name = os.path.splitext(os.path.basename(src_file))[0]
             obj_file = os.path.join(test_bin, src_name) + '.o'
             shell_call([
@@ -982,10 +975,8 @@ class GpuTestCommand(BaseBuildCommand):
             '--trace_path=' + os.path.join(reference_trace_root, 'traces'),
             '--output_path=' + output_path,
             '--reference_path=' + os.path.join(reference_trace_root, 'references'),
-        ] + (['--generate_missing_reference_files']
+            ] + (['--generate_missing_reference_files'] if args['generate_missing_reference_files'] else []) +
-             if args['generate_missing_reference_files'] else []) +
+                (['--update_reference_files'] if args['update_reference_files'] else []) +
-            (['--update_reference_files']
-             if args['update_reference_files'] else []) +
                             pass_args,
                             throw_on_error=False)
         if result:
@@ -1259,8 +1250,6 @@ class StyleCommand(Command):
             print('Style linting completed successfully.')
             return 0
 
-    return 0
-
 
 # TODO(benvanik): merge into linter, or as lint --anal?
 class TidyCommand(Command):
@@ -1287,7 +1276,7 @@ class TidyCommand(Command):
             platform_name = 'windows'
         else:
             platform_name = 'linux'
-    tool_root = 'build/llvm_tools/debug_%s' % (platform_name)
+        tool_root = 'build/llvm_tools/debug_%s' % platform_name
 
         all_files = [file_path for file_path in find_all_source_files()
                      if not file_path.endswith('_test.cc')]
@@ -1323,8 +1312,6 @@ class TidyCommand(Command):
             print('Tidy completed successfully.')
             return 0
 
-    return 0
-
 
 class DevenvCommand(Command):
     """'devenv' command."""
@@ -1356,3 +1343,4 @@ class DevenvCommand(Command):
 
 if __name__ == '__main__':
     main()
+