From 3f9e86e7856729053d43481317b783ad4cf4e3ff Mon Sep 17 00:00:00 2001 From: Triang3l Date: Fri, 20 Nov 2020 21:27:15 +0300 Subject: [PATCH 1/5] [Build] Clone premake to internal storage on Android --- tools/build/premake | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tools/build/premake b/tools/build/premake index 9113958a5..29eab42ac 100644 --- a/tools/build/premake +++ b/tools/build/premake @@ -10,6 +10,7 @@ __author__ = 'ben.vanik@gmail.com (Ben Vanik)' import json import os +import shutil import subprocess import sys import re @@ -17,7 +18,18 @@ import re self_path = os.path.dirname(os.path.abspath(__file__)) root_path = os.path.join(self_path, '..', '..') -premake_path = os.path.join(root_path, 'third_party', 'premake-core') +premake_external_path = os.path.join(root_path, 'third_party', 'premake-core') +# On Android, the repository may be cloned to the external storage, +# which doesn't support executables in it. +# In this case, premake-core needs to be checked out in the internal storage, +# which supports executables, with all the permissions as set in its repository. +# On Termux, the home directory is in the internal storage - use it for executing. +# If xenia-build doesn't have execute permissions, Xenia is in the external storage now. +premake_path = premake_external_path +if 'ANDROID_ROOT' in os.environ: + xb_file = os.path.join(root_path, 'xenia-build') + if os.path.isfile(xb_file) and not os.access(xb_file, os.X_OK) and 'HOME' in os.environ: + premake_path = os.path.join(os.environ['HOME'], 'xenia', 'third_party', 'premake-core') def main(): @@ -58,6 +70,8 @@ def main(): def build_premake(): """Builds premake from source. """ + # Ensure that on Android, premake-core is in the internal storage. + clone_premake_to_internal_storage() cwd = os.getcwd() try: os.chdir(premake_path) @@ -91,6 +105,33 @@ def build_premake(): pass +def clone_premake_to_internal_storage(): + """Clones premake to the Android internal storage so it can be executed. + """ + # premake_path is initialized to a value different than premake_external_path + # if running from the Android external storage, and may not exist yet. + if premake_path == premake_external_path: + return + + # Ensure the submodule has been checked out. + if not os.path.exists(os.path.join(premake_external_path, 'scripts', 'package.lua')): + print('third_party/premake-core was not present; run xb setup...') + sys.exit(1) + return + + # Create or refresh premake-core in the internal storage. + print('Cloning premake5 to the internal storage...') + shutil.rmtree(premake_path, ignore_errors=True) + os.makedirs(premake_path) + shell_call([ + 'git', + 'clone', + '--recurse-submodules', + premake_external_path, + premake_path, + ]) + + def has_bin(bin): """Checks whether the given binary is present. """ From 4786e93c96e322c3bf06b674b9a8e1c5fe4b4056 Mon Sep 17 00:00:00 2001 From: Triang3l Date: Sat, 21 Nov 2020 14:43:10 +0300 Subject: [PATCH 2/5] [Build] Better Android detection in tools/build/premake --- tools/build/premake | 52 +++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/tools/build/premake b/tools/build/premake index 29eab42ac..0e87bcc49 100644 --- a/tools/build/premake +++ b/tools/build/premake @@ -18,18 +18,37 @@ import re self_path = os.path.dirname(os.path.abspath(__file__)) root_path = os.path.join(self_path, '..', '..') -premake_external_path = os.path.join(root_path, 'third_party', 'premake-core') -# On Android, the repository may be cloned to the external storage, -# which doesn't support executables in it. -# In this case, premake-core needs to be checked out in the internal storage, -# which supports executables, with all the permissions as set in its repository. -# On Termux, the home directory is in the internal storage - use it for executing. -# If xenia-build doesn't have execute permissions, Xenia is in the external storage now. -premake_path = premake_external_path -if 'ANDROID_ROOT' in os.environ: - xb_file = os.path.join(root_path, 'xenia-build') - if os.path.isfile(xb_file) and not os.access(xb_file, os.X_OK) and 'HOME' in os.environ: - premake_path = os.path.join(os.environ['HOME'], 'xenia', 'third_party', 'premake-core') +premake_submodule_path = os.path.join(root_path, 'third_party', 'premake-core') +premake_path = premake_submodule_path + + +def setup_premake_path_override(): + global premake_path + premake_path = premake_submodule_path + if sys.platform == 'linux': + # On Android, the repository may be cloned to the external storage, which + # doesn't support executables in it. + # In this case, premake-core needs to be checked out in the internal + # storage, which supports executables, with all the permissions as set in + # its repository. + # On Termux, the home directory is in the internal storage - use it for + # executing. + # If xenia-build doesn't have execute permissions, Xenia is in the external + # storage now. + try: + popen = subprocess.Popen( + ['uname', '-o'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + universal_newlines=True) + if popen.communicate()[0] == 'Android\n': + xb_file = os.path.join(root_path, 'xenia-build') + if (os.path.isfile(xb_file) and not os.access(xb_file, os.X_OK) and + 'HOME' in os.environ): + premake_path = os.path.join( + os.environ['HOME'], 'xenia', 'third_party', 'premake-core') + except Exception: + pass + +setup_premake_path_override() def main(): @@ -108,13 +127,14 @@ def build_premake(): def clone_premake_to_internal_storage(): """Clones premake to the Android internal storage so it can be executed. """ - # premake_path is initialized to a value different than premake_external_path + # premake_path is initialized to a value different than premake_submodule_path # if running from the Android external storage, and may not exist yet. - if premake_path == premake_external_path: + if premake_path == premake_submodule_path: return # Ensure the submodule has been checked out. - if not os.path.exists(os.path.join(premake_external_path, 'scripts', 'package.lua')): + if not os.path.exists( + os.path.join(premake_submodule_path, 'scripts', 'package.lua')): print('third_party/premake-core was not present; run xb setup...') sys.exit(1) return @@ -127,7 +147,7 @@ def clone_premake_to_internal_storage(): 'git', 'clone', '--recurse-submodules', - premake_external_path, + premake_submodule_path, premake_path, ]) From 25606774e146365cbb4ccb8ac76273751a29edcc Mon Sep 17 00:00:00 2001 From: Triang3l Date: Sat, 21 Nov 2020 14:54:57 +0300 Subject: [PATCH 3/5] [Build] xenia-build Android host OS detection --- xenia-build | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/xenia-build b/xenia-build index 89a14c651..4e5ebb889 100755 --- a/xenia-build +++ b/xenia-build @@ -22,6 +22,17 @@ __author__ = 'ben.vanik@gmail.com (Ben Vanik)' self_path = os.path.dirname(os.path.abspath(__file__)) +# Detect if building on Android via Termux. +host_os_is_android = False +if sys.platform == 'linux': + try: + host_os_is_android = subprocess.Popen( + ['uname', '-o'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + universal_newlines=True).communicate()[0] == 'Android\n' + except Exception: + pass + + def main(): # Add self to the root search path. sys.path.insert(0, self_path) From 14157e063ad84379cdbdbc368e41460b13c83676 Mon Sep 17 00:00:00 2001 From: Triang3l Date: Sat, 21 Nov 2020 16:18:50 +0300 Subject: [PATCH 4/5] [Build] Support cross-compilation via xb premake --target_os --- xenia-build | 311 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 181 insertions(+), 130 deletions(-) diff --git a/xenia-build b/xenia-build index 4e5ebb889..6dd704d47 100755 --- a/xenia-build +++ b/xenia-build @@ -23,16 +23,89 @@ self_path = os.path.dirname(os.path.abspath(__file__)) # Detect if building on Android via Termux. -host_os_is_android = False +host_linux_platform_is_android = False if sys.platform == 'linux': try: - host_os_is_android = subprocess.Popen( + host_linux_platform_is_android = subprocess.Popen( ['uname', '-o'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, universal_newlines=True).communicate()[0] == 'Android\n' except Exception: pass +def import_subprocess_environment(args): + popen = subprocess.Popen( + args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + variables, _ = popen.communicate() + envvars_to_save = ( + 'devenvdir', + 'include', + 'lib', + 'libpath', + 'path', + 'pathext', + 'systemroot', + 'temp', + 'tmp', + 'windowssdkdir', + ) + for line in variables.splitlines(): + for envvar in envvars_to_save: + if re.match(envvar + '=', line.lower()): + var, setting = line.split('=', 1) + if envvar == 'path': + setting = os.path.dirname(sys.executable) + os.pathsep + setting + os.environ[var.upper()] = setting + break + + +def import_vs_environment(): + """Finds the installed Visual Studio version and imports + interesting environment variables into os.environ. + + Returns: + A version such as 2015 or None if no installation is found. + """ + + if sys.platform != 'win32': + return None + + version = 0 + install_path = None + env_tool_args = None + + vswhere = subprocess.check_output('third_party/vswhere/vswhere.exe -version "[15,)" -latest -prerelease -format json -utf8', shell=False, universal_newlines=True, encoding="utf-8") + if vswhere: + vswhere = json.loads(vswhere) + if vswhere and len(vswhere) > 0: + version = int(vswhere[0].get("catalog", {}).get("productLineVersion", 2017)) + install_path = vswhere[0].get("installationPath", None) + + if version < 2017: + if 'VS140COMNTOOLS' in os.environ: + version = 2015 + vcvars_path = os.environ['VS140COMNTOOLS'] + vcvars_path = os.path.join(tools_path, '..\\..\\vc\\vcvarsall.bat') + env_tool_args = [vcvars_path, 'x64', '&&', 'set'] + else: + vsdevcmd_path = os.path.join(install_path, 'Common7\\Tools\\VsDevCmd.bat') + if os.path.isfile(vsdevcmd_path) and os.access(vsdevcmd_path, os.X_OK): + env_tool_args = [vsdevcmd_path, '-arch=amd64', '-host_arch=amd64', '&&', 'set'] + else: + vcvars_path = os.path.join(install_path, 'VC\\Auxiliary\\Build\\vcvarsall.bat') + env_tool_args = [vcvars_path, 'x64', '&&', 'set'] + + if version == 0: + return None + + import_subprocess_environment(env_tool_args) + os.environ['VSVERSION'] = str(version) + return version + + +vs_version = import_vs_environment() + + def main(): # Add self to the root search path. sys.path.insert(0, self_path) @@ -57,13 +130,11 @@ def main(): sys.exit(1) # Grab Visual Studio version and execute shell to set up environment. - if sys.platform == 'win32': - vs_version = import_vs_environment() - if vs_version is None: - print('ERROR: Visual Studio not found!') - print('Please refer to the building guide:') - print('https://github.com/xenia-project/xenia/blob/master/docs/building.md') - sys.exit(1) + if sys.platform == 'win32' and vs_version is None: + print('WARNING: Visual Studio not found!') + print('Building for Windows will not be supported.') + print('Please refer to the building guide:') + print('https://github.com/xenia-project/xenia/blob/master/docs/building.md') # Setup main argument parser and common arguments. parser = argparse.ArgumentParser(prog='xenia-build') @@ -109,72 +180,6 @@ def print_box(msg): .format('', msg, len(msg) + 2)) -def import_vs_environment(): - """Finds the installed Visual Studio version and imports - interesting environment variables into os.environ. - - Returns: - A version such as 2015 or None if no installation is found. - """ - version = 0 - install_path = None - env_tool_args = None - - vswhere = subprocess.check_output('third_party/vswhere/vswhere.exe -version "[15,)" -latest -prerelease -format json -utf8', shell=False, universal_newlines=True, encoding="utf-8") - if vswhere: - vswhere = json.loads(vswhere) - if vswhere and len(vswhere) > 0: - version = int(vswhere[0].get("catalog", {}).get("productLineVersion", 2017)) - install_path = vswhere[0].get("installationPath", None) - - if version < 2017: - if 'VS140COMNTOOLS' in os.environ: - version = 2015 - vcvars_path = os.environ['VS140COMNTOOLS'] - vcvars_path = os.path.join(tools_path, '..\\..\\vc\\vcvarsall.bat') - env_tool_args = [vcvars_path, 'x64', '&&', 'set'] - else: - vsdevcmd_path = os.path.join(install_path, 'Common7\\Tools\\VsDevCmd.bat') - if os.path.isfile(vsdevcmd_path) and os.access(vsdevcmd_path, os.X_OK): - env_tool_args = [vsdevcmd_path, '-arch=amd64', '-host_arch=amd64', '&&', 'set'] - else: - vcvars_path = os.path.join(install_path, 'VC\\Auxiliary\\Build\\vcvarsall.bat') - env_tool_args = [vcvars_path, 'x64', '&&', 'set'] - - if version == 0: - return None - - import_subprocess_environment(env_tool_args) - os.environ['VSVERSION'] = str(version) - return version - - -def import_subprocess_environment(args): - popen = subprocess.Popen( - args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - variables, _ = popen.communicate() - envvars_to_save = ( - 'devenvdir', - 'include', - 'lib', - 'libpath', - 'path', - 'pathext', - 'systemroot', - 'temp', - 'tmp', - 'windowssdkdir', - ) - for line in variables.splitlines(): - for envvar in envvars_to_save: - if re.match(envvar + '=', line.lower()): - var, setting = line.split('=', 1) - if envvar == 'path': - setting = os.path.dirname(sys.executable) + os.pathsep + setting - os.environ[var.upper()] = setting - break - - def has_bin(binary): """Checks whether the given binary is present. @@ -347,6 +352,37 @@ def get_clang_format_binary(): sys.exit(1) +def get_premake_target_os(target_os_override=None): + """Gets the target --os to pass to premake, either for the current platform + or for the user-specified cross-compilation target. + + Args: + target_os_override: override specified by the user for cross-compilation, + or None to target the host platform. + + Returns: + Target --os to pass to premake. If a return value of this function valid + for the current configuration is passed to it function again, the same + value will be returned. + """ + if sys.platform == 'darwin': + target_os = 'macosx' + elif sys.platform == 'win32': + target_os = 'windows' + elif host_linux_platform_is_android: + target_os = 'android' + else: + target_os = 'linux' + if target_os_override is not None and target_os_override != target_os: + if target_os_override == 'android': + target_os = target_os_override + else: + print( + 'ERROR: cross-compilation is only supported for Android target') + sys.exit(0) + return target_os + + def run_premake(target_os, action, cc=None): """Runs premake on the main project with the given format. @@ -373,42 +409,26 @@ def run_premake(target_os, action, cc=None): return ret -def run_premake_clean(): - """Runs a premake clean operation. - """ - if sys.platform == 'darwin': - return run_premake('macosx', 'clean') - elif sys.platform == 'win32': - return run_premake('windows', 'clean') - else: - return run_premake('linux', 'clean') - -def run_platform_premake(cc='clang', devenv=None): +def run_platform_premake(target_os_override=None, cc='clang', devenv=None): """Runs all gyp configurations. """ - if sys.platform == 'darwin': - return run_premake('macosx', 'xcode4') - elif sys.platform == 'win32': - vs_version = '2015' - if 'VSVERSION' in os.environ: - vs_version = os.environ['VSVERSION'] - - return run_premake('windows', devenv or ('vs' + vs_version)) - else: - return run_premake('linux', devenv or 'gmake2', cc) - - -def run_premake_export_commands(): - """Runs premake to generate an LLVM compile_commands.json file. - """ - # TODO(benvanik): only do linux? whatever clang-tidy is ok with. - if sys.platform == 'darwin': - run_premake('macosx', 'export-compile-commands') - elif sys.platform == 'win32': - run_premake('windows', 'export-compile-commands') - else: - run_premake('linux', 'export-compile-commands') + target_os = get_premake_target_os(target_os_override) + if devenv is None: + if target_os == 'macosx': + devenv = 'xcode4' + elif target_os == 'windows': + vs_version = '2015' + if 'VSVERSION' in os.environ: + vs_version = os.environ['VSVERSION'] + devenv = 'vs' + vs_version + elif target_os == 'android': + devenv = 'androidmk' + else: + devenv = 'gmake2' + if target_os != 'linux': + cc = None + return run_premake(target_os=target_os, action=devenv, cc=cc) def get_build_bin_path(args): @@ -545,6 +565,9 @@ class SetupCommand(Command): name='setup', help_short='Setup the build environment.', *args, **kwargs) + self.parser.add_argument( + '--target_os', default=None, + help='Target OS passed to premake, for cross-compilation') def execute(self, args, pass_args, cwd): print('Setting up the build environment...') @@ -559,7 +582,7 @@ class SetupCommand(Command): print('') print('- running premake...') - if run_platform_premake() == 0: + if run_platform_premake(target_os_override=args['target_os']) == 0: print('') print('Success!') @@ -575,8 +598,12 @@ class PullCommand(Command): name='pull', help_short='Pulls the repo and all dependencies and rebases changes.', *args, **kwargs) - self.parser.add_argument('--merge', action='store_true', - help='Merges on master instead of rebasing.') + self.parser.add_argument( + '--merge', action='store_true', + help='Merges on master instead of rebasing.') + self.parser.add_argument( + '--target_os', default=None, + help='Target OS passed to premake, for cross-compilation') def execute(self, args, pass_args, cwd): print('Pulling...') @@ -609,7 +636,7 @@ class PullCommand(Command): print('') print('- running premake...') - if run_platform_premake() == 0: + if run_platform_premake(target_os_override=args['target_os']) == 0: print('') print('Success!') @@ -629,12 +656,16 @@ class PremakeCommand(Command): '--cc', default='clang', help='Compiler toolchain passed to premake') self.parser.add_argument( '--devenv', default=None, help='Development environment') + self.parser.add_argument( + '--target_os', default=None, + help='Target OS passed to premake, for cross-compilation') def execute(self, args, pass_args, cwd): # Update premake. If no binary found, it will be built from source. print('Running premake...') print('') - if run_platform_premake(cc=args['cc'], devenv=args['devenv']) == 0: + if run_platform_premake(target_os_override=args['target_os'], + cc=args['cc'], devenv=args['devenv']) == 0: print('Success!') return 0 @@ -667,7 +698,7 @@ class BaseBuildCommand(Command): def execute(self, args, pass_args, cwd): if not args['no_premake']: print('- running premake...') - run_platform_premake(args['cc']) + run_platform_premake(cc=args['cc']) print('') threads = args['j'] @@ -678,21 +709,27 @@ class BaseBuildCommand(Command): 'all' if not len(args['target']) else ', '.join(args['target']), args['config'])) if sys.platform == 'win32': - targets = None - if len(args['target']): - targets = '/t:' + ';'.join(target + (':Rebuild' if args['force'] else '') - for target in args['target']) + if vs_version is None: + print('ERROR: Visual Studio is not installed.'); + result = 1 else: - targets = '/t:Rebuild' if args['force'] else None + targets = None + if len(args['target']): + targets = '/t:' + ';'.join( + target + (':Rebuild' if args['force'] else '') + for target in args['target']) + else: + targets = '/t:Rebuild' if args['force'] else None - result = subprocess.call([ - 'msbuild', - 'build/xenia.sln', - '/nologo', - '/m', - '/v:m', - '/p:Configuration=' + args['config'], - ] + ([targets] if targets is not None else []) + pass_args, shell=False) + result = subprocess.call([ + 'msbuild', + 'build/xenia.sln', + '/nologo', + '/m', + '/v:m', + '/p:Configuration=' + args['config'], + ] + ([targets] if targets is not None else []) + pass_args, + shell=False) elif sys.platform == 'darwin': # TODO(benvanik): other platforms. print('ERROR: don\'t know how to build on this platform.') @@ -1174,13 +1211,16 @@ class CleanCommand(Command): name='clean', help_short='Removes intermediate files and build outputs.', *args, **kwargs) + self.parser.add_argument( + '--target_os', default=None, + help='Target OS passed to premake, for cross-compilation') def execute(self, args, pass_args, cwd): print('Cleaning build artifacts...') print('') print('- premake clean...') - run_premake_clean() + run_premake(get_premake_target_os(args['target_os']), 'clean') print('') print('Success!') @@ -1196,6 +1236,9 @@ class NukeCommand(Command): name='nuke', help_short='Removes all build/ output.', *args, **kwargs) + self.parser.add_argument( + '--target_os', default=None, + help='Target OS passed to premake, for cross-compilation') def execute(self, args, pass_args, cwd): print('Cleaning build artifacts...') @@ -1216,7 +1259,7 @@ class NukeCommand(Command): print('') print('- running premake...') - run_platform_premake() + run_platform_premake(target_os_override=args['target_os']) print('') print('Success!') @@ -1444,10 +1487,15 @@ class TidyCommand(Command): self.parser.add_argument( '--fix', action='store_true', help='Applies suggested fixes, where possible.') + self.parser.add_argument( + '--target_os', default=None, + help='Target OS passed to premake, for cross-compilation') def execute(self, args, pass_args, cwd): # Run premake to generate our compile_commands.json file for clang to use. - run_premake_export_commands() + # TODO(benvanik): only do linux? whatever clang-tidy is ok with. + run_premake(get_premake_target_os(args['target_os']), + 'export-compile-commands') platform_name = '' if sys.platform == 'darwin': @@ -1507,6 +1555,9 @@ class DevenvCommand(Command): devenv = None show_reload_prompt = False if sys.platform == 'win32': + if vs_version is None: + print('ERROR: Visual Studio is not installed.'); + return 1 print('Launching Visual Studio...') elif has_bin('clion') or has_bin('clion.sh'): print('Launching CLion...') From 48c97dd3b467ead5f65bcfc078099ddfcfae4a12 Mon Sep 17 00:00:00 2001 From: Triang3l Date: Sat, 21 Nov 2020 16:26:26 +0300 Subject: [PATCH 5/5] [Base] Android and Arm platform defines --- src/xenia/app/xenia_main.cc | 2 +- src/xenia/base/platform.h | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/xenia/app/xenia_main.cc b/src/xenia/app/xenia_main.cc index bd109681b..97faba005 100644 --- a/src/xenia/app/xenia_main.cc +++ b/src/xenia/app/xenia_main.cc @@ -216,7 +216,7 @@ int xenia_main(const std::vector& args) { if (!cvars::portable && !std::filesystem::exists(storage_root / "portable.txt")) { storage_root = xe::filesystem::GetUserFolder(); -#if defined(XE_PLATFORM_WIN32) || defined(XE_PLATFORM_LINUX) +#if defined(XE_PLATFORM_WIN32) || defined(XE_PLATFORM_GNU_LINUX) storage_root = storage_root / "Xenia"; #else #warning Unhandled platform for the data root. diff --git a/src/xenia/base/platform.h b/src/xenia/base/platform.h index 9b98175c5..441d37750 100644 --- a/src/xenia/base/platform.h +++ b/src/xenia/base/platform.h @@ -31,8 +31,14 @@ #define XE_PLATFORM_MAC 1 #elif defined(WIN32) || defined(_WIN32) #define XE_PLATFORM_WIN32 1 -#else +#elif defined(__ANDROID__) +#define XE_PLATFORM_ANDROID 1 #define XE_PLATFORM_LINUX 1 +#elif defined(__gnu_linux__) +#define XE_PLATFORM_GNU_LINUX 1 +#define XE_PLATFORM_LINUX 1 +#else +#error Unsupported target OS. #endif #if defined(__clang__) @@ -51,8 +57,11 @@ #if defined(_M_AMD64) || defined(__amd64__) #define XE_ARCH_AMD64 1 -#elif defined(_M_IX86) -#error "Xenia is not supported on 32-bit platforms." +#elif defined(_M_ARM64) || defined(__aarch64__) +#define XE_ARCH_ARM64 1 +#elif defined(_M_IX86) || defined(__i386__) || defined(_M_ARM) || \ + defined(__arm__) +#error Xenia is not supported on 32-bit platforms. #elif defined(_M_PPC) || defined(__powerpc__) #define XE_ARCH_PPC 1 #endif