Added API scanner tool

New tool for issue #171 which links to libxenia to dump the API usage
from packaged content

CONTRIBUTORS.md

@@ -4,3 +4,4 @@ Contributors
 Names should be added to this file like so: `Name or Organization <email address>`
 
 * Ben Vanik <ben.vanik@gmail.com>
+* x1nixmzeng <x1nixmzeng@live.co.uk>

src/xenia/kernel/fs/devices/stfs_container_device.cc

@@ -39,7 +39,7 @@ int STFSContainerDevice::Init() {
     return 1;
   }
 
-  stfs_->Dump();
+  //stfs_->Dump();
 
   return 0;
 }

src/xenia/kernel/xam_module.cc

@@ -19,26 +19,36 @@ namespace kernel {
 
 XamModule::XamModule(Emulator* emulator, KernelState* kernel_state)
     : XKernelModule(kernel_state, "xe:\\xam.xex") {
-// Build the export table used for resolution.
+  RegisterExportTable(export_resolver_);
+
+  // Register all exported functions.
+  xam::RegisterContentExports(export_resolver_, kernel_state_);
+  xam::RegisterInfoExports(export_resolver_, kernel_state_);
+  xam::RegisterInputExports(export_resolver_, kernel_state_);
+  xam::RegisterMsgExports(export_resolver_, kernel_state_);
+  xam::RegisterNetExports(export_resolver_, kernel_state_);
+  xam::RegisterNotifyExports(export_resolver_, kernel_state_);
+  xam::RegisterUIExports(export_resolver_, kernel_state_);
+  xam::RegisterUserExports(export_resolver_, kernel_state_);
+  xam::RegisterVideoExports(export_resolver_, kernel_state_);
+  xam::RegisterVoiceExports(export_resolver_, kernel_state_);
+}
+
+void XamModule::RegisterExportTable(ExportResolver* export_resolver) {
+  assert_not_null(export_resolver);
+
+  if (!export_resolver) {
+    return;
+  }
+
+  // Build the export table used for resolution.
 #include "xenia/kernel/util/export_table_pre.inc"
   static KernelExport xam_export_table[] = {
 #include "xenia/kernel/xam_table.inc"
   };
 #include "xenia/kernel/util/export_table_post.inc"
-  export_resolver_->RegisterTable("xam.xex", xam_export_table,
-                                  poly::countof(xam_export_table));
-
-  // Register all exported functions.
-  xam::RegisterContentExports(export_resolver_, kernel_state);
-  xam::RegisterInfoExports(export_resolver_, kernel_state);
-  xam::RegisterInputExports(export_resolver_, kernel_state);
-  xam::RegisterMsgExports(export_resolver_, kernel_state);
-  xam::RegisterNetExports(export_resolver_, kernel_state);
-  xam::RegisterNotifyExports(export_resolver_, kernel_state);
-  xam::RegisterUIExports(export_resolver_, kernel_state);
-  xam::RegisterUserExports(export_resolver_, kernel_state);
-  xam::RegisterVideoExports(export_resolver_, kernel_state);
-  xam::RegisterVoiceExports(export_resolver_, kernel_state);
+  export_resolver->RegisterTable("xam.xex", xam_export_table,
+    poly::countof(xam_export_table));
 }
 
 XamModule::~XamModule() {}

src/xenia/kernel/xam_module.h

@@ -23,6 +23,7 @@ class XamModule : public XKernelModule {
   XamModule(Emulator* emulator, KernelState* kernel_state);
   virtual ~XamModule();
 
+  static void RegisterExportTable(ExportResolver* export_resolver);
  private:
 };
 

src/xenia/kernel/xboxkrnl_module.cc

@@ -26,30 +26,23 @@ namespace kernel {
 XboxkrnlModule::XboxkrnlModule(Emulator* emulator, KernelState* kernel_state)
     : XKernelModule(kernel_state, "xe:\\xboxkrnl.exe"),
       timestamp_timer_(nullptr) {
-// Build the export table used for resolution.
-#include "xenia/kernel/util/export_table_pre.inc"
-  static KernelExport xboxkrnl_export_table[] = {
-#include "xenia/kernel/xboxkrnl_table.inc"
-  };
-#include "xenia/kernel/util/export_table_post.inc"
-  export_resolver_->RegisterTable("xboxkrnl.exe", xboxkrnl_export_table,
-                                  poly::countof(xboxkrnl_export_table));
+  RegisterExportTable(export_resolver_);
 
   // Register all exported functions.
-  xboxkrnl::RegisterAudioExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterAudioXmaExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterDebugExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterHalExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterIoExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterMemoryExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterMiscExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterModuleExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterObExports(export_resolver_, kernel_state);
+  xboxkrnl::RegisterAudioExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterAudioXmaExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterDebugExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterHalExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterIoExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterMemoryExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterMiscExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterModuleExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterObExports(export_resolver_, kernel_state_);
   xboxkrnl::RegisterRtlExports(export_resolver_, kernel_state_);
   xboxkrnl::RegisterStringExports(export_resolver_, kernel_state_);
-  xboxkrnl::RegisterThreadingExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterUsbcamExports(export_resolver_, kernel_state);
-  xboxkrnl::RegisterVideoExports(export_resolver_, kernel_state);
+  xboxkrnl::RegisterThreadingExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterUsbcamExports(export_resolver_, kernel_state_);
+  xboxkrnl::RegisterVideoExports(export_resolver_, kernel_state_);
 
   uint8_t* mem = memory_->membase();
 
@@ -149,6 +142,23 @@ XboxkrnlModule::XboxkrnlModule(Emulator* emulator, KernelState* kernel_state)
                         WT_EXECUTEINTIMERTHREAD);
 }
 
+void XboxkrnlModule::RegisterExportTable(ExportResolver* export_resolver) {
+  assert_not_null(export_resolver);
+
+  if (!export_resolver) {
+    return;
+  }
+
+  // Build the export table used for resolution.
+#include "xenia/kernel/util/export_table_pre.inc"
+  static KernelExport xboxkrnl_export_table[] = {
+#include "xenia/kernel/xboxkrnl_table.inc"
+  };
+#include "xenia/kernel/util/export_table_post.inc"
+  export_resolver->RegisterTable("xboxkrnl.exe", xboxkrnl_export_table,
+    poly::countof(xboxkrnl_export_table));
+}
+
 XboxkrnlModule::~XboxkrnlModule() {
   DeleteTimerQueueTimer(nullptr, timestamp_timer_, nullptr);
 }

src/xenia/kernel/xboxkrnl_module.h

@@ -28,6 +28,8 @@ class XboxkrnlModule : public XKernelModule {
   XboxkrnlModule(Emulator* emulator, KernelState* kernel_state);
   virtual ~XboxkrnlModule();
 
+  static void RegisterExportTable(ExportResolver* export_resolver);
+
   int LaunchModule(const char* path);
 
  private:

src/xenia/kernel/xobject.cc

@@ -23,7 +23,11 @@ XObject::XObject(KernelState* kernel_state, Type type)
       pointer_ref_count_(1),
       type_(type),
       handle_(X_INVALID_HANDLE_VALUE) {
-  kernel_state->object_table()->AddHandle(this, &handle_);
+
+  // Added pointer check to support usage without a kernel_state
+  if (kernel_state != nullptr){
+    kernel_state->object_table()->AddHandle(this, &handle_);
+  }
 }
 
 XObject::~XObject() {
@@ -56,10 +60,15 @@ void XObject::Release() {
 }
 
 X_STATUS XObject::Delete() {
-  if (!name_.empty()) {
-    kernel_state_->object_table()->RemoveNameMapping(name_);
+  if (kernel_state_ == nullptr) {
+    // Fake return value for api-scanner
+    return X_STATUS_SUCCESS;
+  } else {
+    if (!name_.empty()) {
+      kernel_state_->object_table()->RemoveNameMapping(name_);
+    }
+    return kernel_state_->object_table()->RemoveHandle(handle_);
   }
-  return kernel_state_->object_table()->RemoveHandle(handle_);
 }
 
 void XObject::SetAttributes(const uint8_t* obj_attrs_ptr) {

src/xenia/tools/api-scanner/README.md

@@ -0,0 +1,24 @@
+## api-scanner
+
+api-scanner will dump out the API imports from a packaged 360 game
+
+### Usage
+
+Run from the command line
+
+`api-scanner <package_path>`
+
+or:
+
+`api-scanner --target <package_path>`
+
+Output is printed to the console window, so it self-managed.
+
+The suggested usage is to append the output to a local file:
+
+`api-scanner <package_path> >> title_log.txt`
+
+### Issues
+
+- Duplicate code for loading containers
+- Several issues with gflags library - incorrectly prints usage from other files (due to linkage with libxenia)

src/xenia/tools/api-scanner/api_scanner_loader.cc

@@ -0,0 +1,189 @@
+/**
+ ******************************************************************************
+ * api-scanner - Scan for API imports from a packaged 360 game                *
+ ******************************************************************************
+ * Copyright 2015 x1nixmzeng. All rights reserved.                            *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include "api_scanner_loader.h"
+#include "xenia/kernel/xam_module.h"
+#include "xenia/kernel/xboxkrnl_module.h"
+
+namespace xe {
+namespace tools {
+
+void apiscanner_logger::operator()(const LogType type, const char* szMessage) {
+  switch (type) {
+  case LT_WARNING:
+    fprintf(stderr, "[W] %s\n", szMessage);
+    break;
+  case LT_ERROR:
+    fprintf(stderr, "[!] %s\n", szMessage);
+    break;
+  default:
+    break;
+  }
+}
+
+apiscanner_loader::apiscanner_loader()
+  : export_resolver(nullptr)
+  , memory_(nullptr)
+{
+  export_resolver = std::make_unique<ExportResolver>();
+
+  kernel::XamModule::RegisterExportTable(export_resolver.get());
+  kernel::XboxkrnlModule::RegisterExportTable(export_resolver.get());
+
+  memory_ = std::make_unique<Memory>();
+  memory_->Initialize();
+}
+
+apiscanner_loader::~apiscanner_loader()
+{
+  if (export_resolver != nullptr) {
+    export_resolver.reset();
+  }
+
+  if (memory_ != nullptr) {
+    memory_.reset();
+  }
+}
+
+bool apiscanner_loader::LoadTitleImports(const std::wstring& target) {
+  auto type(file_system.InferType(target));
+  int result = file_system.InitializeFromPath(type, target);
+  if (result) {
+    log(log.LT_ERROR, "Could not load target");
+    return false;
+  }
+
+  return ReadTarget();
+}
+
+
+bool apiscanner_loader::ReadTarget() {
+  // XXX Do a wildcard search for all xex files?
+  const char path[] = "game:\\default.xex";
+
+  kernel::XFile* file(nullptr);
+  bool read_result(false);
+  auto fs_entry = file_system.ResolvePath(path);
+  if (!fs_entry) {
+    log(log.LT_WARNING, "Could not resolve xex path");
+    return false;
+  }
+
+  // If the FS supports mapping, map the file in and load from that.
+  if (fs_entry->can_map()) {
+    auto mmap = fs_entry->CreateMemoryMapping(kernel::fs::Mode::READ, 0, 0);
+    if (!mmap) {
+      if (file) {
+        file->Release();
+      }
+      log(log.LT_WARNING, "Could not map filesystem");
+      return false;
+    }
+
+    title res;
+    read_result = ExtractImports(mmap->address(), mmap->length(), res);
+    if (read_result) {
+      loaded_titles.push_back(res);
+    }
+  }
+  else {
+    kernel::XFileInfo file_info;
+    if (fs_entry->QueryInfo(&file_info)) {
+      if (file) {
+        file->Release();
+      }
+      log(log.LT_WARNING, "Could not read xex attributes");
+      return false;
+    }
+
+    // Load into memory
+    std::vector<uint8_t> buffer(file_info.file_length);
+
+    // XXX No kernel state again
+    int result = file_system.Open(std::move(fs_entry), nullptr,
+      kernel::fs::Mode::READ, false, &file);
+    if (result) {
+      if (file) {
+        file->Release();
+      }
+      log(log.LT_WARNING, "Could not open xex file");
+      return false;
+    }
+
+    size_t bytes_read = 0;
+    result = file->Read(buffer.data(), buffer.size(), 0, &bytes_read);
+    if (result) {
+      if (file) {
+        file->Release();
+      }
+      log(log.LT_ERROR, "Could not read xex data");
+      return false;
+    }
+
+    title res;
+    read_result = ExtractImports(buffer.data(), bytes_read, res);
+    if (read_result) {
+      loaded_titles.push_back(res);
+    }
+  }
+
+  if (file) {
+    file->Release();
+  }
+
+  return read_result;
+}
+
+bool apiscanner_loader::ExtractImports(const void* addr, const size_t length,
+  title& info)
+{
+  // Load the XEX into memory and decrypt.
+  xe_xex2_options_t xex_options = { 0 };
+  xe_xex2_ref xex_(xe_xex2_load(memory_.get(), addr, length, xex_options));
+  if (!xex_) {
+    log(log.LT_ERROR, "Failed to parse xex file");
+    return false;
+  }
+
+  const xe_xex2_header_t* header = xe_xex2_get_header(xex_);
+
+  info.title_id = header->execution_info.title_id;
+    
+  // XXX Copy out library versions?
+  for (size_t n = 0; n < header->import_library_count; n++) {
+    const xe_xex2_import_library_t* library = &header->import_libraries[n];
+
+    xe_xex2_import_info_t* import_infos;
+    size_t import_info_count;
+    if (!xe_xex2_get_import_infos(xex_, library, &import_infos,
+      &import_info_count)) {
+
+      for (size_t m = 0; m < import_info_count; m++) {
+        const xe_xex2_import_info_t* import_info = &import_infos[m];
+
+        KernelExport* kernel_export = export_resolver->GetExportByOrdinal(
+          library->name, import_info->ordinal);
+
+        if ((kernel_export && kernel_export->type == KernelExport::Variable)
+          || import_info->thunk_address) {
+          const std::string named(kernel_export ? kernel_export->name : "");
+          info.imports.push_back(named);
+        }
+      }
+    }
+  }
+
+  xe_xex2_dealloc(xex_);
+  std::sort(info.imports.begin(), info.imports.end());
+
+  return true;
+}
+
+} // tools
+} // xe

src/xenia/tools/api-scanner/api_scanner_loader.h

@@ -0,0 +1,69 @@
+/**
+ ******************************************************************************
+ * api-scanner - Scan for API imports from a packaged 360 game                *
+ ******************************************************************************
+ * Copyright 2015 x1nixmzeng. All rights reserved.                            *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include "poly/main.h"
+#include "poly/string.h"
+#include "poly/math.h"
+
+#include "xenia/kernel/fs/filesystem.h"
+#include "xenia/memory.h"
+
+#include "xenia/kernel/objects/xfile.h"
+
+#include "xenia/export_resolver.h"
+#include "xenia/kernel/util/xex2.h"
+#include "xenia/xbox.h"
+
+#include <vector>
+
+namespace xe {
+namespace tools {
+
+  class apiscanner_logger
+  {
+  public:
+    enum LogType {
+      LT_WARNING,
+      LT_ERROR
+    };
+
+    void operator()(const LogType type, const char* szMessage);
+  };
+
+  class apiscanner_loader
+  {
+  private:
+    kernel::fs::FileSystem file_system;
+    apiscanner_logger log;
+    std::unique_ptr<Memory> memory_;
+    std::unique_ptr<ExportResolver> export_resolver;
+  public:
+    apiscanner_loader();
+    ~apiscanner_loader();
+
+    bool LoadTitleImports(const std::wstring& target);
+
+    struct title
+    {
+      uint32_t title_id;
+      std::vector<std::string> imports;
+    };
+
+    const std::vector<title>& GetAllTitles() const { return loaded_titles; }
+
+  private:
+    std::vector<title> loaded_titles;
+
+    bool ReadTarget();
+    bool ExtractImports(const void* addr, const size_t length, title& info);
+  };
+
+} // tools
+} // xe
+

src/xenia/tools/api-scanner/api_scanner_main.cc

@@ -0,0 +1,47 @@
+/**
+ ******************************************************************************
+ * api-scanner - Scan for API imports from a packaged 360 game                *
+ ******************************************************************************
+ * Copyright 2015 x1nixmzeng. All rights reserved.                            *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include <gflags/gflags.h>
+#include "api_scanner_loader.h"
+
+namespace xe {
+namespace tools {
+
+DEFINE_string(target, "", "List of file to extract imports from");
+
+int api_scanner_main(std::vector<std::wstring>& args) {
+
+  // XXX we need gflags to split multiple flags into arrays for us
+
+  if (args.size() == 2 || !FLAGS_target.empty()) {
+    apiscanner_loader loader_;
+    std::wstring target(FLAGS_target.empty() ? args[1] :
+      poly::to_wstring(FLAGS_target));
+
+    std::wstring target_abs = poly::to_absolute_path(target);
+
+    // XXX For each target?
+    if (loader_.LoadTitleImports(target)) {
+      for (const auto title : loader_.GetAllTitles()) {
+        printf("%08x\n", title.title_id);
+        for (const auto import : title.imports) {
+          printf("\t%s\n", import.c_str());
+        }
+      }
+    }
+  }
+
+  return 0;
+}
+
+}  // namespace tools
+}  // namespace xe
+
+DEFINE_ENTRY_POINT(L"api-scanner", L"api-scanner --target=<target file>",
+  xe::tools::api_scanner_main);

src/xenia/tools/api-scanner/sources.gypi

@@ -0,0 +1,8 @@
+# Copyright 2015 x1nixmzeng. All Rights Reserved.
+{
+  'sources': [
+    'api_scanner_loader.cc',
+    'api_scanner_loader.h',
+    'api_scanner_main.cc'
+  ]
+}

xenia.gyp

@@ -493,5 +493,28 @@
         'src/xenia/gpu/trace_viewer_main.cc',
       ],
     },
+
+    {
+      'target_name': 'api-scanner',
+      'type': 'executable',
+
+      'msvs_settings': {
+        'VCLinkerTool': {
+          'SubSystem': '1'
+        },
+      },
+
+      'dependencies': [
+        'libxenia',
+      ],
+
+      'include_dirs': [
+        '.',
+      ],
+
+      'includes': [
+        'src/xenia/tools/api-scanner/sources.gypi',
+      ],
+    },
   ],
 }