Reworking the memory system to not commit 3gb and to properly alloc data.

Now only 512MB is committed on startup. Loaded XEXs are placed into their
required addresses in the 0x8... range. Kernel structures are allocated
from the normal heap like other data. There should no longer be any magical
pointers.

src/xenia/core/memory.cc

@@ -39,8 +39,23 @@
  * We use the host OS to create an entire addressable range for this. That way
  * we don't have to emulate a TLB. It'd be really cool to pass through page
  * sizes or use madvice to let the OS know what to expect.
+ *
+ * We create our own heap of committed memory that lives at XE_MEMORY_HEAP_LOW
+ * to XE_MEMORY_HEAP_HIGH - all normal user allocations come from there. Since
+ * the Xbox has no paging, we know that the size of this heap will never need
+ * to be larger than ~512MB (realistically, smaller than that). We place it far
+ * away from the XEX data and keep the memory around it uncommitted so that we
+ * have some warning if things go astray.
+ *
+ * For XEX/GPU/etc data we allow placement allocations (base_address != 0) and
+ * commit the requested memory as needed. This bypasses the standard heap, but
+ * XEXs should never be overwriting anything so that's fine. We can also query
+ * for previous commits and assert that we really isn't committing twice.
  */
 
+#define XE_MEMORY_HEAP_LOW    0x20000000
+#define XE_MEMORY_HEAP_HIGH   0x40000000
+
 
 struct xe_memory {
   xe_ref_t ref;
@@ -56,14 +71,12 @@ struct xe_memory {
 
 
 xe_memory_ref xe_memory_create(xe_memory_options_t options) {
-  uint32_t offset;
-  uint32_t mspace_size;
 
   xe_memory_ref memory = (xe_memory_ref)xe_calloc(sizeof(xe_memory));
   xe_ref_init((xe_ref)memory);
 
 #if XE_PLATFORM(WIN32)
-  SYSTEM_INFO si;
+  SYSTEM_INFO si;
   GetSystemInfo(&si);
   memory->system_page_size = si.dwPageSize;
 #else
@@ -73,25 +86,38 @@ xe_memory_ref xe_memory_create(xe_memory_options_t options) {
   memory->length = 0xC0000000;
 
 #if XE_PLATFORM(WIN32)
+  // Reserve the entire usable address space.
+  // We're 64-bit, so this should be no problem.
   memory->ptr = VirtualAlloc(0, memory->length,
-                             MEM_COMMIT | MEM_RESERVE,
+                             MEM_RESERVE,
                              PAGE_READWRITE);
+  XEEXPECTNOTNULL(memory->ptr);
 #else
   memory->ptr = mmap(0, memory->length, PROT_READ | PROT_WRITE,
                      MAP_PRIVATE | MAP_ANON, -1, 0);
   XEEXPECT(memory->ptr != MAP_FAILED);
-#endif  // WIN32
   XEEXPECTNOTNULL(memory->ptr);
+#endif  // WIN32
 
-  memory->heap_mutex = xe_mutex_alloc(0);
+  // Lock used around heap allocs/frees.
+  memory->heap_mutex = xe_mutex_alloc(10000);
   XEEXPECTNOTNULL(memory->heap_mutex);
 
+  // Commit the memory where our heap will live.
+  // We don't allocate at 0 to make bad writes easier to find.
+  uint32_t heap_offset = XE_MEMORY_HEAP_LOW;
+  uint32_t heap_size = XE_MEMORY_HEAP_HIGH - XE_MEMORY_HEAP_LOW;
+  uint8_t* heap_ptr = (uint8_t*)memory->ptr + heap_offset;
+#if XE_PLATFORM(WIN32)
+  void* heap_result = VirtualAlloc(heap_ptr, heap_size,
+                                   MEM_COMMIT, PAGE_READWRITE);
+  XEEXPECTNOTNULL(heap_result);
+#else
+#error ?
+#endif  // WIN32
+
   // Allocate the mspace for our heap.
-  // We skip the first page to make writes to 0 easier to find.
+  memory->heap = create_mspace_with_base(heap_ptr, heap_size, 0);
-  offset = 64 * 1024;
-  mspace_size = 512 * 1024 * 1024 - offset;
-  memory->heap = create_mspace_with_base(
-      (uint8_t*)memory->ptr + offset, mspace_size, 0);
 
   return memory;
 
@@ -113,7 +139,8 @@ void xe_memory_dealloc(xe_memory_ref memory) {
   }
 
 #if XE_PLATFORM(WIN32)
-  XEIGNORE(VirtualFree(memory->ptr, memory->length, MEM_RELEASE));
+  // This decommits all pages and releases everything.
+  XEIGNORE(VirtualFree(memory->ptr, 0, MEM_RELEASE));
 #else
   munmap(memory->ptr, memory->length);
 #endif  // WIN32
@@ -161,36 +188,106 @@ uint32_t xe_memory_search_aligned(xe_memory_ref memory, size_t start,
   return 0;
 }
 
-// TODO(benvanik): reserve/commit states/large pages/etc
+uint32_t xe_memory_heap_alloc(
-
+    xe_memory_ref memory, uint32_t base_address, uint32_t size,
-uint32_t xe_memory_heap_alloc(xe_memory_ref memory, uint32_t base_addr,
+    uint32_t flags) {
-                              uint32_t size, uint32_t flags) {
-  XEASSERT(base_addr == 0);
   XEASSERT(flags == 0);
 
-  XEIGNORE(xe_mutex_lock(memory->heap_mutex));
+  // If we were given a base address we are outside of the normal heap and
-  uint8_t* p = (uint8_t*)mspace_malloc(memory->heap, size);
+  // will place wherever asked (so long as it doesn't overlap the heap).
-  XEIGNORE(xe_mutex_unlock(memory->heap_mutex));
+  if (!base_address) {
-  if (!p) {
+    // Normal allocation from the managed heap.
-    return 0;
+    XEIGNORE(xe_mutex_lock(memory->heap_mutex));
-  }
+    uint8_t* p = (uint8_t*)mspace_malloc(memory->heap, size);
+    XEIGNORE(xe_mutex_unlock(memory->heap_mutex));
+    if (!p) {
+      return 0;
+    }
+    return (uint32_t)((uintptr_t)p - (uintptr_t)memory->ptr);
+  } else {
+    if (base_address >= XE_MEMORY_HEAP_LOW &&
+        base_address < XE_MEMORY_HEAP_HIGH) {
+      // Overlapping managed heap.
+      XEASSERTALWAYS();
+      return 0;
+    }
 
-  return (uint32_t)((uintptr_t)p - (uintptr_t)memory->ptr);
+    uint8_t* p = (uint8_t*)memory->ptr + base_address;
+#if XE_PLATFORM(WIN32)
+    // TODO(benvanik): check if address range is in use with a query.
+
+    void* pv = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE);
+    if (!pv) {
+      // Failed.
+      XEASSERTALWAYS();
+      return 0;
+    }
+#else
+#error ?
+#endif  // WIN32
+
+    return base_address;
+  }
 }
 
-uint32_t xe_memory_heap_free(xe_memory_ref memory, uint32_t addr,
+int xe_memory_heap_free(
-                             uint32_t flags) {
+    xe_memory_ref memory, uint32_t address, uint32_t size) {
-  XEASSERT(flags == 0);
+  uint8_t* p = (uint8_t*)memory->ptr + address;
+  if (address >= XE_MEMORY_HEAP_LOW && address < XE_MEMORY_HEAP_HIGH) {
+    // Heap allocated address.
+    size_t real_size = mspace_usable_size(p);
+    if (!real_size) {
+      return 0;
+    }
 
-  uint8_t* p = (uint8_t*)memory->ptr + addr;
+    XEIGNORE(xe_mutex_lock(memory->heap_mutex));
-  size_t real_size = mspace_usable_size(p);
+    mspace_free(memory->heap, p);
-  if (!real_size) {
+    XEIGNORE(xe_mutex_unlock(memory->heap_mutex));
-    return 0;
+    return (uint32_t)real_size;
+  } else {
+    // A placed address. Decommit.
+#if XE_PLATFORM(WIN32)
+    return VirtualFree(p, size, MEM_DECOMMIT) ? 0 : 1;
+#else
+#error decommit
+#endif  // WIN32
   }
-
+}
-  XEIGNORE(xe_mutex_lock(memory->heap_mutex));
+
-  mspace_free(memory->heap, p);
+bool xe_memory_is_valid(xe_memory_ref memory, uint32_t address) {
-  XEIGNORE(xe_mutex_unlock(memory->heap_mutex));
+  uint8_t* p = (uint8_t*)memory->ptr + address;
-
+  if (address >= XE_MEMORY_HEAP_LOW && address < XE_MEMORY_HEAP_HIGH) {
-  return (uint32_t)real_size;
+    // Within heap range, ask dlmalloc.
+    return mspace_usable_size(p) > 0;
+  } else {
+    // Maybe -- could Query here (though that may be expensive).
+    return true;
+  }
+}
+
+int xe_memory_protect(
+    xe_memory_ref memory, uint32_t address, uint32_t size, uint32_t access) {
+  uint8_t* p = (uint8_t*)memory->ptr + address;
+
+#if XE_PLATFORM(WIN32)
+  DWORD new_protect = 0;
+  if (access & XE_MEMORY_ACCESS_WRITE) {
+    new_protect = PAGE_READWRITE;
+  } else if (access & XE_MEMORY_ACCESS_READ) {
+    new_protect = PAGE_READONLY;
+  } else {
+    new_protect = PAGE_NOACCESS;
+  }
+  DWORD old_protect;
+  return VirtualProtect(p, size, new_protect, &old_protect) == TRUE ? 0 : 1;
+#else
+  int prot = 0;
+  if (access & XE_MEMORY_ACCESS_READ) {
+    prot = PROT_READ;
+  }
+  if (access & XE_MEMORY_ACCESS_WRITE) {
+    prot = PROT_WRITE;
+  }
+  return mprotect(p, size, prot);
+#endif  // WIN32
 }

src/xenia/core/memory.h

@@ -28,18 +28,31 @@ xe_memory_ref xe_memory_retain(xe_memory_ref memory);
 void xe_memory_release(xe_memory_ref memory);
 
 size_t xe_memory_get_length(xe_memory_ref memory);
-uint8_t *xe_memory_addr(xe_memory_ref memory, size_t guest_addr);
+uint8_t *xe_memory_addr(xe_memory_ref memory, size_t guest_addr = 0);
 
 uint32_t xe_memory_search_aligned(xe_memory_ref memory, size_t start,
                                   size_t end, const uint32_t *values,
                                   const size_t value_count);
 
-// These methods slice off memory from the virtual address space.
+// These methods slice off memory from the virtual address space, unless
-// They should only be used by kernel modules that know what they are doing.
+// base_address is specified in which case they place into an address in
-uint32_t xe_memory_heap_alloc(xe_memory_ref memory, uint32_t base_addr,
+// memory.
-                              uint32_t size, uint32_t flags);
-uint32_t xe_memory_heap_free(xe_memory_ref memory, uint32_t addr,
-                             uint32_t flags);
 
+enum {
+  XE_MEMORY_FLAG_64KB_PAGES = (1 << 1),
+};
+
+enum {
+  XE_MEMORY_ACCESS_READ     = (1 << 1),
+  XE_MEMORY_ACCESS_WRITE    = (1 << 2)
+};
+
+uint32_t xe_memory_heap_alloc(xe_memory_ref memory, uint32_t base_address,
+                              uint32_t size, uint32_t flags);
+int  xe_memory_heap_free(xe_memory_ref memory, uint32_t addr, uint32_t size);
+
+bool xe_memory_is_valid(xe_memory_ref memory, uint32_t address);
+int xe_memory_protect(xe_memory_ref memory, uint32_t address, uint32_t size,
+                      uint32_t access);
 
 #endif  // XENIA_CORE_MEMORY_H_

src/xenia/kernel/modules/xboxkrnl/module.cc

@@ -58,30 +58,29 @@ XboxkrnlModule::XboxkrnlModule(Runtime* runtime) :
   RegisterThreadingExports(resolver, kernel_state_.get());
   RegisterVideoExports(resolver, kernel_state_.get());
 
-  // TODO(benvanik): alloc heap memory somewhere in user space
+  uint8_t* mem = xe_memory_addr(memory_);
-  // TODO(benvanik): tools for reading/writing to heap memory
-
-  uint8_t* mem = xe_memory_addr(memory_, 0);
 
   // KeDebugMonitorData (?*)
   // Set to a valid value when a remote debugger is attached.
   // Offset 0x18 is a 4b pointer to a handler function that seems to take two
   // arguments. If we wanted to see what would happen we could fake that.
+  uint32_t pKeDebugMonitorData = xe_memory_heap_alloc(memory_, 0, 256, 0);
   resolver->SetVariableMapping(
       "xboxkrnl.exe", ordinals::KeDebugMonitorData,
-      0x80102100);
+      pKeDebugMonitorData);
-  XESETUINT32BE(mem + 0x80102100, 0);
+  XESETUINT32BE(mem + pKeDebugMonitorData, 0);
 
   // XboxHardwareInfo (XboxHardwareInfo_t, 16b)
   // flags       cpu#  ?     ?     ?     ?           ?       ?
   // 0x00000000, 0x06, 0x00, 0x00, 0x00, 0x00000000, 0x0000, 0x0000
   // Games seem to check if bit 26 (0x20) is set, which at least for xbox1
   // was whether an HDD was present. Not sure what the other flags are.
+  uint32_t pXboxHardwareInfo = xe_memory_heap_alloc(memory_, 0, 16, 0);
   resolver->SetVariableMapping(
       "xboxkrnl.exe", ordinals::XboxHardwareInfo,
-      0x80100FED);
+      pXboxHardwareInfo);
-  XESETUINT32BE(mem + 0x80100FED, 0x00000000);  // flags
+  XESETUINT32BE(mem + pXboxHardwareInfo + 0, 0x00000000); // flags
-  XESETUINT8BE(mem  + 0x80100FEE, 0x06);        // cpu count
+  XESETUINT8BE (mem + pXboxHardwareInfo + 4, 0x06);       // cpu count
   // Remaining 11b are zeroes?
 
   // XexExecutableModuleHandle (?**)
@@ -93,21 +92,26 @@ XboxkrnlModule::XboxkrnlModule(Runtime* runtime) :
   // 0x80101000 <- our module structure
   // 0x80101058 <- pointer to xex header
   // 0x80101100 <- xex header base
+  uint32_t ppXexExecutableModuleHandle =
+      xe_memory_heap_alloc(memory_, 0, 4, 0);
   resolver->SetVariableMapping(
       "xboxkrnl.exe", ordinals::XexExecutableModuleHandle,
-      0x80100FFC);
+      ppXexExecutableModuleHandle);
-  XESETUINT32BE(mem + 0x80100FFC, 0x80101000);
+  uint32_t pXexExecutableModuleHandle =
-  XESETUINT32BE(mem + 0x80101058, 0x80101100);
+      xe_memory_heap_alloc(memory_, 0, 256, 0);
+  XESETUINT32BE(mem + ppXexExecutableModuleHandle, pXexExecutableModuleHandle);
+  XESETUINT32BE(mem + pXexExecutableModuleHandle + 0x58, 0x80101100);
 
   // ExLoadedCommandLine (char*)
   // The name of the xex. Not sure this is ever really used on real devices.
   // Perhaps it's how swap disc/etc data is sent?
   // Always set to "default.xex" (with quotes) for now.
+  uint32_t pExLoadedCommandLine = xe_memory_heap_alloc(memory_, 0, 1024, 0);
   resolver->SetVariableMapping(
       "xboxkrnl.exe", ordinals::ExLoadedCommandLine,
-      0x80102000);
+      pExLoadedCommandLine);
   char command_line[] = "\"default.xex\"";
-  xe_copy_memory(mem + 0x80102000, 1024,
+  xe_copy_memory(mem + pExLoadedCommandLine, 1024,
                  command_line, XECOUNT(command_line) + 1);
 }
 

src/xenia/kernel/modules/xboxkrnl/xboxkrnl_video.cc

@@ -159,6 +159,12 @@ SHIM_CALL VdSetGraphicsInterruptCallback_shim(
 // no op?
 
 
+// VdCallGraphicsNotificationRoutines
+// r3 = 1
+// r4 = ?
+// callbacks get 0, r3, r4
+
+
 }  // namespace xboxkrnl
 }  // namespace kernel
 }  // namespace xe

src/xenia/kernel/xex2.cc

@@ -740,6 +740,16 @@ XECLEANUP:
 int xe_xex2_read_image(xe_xex2_ref xex, const uint8_t *xex_addr,
                        const size_t xex_length, xe_memory_ref memory) {
   const xe_xex2_header_t *header = &xex->header;
+
+  // Allocate in-place the XEX memory.
+  uint32_t alloc_result =
+      xe_memory_heap_alloc(memory, header->exe_address, xex_length, 0);
+  if (!alloc_result) {
+    XELOGE("Unable to allocate XEX memory at %.8X-%.8X.",
+           header->exe_address, xex_length);
+    return 2;
+  }
+
   switch (header->file_format_info.compression_type) {
   case XEX_COMPRESSION_NONE:
     return xe_xex2_read_image_uncompressed(