diff --git a/src/xenia/kernel/kernel_state.cc b/src/xenia/kernel/kernel_state.cc
index 57097148e..83f10bbdd 100644
--- a/src/xenia/kernel/kernel_state.cc
+++ b/src/xenia/kernel/kernel_state.cc
@@ -367,8 +367,7 @@ object_ref<UserModule> KernelState::LoadUserModule(const char* raw_name,
     // See if we've already loaded it
     for (auto& existing_module : user_modules_) {
       if (existing_module->path() == path) {
-        existing_module->Retain();
-        return retain_object(existing_module.get());
+        return existing_module;
       }
     }
 
@@ -378,14 +377,13 @@ object_ref<UserModule> KernelState::LoadUserModule(const char* raw_name,
     module = object_ref<UserModule>(new UserModule(this));
     X_STATUS status = module->LoadFromFile(path);
     if (XFAILED(status)) {
-      object_table()->RemoveHandle(module->handle());
+      object_table()->ReleaseHandle(module->handle());
       return nullptr;
     }
 
     global_lock.lock();
 
-    // Retain when putting into the listing.
-    module->Retain();
+    // Putting into the listing automatically retains.
     user_modules_.push_back(module);
   }
 
diff --git a/src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc b/src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc
index 81a44c553..f6d93a9f2 100644
--- a/src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc
+++ b/src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc
@@ -96,8 +96,10 @@ dword_result_t XexLoadImage(lpstring_t module_name, dword_t module_flags,
     // Not found; attempt to load as a user module.
     auto user_module = kernel_state()->LoadUserModule(module_name);
     if (user_module) {
-      user_module->Retain();
-      hmodule = user_module->hmodule_ptr();
+      // Give up object ownership, this reference will be released by the last
+      // XexUnloadImage call
+      auto user_module_raw = user_module.release();
+      hmodule = user_module_raw->hmodule_ptr();
       result = X_STATUS_SUCCESS;
     }
   }