diff --git a/src/xenia/kernel/kernel_state.cc b/src/xenia/kernel/kernel_state.cc
index 57097148e..bf5487521 100644
--- a/src/xenia/kernel/kernel_state.cc
+++ b/src/xenia/kernel/kernel_state.cc
@@ -367,8 +367,7 @@ object_ref<UserModule> KernelState::LoadUserModule(const char* raw_name,
     // See if we've already loaded it
     for (auto& existing_module : user_modules_) {
       if (existing_module->path() == path) {
-        existing_module->Retain();
-        return retain_object(existing_module.get());
+        return existing_module;
       }
     }
 
@@ -384,8 +383,7 @@ object_ref<UserModule> KernelState::LoadUserModule(const char* raw_name,
 
     global_lock.lock();
 
-    // Retain when putting into the listing.
-    module->Retain();
+    // Putting into the listing automatically retains.
     user_modules_.push_back(module);
   }
 
diff --git a/src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc b/src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc
index 44483bcf3..3ae038368 100644
--- a/src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc
+++ b/src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc
@@ -198,8 +198,10 @@ dword_result_t XexLoadImage(lpstring_t module_name, dword_t module_flags,
     // Not found; attempt to load as a user module.
     auto user_module = kernel_state()->LoadUserModule(module_name);
     if (user_module) {
-      user_module->Retain();
-      hmodule = user_module->hmodule_ptr();
+      // Give up object ownership, this reference will be released by the last
+      // XexUnloadImage call
+      auto user_module_raw = user_module.release();
+      hmodule = user_module_raw->hmodule_ptr();
       result = X_STATUS_SUCCESS;
     }
   }