KeEnterCriticalRegion/KeLeaveCriticalRegion.

src/xenia/atomic.h

@@ -28,6 +28,8 @@
     ((void)OSAtomicAdd32Barrier(amount, value))
 #define xe_atomic_sub_32(amount, value) \
     ((void)OSAtomicAdd32Barrier(-amount, value))
+#define xe_atomic_exchange_32(newValue, value) \
+    TODOTODO
 #define xe_atomic_cas_32(oldValue, newValue, value) \
     OSAtomicCompareAndSwap32Barrier(oldValue, newValue, value)
 
@@ -49,6 +51,8 @@ typedef OSQueueHead xe_atomic_stack_t;
     ((void)InterlockedExchangeAdd((volatile LONG*)value, amount))
 #define xe_atomic_sub_32(amount, value) \
     ((void)InterlockedExchangeSubtract((volatile unsigned*)value, amount))
+#define xe_atomic_exchange_32(newValue, value) \
+    InterlockedExchange((volatile LONG*)value, newValue)
 #define xe_atomic_cas_32(oldValue, newValue, value) \
     (InterlockedCompareExchange((volatile LONG*)value, newValue, oldValue) == oldValue)
 
@@ -77,6 +81,8 @@ XEFORCEINLINE void* xe_atomic_stack_dequeue(xe_atomic_stack_t* stack,
     __sync_fetch_and_add(value, amount)
 #define xe_atomic_sub_32(amount, value) \
     __sync_fetch_and_sub(value, amount)
+#define xe_atomic_exchange_32(newValue, value) \
+    TODOTODO
 #define xe_atomic_cas_32(oldValue, newValue, value) \
     __sync_bool_compare_and_swap(value, oldValue, newValue)
 

src/xenia/kernel/modules/xboxkrnl/kernel_state.cc

@@ -10,6 +10,7 @@
 #include <xenia/kernel/modules/xboxkrnl/kernel_state.h>
 
 #include <xenia/kernel/runtime.h>
+#include <xenia/kernel/modules/xboxkrnl/xboxkrnl_private.h>
 #include <xenia/kernel/modules/xboxkrnl/xobject.h>
 #include <xenia/kernel/modules/xboxkrnl/objects/xmodule.h>
 #include <xenia/kernel/modules/xboxkrnl/objects/xthread.h>
@@ -48,6 +49,10 @@ KernelState::~KernelState() {
   xe_memory_release(memory_);
 }
 
+KernelState* KernelState::shared() {
+  return shared_kernel_state_;
+}
+
 Runtime* KernelState::runtime() {
   return runtime_;
 }

src/xenia/kernel/modules/xboxkrnl/kernel_state.h

@@ -42,6 +42,8 @@ public:
   KernelState(Runtime* runtime);
   ~KernelState();
 
+  static KernelState* shared();
+
   Runtime* runtime();
   xe_memory_ref memory();
   cpu::Processor* processor();

src/xenia/kernel/modules/xboxkrnl/objects/xthread.cc

@@ -23,6 +23,8 @@ using namespace xe::kernel::xboxkrnl;
 namespace {
   static uint32_t next_xthread_id = 0;
   static uint32_t current_thread_tls = xeKeTlsAlloc();
+  static xe_mutex_t* critical_region_ = xe_mutex_alloc(10000);
+  static XThread* shared_kernel_thread_ = 0;
 }
 
 
@@ -36,7 +38,8 @@ XThread::XThread(KernelState* kernel_state,
     thread_handle_(0),
     thread_state_address_(0),
     thread_state_(0),
-    event_(NULL) {
+    event_(NULL),
+    irql_(0) {
   creation_params_.stack_size           = stack_size;
   creation_params_.xapi_thread_startup  = xapi_thread_startup;
   creation_params_.start_address        = start_address;
@@ -73,8 +76,26 @@ XThread::~XThread() {
   }
 }
 
+XThread* XThread::GetCurrentThread() {
+  XThread* thread = (XThread*)xeKeTlsGetValue(current_thread_tls);
+  if (!thread) {
+    // Assume this is some shared interrupt thread/etc.
+    XThread::EnterCriticalRegion();
+    thread = shared_kernel_thread_;
+    if (!thread) {
+      thread = new XThread(
+          KernelState::shared(), 32 * 1024, 0, 0, 0, 0);
+      shared_kernel_thread_ = thread;
+      xeKeTlsSetValue(current_thread_tls, (uint64_t)thread);
+    }
+    XThread::LeaveCriticalRegion();
+  }
+  return thread;
+}
+
 uint32_t XThread::GetCurrentThreadHandle() {
-  return xeKeTlsGetValue(current_thread_tls);
+  XThread* thread = XThread::GetCurrentThread();
+  return thread->handle();
 }
 
 uint32_t XThread::GetCurrentThreadId(const uint8_t* thread_state_block) {
@@ -178,8 +199,9 @@ X_STATUS XThread::Exit(int exit_code) {
 
 static uint32_t __stdcall XThreadStartCallbackWin32(void* param) {
   XThread* thread = reinterpret_cast<XThread*>(param);
-  xeKeTlsSetValue(current_thread_tls, thread->handle());
+  xeKeTlsSetValue(current_thread_tls, (uint64_t)thread);
   thread->Execute();
+  xeKeTlsSetValue(current_thread_tls, NULL);
   thread->Release();
   return 0;
 }
@@ -217,8 +239,9 @@ X_STATUS XThread::PlatformExit(int exit_code) {
 
 static void* XThreadStartCallbackPthreads(void* param) {
   XThread* thread = reinterpret_cast<XThread*>(param);
-  xeKeTlsSetValue(current_thread_tls, thread->handle());
+  xeKeTlsSetValue(current_thread_tls, (uint64_t)thread);
   thread->Execute();
+  xeKeTlsSetValue(current_thread_tls, NULL);
   thread->Release();
   return 0;
 }
@@ -299,3 +322,20 @@ X_STATUS XThread::Wait(uint32_t wait_reason, uint32_t processor_mode,
                        uint32_t alertable, uint64_t* opt_timeout) {
   return event_->Wait(wait_reason, processor_mode, alertable, opt_timeout);
 }
+
+void XThread::EnterCriticalRegion() {
+  // Global critical region. This isn't right, but is easy.
+  xe_mutex_lock(critical_region_);
+}
+
+void XThread::LeaveCriticalRegion() {
+  xe_mutex_unlock(critical_region_);
+}
+
+uint32_t XThread::RaiseIrql(uint32_t new_irql) {
+  return xe_atomic_exchange_32(new_irql, &irql_);
+}
+
+void XThread::LowerIrql(uint32_t new_irql) {
+  irql_ = new_irql;
+}

src/xenia/kernel/modules/xboxkrnl/objects/xthread.h

@@ -39,6 +39,7 @@ public:
           uint32_t creation_flags);
   virtual ~XThread();
 
+  static XThread* GetCurrentThread();
   static uint32_t GetCurrentThreadHandle();
   static uint32_t GetCurrentThreadId(const uint8_t* thread_state_block);
 
@@ -54,6 +55,11 @@ public:
   virtual X_STATUS Wait(uint32_t wait_reason, uint32_t processor_mode,
                         uint32_t alertable, uint64_t* opt_timeout);
 
+  static void EnterCriticalRegion();
+  static void LeaveCriticalRegion();
+  uint32_t RaiseIrql(uint32_t new_irql);
+  void LowerIrql(uint32_t new_irql);
+
 private:
   X_STATUS PlatformCreate();
   void PlatformDestroy();
@@ -73,6 +79,8 @@ private:
   uint32_t      thread_state_address_;
   cpu::ThreadState* thread_state_;
 
+  uint32_t      irql_;
+
   XEvent*       event_;
 };
 

src/xenia/kernel/modules/xboxkrnl/xboxkrnl_threading.cc

@@ -296,16 +296,16 @@ SHIM_CALL KeTlsFree_shim(
 
 
 // http://msdn.microsoft.com/en-us/library/ms686812
-uint32_t xeKeTlsGetValue(uint32_t tls_index) {
+uint64_t xeKeTlsGetValue(uint32_t tls_index) {
   // LPVOID
   // _In_  DWORD dwTlsIndex
 
-  uint32_t value = 0;
+  uint64_t value = 0;
 
 #if XE_PLATFORM(WIN32)
-  value = (uint32_t)((uint64_t)TlsGetValue(tls_index));
+  value = (uint64_t)TlsGetValue(tls_index);
 #else
-  value = (uint32_t)((uint64_t)pthread_getspecific(tls_index));
+  value = (uint64_t)pthread_getspecific(tls_index);
 #endif  // WIN32
 
   if (!value) {
@@ -325,13 +325,13 @@ SHIM_CALL KeTlsGetValue_shim(
       "KeTlsGetValue(%.8X)",
       tls_index);
 
-  uint32_t result = xeKeTlsGetValue(tls_index);
+  uint64_t result = xeKeTlsGetValue(tls_index);
   SHIM_SET_RETURN(result);
 }
 
 
 // http://msdn.microsoft.com/en-us/library/ms686818
-int xeKeTlsSetValue(uint32_t tls_index, uint32_t tls_value) {
+int xeKeTlsSetValue(uint32_t tls_index, uint64_t tls_value) {
   // BOOL
   // _In_      DWORD dwTlsIndex,
   // _In_opt_  LPVOID lpTlsValue
@@ -560,11 +560,15 @@ SHIM_CALL NtWaitForSingleObjectEx_shim(
 
 
 uint32_t xeKfAcquireSpinLock(void* lock_ptr) {
+  // Lock.
   while (!xe_atomic_cas_32(0, 1, lock_ptr)) {
     // Spin!
+    // TODO(benvanik): error on deadlock?
   }
-  // TODO(benvanik): set dispatch level.
-  return 0;
+
+  // Raise IRQL to DISPATCH.
+  XThread* thread = XThread::GetCurrentThread();
+  return thread->RaiseIrql(2);
 }
 
 
@@ -583,7 +587,11 @@ SHIM_CALL KfAcquireSpinLock_shim(
 
 
 void xeKfReleaseSpinLock(void* lock_ptr, uint32_t old_irql) {
-  // TODO(benvanik): reset dispatch level.
+  // Restore IRQL.
+  XThread* thread = XThread::GetCurrentThread();
+  thread->LowerIrql(old_irql);
+
+  // Unlock.
   xe_atomic_dec_32(lock_ptr);
 }
 
@@ -602,6 +610,32 @@ SHIM_CALL KfReleaseSpinLock_shim(
 }
 
 
+void xeKeEnterCriticalRegion() {
+  XThread::EnterCriticalRegion();
+}
+
+
+SHIM_CALL KeEnterCriticalRegion_shim(
+  xe_ppc_state_t* ppc_state, KernelState* state) {
+  XELOGD(
+    "KeEnterCriticalRegion()");
+  xeKeEnterCriticalRegion();
+}
+
+
+void xeKeLeaveCriticalRegion() {
+  XThread::LeaveCriticalRegion();
+}
+
+
+SHIM_CALL KeLeaveCriticalRegion_shim(
+  xe_ppc_state_t* ppc_state, KernelState* state) {
+  XELOGD(
+    "KeLeaveCriticalRegion()");
+  xeKeLeaveCriticalRegion();
+}
+
+
 }  // namespace xboxkrnl
 }  // namespace kernel
 }  // namespace xe
@@ -632,4 +666,7 @@ void xe::kernel::xboxkrnl::RegisterThreadingExports(
 
   SHIM_SET_MAPPING("xboxkrnl.exe", KfAcquireSpinLock, state);
   SHIM_SET_MAPPING("xboxkrnl.exe", KfReleaseSpinLock, state);
+
+  SHIM_SET_MAPPING("xboxkrnl.exe", KeEnterCriticalRegion, state);
+  SHIM_SET_MAPPING("xboxkrnl.exe", KeLeaveCriticalRegion, state);
 }

src/xenia/kernel/modules/xboxkrnl/xboxkrnl_threading.h

@@ -35,8 +35,8 @@ void xeKeQuerySystemTime(uint64_t* time_ptr);
 
 uint32_t xeKeTlsAlloc();
 int KeTlsFree(uint32_t tls_index);
-uint32_t xeKeTlsGetValue(uint32_t tls_index);
-int xeKeTlsSetValue(uint32_t tls_index, uint32_t tls_value);
+uint64_t xeKeTlsGetValue(uint32_t tls_index);
+int xeKeTlsSetValue(uint32_t tls_index, uint64_t tls_value);
 
 X_STATUS xeNtCreateEvent(uint32_t* handle_ptr, void* obj_attributes,
                          uint32_t event_type, uint32_t initial_state);
@@ -50,6 +50,9 @@ X_STATUS xeKeWaitForSingleObject(
 uint32_t xeKfAcquireSpinLock(void* lock_ptr);
 void xeKfReleaseSpinLock(void* lock_ptr, uint32_t old_irql);
 
+void xeKeEnterCriticalRegion();
+void xeKeLeaveCriticalRegion();
+
 
 }  // namespace xboxkrnl
 }  // namespace kernel