Implemented suspend/resume kernel Nt routines with the corresponding Ke routines

src/core/kernel/common/types.h

@@ -98,6 +98,8 @@ typedef void* LPSECURITY_ATTRIBUTES;
 #define X_STATUS_FILE_IS_A_DIRECTORY 0xC00000BAL
 #define X_STATUS_END_OF_FILE 0xC0000011L
 #define X_STATUS_INVALID_PAGE_PROTECTION 0xC0000045L
+#define X_STATUS_SUSPEND_COUNT_EXCEEDED 0xC000004AL
+#define X_STATUS_THREAD_IS_TERMINATING 0xC000004BL
 #define X_STATUS_CONFLICTING_ADDRESSES 0xC0000018L
 #define X_STATUS_UNABLE_TO_FREE_VM 0xC000001AL
 #define X_STATUS_FREE_VM_NOT_AT_BASE 0xC000009FL
@@ -1969,6 +1971,8 @@ typedef struct _KTHREAD
 }
 KTHREAD, *PKTHREAD, *RESTRICTED_POINTER PRKTHREAD;
 
+#define X_MAXIMUM_SUSPEND_COUNT 0x7F
+
 // ******************************************************************
 // * ETHREAD
 // ******************************************************************

src/core/kernel/exports/EmuKrnlKe.cpp

@@ -1206,35 +1206,9 @@ XBSYSAPI EXPORTNUM(118) xbox::boolean_xt NTAPI xbox::KeInsertQueueApc
 		Apc->SystemArgument1 = SystemArgument1;
 		Apc->SystemArgument2 = SystemArgument2;
 
-		if (Apc->Inserted) {
-			KfLowerIrql(OldIrql);
-			RETURN(FALSE);
-		}
-		else {
-			KiApcListMtx.lock();
-			InsertTailList(&kThread->ApcState.ApcListHead[Apc->ApcMode], &Apc->ApcListEntry);
-			Apc->Inserted = TRUE;
-			KiApcListMtx.unlock();
-
-			// We can only attempt to execute the queued apc right away if it is been inserted in the current thread, because otherwise the KTHREAD
-			// in the fs selector will not be correct
-			if (kThread == KeGetCurrentThread()) {
-				if (Apc->ApcMode == KernelMode) { // kernel apc
-					// NOTE: this is wrong, we should check the thread state instead of just signaling the kernel apc, but we currently
-					// don't set the appropriate state in kthread
-					kThread->ApcState.KernelApcPending = TRUE;
-					KiExecuteKernelApc();
-				}
-				else if ((kThread->WaitMode == UserMode) && (kThread->Alertable)) { // user apc
-					// NOTE: this should also check the thread state
-					kThread->ApcState.UserApcPending = TRUE;
-					KiExecuteUserApc();
-				}
-			}
-
-			KfLowerIrql(OldIrql);
-			RETURN(TRUE);
-		}
+		boolean_xt result = KiInsertQueueApc(Apc, Increment);
+		KfLowerIrql(OldIrql);
+		RETURN(result);
 	}
 }
 
@@ -1759,11 +1733,20 @@ XBSYSAPI EXPORTNUM(140) xbox::ulong_xt NTAPI xbox::KeResumeThread
 {
 	LOG_FUNC_ONE_ARG(Thread);
 
-	NTSTATUS ret = X_STATUS_SUCCESS;
+	KIRQL OldIrql;
+	KiLockDispatcherDatabase(&OldIrql);
 
-	LOG_UNIMPLEMENTED();
+	char_xt OldCount = Thread->SuspendCount;
+	if (OldCount != 0) {
+		--Thread->SuspendCount;
+		if (Thread->SuspendCount == 0) {
+			++Thread->SuspendSemaphore.Header.SignalState;
+		}
+	}
 
-	RETURN(ret);
+	KiUnlockDispatcherDatabase(OldIrql);
+
+	RETURN(OldCount);
 }
 
 XBSYSAPI EXPORTNUM(141) xbox::PLIST_ENTRY NTAPI xbox::KeRundownQueue
@@ -2105,11 +2088,27 @@ XBSYSAPI EXPORTNUM(152) xbox::ulong_xt NTAPI xbox::KeSuspendThread
 {
 	LOG_FUNC_ONE_ARG(Thread);
 
-	NTSTATUS ret = X_STATUS_SUCCESS;
+	KIRQL OldIrql;
+	KiLockDispatcherDatabase(&OldIrql);
 
-	LOG_UNIMPLEMENTED();
+	char_xt OldCount = Thread->SuspendCount;
+	if (OldCount == X_MAXIMUM_SUSPEND_COUNT) {
+		KiUnlockDispatcherDatabase(OldIrql);
+		RETURN(X_STATUS_SUSPEND_COUNT_EXCEEDED);
+	}
 
-	RETURN(ret);
+	if (Thread->ApcState.ApcQueueable == TRUE) {
+		++Thread->SuspendCount;
+		if (OldCount == 0) {
+			if (KiInsertQueueApc(&Thread->SuspendApc, 0) == FALSE) {
+				--Thread->SuspendSemaphore.Header.SignalState;
+			}
+		}
+	}
+
+	KiUnlockDispatcherDatabase(OldIrql);
+
+	RETURN(OldCount);
 }
 
 // ******************************************************************

src/core/kernel/exports/EmuKrnlKi.cpp

@@ -907,12 +907,13 @@ static xbox::void_xt KiExecuteApc()
 		Apc->Inserted = FALSE;
 		xbox::KiApcListMtx.unlock();
 
-		// NOTE: we never use KernelRoutine because that is only used for kernel APCs, which we currently don't use
+		// This is either KiFreeUserApc, which frees the memory of the apc, or KiSuspendNop, which does nothing
+		(Apc->KernelRoutine)(Apc, &Apc->NormalRoutine, &Apc->NormalContext, &Apc->SystemArgument1, &Apc->SystemArgument2);
+
 		if (Apc->NormalRoutine != xbox::zeroptr) {
 			(Apc->NormalRoutine)(Apc->NormalContext, Apc->SystemArgument1, Apc->SystemArgument2);
 		}
 
-		xbox::ExFreePool(Apc);
 		xbox::KiApcListMtx.lock();
 	}
 
@@ -966,7 +967,8 @@ xbox::PLARGE_INTEGER FASTCALL xbox::KiComputeWaitInterval
 }
 
 // Source: ReactOS
-xbox::void_xt NTAPI xbox::KiSuspendNop(
+xbox::void_xt NTAPI xbox::KiSuspendNop
+(
 	IN PKAPC Apc,
 	IN PKNORMAL_ROUTINE* NormalRoutine,
 	IN PVOID* NormalContext,
@@ -974,7 +976,7 @@ xbox::void_xt NTAPI xbox::KiSuspendNop(
 	IN PVOID* SystemArgument2
 )
 {
-	/* Does nothing */
+	/* Does nothing because the memory of the suspend apc is part of kthread */
 	UNREFERENCED_PARAMETER(Apc);
 	UNREFERENCED_PARAMETER(NormalRoutine);
 	UNREFERENCED_PARAMETER(NormalContext);
@@ -982,8 +984,21 @@ xbox::void_xt NTAPI xbox::KiSuspendNop(
 	UNREFERENCED_PARAMETER(SystemArgument2);
 }
 
+xbox::void_xt NTAPI xbox::KiFreeUserApc
+(
+	IN PKAPC Apc,
+	IN PKNORMAL_ROUTINE *NormalRoutine,
+	IN PVOID *NormalContext,
+	IN PVOID *SystemArgument1,
+	IN PVOID *SystemArgument2
+)
+{
+	ExFreePool(Apc);
+}
+
 // Source: ReactOS
-xbox::void_xt NTAPI xbox::KiSuspendThread(
+xbox::void_xt NTAPI xbox::KiSuspendThread
+(
 	IN PVOID NormalContext,
 	IN PVOID SystemArgument1,
 	IN PVOID SystemArgument2
@@ -1076,3 +1091,40 @@ xbox::void_xt xbox::KiInitializeContextThread(
 	/* Save back the new value of the kernel stack. */
 	Thread->KernelStack = reinterpret_cast<PVOID>(CtxSwitchFrame);
 }
+
+xbox::boolean_xt xbox::KiInsertQueueApc
+(
+	IN PRKAPC Apc,
+	IN KPRIORITY Increment
+)
+{
+	PKTHREAD kThread = Apc->Thread;
+	KiApcListMtx.lock();
+	if (Apc->Inserted) {
+		KiApcListMtx.unlock();
+		return FALSE;
+	}
+	InsertTailList(&kThread->ApcState.ApcListHead[Apc->ApcMode], &Apc->ApcListEntry);
+	Apc->Inserted = TRUE;
+	KiApcListMtx.unlock();
+
+	// We can only attempt to execute the queued apc right away if it is been inserted in the current thread, because otherwise the KTHREAD
+	// in the fs selector will not be correct
+	if (Apc->ApcMode == KernelMode) { // kernel apc
+		kThread->ApcState.KernelApcPending = TRUE;
+		// NOTE: this is wrong, we should check the thread state instead of just signaling the kernel apc, but we currently
+		// don't set the appropriate state in kthread
+		if (kThread == KeGetCurrentThread()) {
+			KiExecuteKernelApc();
+		}
+	}
+	else if ((kThread->WaitMode == UserMode) && (kThread->Alertable)) { // user apc
+		kThread->ApcState.UserApcPending = TRUE;
+		// NOTE: this should also check the thread state
+		if (kThread == KeGetCurrentThread()) {
+			KiExecuteUserApc();
+		}
+	}
+
+	return TRUE;
+}

src/core/kernel/exports/EmuKrnlKi.h

@@ -156,7 +156,8 @@ namespace xbox
 	);
 
 	// Source: ReactOS
-	void_xt NTAPI KiSuspendNop(
+	void_xt NTAPI KiSuspendNop
+	(
 		IN PKAPC Apc,
 		IN PKNORMAL_ROUTINE* NormalRoutine,
 		IN PVOID* NormalContext,
@@ -164,6 +165,15 @@ namespace xbox
 		IN PVOID* SystemArgument2
 	);
 
+	void_xt NTAPI KiFreeUserApc
+	(
+		IN PKAPC Apc,
+		IN PKNORMAL_ROUTINE *NormalRoutine,
+		IN PVOID *NormalContext,
+		IN PVOID *SystemArgument1,
+		IN PVOID *SystemArgument2
+	);
+
 	// Source: ReactOS
 	void_xt NTAPI KiSuspendThread(
 		IN PVOID NormalContext,
@@ -180,6 +190,12 @@ namespace xbox
 		IN PKSTART_ROUTINE StartRoutine,
 		IN PVOID StartContext
 	);
+
+	boolean_xt KiInsertQueueApc
+	(
+		IN PRKAPC Apc,
+		IN KPRIORITY Increment
+	);
 };
 
 extern xbox::KPROCESS KiUniqueProcess;

src/core/kernel/exports/EmuKrnlNt.cpp

@@ -1039,7 +1039,7 @@ XBSYSAPI EXPORTNUM(206) xbox::ntstatus_xt NTAPI xbox::NtQueueApcThread
 
 	PKAPC Apc = static_cast<PKAPC>(ExAllocatePoolWithTag(sizeof(KAPC), 'pasP'));
 	if (Apc != zeroptr) {
-		KeInitializeApc(Apc, &Thread->Tcb, zeroptr, zeroptr, reinterpret_cast<PKNORMAL_ROUTINE>(ApcRoutine), UserMode, ApcRoutineContext);
+		KeInitializeApc(Apc, &Thread->Tcb, KiFreeUserApc, zeroptr, reinterpret_cast<PKNORMAL_ROUTINE>(ApcRoutine), UserMode, ApcRoutineContext);
 		if (!KeInsertQueueApc(Apc, ApcStatusBlock, ApcReserved, 0)) {
 			ExFreePool(Apc);
 			result = X_STATUS_UNSUCCESSFUL;
@@ -1856,15 +1856,20 @@ XBSYSAPI EXPORTNUM(224) xbox::ntstatus_xt NTAPI xbox::NtResumeThread
 		LOG_FUNC_ARG_OUT(PreviousSuspendCount)
 		LOG_FUNC_END;
 
-	if (const auto &nativeHandle = GetNativeHandle(ThreadHandle)) {
-		// Thread handles are created by ob
-		RETURN(NtDll::NtResumeThread(*nativeHandle, (::PULONG)PreviousSuspendCount));
-	}
-	else {
-		RETURN(X_STATUS_INVALID_HANDLE);
+	PETHREAD Thread;
+	ntstatus_xt result = ObReferenceObjectByHandle(ThreadHandle, &PsThreadObjectType, reinterpret_cast<PVOID *>(&Thread));
+	if (!X_NT_SUCCESS(result)) {
+		RETURN(result);
 	}
 
-	// TODO : Once we do our own thread-switching, implement NtResumeThread using KetResumeThread
+	ulong_xt PrevSuspendCount = KeResumeThread(&Thread->Tcb);
+	ObfDereferenceObject(Thread);
+
+	if (PreviousSuspendCount) {
+		*PreviousSuspendCount = PrevSuspendCount;
+	}
+
+	RETURN(X_STATUS_SUCCESS);
 }
 
 // ******************************************************************
@@ -2079,15 +2084,32 @@ XBSYSAPI EXPORTNUM(231) xbox::ntstatus_xt NTAPI xbox::NtSuspendThread
 		LOG_FUNC_ARG_OUT(PreviousSuspendCount)
 		LOG_FUNC_END;
 
-	if (const auto &nativeHandle = GetNativeHandle(ThreadHandle)) {
-		// Thread handles are created by ob
-		RETURN(NtDll::NtSuspendThread(*nativeHandle, (::PULONG)PreviousSuspendCount));
-	}
-	else {
-		RETURN(X_STATUS_INVALID_HANDLE);
+	PETHREAD Thread;
+	ntstatus_xt result = ObReferenceObjectByHandle(ThreadHandle, &PsThreadObjectType, reinterpret_cast<PVOID *>(&Thread));
+	if (!X_NT_SUCCESS(result)) {
+		RETURN(result);
 	}
 
-	// TODO : Once we do our own thread-switching, implement NtSuspendThread using KeSuspendThread
+	if (Thread != PspGetCurrentThread()) {
+		if (Thread->Tcb.HasTerminated) {
+			ObfDereferenceObject(Thread);
+			RETURN(X_STATUS_THREAD_IS_TERMINATING);
+		}
+	}
+
+	ulong_xt PrevSuspendCount = KeSuspendThread(&Thread->Tcb);
+	if (PrevSuspendCount == X_STATUS_SUSPEND_COUNT_EXCEEDED) {
+		ObfDereferenceObject(Thread);
+		RETURN(X_STATUS_SUSPEND_COUNT_EXCEEDED);
+	}
+
+	ObfDereferenceObject(Thread);
+
+	if (PreviousSuspendCount) {
+		*PreviousSuspendCount = PrevSuspendCount;
+	}
+
+	RETURN(X_STATUS_SUCCESS);
 }
 
 // ******************************************************************

src/core/kernel/exports/EmuKrnlPs.cpp

@@ -121,6 +121,8 @@ static unsigned int WINAPI PCSTProxy
 		params.Ethread,
 		params.TlsDataSize);
 
+	xbox::KiExecuteKernelApc();
+
 	auto routine = (xbox::PKSYSTEM_ROUTINE)StartFrame->SystemRoutine;
 	// Debugging notice : When the below line shows up with an Exception dialog and a
 	// message like: "Exception thrown at 0x00026190 in cxbx.exe: 0xC0000005: Access
@@ -409,10 +411,12 @@ XBSYSAPI EXPORTNUM(255) xbox::ntstatus_xt NTAPI xbox::PsCreateSystemThreadEx
 		g_AffinityPolicy->SetAffinityXbox(handle);
 
 		// Now that ThreadId is populated and affinity is changed, resume the thread (unless the guest passed CREATE_SUSPENDED)
-		if (!CreateSuspended) {
-			ResumeThread(handle);
+		if (CreateSuspended) {
+			KeSuspendThread(&eThread->Tcb);
 		}
 
+		ResumeThread(handle);
+
 		// Log ThreadID identical to how GetCurrentThreadID() is rendered :
 		EmuLog(LOG_LEVEL::DEBUG, "Created Xbox proxy thread. Handle : 0x%X, ThreadId : [0x%.4X], Native Handle : 0x%X, Native ThreadId : [0x%.4X]",
 			*ThreadHandle, eThread->UniqueThread, handle, ThreadId);