KThread based debug

This commit starts a big refactor to the original debugger PR by merryhime. The debugger now interfaces with KThreads instead of the ExecutionContext.

The corresponding KThread debug functions properly suspend/resume the thread and call the underlying debug related function in the ExecutionContext.

I also added debugging support to the AppleHV ExecutionContext.

src/ARMeilleure/Instructions/NativeInterface.cs

@@ -175,6 +175,11 @@ namespace ARMeilleure.Instructions
 
             ExecutionContext context = GetContext();
 
+            if (context.DebugStopped == 1)
+            {
+                return false;
+            }
+
             context.CheckInterrupt();
 
             Statistics.ResumeTimer();

src/ARMeilleure/State/ExecutionContext.cs

@@ -101,10 +101,8 @@ namespace ARMeilleure.State
         private readonly ExceptionCallback _supervisorCallback;
         private readonly ExceptionCallback _undefinedCallback;
 
-        internal int _debugState = (int)DebugState.Running;
+        internal int ShouldStep;
-        internal int _shouldStep = 0;
+        internal int DebugStopped;
-        internal ManualResetEvent _debugHalt = new ManualResetEvent(true);
-        internal Barrier _stepBarrier = new Barrier(2);
 
         // This is only valid while debugging is enabled.
         public ulong DebugPc;
@@ -160,34 +158,26 @@ namespace ARMeilleure.State
 
         public void DebugStop()
         {
-            _debugHalt.Reset();
+            if (Interlocked.CompareExchange(ref DebugStopped, 1, 0) == 0)
-            Interlocked.CompareExchange(ref _debugState, (int)DebugState.Stopping, (int)DebugState.Running);
+            {
+                RequestInterrupt();
+            }
         }
 
         public bool DebugStep()
         {
-            if (_debugState != (int)DebugState.Stopped)
+            if (DebugStopped != 1)
             {
                 return false;
             }
 
-            _shouldStep = 1;
+            ShouldStep = 1;
-            _debugHalt.Set();
-            _stepBarrier.SignalAndWait();
-            _debugHalt.Reset();
-            _stepBarrier.SignalAndWait();
-
             return true;
         }
 
         public void DebugContinue()
         {
-            _debugHalt.Set();
+            Interlocked.CompareExchange(ref DebugStopped, 0, 1);
-        }
-
-        public DebugState GetDebugState()
-        {
-            return (DebugState)_debugState;
         }
 
         internal void OnBreak(ulong address, int imm)

src/ARMeilleure/Translation/Translator.cs

@@ -147,21 +147,14 @@ namespace ARMeilleure.Translation
                 {
                     context.DebugPc = ExecuteSingle(context, context.DebugPc);
 
-                    while (context._debugState != (int)DebugState.Running)
+                    while (context.DebugStopped == 1)
                     {
-                        Interlocked.CompareExchange(ref context._debugState, (int)DebugState.Stopped, (int)DebugState.Stopping);
+                        if (Interlocked.CompareExchange(ref context.ShouldStep, 0, 1) == 1)
-                        context._debugHalt.WaitOne();
-                        if (Interlocked.CompareExchange(ref context._shouldStep, 0, 1) == 1)
                         {
                             context.DebugPc = Step(context, context.DebugPc);
-
+                            context.RequestInterrupt();
-                            context._stepBarrier.SignalAndWait();
-                            context._stepBarrier.SignalAndWait();
-                        }
-                        else
-                        {
-                            Interlocked.CompareExchange(ref context._debugState, (int)DebugState.Running, (int)DebugState.Stopped);
                         }
+                        context.CheckInterrupt();
                     }
                 }
                 while (context.Running && context.DebugPc != 0);
@@ -222,7 +215,7 @@ namespace ARMeilleure.Translation
             return nextAddr;
         }
 
-        public ulong Step(State.ExecutionContext context, ulong address)
+        private ulong Step(State.ExecutionContext context, ulong address)
         {
             TranslatedFunction func = Translate(address, context.ExecutionMode, highCq: false, singleStep: true);
 

src/Ryujinx.Cpu/AppleHv/HvExecutionContext.cs

@@ -70,6 +70,8 @@ namespace Ryujinx.Cpu.AppleHv
         private readonly ICounter _counter;
         private readonly IHvExecutionContext _shadowContext;
         private IHvExecutionContext _impl;
+        private int _shouldStep;
+        private int _debugStopped;
 
         private readonly ExceptionCallbacks _exceptionCallbacks;
 
@@ -131,22 +133,37 @@ namespace Ryujinx.Cpu.AppleHv
         }
 
         /// <inheritdoc/>
-        public void DebugStop() => _impl.DebugStop();
+        public void DebugStop()
+        {
+            if (Interlocked.CompareExchange(ref _debugStopped, 1, 0) == 0)
+            {
+                RequestInterrupt();
+            }
+        }
 
         /// <inheritdoc/>
-        public bool DebugStep() => _impl.DebugStep();
+        public bool DebugStep()
+        {
+            if (_debugStopped != 1)
+            {
+                return false;
+            }
+            
+            _shouldStep = 1;
+            return true;
+        }
 
         /// <inheritdoc/>
-        public void DebugContinue() => _impl.DebugContinue();
+        public void DebugContinue()
-
+        {
-        /// <inheritdoc/>
+            Interlocked.CompareExchange(ref _debugStopped, 0, 1);
-        public DebugState GetDebugState() => _impl.GetDebugState();
+        }
 
         /// <inheritdoc/>
         public ulong DebugPc
         {
-            get => _impl.DebugPc;
+            get => Pc;
-            set => _impl.DebugPc = value;
+            set => Pc = value;
         }
 
         /// <inheritdoc/>
@@ -164,6 +181,18 @@ namespace Ryujinx.Cpu.AppleHv
 
             while (Running)
             {
+                if (Interlocked.CompareExchange(ref _shouldStep, 0, 1) == 1)
+                {
+                    uint currentEl = Pstate & ~(0xfffffff0);
+                    if (currentEl == 0b0101)
+                    {
+                        HvApi.hv_vcpu_get_sys_reg(vcpu.Handle, HvSysReg.SPSR_EL1, out ulong spsr).ThrowOnError();
+                        spsr |= (1 << 21);
+                        HvApi.hv_vcpu_set_sys_reg(vcpu.Handle, HvSysReg.SPSR_EL1, spsr);
+                    }
+                    HvApi.hv_vcpu_set_sys_reg(vcpu.Handle, HvSysReg.MDSCR_EL1, 1);
+                }
+
                 HvApi.hv_vcpu_run(vcpu.Handle).ThrowOnError();
 
                 HvExitReason reason = vcpu.ExitInfo->Reason;
@@ -231,6 +260,21 @@ namespace Ryujinx.Cpu.AppleHv
                     SupervisorCallHandler(elr - 4UL, id);
                     vcpu = RentFromPool(memoryManager.AddressSpace, vcpu);
                     break;
+                case ExceptionClass.SoftwareStepLowerEl:
+                    HvApi.hv_vcpu_get_sys_reg(vcpuHandle, HvSysReg.SPSR_EL1, out ulong spsr).ThrowOnError();
+                    spsr &= ~((ulong)(1 << 21));
+                    HvApi.hv_vcpu_set_sys_reg(vcpuHandle, HvSysReg.SPSR_EL1,  spsr).ThrowOnError();
+                    HvApi.hv_vcpu_set_sys_reg(vcpuHandle, HvSysReg.MDSCR_EL1, 0);
+                    ReturnToPool(vcpu);
+                    InterruptHandler();
+                    vcpu = RentFromPool(memoryManager.AddressSpace, vcpu);
+                    break;
+                case ExceptionClass.BrkAarch64:
+                    ReturnToPool(vcpu);
+                    BreakHandler(elr, (ushort)esr);
+                    InterruptHandler();
+                    vcpu = RentFromPool(memoryManager.AddressSpace, vcpu);
+                    break;
                 default:
                     throw new Exception($"Unhandled guest exception {ec}.");
             }
@@ -241,10 +285,7 @@ namespace Ryujinx.Cpu.AppleHv
                 // TODO: Invalidate only the range that was modified?
                 return HvAddressSpace.KernelRegionTlbiEretAddress;
             }
-            else
+            return HvAddressSpace.KernelRegionEretAddress;
-            {
-                return HvAddressSpace.KernelRegionEretAddress;
-            }
         }
 
         private static void DataAbort(MemoryTracking tracking, ulong vcpu, uint esr)

src/Ryujinx.Cpu/AppleHv/HvExecutionContextShadow.cs

@@ -23,8 +23,6 @@ namespace Ryujinx.Cpu.AppleHv
         private readonly ulong[] _x;
         private readonly V128[] _v;
 
-        public ulong DebugPc { get; set; }
-
         public HvExecutionContextShadow()
         {
             _x = new ulong[32];
@@ -55,24 +53,6 @@ namespace Ryujinx.Cpu.AppleHv
         {
         }
 
-        public void DebugStop()
-        {
-        }
-
-        public bool DebugStep()
-        {
-            return false;
-        }
-
-        public void DebugContinue()
-        {
-        }
-
-        public DebugState GetDebugState()
-        {
-            return DebugState.Stopped;
-        }
-
         public bool GetAndClearInterruptRequested()
         {
             return false;

src/Ryujinx.Cpu/AppleHv/HvExecutionContextVcpu.cs

@@ -13,14 +13,6 @@ namespace Ryujinx.Cpu.AppleHv
         private static readonly SetSimdFpReg _setSimdFpReg;
         private static readonly IntPtr _setSimdFpRegNativePtr;
 
-        internal int _debugState = (int)DebugState.Running;
-        internal int _shouldStep = 0;
-        internal ManualResetEvent _debugHalt = new ManualResetEvent(true);
-        internal Barrier _stepBarrier = new Barrier(2);
-
-        // This is only valid while debugging is enabled.
-        public ulong DebugPc { get; set; }
-
         public ulong ThreadUid { get; set; }
 
         static HvExecutionContextVcpu()
@@ -198,41 +190,6 @@ namespace Ryujinx.Cpu.AppleHv
             }
         }
 
-        public void DebugStop()
-        {
-            _debugHalt.Reset();
-            Interlocked.CompareExchange(ref _debugState, (int)DebugState.Stopping, (int)DebugState.Running);
-            ulong vcpu = _vcpu;
-            HvApi.hv_vcpus_exit(ref vcpu, 1);
-        }
-
-        public bool DebugStep()
-        {
-            if (_debugState != (int)DebugState.Stopped)
-            {
-                return false;
-            }
-
-            _shouldStep = 1;
-            _debugHalt.Set();
-            _stepBarrier.SignalAndWait();
-            _debugHalt.Reset();
-            _stepBarrier.SignalAndWait();
-
-            return true;
-        }
-
-        public void DebugContinue()
-        {
-            _debugHalt.Set();
-            HvApi.hv_vcpu_run(_vcpu);
-        }
-
-        public DebugState GetDebugState()
-        {
-            return (DebugState)_debugState;
-        }
-
         public bool GetAndClearInterruptRequested()
         {
             return Interlocked.Exchange(ref _interruptRequested, 0) != 0;

src/Ryujinx.Cpu/AppleHv/IHvExecutionContext.cs

@@ -42,13 +42,5 @@ namespace Ryujinx.Cpu.AppleHv
 
         void RequestInterrupt();
         bool GetAndClearInterruptRequested();
-
-        // TODO: comments
-        void DebugStop();
-        bool DebugStep();
-        void DebugContinue();
-        DebugState GetDebugState();
-
-        ulong DebugPc { get; set; }
     }
 }

src/Ryujinx.Cpu/IExecutionContext.cs

@@ -118,7 +118,6 @@ namespace Ryujinx.Cpu
         void DebugStop();
         bool DebugStep();
         void DebugContinue();
-        DebugState GetDebugState();
 
         ulong DebugPc { get; set; }
     }

src/Ryujinx.Cpu/Jit/JitExecutionContext.cs

@@ -125,9 +125,6 @@ namespace Ryujinx.Cpu.Jit
         /// <inheritdoc/>
         public void DebugContinue() => _impl.DebugContinue();
 
-        /// <inheritdoc/>
-        public DebugState GetDebugState() => _impl.GetDebugState();
-
         /// <inheritdoc/>
         public ulong DebugPc
         {

src/Ryujinx.HLE/Debugger/DebugState.cs

@@ -1,4 +1,4 @@
-namespace ARMeilleure.State
+namespace Ryujinx.HLE.Debugger
 {
     public enum DebugState
     {

src/Ryujinx.HLE/Debugger/Debugger.cs

@@ -2,6 +2,8 @@ using ARMeilleure.State;
 using Ryujinx.Common;
 using Ryujinx.Common.Logging;
 using Ryujinx.Memory;
+using Ryujinx.HLE.HOS.Kernel;
+using Ryujinx.HLE.HOS.Kernel.Threading;
 using System;
 using System.Collections.Concurrent;
 using System.Linq;
@@ -46,8 +48,8 @@ namespace Ryujinx.HLE.Debugger
 
         private void HaltApplication() => Device.System.DebugGetApplicationProcess().DebugStopAllThreads();
         private ulong[] GetThreadIds() => Device.System.DebugGetApplicationProcess().DebugGetThreadUids();
-        private Ryujinx.Cpu.IExecutionContext GetThread(ulong threadUid) => Device.System.DebugGetApplicationProcess().DebugGetThreadContext(threadUid);
+        private KThread GetThread(ulong threadUid) => Device.System.DebugGetApplicationProcess().DebugGetThread(threadUid);
-        private Ryujinx.Cpu.IExecutionContext[] GetThreads() => GetThreadIds().Select(x => GetThread(x)).ToArray();
+        private KThread[] GetThreads() => GetThreadIds().Select(x => GetThread(x)).ToArray();
         private IVirtualMemoryManager GetMemory() => Device.System.DebugGetApplicationProcess().CpuMemory;
         private void InvalidateCacheRegion(ulong address, ulong size) =>
             Device.System.DebugGetApplicationProcess().InvalidateCacheRegion(address, size);
@@ -304,8 +306,8 @@ namespace Ryujinx.HLE.Debugger
                             break;
                         }
 
-                        IExecutionContext ctx = GetThread(threadId.Value);
+                        KThread thread = GetThread(threadId.Value);
-                        if (ctx.GetDebugState() == DebugState.Stopped)
+                        if (thread.GetDebugState() == DebugState.Stopped)
                         {
                             Reply(ToHex("Stopped"));
                         }
@@ -387,7 +389,7 @@ namespace Ryujinx.HLE.Debugger
                     return;
                 }
 
-                GetThread(cThread.Value).DebugPc = newPc.Value;
+                GetThread(cThread.Value).Context.DebugPc = newPc.Value;
             }
 
             foreach (var thread in GetThreads())
@@ -410,7 +412,7 @@ namespace Ryujinx.HLE.Debugger
                 return;
             }
 
-            var ctx = GetThread(gThread.Value);
+            var ctx = GetThread(gThread.Value).Context;
             string registers = "";
             for (int i = 0; i < GdbRegisterCount; i++)
             {
@@ -428,7 +430,7 @@ namespace Ryujinx.HLE.Debugger
                 return;
             }
 
-            var ctx = GetThread(gThread.Value);
+            var ctx = GetThread(gThread.Value).Context;
             for (int i = 0; i < GdbRegisterCount; i++)
             {
                 if (!GdbWriteRegister(ctx, i, ss))
@@ -513,7 +515,7 @@ namespace Ryujinx.HLE.Debugger
                 return;
             }
 
-            var ctx = GetThread(gThread.Value);
+            var ctx = GetThread(gThread.Value).Context;
             string result = GdbReadRegister(ctx, gdbRegId);
             if (result != null)
             {
@@ -533,7 +535,7 @@ namespace Ryujinx.HLE.Debugger
                 return;
             }
 
-            var ctx = GetThread(gThread.Value);
+            var ctx = GetThread(gThread.Value).Context;
             if (GdbWriteRegister(ctx, gdbRegId, ss) && ss.IsEmpty())
             {
                 ReplyOK();
@@ -552,15 +554,16 @@ namespace Ryujinx.HLE.Debugger
                 return;
             }
 
-            var ctx = GetThread(cThread.Value);
+            var thread = GetThread(cThread.Value);
 
             if (newPc.HasValue)
             {
-                ctx.DebugPc = newPc.Value;
+                thread.Context.DebugPc = newPc.Value;
             }
 
-            ctx.DebugStep();
+            thread.DebugStep();
-            Reply($"T05thread:{ctx.ThreadUid:x};");
+
+            Reply($"T05thread:{thread.ThreadUid:x};");
         }
 
         private void CommandIsAlive(ulong? threadId)
@@ -721,7 +724,9 @@ namespace Ryujinx.HLE.Debugger
 
         public void ThreadBreak(IExecutionContext ctx, ulong address, int imm)
         {
-            ctx.DebugStop();
+            KThread thread = GetThread(ctx.ThreadUid);
+
+            thread.DebugStop();
 
             Logger.Notice.Print(LogClass.GdbStub, $"Break hit on thread {ctx.ThreadUid} at pc {address:x016}");
 

src/Ryujinx.HLE/Debugger/IDebuggableProcess.cs

@@ -1,12 +1,13 @@
 using Ryujinx.Memory;
+using Ryujinx.HLE.HOS.Kernel.Threading;
 
 namespace Ryujinx.HLE.Debugger
 {
-    public interface IDebuggableProcess
+    internal interface IDebuggableProcess
     {
         void DebugStopAllThreads();
         ulong[] DebugGetThreadUids();
-        Ryujinx.Cpu.IExecutionContext DebugGetThreadContext(ulong threadUid);
+        public KThread DebugGetThread(ulong threadUid);
         IVirtualMemoryManager CpuMemory { get; }
         void InvalidateCacheRegion(ulong address, ulong size);
     }

src/Ryujinx.HLE/HOS/Horizon.cs

@@ -475,7 +475,7 @@ namespace Ryujinx.HLE.HOS
             IsPaused = pause;
         }
 
-        public IDebuggableProcess DebugGetApplicationProcess()
+        internal IDebuggableProcess DebugGetApplicationProcess()
         {
             lock (KernelContext.Processes)
             {

src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs

@@ -12,6 +12,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
+using ExceptionCallback = Ryujinx.Cpu.ExceptionCallback;
 
 namespace Ryujinx.HLE.HOS.Kernel.Process
 {
@@ -748,6 +749,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
         {
             KThread currentThread = KernelStatic.GetCurrentThread();
 
+            if (currentThread.GetDebugState() != DebugState.Running)
+            {
+                KernelContext.CriticalSection.Enter();
+                currentThread.Suspend(ThreadSchedState.ThreadPauseFlag);
+                currentThread.DebugHalt.Set();
+                KernelContext.CriticalSection.Leave();
+            }
+
             if (currentThread.Context.Running &&
                 currentThread.Owner != null &&
                 currentThread.GetUserDisableCount() != 0 &&
@@ -1200,7 +1209,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
                 {
                     foreach (KThread thread in _parent._threads)
                     {
-                        thread.Context.DebugStop();
+                        thread.DebugStop();
                     }
                 }
             }
@@ -1213,11 +1222,11 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
                 }
             }
 
-            public Ryujinx.Cpu.IExecutionContext DebugGetThreadContext(ulong threadUid)
+            public KThread DebugGetThread(ulong threadUid)
             {
                 lock (_parent._threadingLock)
                 {
-                    return _parent._threads.FirstOrDefault(x => x.ThreadUid == threadUid)?.Context;
+                    return _parent._threads.FirstOrDefault(x => x.ThreadUid == threadUid);
                 }
             }
 

src/Ryujinx.HLE/HOS/Kernel/Process/ProcessExecutionContext.cs

@@ -48,11 +48,6 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
         {
         }
 
-        public DebugState GetDebugState()
-        {
-            return DebugState.Stopped;
-        }
-
         public void StopRunning()
         {
             Running = false;

src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs

@@ -296,6 +296,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
             currentThread.SchedulerWaitEvent.Reset();
             currentThread.ThreadContext.Unlock();
+            currentThread.DebugHalt.Set();
 
             // Wake all the threads that might be waiting until this thread context is unlocked.
             for (int core = 0; core < CpuCoresCount; core++)

src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs

@@ -1,5 +1,6 @@
 using Ryujinx.Common.Logging;
 using Ryujinx.Cpu;
+using Ryujinx.HLE.Debugger;
 using Ryujinx.HLE.HOS.Kernel.Common;
 using Ryujinx.HLE.HOS.Kernel.Process;
 using Ryujinx.HLE.HOS.Kernel.SupervisorCall;
@@ -114,6 +115,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
         private readonly object _activityOperationLock = new();
 
+        private int _debugState = (int)DebugState.Running;
+        internal readonly ManualResetEvent DebugHalt = new(false);
+
         public KThread(KernelContext context) : base(context)
         {
             WaitSyncObjects = new KSynchronizationObject[MaxWaitSyncObjects];
@@ -1432,5 +1436,49 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
         {
             Owner.CpuMemory.Write<ushort>(_tlsAddress + TlsUserInterruptFlagOffset, 0);
         }
+        
+        public bool DebugStep()
+        {
+            if (_debugState != (int)DebugState.Stopped || !Context.DebugStep())
+            {
+                return false;
+            }
+
+            DebugHalt.Reset();
+            SetActivity(false);
+            DebugHalt.WaitOne();
+            return true;
+        }
+
+        public void DebugStop()
+        {
+            if (Interlocked.CompareExchange(ref _debugState, (int)DebugState.Stopping,
+                    (int)DebugState.Running) != (int)DebugState.Running)
+            {
+                return;
+            }
+
+            Context.DebugStop();
+            DebugHalt.WaitOne();
+            DebugHalt.Reset();
+            _debugState = (int)DebugState.Stopped;
+        }
+
+        public void DebugContinue()
+        {
+            if (Interlocked.CompareExchange(ref _debugState, (int)DebugState.Running,
+                    (int)DebugState.Stopped) != (int)DebugState.Stopped)
+            {
+                return;
+            }
+
+            Context.DebugContinue();
+            SetActivity(false);
+        }
+
+        public DebugState GetDebugState()
+        {
+            return (DebugState)_debugState;
+        }
     }
 }