From a13f3f48a289e71cc00e6f56714e3fa80ba98fae Mon Sep 17 00:00:00 2001
From: Erik Abair <erik.abair@gmail.com>
Date: Fri, 21 Jan 2022 14:26:27 -0800
Subject: [PATCH] nv2a: Improve RenderDoc triggering

---
 hw/xbox/nv2a/debug.c | 46 ++++++++++++++++++++++++++++++++++++++++++++
 hw/xbox/nv2a/debug.h |  8 ++++++++
 ui/xemu-hud.cc       | 22 +++++++++++++++++++++
 3 files changed, 76 insertions(+)

diff --git a/hw/xbox/nv2a/debug.c b/hw/xbox/nv2a/debug.c
index 2d24c8d222..b570cf3168 100644
--- a/hw/xbox/nv2a/debug.c
+++ b/hw/xbox/nv2a/debug.c
@@ -27,6 +27,14 @@
 #include <stdarg.h>
 #include <assert.h>
 
+#ifdef ENABLE_RENDERDOC
+#include <renderdoc_app.h>
+#include <dlfcn.h>
+
+static RENDERDOC_API_1_1_2 *rdoc_api = NULL;
+static int32_t renderdoc_capture_frames = 0;
+#endif
+
 static bool has_GL_GREMEDY_frame_terminator = false;
 static bool has_GL_KHR_debug = false;
 
@@ -52,6 +60,17 @@ void gl_debug_initialize(void)
        assert(glGetError() == GL_NO_ERROR);
 #endif
     }
+
+#ifdef ENABLE_RENDERDOC
+    void* renderdoc = dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
+    if (renderdoc) {
+        pRENDERDOC_GetAPI RENDERDOC_GetAPI = (pRENDERDOC_GetAPI)dlsym(
+            renderdoc, "RENDERDOC_GetAPI");
+        int ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_1_2,
+                                   (void **)&rdoc_api);
+        assert(ret == 1 && "Failed to retrieve RenderDoc API.");
+    }
+#endif
 }
 
 void gl_debug_message(bool cc, const char *fmt, ...)
@@ -121,10 +140,26 @@ void gl_debug_label(GLenum target, GLuint name, const char *fmt, ...)
     va_end(ap);
 
     glObjectLabel(target, name, n, buffer);
+
+    GLenum err = glGetError();
+    assert(err == GL_NO_ERROR);
 }
 
 void gl_debug_frame_terminator(void)
 {
+#ifdef ENABLE_RENDERDOC
+    if (rdoc_api) {
+        if (rdoc_api->IsTargetControlConnected()) {
+            if (rdoc_api->IsFrameCapturing()) {
+                rdoc_api->EndFrameCapture(NULL, NULL);
+            }
+            if (renderdoc_capture_frames) {
+                rdoc_api->StartFrameCapture(NULL, NULL);
+                --renderdoc_capture_frames;
+            }
+        }
+    }
+#endif
     if (!has_GL_GREMEDY_frame_terminator) {
         return;
     }
@@ -132,4 +167,15 @@ void gl_debug_frame_terminator(void)
     glFrameTerminatorGREMEDY();
 }
 
+#ifdef ENABLE_RENDERDOC
+
+bool nv2a_dbg_renderdoc_available(void) {
+  return rdoc_api != NULL;
+}
+
+void nv2a_dbg_renderdoc_capture_frames(uint32_t num_frames) {
+  renderdoc_capture_frames = num_frames;
+}
+#endif // ENABLE_RENDERDOC
+
 #endif
diff --git a/hw/xbox/nv2a/debug.h b/hw/xbox/nv2a/debug.h
index ed1491a246..73c8bd3f4f 100644
--- a/hw/xbox/nv2a/debug.h
+++ b/hw/xbox/nv2a/debug.h
@@ -40,6 +40,9 @@
 // #define DEBUG_NV2A_GL
 #ifdef DEBUG_NV2A_GL
 
+// Improve frame capture boundaries with RenderDoc.
+// #define ENABLE_RENDERDOC
+
 #include <stdbool.h>
 #include "gl/gloffscreen.h"
 
@@ -149,6 +152,11 @@ extern NV2AStats g_nv2a_stats;
 const char *nv2a_profile_get_counter_name(unsigned int cnt);
 int nv2a_profile_get_counter_value(unsigned int cnt);
 
+#ifdef ENABLE_RENDERDOC
+bool nv2a_dbg_renderdoc_available(void);
+void nv2a_dbg_renderdoc_capture_frames(uint32_t num_frames);
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/ui/xemu-hud.cc b/ui/xemu-hud.cc
index 0154eb28dd..6054a5a5ec 100644
--- a/ui/xemu-hud.cc
+++ b/ui/xemu-hud.cc
@@ -1898,6 +1898,10 @@ static AutoUpdateWindow update_window;
 #endif
 static std::deque<const char *> g_errors;
 
+#ifdef ENABLE_RENDERDOC
+static bool capture_renderdoc_frame = false;
+#endif
+
 class FirstBootWindow
 {
 public:
@@ -2058,6 +2062,12 @@ static void process_keyboard_shortcuts(void)
     if (is_key_pressed(SDL_SCANCODE_GRAVE)) {
         monitor_window.toggle_open();
     }
+
+#ifdef ENABLE_RENDERDOC
+    if (is_key_pressed(SDL_SCANCODE_F10)) {
+        nv2a_dbg_renderdoc_capture_frames(1);
+    }
+#endif
 }
 
 #if defined(__APPLE__)
@@ -2136,6 +2146,11 @@ static void ShowMainMenu()
             ImGui::MenuItem("Monitor", "~", &monitor_window.is_open);
             ImGui::MenuItem("Audio", NULL, &apu_window.is_open);
             ImGui::MenuItem("Video", NULL, &video_window.is_open);
+#ifdef ENABLE_RENDERDOC
+            if (nv2a_dbg_renderdoc_available()) {
+                ImGui::MenuItem("RenderDoc: Capture", NULL, &capture_renderdoc_frame);
+            }
+#endif
             ImGui::EndMenu();
         }
 
@@ -2413,6 +2428,13 @@ void xemu_hud_render(void)
     ImGui::NewFrame();
     process_keyboard_shortcuts();
 
+#ifdef ENABLE_RENDERDOC
+    if (capture_renderdoc_frame) {
+        nv2a_dbg_renderdoc_capture_frames(1);
+        capture_renderdoc_frame = false;
+    }
+#endif
+
     bool show_main_menu = true;
 
     if (first_boot_window.is_open) {