From 0523deaa938dd267a17bcc5102a49dd58ebffe79 Mon Sep 17 00:00:00 2001
From: Matt Borgerson <contact@mborgerson.com>
Date: Thu, 28 Jun 2018 00:27:09 -0700
Subject: [PATCH] nv2a: Recycle FIFO command queue memory

This patch adds an additional "retired" queue in which FIFO command
entry objects are placed after execution. This queue of objects is then
returned to the pusher's new "available" queue for re-use.

This improves the performance of the system by avoiding the costly
overhead associated with the general-purpose use of `malloc` and `free`
for previous allocation of FIFO command queue objects.
---
 capstone                  |  2 +-
 hw/xbox/nv2a/nv2a.c       | 18 ++++++++++++++++++
 hw/xbox/nv2a/nv2a.h       |  3 +++
 hw/xbox/nv2a/nv2a_pfifo.c | 32 ++++++++++++++++++++++++++++++--
 4 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/capstone b/capstone
index 22ead3e0bf..5f173b0562 160000
--- a/capstone
+++ b/capstone
@@ -1 +1 @@
-Subproject commit 22ead3e0bfdb87516656453336160e0a37b066bf
+Subproject commit 5f173b0562ae2bf875cfdd71691776c2d9ed177f
diff --git a/hw/xbox/nv2a/nv2a.c b/hw/xbox/nv2a/nv2a.c
index 6feb587385..d98d40e92f 100644
--- a/hw/xbox/nv2a/nv2a.c
+++ b/hw/xbox/nv2a/nv2a.c
@@ -507,10 +507,21 @@ static void nv2a_realize(PCIDevice *dev, Error **errp)
     }
 
     /* init fifo cache1 */
+    qemu_spin_init(&d->pfifo.cache1.alloc_lock);
     qemu_mutex_init(&d->pfifo.cache1.cache_lock);
     qemu_cond_init(&d->pfifo.cache1.cache_cond);
     QSIMPLEQ_INIT(&d->pfifo.cache1.cache);
     QSIMPLEQ_INIT(&d->pfifo.cache1.working_cache);
+    QSIMPLEQ_INIT(&d->pfifo.cache1.available_entries);
+    QSIMPLEQ_INIT(&d->pfifo.cache1.retired_entries);
+
+    /* Pre-allocate memory for CacheEntry objects */
+    for (i=0; i < 100000; i++) {
+        CacheEntry *command = g_malloc0(sizeof(CacheEntry));
+        assert(command != NULL);
+        QSIMPLEQ_INSERT_TAIL(&d->pfifo.cache1.available_entries,
+                             command, entry);
+    }
 }
 
 static void nv2a_exitfn(PCIDevice *dev)
@@ -525,6 +536,13 @@ static void nv2a_exitfn(PCIDevice *dev)
     qemu_mutex_destroy(&d->pfifo.cache1.cache_lock);
     qemu_cond_destroy(&d->pfifo.cache1.cache_cond);
 
+    /* Release allocated CacheEntry objects */
+    while (!QSIMPLEQ_EMPTY(&d->pfifo.cache1.available_entries)) {
+        CacheEntry *entry = QSIMPLEQ_FIRST(&d->pfifo.cache1.available_entries);
+        QSIMPLEQ_REMOVE_HEAD(&d->pfifo.cache1.available_entries, entry);
+        free(entry);
+    }
+
     pgraph_destroy(&d->pgraph);
 }
 
diff --git a/hw/xbox/nv2a/nv2a.h b/hw/xbox/nv2a/nv2a.h
index e015020363..fc517e6ddd 100644
--- a/hw/xbox/nv2a/nv2a.h
+++ b/hw/xbox/nv2a/nv2a.h
@@ -348,10 +348,13 @@ typedef struct Cache1State {
     enum FIFOEngine last_engine;
 
     /* The actual command queue */
+    QemuSpin alloc_lock;
     QemuMutex cache_lock;
     QemuCond cache_cond;
     QSIMPLEQ_HEAD(, CacheEntry) cache;
     QSIMPLEQ_HEAD(, CacheEntry) working_cache;
+    QSIMPLEQ_HEAD(, CacheEntry) available_entries;
+    QSIMPLEQ_HEAD(, CacheEntry) retired_entries;
 } Cache1State;
 
 typedef struct ChannelControl {
diff --git a/hw/xbox/nv2a/nv2a_pfifo.c b/hw/xbox/nv2a/nv2a_pfifo.c
index 748f29bf7a..80c9de6107 100644
--- a/hw/xbox/nv2a/nv2a_pfifo.c
+++ b/hw/xbox/nv2a/nv2a_pfifo.c
@@ -234,6 +234,24 @@ void pfifo_write(void *opaque, hwaddr addr, uint64_t val, unsigned int size)
     }
 }
 
+static CacheEntry *alloc_entry(Cache1State *state)
+{
+    CacheEntry *entry;
+
+    qemu_spin_lock(&state->alloc_lock);
+    if (QSIMPLEQ_EMPTY(&state->available_entries)) {
+        qemu_spin_unlock(&state->alloc_lock);
+        entry = g_malloc0(sizeof(CacheEntry));
+        assert(entry != NULL);
+    } else {
+        entry = QSIMPLEQ_FIRST(&state->available_entries);
+        QSIMPLEQ_REMOVE_HEAD(&state->available_entries, entry);
+        qemu_spin_unlock(&state->alloc_lock);
+        memset(entry, 0, sizeof(CacheEntry));
+    }
+
+    return entry;
+}
 
 /* pusher should be fine to run from a mimo handler
  * whenever's it's convenient */
@@ -287,11 +305,12 @@ static void pfifo_run_pusher(NV2AState *d) {
             /* data word of methods command */
             state->data_shadow = word;
 
-            command = (CacheEntry*)g_malloc0(sizeof(CacheEntry));
+            command = alloc_entry(state);
             command->method = state->method;
             command->subchannel = state->subchannel;
             command->nonincreasing = state->method_nonincreasing;
             command->parameter = word;
+
             qemu_mutex_lock(&state->cache_lock);
             QSIMPLEQ_INSERT_TAIL(&state->cache, command, entry);
             qemu_cond_signal(&state->cache_cond);
@@ -381,6 +400,14 @@ void *pfifo_puller_thread(void *opaque)
 
     while (true) {
         qemu_mutex_lock(&state->cache_lock);
+
+        /* Return any retired command entry objects back to the available
+         * queue for re-use by the pusher.
+         */
+        qemu_spin_lock(&state->alloc_lock);
+        QSIMPLEQ_CONCAT(&state->available_entries, &state->retired_entries);
+        qemu_spin_unlock(&state->alloc_lock);
+
         while (QSIMPLEQ_EMPTY(&state->cache) || !state->pull_enabled) {
             qemu_cond_wait(&state->cache_cond, &state->cache_lock);
 
@@ -459,7 +486,8 @@ void *pfifo_puller_thread(void *opaque)
                 // qemu_mutex_unlock(&state->cache_lock);
             }
 
-            g_free(command);
+            /* Hang onto the command object to recycle its memory later */
+            QSIMPLEQ_INSERT_TAIL(&state->retired_entries, command, entry);
         }
 
         qemu_mutex_unlock(&d->pgraph.lock);