nv2a: Recycle FIFO command queue memory

This patch adds an additional "retired" queue in which FIFO command
entry objects are placed after execution. This queue of objects is then
returned to the pusher's new "available" queue for re-use.

This improves the performance of the system by avoiding the costly
overhead associated with the general-purpose use of `malloc` and `free`
for previous allocation of FIFO command queue objects.

capstone

@@ -1 +1 @@
-Subproject commit 22ead3e0bfdb87516656453336160e0a37b066bf
+Subproject commit 5f173b0562ae2bf875cfdd71691776c2d9ed177f

hw/xbox/nv2a/nv2a.c

@@ -507,10 +507,21 @@ static void nv2a_realize(PCIDevice *dev, Error **errp)
     }
 
     /* init fifo cache1 */
+    qemu_spin_init(&d->pfifo.cache1.alloc_lock);
     qemu_mutex_init(&d->pfifo.cache1.cache_lock);
     qemu_cond_init(&d->pfifo.cache1.cache_cond);
     QSIMPLEQ_INIT(&d->pfifo.cache1.cache);
     QSIMPLEQ_INIT(&d->pfifo.cache1.working_cache);
+    QSIMPLEQ_INIT(&d->pfifo.cache1.available_entries);
+    QSIMPLEQ_INIT(&d->pfifo.cache1.retired_entries);
+
+    /* Pre-allocate memory for CacheEntry objects */
+    for (i=0; i < 100000; i++) {
+        CacheEntry *command = g_malloc0(sizeof(CacheEntry));
+        assert(command != NULL);
+        QSIMPLEQ_INSERT_TAIL(&d->pfifo.cache1.available_entries,
+                             command, entry);
+    }
 }
 
 static void nv2a_exitfn(PCIDevice *dev)
@@ -525,6 +536,13 @@ static void nv2a_exitfn(PCIDevice *dev)
     qemu_mutex_destroy(&d->pfifo.cache1.cache_lock);
     qemu_cond_destroy(&d->pfifo.cache1.cache_cond);
 
+    /* Release allocated CacheEntry objects */
+    while (!QSIMPLEQ_EMPTY(&d->pfifo.cache1.available_entries)) {
+        CacheEntry *entry = QSIMPLEQ_FIRST(&d->pfifo.cache1.available_entries);
+        QSIMPLEQ_REMOVE_HEAD(&d->pfifo.cache1.available_entries, entry);
+        free(entry);
+    }
+
     pgraph_destroy(&d->pgraph);
 }
 

hw/xbox/nv2a/nv2a.h

@@ -348,10 +348,13 @@ typedef struct Cache1State {
     enum FIFOEngine last_engine;
 
     /* The actual command queue */
+    QemuSpin alloc_lock;
     QemuMutex cache_lock;
     QemuCond cache_cond;
     QSIMPLEQ_HEAD(, CacheEntry) cache;
     QSIMPLEQ_HEAD(, CacheEntry) working_cache;
+    QSIMPLEQ_HEAD(, CacheEntry) available_entries;
+    QSIMPLEQ_HEAD(, CacheEntry) retired_entries;
 } Cache1State;
 
 typedef struct ChannelControl {

hw/xbox/nv2a/nv2a_pfifo.c

@@ -234,6 +234,24 @@ void pfifo_write(void *opaque, hwaddr addr, uint64_t val, unsigned int size)
     }
 }
 
+static CacheEntry *alloc_entry(Cache1State *state)
+{
+    CacheEntry *entry;
+
+    qemu_spin_lock(&state->alloc_lock);
+    if (QSIMPLEQ_EMPTY(&state->available_entries)) {
+        qemu_spin_unlock(&state->alloc_lock);
+        entry = g_malloc0(sizeof(CacheEntry));
+        assert(entry != NULL);
+    } else {
+        entry = QSIMPLEQ_FIRST(&state->available_entries);
+        QSIMPLEQ_REMOVE_HEAD(&state->available_entries, entry);
+        qemu_spin_unlock(&state->alloc_lock);
+        memset(entry, 0, sizeof(CacheEntry));
+    }
+
+    return entry;
+}
 
 /* pusher should be fine to run from a mimo handler
  * whenever's it's convenient */
@@ -287,11 +305,12 @@ static void pfifo_run_pusher(NV2AState *d) {
             /* data word of methods command */
             state->data_shadow = word;
 
-            command = (CacheEntry*)g_malloc0(sizeof(CacheEntry));
+            command = alloc_entry(state);
             command->method = state->method;
             command->subchannel = state->subchannel;
             command->nonincreasing = state->method_nonincreasing;
             command->parameter = word;
+
             qemu_mutex_lock(&state->cache_lock);
             QSIMPLEQ_INSERT_TAIL(&state->cache, command, entry);
             qemu_cond_signal(&state->cache_cond);
@@ -381,6 +400,14 @@ void *pfifo_puller_thread(void *opaque)
 
     while (true) {
         qemu_mutex_lock(&state->cache_lock);
+
+        /* Return any retired command entry objects back to the available
+         * queue for re-use by the pusher.
+         */
+        qemu_spin_lock(&state->alloc_lock);
+        QSIMPLEQ_CONCAT(&state->available_entries, &state->retired_entries);
+        qemu_spin_unlock(&state->alloc_lock);
+
         while (QSIMPLEQ_EMPTY(&state->cache) || !state->pull_enabled) {
             qemu_cond_wait(&state->cache_cond, &state->cache_lock);
 
@@ -459,7 +486,8 @@ void *pfifo_puller_thread(void *opaque)
                 // qemu_mutex_unlock(&state->cache_lock);
             }
 
-            g_free(command);
+            /* Hang onto the command object to recycle its memory later */
+            QSIMPLEQ_INSERT_TAIL(&state->retired_entries, command, entry);
         }
 
         qemu_mutex_unlock(&d->pgraph.lock);