From f8d3e0c82ff0c77a70568ce90224d8415fcd5192 Mon Sep 17 00:00:00 2001
From: Triang3l <triang3l@yandex.ru>
Date: Sun, 21 Oct 2018 22:14:48 +0300
Subject: [PATCH] [D3D12] ROV: Small stencil-related cleanup

---
 src/xenia/gpu/d3d12/d3d12_command_processor.cc | 17 ++++++++++-------
 src/xenia/gpu/dxbc_shader_translator.cc        |  4 ++--
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/xenia/gpu/d3d12/d3d12_command_processor.cc b/src/xenia/gpu/d3d12/d3d12_command_processor.cc
index e14f39cbf..5b7c3c031 100644
--- a/src/xenia/gpu/d3d12/d3d12_command_processor.cc
+++ b/src/xenia/gpu/d3d12/d3d12_command_processor.cc
@@ -1128,8 +1128,11 @@ bool D3D12CommandProcessor::IssueDraw(PrimitiveType primitive_type,
   }
 
   uint32_t color_mask = GetCurrentColorMask(pixel_shader);
-  if (!color_mask && !(regs[XE_GPU_REG_RB_DEPTHCONTROL].u32 & (0x1 | 0x4))) {
-    // Not writing to color, depth or doing stencil test, so doesn't draw.
+  uint32_t rb_depthcontrol = regs[XE_GPU_REG_RB_DEPTHCONTROL].u32;
+  uint32_t rb_stencilrefmask = regs[XE_GPU_REG_RB_STENCILREFMASK].u32;
+  if (!color_mask && ((rb_depthcontrol & (0x2 | 0x4)) != (0x2 | 0x4)) &&
+      (!(rb_depthcontrol & 0x1) || !(rb_stencilrefmask & (0xFF << 16)))) {
+    // Not writing to color, depth or stencil, so doesn't draw.
     return true;
   }
 
@@ -1601,6 +1604,7 @@ void D3D12CommandProcessor::UpdateSystemConstantValues(
   uint32_t vgt_indx_offset = regs[XE_GPU_REG_VGT_INDX_OFFSET].u32;
   uint32_t pa_cl_vte_cntl = regs[XE_GPU_REG_PA_CL_VTE_CNTL].u32;
   uint32_t rb_depthcontrol = regs[XE_GPU_REG_RB_DEPTHCONTROL].u32;
+  uint32_t rb_stencilrefmask = regs[XE_GPU_REG_RB_STENCILREFMASK].u32;
   uint32_t rb_depth_info = regs[XE_GPU_REG_RB_DEPTH_INFO].u32;
   uint32_t pa_cl_clip_cntl = regs[XE_GPU_REG_PA_CL_CLIP_CNTL].u32;
   uint32_t pa_su_vtx_cntl = regs[XE_GPU_REG_PA_SU_VTX_CNTL].u32;
@@ -1677,10 +1681,10 @@ void D3D12CommandProcessor::UpdateSystemConstantValues(
                  DxbcShaderTranslator::kSysFlag_DepthPassIfGreater;
       }
       if (rb_depthcontrol & 0x1) {
-        // Stencil test may modify the stencil buffer arbitrarily, so enable
-        // writing.
-        flags |= DxbcShaderTranslator::kSysFlag_StencilTest |
-                 DxbcShaderTranslator::kSysFlag_DepthStencilWrite;
+        flags |= DxbcShaderTranslator::kSysFlag_StencilTest;
+        if (rb_stencilrefmask & (0xFF << 16)) {
+          flags |= DxbcShaderTranslator::kSysFlag_DepthStencilWrite;
+        }
       }
     }
   }
@@ -1949,7 +1953,6 @@ void D3D12CommandProcessor::UpdateSystemConstantValues(
     system_constants_.edram_depth_base_dwords = depth_base_dwords;
 
     if (rb_depthcontrol & 0x1) {
-      uint32_t rb_stencilrefmask = regs[XE_GPU_REG_RB_STENCILREFMASK].u32;
       uint32_t stencil_value;
 
       stencil_value = rb_stencilrefmask & 0xFF;
diff --git a/src/xenia/gpu/dxbc_shader_translator.cc b/src/xenia/gpu/dxbc_shader_translator.cc
index bcc3a4328..8fb107456 100644
--- a/src/xenia/gpu/dxbc_shader_translator.cc
+++ b/src/xenia/gpu/dxbc_shader_translator.cc
@@ -4224,7 +4224,7 @@ void DxbcShaderTranslator::CompletePixelShader_WriteToROV() {
   // can mark the pixel to be discarded if the stencil test has failed - by
   // setting that whole depth/stencil test has failed. The original depth value
   // will be restored in this case, and after writing the new stencil, the pixel
-  // will be discared.
+  // will be discarded.
   shader_code_.push_back(ENCODE_D3D10_SB_OPCODE_TYPE(D3D10_SB_OPCODE_AND) |
                          ENCODE_D3D10_SB_TOKENIZED_INSTRUCTION_LENGTH(7));
   shader_code_.push_back(
@@ -4370,7 +4370,7 @@ void DxbcShaderTranslator::CompletePixelShader_WriteToROV() {
       EncodeVectorMaskedOperand(D3D10_SB_OPERAND_TYPE_TEMP, 0b0010, 1));
   shader_code_.push_back(depth_stencil_test_temp);
   shader_code_.push_back(
-      EncodeVectorReplicatedOperand(D3D10_SB_OPERAND_TYPE_TEMP, 2, 1));
+      EncodeVectorSelectOperand(D3D10_SB_OPERAND_TYPE_TEMP, 2, 1));
   shader_code_.push_back(depth_stencil_test_result_temp);
   shader_code_.push_back(
       EncodeScalarOperand(D3D10_SB_OPERAND_TYPE_IMMEDIATE32, 0));