Render target deduplication

not sure if this is working

src/Ryujinx.Graphics.Metal/EncoderState.cs

@@ -98,6 +98,9 @@ namespace Ryujinx.Graphics.Metal
         // Changes to attachments take recreation!
         public Texture DepthStencil = default;
         public Texture[] RenderTargets = new Texture[Constants.MaxColorAttachments];
+        public ITexture PreMaskDepthStencil = default;
+        public ITexture[] PreMaskRenderTargets;
+        public bool FramebufferUsingColorWriteMask;
 
         public MTLColorWriteMask[] RenderTargetMasks = Enumerable.Repeat(MTLColorWriteMask.All, Constants.MaxColorAttachments).ToArray();
         public BlendDescriptor?[] BlendDescriptors = new BlendDescriptor?[Constants.MaxColorAttachments];

src/Ryujinx.Graphics.Metal/EncoderStateManager.cs

@@ -429,6 +429,88 @@ namespace Ryujinx.Graphics.Metal
 
         public void UpdateRenderTargets(ITexture[] colors, ITexture depthStencil)
         {
+            _currentState.FramebufferUsingColorWriteMask = false;
+            UpdateRenderTargetsInternal(colors, depthStencil);
+        }
+
+        public void UpdateRenderTargetColorMasks(ReadOnlySpan<uint> componentMask)
+        {
+            _currentState.RenderTargetMasks = new MTLColorWriteMask[Constants.MaxColorAttachments];
+
+            for (int i = 0; i < componentMask.Length; i++)
+            {
+                bool red = (componentMask[i] & (0x1 << 0)) != 0;
+                bool green = (componentMask[i] & (0x1 << 1)) != 0;
+                bool blue = (componentMask[i] & (0x1 << 2)) != 0;
+                bool alpha = (componentMask[i] & (0x1 << 3)) != 0;
+
+                var mask = MTLColorWriteMask.None;
+
+                mask |= red ? MTLColorWriteMask.Red : 0;
+                mask |= green ? MTLColorWriteMask.Green : 0;
+                mask |= blue ? MTLColorWriteMask.Blue : 0;
+                mask |= alpha ? MTLColorWriteMask.Alpha : 0;
+
+                _currentState.RenderTargetMasks[i] = mask;
+            }
+
+            if (_currentState.FramebufferUsingColorWriteMask)
+            {
+                UpdateRenderTargetsInternal(_currentState.PreMaskRenderTargets, _currentState.PreMaskDepthStencil);
+            }
+            else
+            {
+                // Requires recreating pipeline
+                if (_pipeline.CurrentEncoderType == EncoderType.Render)
+                {
+                    _pipeline.EndCurrentPass();
+                }
+            }
+        }
+
+        private void UpdateRenderTargetsInternal(ITexture[] colors, ITexture depthStencil)
+        {
+            // TBDR GPUs don't work properly if the same attachment is bound to multiple targets,
+            // due to each attachment being a copy of the real attachment, rather than a direct write.
+            //
+            // Just try to remove duplicate attachments.
+            // Save a copy of the array to rebind when mask changes.
+
+            // Look for textures that are masked out.
+
+            for (int i = 0; i < colors.Length; i++)
+            {
+                if (colors[i] == null)
+                {
+                    continue;
+                }
+
+                ref var mtlMask = ref _currentState.RenderTargetMasks[i];
+
+                for (int j = 0; j < i; j++)
+                {
+                    // Check each binding for a duplicate binding before it.
+
+                    if (colors[i] == colors[j])
+                    {
+                        // Prefer the binding with no write mask.
+
+                        ref var mtlMask2 = ref _currentState.RenderTargetMasks[j];
+
+                        if (mtlMask == 0)
+                        {
+                            colors[i] = null;
+                            MaskOut(colors, depthStencil);
+                        }
+                        else if (mtlMask2 == 0)
+                        {
+                            colors[j] = null;
+                            MaskOut(colors, depthStencil);
+                        }
+                    }
+                }
+            }
+
             _currentState.RenderTargets = new Texture[Constants.MaxColorAttachments];
 
             for (int i = 0; i < colors.Length; i++)
@@ -457,32 +539,16 @@ namespace Ryujinx.Graphics.Metal
             }
         }
 
-        public void UpdateRenderTargetColorMasks(ReadOnlySpan<uint> componentMask)
+        private void MaskOut(ITexture[] colors, ITexture depthStencil)
         {
-            _currentState.RenderTargetMasks = new MTLColorWriteMask[Constants.MaxColorAttachments];
-
-            for (int i = 0; i < componentMask.Length; i++)
+            if (!_currentState.FramebufferUsingColorWriteMask)
             {
-                bool red = (componentMask[i] & (0x1 << 0)) != 0;
-                bool green = (componentMask[i] & (0x1 << 1)) != 0;
-                bool blue = (componentMask[i] & (0x1 << 2)) != 0;
-                bool alpha = (componentMask[i] & (0x1 << 3)) != 0;
-
-                var mask = MTLColorWriteMask.None;
-
-                mask |= red ? MTLColorWriteMask.Red : 0;
-                mask |= green ? MTLColorWriteMask.Green : 0;
-                mask |= blue ? MTLColorWriteMask.Blue : 0;
-                mask |= alpha ? MTLColorWriteMask.Alpha : 0;
-
-                _currentState.RenderTargetMasks[i] = mask;
+                _currentState.PreMaskRenderTargets = colors;
+                _currentState.PreMaskDepthStencil = depthStencil;
             }
 
-            // Requires recreating pipeline
-            if (_pipeline.CurrentEncoderType == EncoderType.Render)
-            {
-                _pipeline.EndCurrentPass();
-            }
+            // If true, then the framebuffer must be recreated when the mask changes.
+            _currentState.FramebufferUsingColorWriteMask = true;
         }
 
         public void UpdateVertexAttribs(ReadOnlySpan<VertexAttribDescriptor> vertexAttribs)