Return mapped buffer pointer directly for flush, WriteableRegion for textures (#2494)

* Return mapped buffer pointer directly for flush, WriteableRegion for textures

A few changes here to generally improve performance, even for platforms not using the persistent buffer flush.

- Texture and buffer flush now return a ReadOnlySpan<byte>. It's guaranteed that this span is pinned in memory, but it will be overwritten on the next flush from that thread, so it is expected that the data is used before calling again.
- As a result, persistent mappings no longer copy to a new array - rather the persistent map is returned directly as a Span<>. A similar host array is used for the glGet flushes instead of allocating new arrays each time.
- Texture flushes now do their layout conversion into a WriteableRegion when the texture is not MultiRange, which allows the flush to happen directly into guest memory rather than into a temporary span, then copied over. This avoids another copy when doing layout conversion.

Overall, this saves 1 data copy for buffer flush, 1 copy for linear textures with matching source/target stride, and 2 copies for block textures or linear textures with mismatching strides.

* Fix tests

* Fix array pointer for Mesa/Intel path

* Address some feedback

* Update method for getting array pointer.
This commit is contained in:
riperiperi 2021-07-19 23:10:54 +01:00 committed by GitHub
parent 10e17ab423
commit 4b60371e64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 143 additions and 66 deletions

View File

@ -237,7 +237,7 @@ namespace Ryujinx.Cpu
}
/// <inheritdoc/>
public unsafe WritableRegion GetWritableRegion(ulong va, int size)
public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{
if (size == 0)
{
@ -246,6 +246,11 @@ namespace Ryujinx.Cpu
if (IsContiguousAndMapped(va, size))
{
if (tracked)
{
SignalMemoryTracking(va, (ulong)size, true);
}
return new WritableRegion(null, va, new NativeMemoryManager<byte>((byte*)GetHostAddress(va), size).Memory);
}
else
@ -254,7 +259,7 @@ namespace Ryujinx.Cpu
GetSpan(va, size).CopyTo(memory.Span);
return new WritableRegion(this, va, memory);
return new WritableRegion(this, va, memory, tracked);
}
}

View File

@ -285,9 +285,16 @@ namespace Ryujinx.Cpu
}
/// <inheritdoc/>
public WritableRegion GetWritableRegion(ulong va, int size)
public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{
AssertMapped(va, (ulong)size);
if (tracked)
{
SignalMemoryTracking(va, (ulong)size, true);
}
else
{
AssertMapped(va, (ulong)size);
}
return _addressSpaceMirror.GetWritableRegion(va, size);
}

View File

@ -27,7 +27,7 @@ namespace Ryujinx.Graphics.GAL
void DeleteBuffer(BufferHandle buffer);
byte[] GetBufferData(BufferHandle buffer, int offset, int size);
ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size);
Capabilities GetCapabilities();

View File

@ -14,7 +14,7 @@ namespace Ryujinx.Graphics.GAL
ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel);
byte[] GetData();
ReadOnlySpan<byte> GetData();
void SetData(ReadOnlySpan<byte> data);
void SetData(ReadOnlySpan<byte> data, int layer, int level);

View File

@ -4,6 +4,7 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Memory;
using Ryujinx.Graphics.Texture;
using Ryujinx.Graphics.Texture.Astc;
using Ryujinx.Memory;
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
@ -821,14 +822,7 @@ namespace Ryujinx.Graphics.Gpu.Image
return; // Flushing this format is not supported, as it may have been converted to another host format.
}
if (tracked)
{
_physicalMemory.Write(Range, GetTextureDataFromGpu(tracked));
}
else
{
_physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(tracked));
}
FlushTextureDataToGuest(tracked);
}
/// <summary>
@ -864,10 +858,44 @@ namespace Ryujinx.Graphics.Gpu.Image
texture = _flushHostTexture = GetScaledHostTexture(1f, _flushHostTexture);
}
_physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(false, texture));
FlushTextureDataToGuest(false, texture);
});
}
/// <summary>
/// Gets data from the host GPU, and flushes it to guest memory.
/// </summary>
/// <remarks>
/// This method should be used to retrieve data that was modified by the host GPU.
/// This is not cheap, avoid doing that unless strictly needed.
/// When possible, the data is written directly into guest memory, rather than copied.
/// </remarks>
/// <param name="tracked">True if writing the texture data is tracked, false otherwise</param>
/// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
private void FlushTextureDataToGuest(bool tracked, ITexture texture = null)
{
if (Range.Count == 1)
{
MemoryRange subrange = Range.GetSubRange(0);
using (WritableRegion region = _physicalMemory.GetWritableRegion(subrange.Address, (int)subrange.Size, tracked))
{
GetTextureDataFromGpu(region.Memory.Span, tracked, texture);
}
}
else
{
if (tracked)
{
_physicalMemory.Write(Range, GetTextureDataFromGpu(Span<byte>.Empty, true, texture));
}
else
{
_physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(Span<byte>.Empty, false, texture));
}
}
}
/// <summary>
/// Gets data from the host GPU.
/// </summary>
@ -875,8 +903,11 @@ namespace Ryujinx.Graphics.Gpu.Image
/// This method should be used to retrieve data that was modified by the host GPU.
/// This is not cheap, avoid doing that unless strictly needed.
/// </remarks>
/// <returns>Host texture data</returns>
private ReadOnlySpan<byte> GetTextureDataFromGpu(bool blacklist, ITexture texture = null)
/// <param name="output">An output span to place the texture data into. If empty, one is generated</param>
/// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param>
/// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
/// <returns>The span containing the texture data</returns>
private ReadOnlySpan<byte> GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null)
{
ReadOnlySpan<byte> data;
@ -909,6 +940,7 @@ namespace Ryujinx.Graphics.Gpu.Image
if (Info.IsLinear)
{
data = LayoutConverter.ConvertLinearToLinearStrided(
output,
Info.Width,
Info.Height,
Info.FormatInfo.BlockWidth,
@ -920,6 +952,7 @@ namespace Ryujinx.Graphics.Gpu.Image
else
{
data = LayoutConverter.ConvertLinearToBlockLinear(
output,
Info.Width,
Info.Height,
_depth,

View File

@ -412,7 +412,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
int offset = (int)(address - Address);
byte[] data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
ReadOnlySpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
// TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers.
_physicalMemory.WriteUntracked(address, data);

View File

@ -128,10 +128,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes to be range</param>
/// <param name="tracked">True if write tracking is triggered on the span</param>
/// <returns>A writable region with the data at the specified memory location</returns>
public WritableRegion GetWritableRegion(ulong address, int size)
public WritableRegion GetWritableRegion(ulong address, int size, bool tracked = false)
{
return _cpuMemory.GetWritableRegion(address, size);
return _cpuMemory.GetWritableRegion(address, size, tracked);
}
/// <summary>

View File

@ -55,15 +55,22 @@ namespace Ryujinx.Graphics.OpenGL
(IntPtr)size);
}
public static byte[] GetData(BufferHandle buffer, int offset, int size)
public static unsafe ReadOnlySpan<byte> GetData(Renderer renderer, BufferHandle buffer, int offset, int size)
{
GL.BindBuffer(BufferTarget.CopyReadBuffer, buffer.ToInt32());
if (HwCapabilities.UsePersistentBufferForFlush)
{
return renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size);
}
else
{
IntPtr target = renderer.PersistentBuffers.Default.GetHostArray(size);
byte[] data = new byte[size];
GL.BindBuffer(BufferTarget.CopyReadBuffer, buffer.ToInt32());
GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, data);
GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, target);
return data;
return new ReadOnlySpan<byte>(target.ToPointer(), size);
}
}
public static void Resize(BufferHandle handle, int size)

View File

@ -38,9 +38,9 @@ namespace Ryujinx.Graphics.OpenGL.Image
throw new NotSupportedException();
}
public byte[] GetData()
public ReadOnlySpan<byte> GetData()
{
return Buffer.GetData(_buffer, _bufferOffset, _bufferSize);
return Buffer.GetData(_renderer, _buffer, _bufferOffset, _bufferSize);
}
public void SetData(ReadOnlySpan<byte> data)

View File

@ -119,7 +119,7 @@ namespace Ryujinx.Graphics.OpenGL.Image
_renderer.TextureCopy.Copy(this, (TextureView)destination, srcRegion, dstRegion, linearFilter);
}
public byte[] GetData()
public unsafe ReadOnlySpan<byte> GetData()
{
int size = 0;
@ -134,17 +134,11 @@ namespace Ryujinx.Graphics.OpenGL.Image
}
else
{
byte[] data = new byte[size];
IntPtr target = _renderer.PersistentBuffers.Default.GetHostArray(size);
unsafe
{
fixed (byte* ptr = data)
{
WriteTo((IntPtr)ptr);
}
}
WriteTo(target);
return data;
return new ReadOnlySpan<byte>(target.ToPointer(), size);
}
}

View File

@ -1,4 +1,5 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using OpenTK.Graphics.OpenGL;
using Ryujinx.Common.Logging;
@ -27,6 +28,9 @@ namespace Ryujinx.Graphics.OpenGL
private int _copyBufferHandle;
private int _copyBufferSize;
private byte[] _data;
private IntPtr _dataMap;
private void EnsureBuffer(int requiredSize)
{
if (_copyBufferSize < requiredSize && _copyBufferHandle != 0)
@ -48,6 +52,18 @@ namespace Ryujinx.Graphics.OpenGL
}
}
public unsafe IntPtr GetHostArray(int requiredSize)
{
if (_data == null || _data.Length < requiredSize)
{
_data = GC.AllocateUninitializedArray<byte>(requiredSize, true);
_dataMap = (IntPtr)Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(_data));
}
return _dataMap;
}
private void Sync()
{
GL.MemoryBarrier(MemoryBarrierFlags.ClientMappedBufferBarrierBit);
@ -63,7 +79,7 @@ namespace Ryujinx.Graphics.OpenGL
GL.DeleteSync(sync);
}
public byte[] GetTextureData(TextureView view, int size)
public unsafe ReadOnlySpan<byte> GetTextureData(TextureView view, int size)
{
EnsureBuffer(size);
@ -73,16 +89,12 @@ namespace Ryujinx.Graphics.OpenGL
GL.BindBuffer(BufferTarget.PixelPackBuffer, 0);
byte[] data = new byte[size];
Sync();
Marshal.Copy(_bufferMap, data, 0, size);
return data;
return new ReadOnlySpan<byte>(_bufferMap.ToPointer(), size);
}
public byte[] GetBufferData(BufferHandle buffer, int offset, int size)
public unsafe ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{
EnsureBuffer(size);
@ -93,13 +105,9 @@ namespace Ryujinx.Graphics.OpenGL
GL.BindBuffer(BufferTarget.CopyWriteBuffer, 0);
byte[] data = new byte[size];
Sync();
Marshal.Copy(_bufferMap, data, 0, size);
return data;
return new ReadOnlySpan<byte>(_bufferMap.ToPointer(), size);
}
public void Dispose()

View File

@ -91,16 +91,9 @@ namespace Ryujinx.Graphics.OpenGL
Buffer.Delete(buffer);
}
public byte[] GetBufferData(BufferHandle buffer, int offset, int size)
public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{
if (HwCapabilities.UsePersistentBufferForFlush)
{
return PersistentBuffers.Default.GetBufferData(buffer, offset, size);
}
else
{
return Buffer.GetData(buffer, offset, size);
}
return Buffer.GetData(this, buffer, offset, size);
}
public Capabilities GetCapabilities()

View File

@ -359,6 +359,7 @@ namespace Ryujinx.Graphics.Texture
}
public static ReadOnlySpan<byte> ConvertLinearToBlockLinear(
Span<byte> output,
int width,
int height,
int depth,
@ -373,7 +374,10 @@ namespace Ryujinx.Graphics.Texture
SizeInfo sizeInfo,
ReadOnlySpan<byte> data)
{
Span<byte> output = new byte[sizeInfo.TotalSize];
if (output.Length == 0)
{
output = new byte[sizeInfo.TotalSize];
}
int inOffs = 0;
@ -500,6 +504,7 @@ namespace Ryujinx.Graphics.Texture
}
public static ReadOnlySpan<byte> ConvertLinearToLinearStrided(
Span<byte> output,
int width,
int height,
int blockWidth,
@ -516,10 +521,21 @@ namespace Ryujinx.Graphics.Texture
if (inStride == stride)
{
return data;
if (output.Length != 0)
{
data.CopyTo(output);
return output;
}
else
{
return data;
}
}
Span<byte> output = new byte[h * stride];
if (output.Length == 0)
{
output = new byte[h * stride];
}
int inOffs = 0;
int outOffs = 0;

View File

@ -49,7 +49,7 @@ namespace Ryujinx.Memory.Tests
throw new NotImplementedException();
}
public WritableRegion GetWritableRegion(ulong va, int size)
public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{
throw new NotImplementedException();
}

View File

@ -207,9 +207,10 @@ namespace Ryujinx.Memory
/// </remarks>
/// <param name="va">Virtual address of the data</param>
/// <param name="size">Size of the data</param>
/// <param name="tracked">True if write tracking is triggered on the span</param>
/// <returns>A writable region of memory containing the data</returns>
/// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
public unsafe WritableRegion GetWritableRegion(ulong va, int size)
public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{
if (size == 0)
{

View File

@ -87,9 +87,10 @@ namespace Ryujinx.Memory
/// </summary>
/// <param name="va">Virtual address of the data</param>
/// <param name="size">Size of the data</param>
/// <param name="tracked">True if write tracking is triggered on the span</param>
/// <returns>A writable region of memory containing the data</returns>
/// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
WritableRegion GetWritableRegion(ulong va, int size);
WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false);
/// <summary>
/// Gets a reference for the given type at the specified virtual memory address.

View File

@ -5,5 +5,7 @@ namespace Ryujinx.Memory
public interface IWritableBlock
{
void Write(ulong va, ReadOnlySpan<byte> data);
void WriteUntracked(ulong va, ReadOnlySpan<byte> data) => Write(va, data);
}
}

View File

@ -6,15 +6,17 @@ namespace Ryujinx.Memory
{
private readonly IWritableBlock _block;
private readonly ulong _va;
private readonly bool _tracked;
private bool NeedsWriteback => _block != null;
public Memory<byte> Memory { get; }
public WritableRegion(IWritableBlock block, ulong va, Memory<byte> memory)
public WritableRegion(IWritableBlock block, ulong va, Memory<byte> memory, bool tracked = false)
{
_block = block;
_va = va;
_tracked = tracked;
Memory = memory;
}
@ -22,7 +24,14 @@ namespace Ryujinx.Memory
{
if (NeedsWriteback)
{
_block.Write(_va, Memory.Span);
if (_tracked)
{
_block.Write(_va, Memory.Span);
}
else
{
_block.WriteUntracked(_va, Memory.Span);
}
}
}
}