deadcade/Ryujinx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Vulkan: Migrate buffers between memory types to improve GPU performance (#4540)

Browse source 
* Initial implementation of migration between memory heaps

- Missing OOM handling
- Missing `_map` data safety when remapping
  - Copy may not have completed yet (needs some kind of fence)
  - Map may be unmapped before it is done being used. (needs scoped access)
- SSBO accesses are all "writes" - maybe pass info in another way.
- Missing keeping map type when resizing buffers (should this be done?)

* Ensure migrated data is in place before flushing.

* Fix issue where old waitable would be signalled.

- There is a real issue where existing Auto<> references need to be replaced.

* Swap bound Auto<> instances when swapping buffer backing

* Fix conversion buffers

* Don't try move buffers if the host has shared memory.

* Make GPU methods return PinnedSpan with scope

* Storage Hint

* Fix stupidity

* Fix rebase

* Tweak rules

Attempt to sidestep BOTW slowdown

* Remove line

* Migrate only when command buffers flush

* Change backing swap log to debug

* Address some feedback

* Disallow backing swap when the flush lock is held by the current thread

* Make PinnedSpan from ReadOnlySpan explicitly unsafe

* Fix some small issues

- Index buffer swap fixed
- Allocate DeviceLocal buffers using a separate block list to images.

* Remove alternative flags

* Address feedback
This commit is contained in:
riperiperi 2023-03-19 20:56:48 +00:00 committed by GitHub
parent 67b4e63cff
commit 9f1cf6458c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 660 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Ryujinx.Graphics.GAL/IRenderer.cs
View file

@ -15,7 +15,12 @@ namespace Ryujinx.Graphics.GAL
void BackgroundContextAction(Action action, bool alwaysBackground = false);
BufferHandle CreateBuffer(int size);
BufferHandle CreateBuffer(int size, BufferHandle storageHint);
BufferHandle CreateBuffer(int size)
{
return CreateBuffer(size, BufferHandle.Null);
}
IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info);
@ -26,7 +31,7 @@ namespace Ryujinx.Graphics.GAL
void DeleteBuffer(BufferHandle buffer);
ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size);
PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size);
Capabilities GetCapabilities();
ulong GetCurrentSync();

4
Ryujinx.Graphics.GAL/ITexture.cs
View file

@ -15,8 +15,8 @@ namespace Ryujinx.Graphics.GAL
ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel);
ReadOnlySpan<byte> GetData();
ReadOnlySpan<byte> GetData(int layer, int level);
PinnedSpan<byte> GetData();
PinnedSpan<byte> GetData(int layer, int level);
void SetData(SpanOrArray<byte> data);
void SetData(SpanOrArray<byte> data, int layer, int level);

4
Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetDataCommand.cs
View file

@ -21,9 +21,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Buffer
public static void Run(ref BufferGetDataCommand command, ThreadedRenderer threaded, IRenderer renderer)
{
ReadOnlySpan<byte> result = renderer.GetBufferData(threaded.Buffers.MapBuffer(command._buffer), command._offset, command._size);
PinnedSpan<byte> result = renderer.GetBufferData(threaded.Buffers.MapBuffer(command._buffer), command._offset, command._size);
command._result.Get(threaded).Result = new PinnedSpan<byte>(result);
command._result.Get(threaded).Result = result;
}
}
}

13
Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs
View file

@ -5,16 +5,25 @@
public CommandType CommandType => CommandType.CreateBuffer;
private BufferHandle _threadedHandle;
private int _size;
private BufferHandle _storageHint;
public void Set(BufferHandle threadedHandle, int size)
public void Set(BufferHandle threadedHandle, int size, BufferHandle storageHint)
{
_threadedHandle = threadedHandle;
_size = size;
_storageHint = storageHint;
}
public static void Run(ref CreateBufferCommand command, ThreadedRenderer threaded, IRenderer renderer)
{
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size));
BufferHandle hint = BufferHandle.Null;
if (command._storageHint != BufferHandle.Null)
{
hint = threaded.Buffers.MapBuffer(command._storageHint);
}
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, hint));
}
}
}

4
Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataCommand.cs
View file

@ -18,9 +18,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Texture
public static void Run(ref TextureGetDataCommand command, ThreadedRenderer threaded, IRenderer renderer)
{
ReadOnlySpan<byte> result = command._texture.Get(threaded).Base.GetData();
PinnedSpan<byte> result = command._texture.Get(threaded).Base.GetData();
command._result.Get(threaded).Result = new PinnedSpan<byte>(result);
command._result.Get(threaded).Result = result;
}
}
}

4
Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataSliceCommand.cs
View file

@ -22,9 +22,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Texture
public static void Run(ref TextureGetDataSliceCommand command, ThreadedRenderer threaded, IRenderer renderer)
{
ReadOnlySpan<byte> result = command._texture.Get(threaded).Base.GetData(command._layer, command._level);
PinnedSpan<byte> result = command._texture.Get(threaded).Base.GetData(command._layer, command._level);
command._result.Get(threaded).Result = new PinnedSpan<byte>(result);
command._result.Get(threaded).Result = result;
}
}
}

23
Ryujinx.Graphics.GAL/Multithreading/Model/PinnedSpan.cs
View file

@ -1,23 +0,0 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.GAL.Multithreading.Model
{
unsafe struct PinnedSpan<T> where T : unmanaged
{
private void* _ptr;
private int _size;
public PinnedSpan(ReadOnlySpan<T> span)
{
_ptr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(span));
_size = span.Length;
}
public ReadOnlySpan<T> Get()
{
return new ReadOnlySpan<T>(_ptr, _size * Unsafe.SizeOf<T>());
}
}
}

8
Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs
View file

@ -72,7 +72,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
return newTex;
}
public ReadOnlySpan<byte> GetData()
public PinnedSpan<byte> GetData()
{
if (_renderer.IsGpuThread())
{
@ -80,7 +80,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
_renderer.New<TextureGetDataCommand>().Set(Ref(this), Ref(box));
_renderer.InvokeCommand();
return box.Result.Get();
return box.Result;
}
else
{
@ -90,7 +90,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
}
}
public ReadOnlySpan<byte> GetData(int layer, int level)
public PinnedSpan<byte> GetData(int layer, int level)
{
if (_renderer.IsGpuThread())
{
@ -98,7 +98,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
_renderer.New<TextureGetDataSliceCommand>().Set(Ref(this), Ref(box), layer, level);
_renderer.InvokeCommand();
return box.Result.Get();
return box.Result;
}
else
{

8
Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs
View file

@ -265,10 +265,10 @@ namespace Ryujinx.Graphics.GAL.Multithreading
}
}
public BufferHandle CreateBuffer(int size)
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{
BufferHandle handle = Buffers.CreateBufferHandle();
New<CreateBufferCommand>().Set(handle, size);
New<CreateBufferCommand>().Set(handle, size, storageHint);
QueueCommand();
return handle;
@ -329,7 +329,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
QueueCommand();
}
public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{
if (IsGpuThread())
{
@ -337,7 +337,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
New<BufferGetDataCommand>().Set(buffer, offset, size, Ref(box));
InvokeCommand();
return box.Result.Get();
return box.Result;
}
else
{

53
Ryujinx.Graphics.GAL/PinnedSpan.cs Normal file
View file

@ -0,0 +1,53 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.GAL
{
public unsafe struct PinnedSpan<T> : IDisposable where T : unmanaged
{
private void* _ptr;
private int _size;
private Action _disposeAction;
/// <summary>
/// Creates a new PinnedSpan from an existing ReadOnlySpan. The span *must* be pinned in memory.
/// The data must be guaranteed to live until disposeAction is called.
/// </summary>
/// <param name="span">Existing span</param>
/// <param name="disposeAction">Action to call on dispose</param>
/// <remarks>
/// If a dispose action is not provided, it is safe to assume the resource will be available until the next call.
/// </remarks>
public static PinnedSpan<T> UnsafeFromSpan(ReadOnlySpan<T> span, Action disposeAction = null)
{
return new PinnedSpan<T>(Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)), span.Length, disposeAction);
}
/// <summary>
/// Creates a new PinnedSpan from an existing unsafe region. The data must be guaranteed to live until disposeAction is called.
/// </summary>
/// <param name="ptr">Pointer to the region</param>
/// <param name="size">The total items of T the region contains</param>
/// <param name="disposeAction">Action to call on dispose</param>
/// <remarks>
/// If a dispose action is not provided, it is safe to assume the resource will be available until the next call.
/// </remarks>
public PinnedSpan(void* ptr, int size, Action disposeAction = null)
{
_ptr = ptr;
_size = size;
_disposeAction = disposeAction;
}
public ReadOnlySpan<T> Get()
{
return new ReadOnlySpan<T>(_ptr, _size * Unsafe.SizeOf<T>());
}
public void Dispose()
{
_disposeAction?.Invoke();
}
}
}

20
Ryujinx.Graphics.Gpu/Image/Texture.cs
View file

@ -1022,13 +1022,12 @@ namespace Ryujinx.Graphics.Gpu.Image
/// This method should be used to retrieve data that was modified by the host GPU.
/// This is not cheap, avoid doing that unless strictly needed.
/// </remarks>
/// <param name="output">An output span to place the texture data into. If empty, one is generated</param>
/// <param name="output">An output span to place the texture data into</param>
/// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param>
/// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
/// <returns>The span containing the texture data</returns>
private ReadOnlySpan<byte> GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null)
private void GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null)
{
ReadOnlySpan<byte> data;
PinnedSpan<byte> data;
if (texture != null)
{
@ -1054,9 +1053,9 @@ namespace Ryujinx.Graphics.Gpu.Image
}
}
data = ConvertFromHostCompatibleFormat(output, data);
ConvertFromHostCompatibleFormat(output, data.Get());
return data;
data.Dispose();
}
/// <summary>
@ -1071,10 +1070,9 @@ namespace Ryujinx.Graphics.Gpu.Image
/// <param name="level">The level of the texture to flush</param>
/// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param>
/// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
/// <returns>The span containing the texture data</returns>
public ReadOnlySpan<byte> GetTextureDataSliceFromGpu(Span<byte> output, int layer, int level, bool blacklist, ITexture texture = null)
public void GetTextureDataSliceFromGpu(Span<byte> output, int layer, int level, bool blacklist, ITexture texture = null)
{
ReadOnlySpan<byte> data;
PinnedSpan<byte> data;
if (texture != null)
{
@ -1100,9 +1098,9 @@ namespace Ryujinx.Graphics.Gpu.Image
}
}
data = ConvertFromHostCompatibleFormat(output, data, level, true);
ConvertFromHostCompatibleFormat(output, data.Get(), level, true);
return data;
data.Dispose();
}
/// <summary>

6
Ryujinx.Graphics.Gpu/Memory/Buffer.cs
View file

@ -82,7 +82,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
Address = address;
Size = size;
Handle = context.Renderer.CreateBuffer((int)size);
Handle = context.Renderer.CreateBuffer((int)size, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
_useGranular = size > GranularBufferThreshold;
@ -415,10 +415,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
int offset = (int)(address - Address);
ReadOnlySpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
// TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers.
_physicalMemory.WriteUntracked(address, data);
_physicalMemory.WriteUntracked(address, data.Get());
}
/// <summary>

9
Ryujinx.Graphics.OpenGL/Buffer.cs
View file

@ -55,11 +55,14 @@ namespace Ryujinx.Graphics.OpenGL
(IntPtr)size);
}
public static unsafe ReadOnlySpan<byte> GetData(OpenGLRenderer renderer, BufferHandle buffer, int offset, int size)
public static unsafe PinnedSpan<byte> GetData(OpenGLRenderer renderer, BufferHandle buffer, int offset, int size)
{
// Data in the persistent buffer and host array is guaranteed to be available
// until the next time the host thread requests data.
if (HwCapabilities.UsePersistentBufferForFlush)
{
return renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size);
return PinnedSpan<byte>.UnsafeFromSpan(renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size));
}
else
{
@ -69,7 +72,7 @@ namespace Ryujinx.Graphics.OpenGL
GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, target);
return new ReadOnlySpan<byte>(target.ToPointer(), size);
return new PinnedSpan<byte>(target.ToPointer(), size);
}
}

4
Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs
View file

@ -39,12 +39,12 @@ namespace Ryujinx.Graphics.OpenGL.Image
throw new NotSupportedException();
}
public ReadOnlySpan<byte> GetData()
public PinnedSpan<byte> GetData()
{
return Buffer.GetData(_renderer, _buffer, _bufferOffset, _bufferSize);
}
public ReadOnlySpan<byte> GetData(int layer, int level)
public PinnedSpan<byte> GetData(int layer, int level)
{
return GetData();
}

10
Ryujinx.Graphics.OpenGL/Image/TextureView.cs
View file

@ -166,7 +166,7 @@ namespace Ryujinx.Graphics.OpenGL.Image
_renderer.TextureCopy.Copy(this, (TextureView)destination, srcRegion, dstRegion, linearFilter);
}
public unsafe ReadOnlySpan<byte> GetData()
public unsafe PinnedSpan<byte> GetData()
{
int size = 0;
int levels = Info.GetLevelsClamped();
@ -196,16 +196,16 @@ namespace Ryujinx.Graphics.OpenGL.Image
data = FormatConverter.ConvertD24S8ToS8D24(data);
}
return data;
return PinnedSpan<byte>.UnsafeFromSpan(data);
}
public unsafe ReadOnlySpan<byte> GetData(int layer, int level)
public unsafe PinnedSpan<byte> GetData(int layer, int level)
{
int size = Info.GetMipSize(level);
if (HwCapabilities.UsePersistentBufferForFlush)
{
return _renderer.PersistentBuffers.Default.GetTextureData(this, size, layer, level);
return PinnedSpan<byte>.UnsafeFromSpan(_renderer.PersistentBuffers.Default.GetTextureData(this, size, layer, level));
}
else
{
@ -213,7 +213,7 @@ namespace Ryujinx.Graphics.OpenGL.Image
int offset = WriteTo2D(target, layer, level);
return new ReadOnlySpan<byte>(target.ToPointer(), size).Slice(offset);
return new PinnedSpan<byte>((byte*)target.ToPointer() + offset, size);
}
}

4
Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs
View file

@ -57,7 +57,7 @@ namespace Ryujinx.Graphics.OpenGL
ResourcePool = new ResourcePool();
}
public BufferHandle CreateBuffer(int size)
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{
BufferCount++;
@ -96,7 +96,7 @@ namespace Ryujinx.Graphics.OpenGL
return new HardwareInfo(GpuVendor, GpuRenderer);
}
public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{
return Buffer.GetData(this, buffer, offset, size);
}

12
Ryujinx.Graphics.Vulkan/BufferAllocationType.cs Normal file
View file

@ -0,0 +1,12 @@
namespace Ryujinx.Graphics.Vulkan
{
internal enum BufferAllocationType
{
Auto = 0,
HostMappedNoCache,
HostMapped,
DeviceLocal,
DeviceLocalMapped
}
}

302
Ryujinx.Graphics.Vulkan/BufferHolder.cs
View file

@ -1,7 +1,10 @@
using Ryujinx.Graphics.GAL;
using Ryujinx.Common.Logging;
using Ryujinx.Graphics.GAL;
using Silk.NET.Vulkan;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using VkBuffer = Silk.NET.Vulkan.Buffer;
using VkFormat = Silk.NET.Vulkan.Format;
@ -11,6 +14,12 @@ namespace Ryujinx.Graphics.Vulkan
{
private const int MaxUpdateBufferSize = 0x10000;
private const int SetCountThreshold = 100;
private const int WriteCountThreshold = 50;
private const int FlushCountThreshold = 5;
public const int DeviceLocalSizeThreshold = 256 * 1024; // 256kb
public const AccessFlags DefaultAccessFlags =
AccessFlags.IndirectCommandReadBit |
AccessFlags.ShaderReadBit |
@ -21,10 +30,10 @@ namespace Ryujinx.Graphics.Vulkan
private readonly VulkanRenderer _gd;
private readonly Device _device;
private readonly MemoryAllocation _allocation;
private readonly Auto<DisposableBuffer> _buffer;
private readonly Auto<MemoryAllocation> _allocationAuto;
private readonly ulong _bufferHandle;
private MemoryAllocation _allocation;
private Auto<DisposableBuffer> _buffer;
private Auto<MemoryAllocation> _allocationAuto;
private ulong _bufferHandle;
private CacheByRange<BufferHolder> _cachedConvertedBuffers;
@ -32,11 +41,28 @@ namespace Ryujinx.Graphics.Vulkan
private IntPtr _map;
private readonly MultiFenceHolder _waitable;
private MultiFenceHolder _waitable;
private bool _lastAccessIsWrite;
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size)
private BufferAllocationType _baseType;
private BufferAllocationType _currentType;
private bool _swapQueued;
public BufferAllocationType DesiredType { get; private set; }
private int _setCount;
private int _writeCount;
private int _flushCount;
private int _flushTemp;
private ReaderWriterLock _flushLock;
private FenceHolder _flushFence;
private int _flushWaiting;
private List<Action> _swapActions;
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size, BufferAllocationType type, BufferAllocationType currentType)
{
_gd = gd;
_device = device;
@ -47,9 +73,153 @@ namespace Ryujinx.Graphics.Vulkan
_bufferHandle = buffer.Handle;
Size = size;
_map = allocation.HostPointer;
_baseType = type;
_currentType = currentType;
DesiredType = currentType;
_flushLock = new ReaderWriterLock();
}
public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size)
public bool TryBackingSwap(ref CommandBufferScoped? cbs)
{
if (_swapQueued && DesiredType != _currentType)
{
// Only swap if the buffer is not used in any queued command buffer.
bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool);
if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld)
{
var currentAllocation = _allocationAuto;
var currentBuffer = _buffer;
IntPtr currentMap = _map;
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, _currentType);
if (buffer.Handle != 0)
{
_flushLock.AcquireWriterLock(Timeout.Infinite);
ClearFlushFence();
_waitable = new MultiFenceHolder(Size);
_allocation = allocation;
_allocationAuto = new Auto<MemoryAllocation>(allocation);
_buffer = new Auto<DisposableBuffer>(new DisposableBuffer(_gd.Api, _device, buffer), _waitable, _allocationAuto);
_bufferHandle = buffer.Handle;
_map = allocation.HostPointer;
if (_map != IntPtr.Zero && currentMap != IntPtr.Zero)
{
// Copy data directly. Readbacks don't have to wait if this is done.
unsafe
{
new Span<byte>((void*)currentMap, Size).CopyTo(new Span<byte>((void*)_map, Size));
}
}
else
{
if (cbs == null)
{
cbs = _gd.CommandBufferPool.Rent();
}
CommandBufferScoped cbsV = cbs.Value;
Copy(_gd, cbsV, currentBuffer, _buffer, 0, 0, Size);
// Need to wait for the data to reach the new buffer before data can be flushed.
_flushFence = _gd.CommandBufferPool.GetFence(cbsV.CommandBufferIndex);
_flushFence.Get();
}
Logger.Debug?.PrintMsg(LogClass.Gpu, $"Converted {Size} buffer {_currentType} to {resultType}");
_currentType = resultType;
if (_swapActions != null)
{
foreach (var action in _swapActions)
{
action();
}
_swapActions.Clear();
}
currentBuffer.Dispose();
currentAllocation.Dispose();
_gd.PipelineInternal.SwapBuffer(currentBuffer, _buffer);
_flushLock.ReleaseWriterLock();
}
_swapQueued = false;
return true;
}
else
{
return false;
}
}
else
{
_swapQueued = false;
return true;
}
}
private void ConsiderBackingSwap()
{
if (_baseType == BufferAllocationType.Auto)
{
if (_writeCount >= WriteCountThreshold || _setCount >= SetCountThreshold || _flushCount >= FlushCountThreshold)
{
if (_flushCount > 0 || _flushTemp-- > 0)
{
// Buffers that flush should ideally be mapped in host address space for easy copies.
// If the buffer is large it will do better on GPU memory, as there will be more writes than data flushes (typically individual pages).
// If it is small, then it's likely most of the buffer will be flushed so we want it on host memory, as access is cached.
DesiredType = Size > DeviceLocalSizeThreshold ? BufferAllocationType.DeviceLocalMapped : BufferAllocationType.HostMapped;
// It's harder for a buffer that is flushed to revert to another type of mapping.
if (_flushCount > 0)
{
_flushTemp = 1000;
}
}
else if (_writeCount >= WriteCountThreshold)
{
// Buffers that are written often should ideally be in the device local heap. (Storage buffers)
DesiredType = BufferAllocationType.DeviceLocal;
}
else if (_setCount > SetCountThreshold)
{
// Buffers that have their data set often should ideally be host mapped. (Constant buffers)
DesiredType = BufferAllocationType.HostMapped;
}
_flushCount = 0;
_writeCount = 0;
_setCount = 0;
}
if (!_swapQueued && DesiredType != _currentType)
{
_swapQueued = true;
_gd.PipelineInternal.AddBackingSwap(this);
}
}
}
public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size, Action invalidateView)
{
var bufferViewCreateInfo = new BufferViewCreateInfo()
{
@ -62,9 +232,19 @@ namespace Ryujinx.Graphics.Vulkan
_gd.Api.CreateBufferView(_device, bufferViewCreateInfo, null, out var bufferView).ThrowOnError();
(_swapActions ??= new List<Action>()).Add(invalidateView);
return new Auto<DisposableBufferView>(new DisposableBufferView(_gd.Api, _device, bufferView), _waitable, _buffer);
}
public void InheritMetrics(BufferHolder other)
{
_setCount = other._setCount;
_writeCount = other._writeCount;
_flushCount = other._flushCount;
_flushTemp = other._flushTemp;
}
public unsafe void InsertBarrier(CommandBuffer commandBuffer, bool isWrite)
{
// If the last access is write, we always need a barrier to be sure we will read or modify
@ -104,12 +284,22 @@ namespace Ryujinx.Graphics.Vulkan
return _buffer;
}
public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, bool isWrite = false)
public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, bool isWrite = false, bool isSSBO = false)
{
if (isWrite)
{
_writeCount++;
SignalWrite(0, Size);
}
else if (isSSBO)
{
// Always consider SSBO access for swapping to device local memory.
_writeCount++;
ConsiderBackingSwap();
}
return _buffer;
}
@ -118,6 +308,8 @@ namespace Ryujinx.Graphics.Vulkan
{
if (isWrite)
{
_writeCount++;
SignalWrite(offset, size);
}
@ -126,6 +318,8 @@ namespace Ryujinx.Graphics.Vulkan
public void SignalWrite(int offset, int size)
{
ConsiderBackingSwap();
if (offset == 0 && size == Size)
{
_cachedConvertedBuffers.Clear();
@ -147,11 +341,76 @@ namespace Ryujinx.Graphics.Vulkan
return _map;
}
public unsafe ReadOnlySpan<byte> GetData(int offset, int size)
private void ClearFlushFence()
{
// Asusmes _flushLock is held as writer.
if (_flushFence != null)
{
if (_flushWaiting == 0)
{
_flushFence.Put();
}
_flushFence = null;
}
}
private void WaitForFlushFence()
{
// Assumes the _flushLock is held as reader, returns in same state.
if (_flushFence != null)
{
// If storage has changed, make sure the fence has been reached so that the data is in place.
var cookie = _flushLock.UpgradeToWriterLock(Timeout.Infinite);
if (_flushFence != null)
{
var fence = _flushFence;
Interlocked.Increment(ref _flushWaiting);
// Don't wait in the lock.
var restoreCookie = _flushLock.ReleaseLock();
fence.Wait();
_flushLock.RestoreLock(ref restoreCookie);
if (Interlocked.Decrement(ref _flushWaiting) == 0)
{
fence.Put();
}
_flushFence = null;
}
_flushLock.DowngradeFromWriterLock(ref cookie);
}
}
public unsafe PinnedSpan<byte> GetData(int offset, int size)
{
_flushLock.AcquireReaderLock(Timeout.Infinite);
WaitForFlushFence();
_flushCount++;
Span<byte> result;
if (_map != IntPtr.Zero)
{
return GetDataStorage(offset, size);
result = GetDataStorage(offset, size);
// Need to be careful here, the buffer can't be unmapped while the data is being used.
_buffer.IncrementReferenceCount();
_flushLock.ReleaseReaderLock();
return PinnedSpan<byte>.UnsafeFromSpan(result, _buffer.DecrementReferenceCount);
}
else
{
@ -161,12 +420,17 @@ namespace Ryujinx.Graphics.Vulkan
{
_gd.FlushAllCommands();
return resource.GetFlushBuffer().GetBufferData(_gd.CommandBufferPool, this, offset, size);
result = resource.GetFlushBuffer().GetBufferData(_gd.CommandBufferPool, this, offset, size);
}
else
{
return resource.GetFlushBuffer().GetBufferData(resource.GetPool(), this, offset, size);
result = resource.GetFlushBuffer().GetBufferData(resource.GetPool(), this, offset, size);
}
_flushLock.ReleaseReaderLock();
// Flush buffer is pinned until the next GetBufferData on the thread, which is fine for current uses.
return PinnedSpan<byte>.UnsafeFromSpan(result);
}
}
@ -190,6 +454,8 @@ namespace Ryujinx.Graphics.Vulkan
return;
}
_setCount++;
if (_map != IntPtr.Zero)
{
// If persistently mapped, set the data directly if the buffer is not currently in use.
@ -268,6 +534,8 @@ namespace Ryujinx.Graphics.Vulkan
var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length).Value;
_writeCount--;
InsertBufferBarrier(
_gd,
cbs.CommandBuffer,
@ -502,11 +770,19 @@ namespace Ryujinx.Graphics.Vulkan
public void Dispose()
{
_swapQueued = false;
_gd.PipelineInternal?.FlushCommandsIfWeightExceeding(_buffer, (ulong)Size);
_buffer.Dispose();
_allocationAuto.Dispose();
_cachedConvertedBuffers.Dispose();
_flushLock.AcquireWriterLock(Timeout.Infinite);
ClearFlushFence();
_flushLock.ReleaseWriterLock();
}
}
}

122
Ryujinx.Graphics.Vulkan/BufferManager.cs
View file

@ -4,6 +4,7 @@ using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using VkFormat = Silk.NET.Vulkan.Format;
using VkBuffer = Silk.NET.Vulkan.Buffer;
namespace Ryujinx.Graphics.Vulkan
{
@ -16,17 +17,17 @@ namespace Ryujinx.Graphics.Vulkan
// Some drivers don't expose a "HostCached" memory type,
// so we need those alternative flags for the allocation to succeed there.
private const MemoryPropertyFlags DefaultBufferMemoryAltFlags =
private const MemoryPropertyFlags DefaultBufferMemoryNoCacheFlags =
MemoryPropertyFlags.HostVisibleBit |
MemoryPropertyFlags.HostCoherentBit;
private const MemoryPropertyFlags DeviceLocalBufferMemoryFlags =
MemoryPropertyFlags.DeviceLocalBit;
private const MemoryPropertyFlags FlushableDeviceLocalBufferMemoryFlags =
private const MemoryPropertyFlags DeviceLocalMappedBufferMemoryFlags =
MemoryPropertyFlags.DeviceLocalBit |
MemoryPropertyFlags.HostVisibleBit |
MemoryPropertyFlags.HostCoherentBit |
MemoryPropertyFlags.DeviceLocalBit;
MemoryPropertyFlags.HostCoherentBit;
private const BufferUsageFlags DefaultBufferUsageFlags =
BufferUsageFlags.TransferSrcBit |
@ -54,14 +55,14 @@ namespace Ryujinx.Graphics.Vulkan
StagingBuffer = new StagingBuffer(gd, this);
}
public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, bool deviceLocal)
public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default)
{
return CreateWithHandle(gd, size, deviceLocal, out _);
return CreateWithHandle(gd, size, out _, baseType, storageHint);
}
public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, bool deviceLocal, out BufferHolder holder)
public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, out BufferHolder holder, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default)
{
holder = Create(gd, size, deviceLocal: deviceLocal);
holder = Create(gd, size, baseType: baseType, storageHint: storageHint);
if (holder == null)
{
return BufferHandle.Null;
@ -74,7 +75,12 @@ namespace Ryujinx.Graphics.Vulkan
return Unsafe.As<ulong, BufferHandle>(ref handle64);
}
public unsafe BufferHolder Create(VulkanRenderer gd, int size, bool forConditionalRendering = false, bool deviceLocal = false)
public unsafe (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) CreateBacking(
VulkanRenderer gd,
int size,
BufferAllocationType type,
bool forConditionalRendering = false,
BufferAllocationType fallbackType = BufferAllocationType.Auto)
{
var usage = DefaultBufferUsageFlags;
@ -98,48 +104,106 @@ namespace Ryujinx.Graphics.Vulkan
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements);
MemoryPropertyFlags allocateFlags;
MemoryPropertyFlags allocateFlagsAlt;
MemoryAllocation allocation;
if (deviceLocal)
do
{
allocateFlags = DeviceLocalBufferMemoryFlags;
allocateFlagsAlt = DeviceLocalBufferMemoryFlags;
}
else
var allocateFlags = type switch
{
allocateFlags = DefaultBufferMemoryFlags;
allocateFlagsAlt = DefaultBufferMemoryAltFlags;
}
BufferAllocationType.HostMappedNoCache => DefaultBufferMemoryNoCacheFlags,
BufferAllocationType.HostMapped => DefaultBufferMemoryFlags,
BufferAllocationType.DeviceLocal => DeviceLocalBufferMemoryFlags,
BufferAllocationType.DeviceLocalMapped => DeviceLocalMappedBufferMemoryFlags,
_ => DefaultBufferMemoryFlags
};
var allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, allocateFlagsAlt);
// If an allocation with this memory type fails, fall back to the previous one.
try
{
allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, true);
}
catch (VulkanException)
{
allocation = default;
}
}
while (allocation.Memory.Handle == 0 && (--type != fallbackType));
if (allocation.Memory.Handle == 0UL)
{
gd.Api.DestroyBuffer(_device, buffer, null);
return null;
return default;
}
gd.Api.BindBufferMemory(_device, buffer, allocation.Memory, allocation.Offset);
return new BufferHolder(gd, _device, buffer, allocation, size);
return (buffer, allocation, type);
}
public Auto<DisposableBufferView> CreateView(BufferHandle handle, VkFormat format, int offset, int size)
public unsafe BufferHolder Create(
VulkanRenderer gd,
int size,
bool forConditionalRendering = false,
BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default)
{
if (TryGetBuffer(handle, out var holder))
BufferAllocationType type = baseType;
BufferHolder storageHintHolder = null;
if (baseType == BufferAllocationType.Auto)
{
return holder.CreateView(format, offset, size);
if (gd.IsSharedMemory)
{
baseType = BufferAllocationType.HostMapped;
type = baseType;
}
else
{
type = size >= BufferHolder.DeviceLocalSizeThreshold ? BufferAllocationType.DeviceLocal : BufferAllocationType.HostMapped;
}
if (storageHint != BufferHandle.Null)
{
if (TryGetBuffer(storageHint, out storageHintHolder))
{
type = storageHintHolder.DesiredType;
}
}
}
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) =
CreateBacking(gd, size, type, forConditionalRendering);
if (buffer.Handle != 0)
{
var holder = new BufferHolder(gd, _device, buffer, allocation, size, baseType, resultType);
if (storageHintHolder != null)
{
holder.InheritMetrics(storageHintHolder);
}
return holder;
}
return null;
}
public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite)
public Auto<DisposableBufferView> CreateView(BufferHandle handle, VkFormat format, int offset, int size, Action invalidateView)
{
if (TryGetBuffer(handle, out var holder))
{
return holder.GetBuffer(commandBuffer, isWrite);
return holder.CreateView(format, offset, size, invalidateView);
}
return null;
}
public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite, bool isSSBO = false)
{
if (TryGetBuffer(handle, out var holder))
{
return holder.GetBuffer(commandBuffer, isWrite, isSSBO);
}
return null;
@ -332,14 +396,14 @@ namespace Ryujinx.Graphics.Vulkan
return null;
}
public ReadOnlySpan<byte> GetData(BufferHandle handle, int offset, int size)
public PinnedSpan<byte> GetData(BufferHandle handle, int offset, int size)
{
if (TryGetBuffer(handle, out var holder))
{
return holder.GetData(offset, size);
}
return ReadOnlySpan<byte>.Empty;
return new PinnedSpan<byte>();
}
public void SetData<T>(BufferHandle handle, int offset, ReadOnlySpan<T> data) where T : unmanaged

15
Ryujinx.Graphics.Vulkan/BufferState.cs
View file

@ -2,14 +2,14 @@
namespace Ryujinx.Graphics.Vulkan
{
readonly struct BufferState : IDisposable
struct BufferState : IDisposable
{
public static BufferState Null => new BufferState(null, 0, 0);
private readonly int _offset;
private readonly int _size;
private readonly Auto<DisposableBuffer> _buffer;
private Auto<DisposableBuffer> _buffer;
public BufferState(Auto<DisposableBuffer> buffer, int offset, int size)
{
@ -29,6 +29,17 @@ namespace Ryujinx.Graphics.Vulkan
}
}
public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
if (_buffer == from)
{
_buffer.DecrementReferenceCount();
to.IncrementReferenceCount();
_buffer = to;
}
}
public void Dispose()
{
_buffer?.DecrementReferenceCount();

21
Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs
View file

@ -94,7 +94,7 @@ namespace Ryujinx.Graphics.Vulkan
else
{
// If null descriptors are not supported, we need to pass the handle of a dummy buffer on unused bindings.
_dummyBuffer = gd.BufferManager.Create(gd, 0x10000, forConditionalRendering: false, deviceLocal: true);
_dummyBuffer = gd.BufferManager.Create(gd, 0x10000, forConditionalRendering: false, baseType: BufferAllocationType.DeviceLocal);
}
_dummyTexture = gd.CreateTextureView(new TextureCreateInfo(
@ -178,7 +178,7 @@ namespace Ryujinx.Graphics.Vulkan
var buffer = assignment.Range;
int index = assignment.Binding;
Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false);
Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false, isSSBO: true);
ref Auto<DisposableBuffer> currentVkBuffer = ref _storageBufferRefs[index];
DescriptorBufferInfo info = new DescriptorBufferInfo()
@ -640,6 +640,23 @@ namespace Ryujinx.Graphics.Vulkan
Array.Clear(_storageSet);
}
private void SwapBuffer(Auto<DisposableBuffer>[] list, Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
for (int i = 0; i < list.Length; i++)
{
if (list[i] == from)
{
list[i] = to;
}
}
}
public void SwapBuffer(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
SwapBuffer(_uniformBufferRefs, from, to);
SwapBuffer(_storageBufferRefs, from, to);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)

4
Ryujinx.Graphics.Vulkan/Effects/FsrScalingFilter.cs
View file

@ -156,11 +156,11 @@ namespace Ryujinx.Graphics.Vulkan.Effects
};
int rangeSize = dimensionsBuffer.Length * sizeof(float);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize);
_renderer.BufferManager.SetData(bufferHandle, 0, dimensionsBuffer);
ReadOnlySpan<float> sharpeningBuffer = stackalloc float[] { 1.5f - (Level * 0.01f * 1.5f)};
var sharpeningBufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, sizeof(float), false);
var sharpeningBufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, sizeof(float));
_renderer.BufferManager.SetData(sharpeningBufferHandle, 0, sharpeningBuffer);
int threadGroupWorkRegionDim = 16;

2
Ryujinx.Graphics.Vulkan/Effects/FxaaPostProcessingEffect.cs
View file

@ -87,7 +87,7 @@ namespace Ryujinx.Graphics.Vulkan.Effects
ReadOnlySpan<float> resolutionBuffer = stackalloc float[] { view.Width, view.Height };
int rangeSize = resolutionBuffer.Length * sizeof(float);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize);
_renderer.BufferManager.SetData(bufferHandle, 0, resolutionBuffer);

2
Ryujinx.Graphics.Vulkan/Effects/SmaaPostProcessingEffect.cs
View file

@ -266,7 +266,7 @@ namespace Ryujinx.Graphics.Vulkan.Effects
ReadOnlySpan<float> resolutionBuffer = stackalloc float[] { view.Width, view.Height };
int rangeSize = resolutionBuffer.Length * sizeof(float);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize);
_renderer.BufferManager.SetData(bufferHandle, 0, resolutionBuffer);
var bufferRanges = new BufferRange(bufferHandle, 0, rangeSize);

18
Ryujinx.Graphics.Vulkan/HelperShader.cs
View file

@ -394,7 +394,7 @@ namespace Ryujinx.Graphics.Vulkan
(region[2], region[3]) = (region[3], region[2]);
}
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize);
gd.BufferManager.SetData<float>(bufferHandle, 0, region);
@ -495,7 +495,7 @@ namespace Ryujinx.Graphics.Vulkan
(region[2], region[3]) = (region[3], region[2]);
}
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize);
gd.BufferManager.SetData<float>(bufferHandle, 0, region);
@ -649,7 +649,7 @@ namespace Ryujinx.Graphics.Vulkan
_pipeline.SetCommandBuffer(cbs);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ClearColorBufferSize, false);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ClearColorBufferSize);
gd.BufferManager.SetData<float>(bufferHandle, 0, clearColor);
@ -726,7 +726,7 @@ namespace Ryujinx.Graphics.Vulkan
(region[2], region[3]) = (region[3], region[2]);
}
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize);
gd.BufferManager.SetData<float>(bufferHandle, 0, region);
@ -802,7 +802,7 @@ namespace Ryujinx.Graphics.Vulkan
shaderParams[2] = size;
shaderParams[3] = srcOffset;
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
@ -958,7 +958,7 @@ namespace Ryujinx.Graphics.Vulkan
shaderParams[0] = BitOperations.Log2((uint)ratio);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
@ -1050,7 +1050,7 @@ namespace Ryujinx.Graphics.Vulkan
(shaderParams[0], shaderParams[1]) = GetSampleCountXYLog2(samples);
(shaderParams[2], shaderParams[3]) = GetSampleCountXYLog2((int)TextureStorage.ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)samples));
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
@ -1133,7 +1133,7 @@ namespace Ryujinx.Graphics.Vulkan
(shaderParams[0], shaderParams[1]) = GetSampleCountXYLog2(samples);
(shaderParams[2], shaderParams[3]) = GetSampleCountXYLog2((int)TextureStorage.ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)samples));
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
@ -1407,7 +1407,7 @@ namespace Ryujinx.Graphics.Vulkan
pattern.OffsetIndex.CopyTo(shaderParams.Slice(0, pattern.OffsetIndex.Length));
var patternBufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false, out var patternBuffer);
var patternBufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, out var patternBuffer);
var patternBufferAuto = patternBuffer.GetBuffer();
gd.BufferManager.SetData<int>(patternBufferHandle, 0, shaderParams);

2
Ryujinx.Graphics.Vulkan/IndexBufferPattern.cs
View file

@ -82,7 +82,7 @@ namespace Ryujinx.Graphics.Vulkan
}
// Expand the repeating pattern to the number of requested primitives.
BufferHandle newBuffer = _gd.CreateBuffer(expectedSize * sizeof(int));
BufferHandle newBuffer = _gd.BufferManager.CreateWithHandle(_gd, expectedSize * sizeof(int));
// Copy the old data to the new one.
if (_repeatingBuffer != BufferHandle.Null)

11
Ryujinx.Graphics.Vulkan/IndexBufferState.cs
View file

@ -146,5 +146,16 @@ namespace Ryujinx.Graphics.Vulkan
{
return _buffer == buffer;
}
public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
if (_buffer == from)
{
_buffer.DecrementReferenceCount();
to.IncrementReferenceCount();
_buffer = to;
}
}
}
}

49
Ryujinx.Graphics.Vulkan/MemoryAllocator.cs
View file

@ -28,32 +28,25 @@ namespace Ryujinx.Graphics.Vulkan
public MemoryAllocation AllocateDeviceMemory(
MemoryRequirements requirements,
MemoryPropertyFlags flags = 0)
MemoryPropertyFlags flags = 0,
bool isBuffer = false)
{
return AllocateDeviceMemory(requirements, flags, flags);
}
public MemoryAllocation AllocateDeviceMemory(
MemoryRequirements requirements,
MemoryPropertyFlags flags,
MemoryPropertyFlags alternativeFlags)
{
int memoryTypeIndex = FindSuitableMemoryTypeIndex(requirements.MemoryTypeBits, flags, alternativeFlags);
int memoryTypeIndex = FindSuitableMemoryTypeIndex(requirements.MemoryTypeBits, flags);
if (memoryTypeIndex < 0)
{
return default;
}
bool map = flags.HasFlag(MemoryPropertyFlags.HostVisibleBit);
return Allocate(memoryTypeIndex, requirements.Size, requirements.Alignment, map);
return Allocate(memoryTypeIndex, requirements.Size, requirements.Alignment, map, isBuffer);
}
private MemoryAllocation Allocate(int memoryTypeIndex, ulong size, ulong alignment, bool map)
private MemoryAllocation Allocate(int memoryTypeIndex, ulong size, ulong alignment, bool map, bool isBuffer)
{
for (int i = 0; i < _blockLists.Count; i++)
{
var bl = _blockLists[i];
if (bl.MemoryTypeIndex == memoryTypeIndex)
if (bl.MemoryTypeIndex == memoryTypeIndex && bl.ForBuffer == isBuffer)
{
lock (bl)
{
@ -62,18 +55,15 @@ namespace Ryujinx.Graphics.Vulkan
}
}
var newBl = new MemoryAllocatorBlockList(_api, _device, memoryTypeIndex, _blockAlignment);
var newBl = new MemoryAllocatorBlockList(_api, _device, memoryTypeIndex, _blockAlignment, isBuffer);
_blockLists.Add(newBl);
return newBl.Allocate(size, alignment, map);
}
private int FindSuitableMemoryTypeIndex(
uint memoryTypeBits,
MemoryPropertyFlags flags,
MemoryPropertyFlags alternativeFlags)
MemoryPropertyFlags flags)
{
int bestCandidateIndex = -1;
for (int i = 0; i < _physicalDeviceMemoryProperties.MemoryTypeCount; i++)
{
var type = _physicalDeviceMemoryProperties.MemoryTypes[i];
@ -84,14 +74,27 @@ namespace Ryujinx.Graphics.Vulkan
{
return i;
}
else if (type.PropertyFlags.HasFlag(alternativeFlags))
{
bestCandidateIndex = i;
}
}
}
return bestCandidateIndex;
return -1;
}
public static bool IsDeviceMemoryShared(Vk api, PhysicalDevice physicalDevice)
{
// The device is regarded as having shared memory if all heaps have the device local bit.
api.GetPhysicalDeviceMemoryProperties(physicalDevice, out var properties);
for (int i = 0; i < properties.MemoryHeapCount; i++)
{
if (!properties.MemoryHeaps[i].Flags.HasFlag(MemoryHeapFlags.DeviceLocalBit))
{
return false;
}
}
return true;
}
public void Dispose()

4
Ryujinx.Graphics.Vulkan/MemoryAllocatorBlockList.cs
View file

@ -162,15 +162,17 @@ namespace Ryujinx.Graphics.Vulkan
private readonly Device _device;
public int MemoryTypeIndex { get; }
public bool ForBuffer { get; }
private readonly int _blockAlignment;
public MemoryAllocatorBlockList(Vk api, Device device, int memoryTypeIndex, int blockAlignment)
public MemoryAllocatorBlockList(Vk api, Device device, int memoryTypeIndex, int blockAlignment, bool forBuffer)
{
_blocks = new List<Block>();
_api = api;
_device = device;
MemoryTypeIndex = memoryTypeIndex;
ForBuffer = forBuffer;
_blockAlignment = blockAlignment;
}

19
Ryujinx.Graphics.Vulkan/PipelineBase.cs
View file

@ -1297,6 +1297,25 @@ namespace Ryujinx.Graphics.Vulkan
SignalStateChange();
}
public void SwapBuffer(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
_indexBuffer.Swap(from, to);
for (int i = 0; i < _vertexBuffers.Length; i++)
{
_vertexBuffers[i].Swap(from, to);
}
for (int i = 0; i < _transformFeedbackBuffers.Length; i++)
{
_transformFeedbackBuffers[i].Swap(from, to);
}
_descriptorSetUpdater.SwapBuffer(from, to);
SignalCommandBufferChange();
}
public unsafe void TextureBarrier()
{
MemoryBarrier memoryBarrier = new MemoryBarrier()

19
Ryujinx.Graphics.Vulkan/PipelineFull.cs
View file

@ -17,10 +17,13 @@ namespace Ryujinx.Graphics.Vulkan
private ulong _byteWeight;
private List<BufferHolder> _backingSwaps;
public PipelineFull(VulkanRenderer gd, Device device) : base(gd, device)
{
_activeQueries = new List<(QueryPool, bool)>();
_pendingQueryCopies = new();
_backingSwaps = new();
CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer;
}
@ -185,6 +188,20 @@ namespace Ryujinx.Graphics.Vulkan
}
}
private void TryBackingSwaps()
{
CommandBufferScoped? cbs = null;
_backingSwaps.RemoveAll((holder) => holder.TryBackingSwap(ref cbs));
cbs?.Dispose();
}
public void AddBackingSwap(BufferHolder holder)
{
_backingSwaps.Add(holder);
}
public void Restore()
{
if (Pipeline != null)
@ -230,6 +247,8 @@ namespace Ryujinx.Graphics.Vulkan
Gd.ResetCounterPool();
TryBackingSwaps();
Restore();
}

8
Ryujinx.Graphics.Vulkan/TextureBuffer.cs
View file

@ -57,12 +57,12 @@ namespace Ryujinx.Graphics.Vulkan
throw new NotSupportedException();
}
public ReadOnlySpan<byte> GetData()
public PinnedSpan<byte> GetData()
{
return _gd.GetBufferData(_bufferHandle, _offset, _size);
}
public ReadOnlySpan<byte> GetData(int layer, int level)
public PinnedSpan<byte> GetData(int layer, int level)
{
return GetData();
}
@ -128,7 +128,7 @@ namespace Ryujinx.Graphics.Vulkan
{
if (_bufferView == null)
{
_bufferView = _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size);
_bufferView = _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size, ReleaseImpl);
}
return _bufferView?.Get(cbs, _offset, _size).Value ?? default;
@ -147,7 +147,7 @@ namespace Ryujinx.Graphics.Vulkan
return bufferView.Get(cbs, _offset, _size).Value;
}
bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size);
bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size, ReleaseImpl);
if (bufferView != null)
{

12
Ryujinx.Graphics.Vulkan/TextureView.cs
View file

@ -531,7 +531,7 @@ namespace Ryujinx.Graphics.Vulkan
return bitmap;
}
public ReadOnlySpan<byte> GetData()
public PinnedSpan<byte> GetData()
{
BackgroundResource resources = _gd.BackgroundResources.Get();
@ -539,15 +539,15 @@ namespace Ryujinx.Graphics.Vulkan
{
_gd.FlushAllCommands();
return GetData(_gd.CommandBufferPool, resources.GetFlushBuffer());
return PinnedSpan<byte>.UnsafeFromSpan(GetData(_gd.CommandBufferPool, resources.GetFlushBuffer()));
}
else
{
return GetData(resources.GetPool(), resources.GetFlushBuffer());
return PinnedSpan<byte>.UnsafeFromSpan(GetData(resources.GetPool(), resources.GetFlushBuffer()));
}
}
public ReadOnlySpan<byte> GetData(int layer, int level)
public PinnedSpan<byte> GetData(int layer, int level)
{
BackgroundResource resources = _gd.BackgroundResources.Get();
@ -555,11 +555,11 @@ namespace Ryujinx.Graphics.Vulkan
{
_gd.FlushAllCommands();
return GetData(_gd.CommandBufferPool, resources.GetFlushBuffer(), layer, level);
return PinnedSpan<byte>.UnsafeFromSpan(GetData(_gd.CommandBufferPool, resources.GetFlushBuffer(), layer, level));
}
else
{
return GetData(resources.GetPool(), resources.GetFlushBuffer(), layer, level);
return PinnedSpan<byte>.UnsafeFromSpan(GetData(resources.GetPool(), resources.GetFlushBuffer(), layer, level));
}
}

11
Ryujinx.Graphics.Vulkan/VertexBufferState.cs
View file

@ -129,6 +129,17 @@ namespace Ryujinx.Graphics.Vulkan
return _buffer == buffer;
}
public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
if (_buffer == from)
{
_buffer.DecrementReferenceCount();
to.IncrementReferenceCount();
_buffer = to;
}
}
public void Dispose()
{
// Only dispose if this buffer is not refetched on each bind.

9
Ryujinx.Graphics.Vulkan/VulkanRenderer.cs
View file

@ -80,6 +80,7 @@ namespace Ryujinx.Graphics.Vulkan
internal bool IsAmdGcn { get; private set; }
internal bool IsMoltenVk { get; private set; }
internal bool IsTBDR { get; private set; }
internal bool IsSharedMemory { get; private set; }
public string GpuVendor { get; private set; }
public string GpuRenderer { get; private set; }
public string GpuVersion { get; private set; }
@ -313,6 +314,8 @@ namespace Ryujinx.Graphics.Vulkan
portabilityFlags,
vertexBufferAlignment);
IsSharedMemory = MemoryAllocator.IsDeviceMemoryShared(Api, _physicalDevice);
MemoryAllocator = new MemoryAllocator(Api, _physicalDevice, _device, properties.Limits.MaxMemoryAllocationCount);
CommandBufferPool = VulkanInitialization.CreateCommandBufferPool(Api, _device, Queue, QueueLock, queueFamilyIndex);
@ -373,9 +376,9 @@ namespace Ryujinx.Graphics.Vulkan
_initialized = true;
}
public BufferHandle CreateBuffer(int size)
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{
return BufferManager.CreateWithHandle(this, size, false);
return BufferManager.CreateWithHandle(this, size, BufferAllocationType.Auto, storageHint);
}
public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info)
@ -439,7 +442,7 @@ namespace Ryujinx.Graphics.Vulkan
_syncManager.RegisterFlush();
}
public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{
return BufferManager.GetData(buffer, offset, size);
}