mirrors/kenji-nx
1
0
Fork 0
mirror of https://git.ryujinx.app/kenji-nx/ryujinx.git synced 2025-12-14 16:37:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Memory changes 2.2

Browse source 
A few more internal changes to the RangeList systems.

* No longer using a QuickAccess dictionary.

  * The performance of the dictionary wasn't much faster than just doing binary searches.

  * Using just binary searches allows us to take advantage of span and array returns as they're are faster than linked lists when iterating or copying the overlaps.

Small code optimizations.

Fixes a few leftover crashes.
This commit is contained in:
LotP 2025-09-06 11:10:55 -05:00 • committed by KeatonTheBot
parent e3ef1e1fde
commit 65caa1e3f2
14 changed files with 240 additions and 237 deletions
Download patch file Download diff file Expand all files Collapse all files

25
src/Ryujinx.Graphics.Device/DeviceState.cs
View file

@ -96,7 +96,7 @@ namespace Ryujinx.Graphics.Device
uint alignedOffset = index * RegisterSize; uint alignedOffset = index * RegisterSize;
DebugWrite(alignedOffset, data); DebugWrite(alignedOffset, data);
GetRefIntAlignedUncheck(index) = data; SetIntAlignedUncheck(index, data);
_writeCallbacks[index]?.Invoke(data); _writeCallbacks[index]?.Invoke(data);
} }
@ -111,9 +111,7 @@ namespace Ryujinx.Graphics.Device
uint alignedOffset = index * RegisterSize; uint alignedOffset = index * RegisterSize;
DebugWrite(alignedOffset, data); DebugWrite(alignedOffset, data);
ref var storage = ref GetRefIntAlignedUncheck(index); changed = SetIntAlignedUncheckChanged(index, data);
changed = storage != data;
storage = data;
_writeCallbacks[index]?.Invoke(data); _writeCallbacks[index]?.Invoke(data);
} }
@ -153,5 +151,24 @@ namespace Ryujinx.Graphics.Device
{ {
return ref Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (IntPtr)index); return ref Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (IntPtr)index);
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void SetIntAlignedUncheck(ulong index, int data)
{
Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (nint)index) = data;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool SetIntAlignedUncheckChanged(ulong index, int data)
{
ref int val = ref Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (nint)index);
if (val == data)
{
return false;
}
val = data;
return true;
}
} }
} }

12
src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
View file

@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong size, ulong size,
BufferStage stage, BufferStage stage,
bool sparseCompatible, bool sparseCompatible,
List<Buffer> baseBuffers) RangeItem<Buffer>[] baseBuffers)
{ {
_context = context; _context = context;
_physicalMemory = physicalMemory; _physicalMemory = physicalMemory;
@ -128,18 +128,18 @@ namespace Ryujinx.Graphics.Gpu.Memory
List<IRegionHandle> baseHandles = null; List<IRegionHandle> baseHandles = null;
if (baseBuffers.Count != 0) if (baseBuffers.Length != 0)
{ {
baseHandles = new List<IRegionHandle>(); baseHandles = new List<IRegionHandle>();
foreach (Buffer buffer in baseBuffers) foreach (RangeItem<Buffer> item in baseBuffers)
{ {
if (buffer._useGranular) if (item.Value._useGranular)
{ {
baseHandles.AddRange((buffer._memoryTrackingGranular.GetHandles())); baseHandles.AddRange((item.Value._memoryTrackingGranular.GetHandles()));
} }
else else
{ {
baseHandles.Add(buffer._memoryTracking); baseHandles.Add(item.Value._memoryTracking);
} }
} }
} }

11
src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs
View file

@ -1,4 +1,5 @@
using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="parent">Parent buffer</param> /// <param name="parent">Parent buffer</param>
/// <param name="stage">Initial buffer stage</param> /// <param name="stage">Initial buffer stage</param>
/// <param name="baseBuffers">Buffers to inherit state from</param> /// <param name="baseBuffers">Buffers to inherit state from</param>
public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, List<Buffer> baseBuffers) public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, RangeItem<Buffer>[] baseBuffers)
{ {
_size = (int)parent.Size; _size = (int)parent.Size;
_systemMemoryType = context.Capabilities.MemoryType; _systemMemoryType = context.Capabilities.MemoryType;
@ -72,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferStage storageFlags = stage & BufferStage.StorageMask; BufferStage storageFlags = stage & BufferStage.StorageMask;
if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Count == 0) if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Length == 0)
{ {
_desiredType = BufferBackingType.DeviceMemory; _desiredType = BufferBackingType.DeviceMemory;
} }
@ -100,11 +101,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
// TODO: Might be nice to force atomic access to be device local for any stage. // TODO: Might be nice to force atomic access to be device local for any stage.
} }
if (baseBuffers.Count != 0) if (baseBuffers.Length != 0)
{ {
foreach (Buffer buffer in baseBuffers) foreach (RangeItem<Buffer> item in baseBuffers)
{ {
CombineState(buffer.BackingState); CombineState(item.Value.BackingState);
} }
} }
} }

95
src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
View file

@ -81,13 +81,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
MemoryRange subRange = range.GetSubRange(index); MemoryRange subRange = range.GetSubRange(index);
_buffers.Lock.EnterReadLock(); _buffers.Lock.EnterReadLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(subRange.Address, subRange.Size); Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(subRange.Address, subRange.Size);
RangeItem<Buffer> current = first; for (int i = 0; i < overlaps.Length; i++)
while (last != null && current != last.Next)
{ {
current.Value.Unmapped(subRange.Address, subRange.Size); overlaps[i].Value.Unmapped(subRange.Address, subRange.Size);
current = current.Next;
} }
_buffers.Lock.ExitReadLock(); _buffers.Lock.ExitReadLock();
@ -490,9 +488,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) private void CreateBufferAligned(ulong address, ulong size, BufferStage stage)
{ {
_buffers.Lock.EnterWriteLock(); _buffers.Lock.EnterWriteLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(address, size); Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(address, size);
if (first is not null) if (overlaps.Length > 0)
{ {
// The buffer already exists. We can just return the existing buffer // The buffer already exists. We can just return the existing buffer
// if the buffer we need is fully contained inside the overlapping buffer. // if the buffer we need is fully contained inside the overlapping buffer.
@ -502,7 +500,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size; ulong endAddress = address + size;
if (first.Address > address || first.EndAddress < endAddress) if (overlaps[0].Address > address || overlaps[0].EndAddress < endAddress)
{ {
bool anySparseCompatible = false; bool anySparseCompatible = false;
@ -515,39 +513,40 @@ namespace Ryujinx.Graphics.Gpu.Memory
// sequential memory. // sequential memory.
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the // Allowing for 2 pages (rather than just one) is necessary to catch cases where the
// range crosses a page, and after alignment, ends having a size of 2 pages. // range crosses a page, and after alignment, ends having a size of 2 pages.
if (first == last && if (overlaps.Length == 1 &&
address >= first.Address && address >= overlaps[0].Address &&
endAddress - first.EndAddress <= BufferAlignmentSize * 2) endAddress - overlaps[0].EndAddress <= BufferAlignmentSize * 2)
{ {
// Try to grow the buffer by 1.5x of its current size. // Try to grow the buffer by 1.5x of its current size.
// This improves performance in the cases where the buffer is resized often by small amounts. // This improves performance in the cases where the buffer is resized often by small amounts.
ulong existingSize = first.Value.Size; ulong existingSize = overlaps[0].Value.Size;
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
size = Math.Max(size, growthSize); size = Math.Max(size, growthSize);
endAddress = address + size; endAddress = address + size;
(first, last) = _buffers.FindOverlaps(address, size); overlaps = _buffers.FindOverlapsAsSpan(address, size);
} }
address = Math.Min(address, first.Address); address = Math.Min(address, overlaps[0].Address);
endAddress = Math.Max(endAddress, last.EndAddress); endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
List<Buffer> overlaps = []; RangeItem<Buffer>[] overlapsArray = overlaps.ToArray();
RangeItem<Buffer> current = first; for (int i = 0; i < overlaps.Length; i++)
while (current != last.Next)
{ {
anySparseCompatible |= current.Value.SparseCompatible; anySparseCompatible |= overlaps[i].Value.SparseCompatible;
overlaps.Add(current.Value);
_buffers.Remove(current.Value);
current = current.Next;
} }
_buffers.RemoveRange(overlaps[0], overlaps[^1]);
_buffers.Lock.ExitWriteLock();
ulong newSize = endAddress - address; ulong newSize = endAddress - address;
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps); Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlapsArray);
_buffers.Lock.EnterWriteLock();
_buffers.Add(newBuffer); _buffers.Add(newBuffer);
} }
@ -577,19 +576,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
bool sparseAligned = alignment >= SparseBufferAlignmentSize; bool sparseAligned = alignment >= SparseBufferAlignmentSize;
_buffers.Lock.EnterWriteLock(); _buffers.Lock.EnterWriteLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(address, size); Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(address, size);
if (first is not null) if (overlaps.Length > 0)
{ {
// If the buffer already exists, make sure if covers the entire range, // If the buffer already exists, make sure if covers the entire range,
// and make sure it is properly aligned, otherwise sparse mapping may fail. // and make sure it is properly aligned, otherwise sparse mapping may fail.
ulong endAddress = address + size; ulong endAddress = address + size;
if (first.Address > address || if (overlaps[0].Address > address ||
first.EndAddress < endAddress || overlaps[0].EndAddress < endAddress ||
(first.Address & (alignment - 1)) != 0 || (overlaps[0].Address & (alignment - 1)) != 0 ||
(!first.Value.SparseCompatible && sparseAligned)) (!overlaps[0].Value.SparseCompatible && sparseAligned))
{ {
// We need to make sure the new buffer is properly aligned. // We need to make sure the new buffer is properly aligned.
// However, after the range is aligned, it is possible that it // However, after the range is aligned, it is possible that it
@ -597,33 +596,30 @@ namespace Ryujinx.Graphics.Gpu.Memory
// and ensure we cover all overlaps. // and ensure we cover all overlaps.
RangeItem<Buffer> oldFirst; RangeItem<Buffer> oldFirst;
endAddress = Math.Max(endAddress, last.EndAddress); endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
do do
{ {
address = Math.Min(address, first.Address); address = Math.Min(address, overlaps[0].Address);
address &= ~(alignment - 1); address &= ~(alignment - 1);
oldFirst = first; oldFirst = overlaps[0];
(first, last) = _buffers.FindOverlaps(address, endAddress - address); overlaps = _buffers.FindOverlapsAsSpan(address, endAddress - address);
} }
while (oldFirst != first); while (oldFirst != overlaps[0]);
ulong newSize = endAddress - address; ulong newSize = endAddress - address;
List<Buffer> overlaps = []; RangeItem<Buffer>[] overlapsArray = overlaps.ToArray();
RangeItem<Buffer> current = first; _buffers.RemoveRange(overlaps[0], overlaps[^1]);
while (current != last.Next)
{
overlaps.Add(current.Value);
_buffers.Remove(current.Value);
current = current.Next;
}
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps); _buffers.Lock.ExitWriteLock();
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlapsArray);
_buffers.Lock.EnterWriteLock();
_buffers.Add(newBuffer); _buffers.Add(newBuffer);
} }
@ -635,6 +631,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_buffers.Add(buffer); _buffers.Add(buffer);
} }
_buffers.Lock.ExitWriteLock(); _buffers.Lock.ExitWriteLock();
} }
@ -648,13 +645,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="stage">The type of usage that created the buffer</param> /// <param name="stage">The type of usage that created the buffer</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param> /// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="overlaps">Buffers overlapping the range</param> /// <param name="overlaps">Buffers overlapping the range</param>
private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, List<Buffer> overlaps) private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, RangeItem<Buffer>[] overlaps)
{ {
Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps);
for (int index = 0; index < overlaps.Count; index++) for (int index = 0; index < overlaps.Length; index++)
{ {
Buffer buffer = overlaps[index]; Buffer buffer = overlaps[index].Value;
int dstOffset = (int)(buffer.Address - newBuffer.Address); int dstOffset = (int)(buffer.Address - newBuffer.Address);
@ -930,7 +927,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (size != 0) if (size != 0)
{ {
buffer = _buffers.FindOverlapFast(address, size).Value; buffer = _buffers.FindOverlap(address, size).Value;
buffer.CopyFromDependantVirtualBuffers(); buffer.CopyFromDependantVirtualBuffers();
buffer.SynchronizeMemory(address, size); buffer.SynchronizeMemory(address, size);

60
src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
View file

@ -80,8 +80,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
private BufferMigration _source; private BufferMigration _source;
private BufferModifiedRangeList _migrationTarget; private BufferModifiedRangeList _migrationTarget;
private List<RangeItem<BufferModifiedRange>> _overlaps;
/// <summary> /// <summary>
/// Whether the modified range list has any entries or not. /// Whether the modified range list has any entries or not.
@ -108,7 +106,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
_context = context; _context = context;
_parent = parent; _parent = parent;
_flushAction = flushAction; _flushAction = flushAction;
_overlaps = [];
} }
/// <summary> /// <summary>
@ -122,12 +119,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Slices a given region using the modified regions in the list. Calls the action for the new slices. // Slices a given region using the modified regions in the list. Calls the action for the new slices.
Lock.EnterReadLock(); Lock.EnterReadLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size); Span<RangeItem<BufferModifiedRange>> overlaps = FindOverlapsAsSpan(address, size);
RangeItem<BufferModifiedRange> current = first; for (int i = 0; i < overlaps.Length; i++)
while (last != null && current != last.Next)
{ {
BufferModifiedRange overlap = current.Value; BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.Address > address) if (overlap.Address > address)
{ {
@ -138,7 +134,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Remaining region is after this overlap. // Remaining region is after this overlap.
size -= overlap.EndAddress - address; size -= overlap.EndAddress - address;
address = overlap.EndAddress; address = overlap.EndAddress;
current = current.Next;
} }
Lock.ExitReadLock(); Lock.ExitReadLock();
@ -158,12 +153,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size of the modified region in bytes</param> /// <param name="size">Size of the modified region in bytes</param>
public void SignalModified(ulong address, ulong size) public void SignalModified(ulong address, ulong size)
{ {
// We may overlap with some existing modified regions. They must be cut into by the new entry.
Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
ulong endAddress = address + size; ulong endAddress = address + size;
ulong syncNumber = _context.SyncNumber; ulong syncNumber = _context.SyncNumber;
// We may overlap with some existing modified regions. They must be cut into by the new entry.
Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlapsAsNodes(address, size);
if (first is null) if (first is null)
{ {
@ -172,8 +166,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
return; return;
} }
if (first == last) if (first == last)
{ {
if (first.Address == address && first.EndAddress == endAddress) if (first.Address == address && first.EndAddress == endAddress)
@ -260,19 +252,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action<ulong, ulong> rangeAction) public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action<ulong, ulong> rangeAction)
{ {
Lock.EnterReadLock(); Lock.EnterReadLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size); Span<RangeItem<BufferModifiedRange>> overlaps = FindOverlapsAsSpan(address, size);
RangeItem<BufferModifiedRange> current = first; for (int i = 0; i < overlaps.Length; i++)
while (last != null && current != last.Next)
{ {
BufferModifiedRange overlap = current.Value; BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.SyncNumber == syncNumber) if (overlap.SyncNumber == syncNumber)
{ {
rangeAction(overlap.Address, overlap.Size); rangeAction(overlap.Address, overlap.Size);
} }
current = current.Next;
} }
Lock.ExitReadLock(); Lock.ExitReadLock();
@ -288,22 +277,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
// We use the non-span method here because keeping the lock will cause a deadlock. // We use the non-span method here because keeping the lock will cause a deadlock.
Lock.EnterReadLock(); Lock.EnterReadLock();
RangeItem<BufferModifiedRange>[] overlaps = FindOverlapsAsArray(address, size);
_overlaps.Clear();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
RangeItem<BufferModifiedRange> current = first;
while (last != null && current != last.Next)
{
_overlaps.Add(current);
current = current.Next;
}
Lock.ExitReadLock(); Lock.ExitReadLock();
for (int i = 0; i < _overlaps.Count; i++) for (int i = 0; i < overlaps.Length; i++)
{ {
BufferModifiedRange overlap = _overlaps[i].Value; BufferModifiedRange overlap = overlaps[i].Value;
rangeAction(overlap.Address, overlap.Size); rangeAction(overlap.Address, overlap.Size);
} }
} }
@ -404,8 +383,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size; ulong endAddress = address + size;
ulong currentSync = _context.SyncNumber; ulong currentSync = _context.SyncNumber;
List<RangeItem<BufferModifiedRange>> overlaps = [];
// Range list must be consistent for this operation // Range list must be consistent for this operation
if (_migrationTarget != null) if (_migrationTarget != null)
{ {
@ -416,16 +393,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
Lock.EnterWriteLock(); Lock.EnterWriteLock();
// We use the non-span method here because the array is partially modified by the code, which would invalidate a span. // We use the non-span method here because the array is partially modified by the code, which would invalidate a span.
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size); RangeItem<BufferModifiedRange>[] overlaps = FindOverlapsAsArray(address, size);
RangeItem<BufferModifiedRange> current = first;
while (last != null && current != last.Next)
{
overlaps.Add(current);
current = current.Next;
}
int rangeCount = overlaps.Count; int rangeCount = overlaps.Length;
if (rangeCount == 0) if (rangeCount == 0)
{ {
@ -582,7 +552,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
ulong endAddress = address + size; ulong endAddress = address + size;
Lock.EnterWriteLock(); Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size); (RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlapsAsNodes(address, size);
if (first is null) if (first is null)
{ {

2
src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs
View file

@ -122,7 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong originalVa = gpuVa; ulong originalVa = gpuVa;
_virtualRanges.Lock.EnterWriteLock(); _virtualRanges.Lock.EnterWriteLock();
(RangeItem<VirtualRange> first, RangeItem<VirtualRange> last) = _virtualRanges.FindOverlaps(gpuVa, size); (RangeItem<VirtualRange> first, RangeItem<VirtualRange> last) = _virtualRanges.FindOverlapsAsNodes(gpuVa, size);
if (first is not null) if (first is not null)
{ {

2
src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs
View file

@ -850,7 +850,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
{ {
lock (_threadingLock) lock (_threadingLock)
{ {
thread.ProcessListNode = _threads.AddLast(thread); _threads.AddLast(thread.ProcessListNode);
} }
} }

14
src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs
View file

@ -13,16 +13,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
Monitor.Exit(mutex); Monitor.Exit(mutex);
currentThread.Withholder = threadList;
currentThread.Reschedule(ThreadSchedState.Paused);
currentThread.WithholderNode = threadList.AddLast(currentThread);
if (currentThread.TerminationRequested) if (currentThread.TerminationRequested)
{ {
threadList.Remove(currentThread.WithholderNode);
currentThread.Reschedule(ThreadSchedState.Running); currentThread.Reschedule(ThreadSchedState.Running);
currentThread.Withholder = null; currentThread.Withholder = null;
@ -31,6 +23,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
} }
else else
{ {
currentThread.Withholder = threadList;
currentThread.Reschedule(ThreadSchedState.Paused);
threadList.AddLast(currentThread.WithholderNode);
if (timeout > 0) if (timeout > 0)
{ {
context.TimeManager.ScheduleFutureInvocation(currentThread, timeout); context.TimeManager.ScheduleFutureInvocation(currentThread, timeout);

2
src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs
View file

@ -50,7 +50,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
// even if they are not scheduled on guest cores. // even if they are not scheduled on guest cores.
if (currentThread != null && !currentThread.IsSchedulable && currentThread.Context.Running) if (currentThread != null && !currentThread.IsSchedulable && currentThread.Context.Running)
{ {
currentThread.SchedulerWaitEvent.WaitOne(); currentThread.SchedulerWaitEvent.Wait();
} }
} }
} }

8
src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs
View file

@ -194,7 +194,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return; return;
} }
thread.SiblingsPerCore[core] = SuggestedQueue(prio, core).AddFirst(thread); SuggestedQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]);
_suggestedPrioritiesPerCore[core] |= 1L << prio; _suggestedPrioritiesPerCore[core] |= 1L << prio;
} }
@ -223,7 +223,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return; return;
} }
thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddLast(thread); ScheduledQueue(prio, core).AddLast(thread.SiblingsPerCore[core]);
_scheduledPrioritiesPerCore[core] |= 1L << prio; _scheduledPrioritiesPerCore[core] |= 1L << prio;
} }
@ -235,7 +235,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return; return;
} }
thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddFirst(thread); ScheduledQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]);
_scheduledPrioritiesPerCore[core] |= 1L << prio; _scheduledPrioritiesPerCore[core] |= 1L << prio;
} }
@ -251,7 +251,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
queue.Remove(thread.SiblingsPerCore[core]); queue.Remove(thread.SiblingsPerCore[core]);
thread.SiblingsPerCore[core] = queue.AddLast(thread); queue.AddLast(thread.SiblingsPerCore[core]);
return queue.First?.Value; return queue.First?.Value;
} }

5
src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs
View file

@ -320,11 +320,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
if (nextThread == null) if (nextThread == null)
{ {
ActivateIdleThread(); ActivateIdleThread();
currentThread.SchedulerWaitEvent.WaitOne(); currentThread.SchedulerWaitEvent.Wait();
} }
else else
{ {
WaitHandle.SignalAndWait(nextThread.SchedulerWaitEvent, currentThread.SchedulerWaitEvent); nextThread.SchedulerWaitEvent.Set();
currentThread.SchedulerWaitEvent.Wait();
} }
} }
else else

23
src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs
View file

@ -19,9 +19,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
public const int MaxWaitSyncObjects = 64; public const int MaxWaitSyncObjects = 64;
private ManualResetEvent _schedulerWaitEvent; private ManualResetEventSlim _schedulerWaitEvent;
public ManualResetEvent SchedulerWaitEvent => _schedulerWaitEvent; public ManualResetEventSlim SchedulerWaitEvent => _schedulerWaitEvent;
public Thread HostThread { get; private set; } public Thread HostThread { get; private set; }
@ -73,6 +73,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
private LinkedListNode<KThread> _mutexWaiterNode; private LinkedListNode<KThread> _mutexWaiterNode;
private readonly LinkedList<KThread> _pinnedWaiters; private readonly LinkedList<KThread> _pinnedWaiters;
private LinkedListNode<KThread> _pinnedWaiterNode;
public KThread MutexOwner { get; private set; } public KThread MutexOwner { get; private set; }
@ -122,8 +123,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount]; SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
for (int i = 0; i < SiblingsPerCore.Length; i++)
{
SiblingsPerCore[i] = new LinkedListNode<KThread>(this);
}
_mutexWaiters = []; _mutexWaiters = [];
_pinnedWaiters = []; _pinnedWaiters = [];
WithholderNode = new LinkedListNode<KThread>(this);
ProcessListNode = new LinkedListNode<KThread>(this);
_mutexWaiterNode = new LinkedListNode<KThread>(this);
_pinnedWaiterNode = new LinkedListNode<KThread>(this);
} }
public Result Initialize( public Result Initialize(
@ -605,7 +616,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
break; break;
} }
_pinnedWaiters.AddLast(currentThread); _pinnedWaiters.AddLast(_pinnedWaiterNode);
currentThread.Reschedule(ThreadSchedState.Paused); currentThread.Reschedule(ThreadSchedState.Paused);
} }
@ -822,7 +833,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return KernelResult.ThreadTerminating; return KernelResult.ThreadTerminating;
} }
_pinnedWaiters.AddLast(currentThread); _pinnedWaiters.AddLast(_pinnedWaiterNode);
currentThread.Reschedule(ThreadSchedState.Paused); currentThread.Reschedule(ThreadSchedState.Paused);
} }
@ -1236,7 +1247,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
{ {
if (_schedulerWaitEvent == null) if (_schedulerWaitEvent == null)
{ {
var schedulerWaitEvent = new ManualResetEvent(false); var schedulerWaitEvent = new ManualResetEventSlim(false);
if (Interlocked.Exchange(ref _schedulerWaitEvent, schedulerWaitEvent) == null) if (Interlocked.Exchange(ref _schedulerWaitEvent, schedulerWaitEvent) == null)
{ {
@ -1251,7 +1262,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
private void ThreadStart() private void ThreadStart()
{ {
_schedulerWaitEvent.WaitOne(); _schedulerWaitEvent.Wait();
KernelStatic.SetKernelContext(KernelContext, this); KernelStatic.SetKernelContext(KernelContext, this);
if (_customThreadStart != null) if (_customThreadStart != null)

184
src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs
View file

@ -12,8 +12,8 @@ namespace Ryujinx.Memory.Range
/// <typeparam name="T">Type of the range.</typeparam> /// <typeparam name="T">Type of the range.</typeparam>
public unsafe class NonOverlappingRangeList<T> : RangeListBase<T> where T : class, INonOverlappingRange public unsafe class NonOverlappingRangeList<T> : RangeListBase<T> where T : class, INonOverlappingRange
{ {
private readonly Dictionary<ulong, RangeItem<T>> _quickAccess = new(AddressEqualityComparer.Comparer); // private readonly Dictionary<ulong, RangeItem<T>> _quickAccess = new(AddressEqualityComparer.Comparer);
private readonly Dictionary<ulong, RangeItem<T>> _fastQuickAccess = new(AddressEqualityComparer.Comparer); // private readonly Dictionary<ulong, RangeItem<T>> _fastQuickAccess = new(AddressEqualityComparer.Comparer);
public readonly ReaderWriterLockSlim Lock = new(); public readonly ReaderWriterLockSlim Lock = new();
@ -45,7 +45,7 @@ namespace Ryujinx.Memory.Range
Insert(index, rangeItem); Insert(index, rangeItem);
_quickAccess.Add(item.Address, rangeItem); // _quickAccess.Add(item.Address, rangeItem);
} }
/// <summary> /// <summary>
@ -71,15 +71,15 @@ namespace Ryujinx.Memory.Range
Items[index + 1].Previous = rangeItem; Items[index + 1].Previous = rangeItem;
} }
foreach (ulong addr in Items[index].QuickAccessAddresses) // foreach (ulong addr in Items[index].QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
Items[index] = rangeItem; Items[index] = rangeItem;
_quickAccess[item.Address] = rangeItem; // _quickAccess[item.Address] = rangeItem;
return true; return true;
} }
@ -108,18 +108,18 @@ namespace Ryujinx.Memory.Range
Items[index + 1].Previous = rangeItem; Items[index + 1].Previous = rangeItem;
} }
foreach (ulong addr in item.QuickAccessAddresses) // foreach (ulong addr in item.QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
Items[index] = rangeItem; Items[index] = rangeItem;
if (item.Address != rangeItem.Address) // if (item.Address != rangeItem.Address)
_quickAccess.Remove(item.Address); // _quickAccess.Remove(item.Address);
//
_quickAccess[rangeItem.Address] = rangeItem; // _quickAccess[rangeItem.Address] = rangeItem;
return true; return true;
} }
@ -196,13 +196,13 @@ namespace Ryujinx.Memory.Range
if (index >= 0 && Items[index].Value.Equals(item)) if (index >= 0 && Items[index].Value.Equals(item))
{ {
_quickAccess.Remove(item.Address); // _quickAccess.Remove(item.Address);
//
foreach (ulong addr in Items[index].QuickAccessAddresses) // foreach (ulong addr in Items[index].QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
RemoveAt(index); RemoveAt(index);
@ -232,15 +232,15 @@ namespace Ryujinx.Memory.Range
(int startIndex, int endIndex) = BinarySearchEdges(startItem.Address, endItem.EndAddress); (int startIndex, int endIndex) = BinarySearchEdges(startItem.Address, endItem.EndAddress);
for (int i = startIndex; i < endIndex; i++) // for (int i = startIndex; i < endIndex; i++)
{ // {
_quickAccess.Remove(Items[i].Address); // _quickAccess.Remove(Items[i].Address);
foreach (ulong addr in Items[i].QuickAccessAddresses) // foreach (ulong addr in Items[i].QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
} // }
if (endIndex < Count) if (endIndex < Count)
{ {
@ -279,12 +279,12 @@ namespace Ryujinx.Memory.Range
while (Items[endIndex] is not null && Items[endIndex].Address < address + size) while (Items[endIndex] is not null && Items[endIndex].Address < address + size)
{ {
_quickAccess.Remove(Items[endIndex].Address); // _quickAccess.Remove(Items[endIndex].Address);
foreach (ulong addr in Items[endIndex].QuickAccessAddresses) // foreach (ulong addr in Items[endIndex].QuickAccessAddresses)
{ // {
_quickAccess.Remove(addr); // _quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr); // _fastQuickAccess.Remove(addr);
} // }
if (endIndex == Count - 1) if (endIndex == Count - 1)
{ {
@ -321,9 +321,9 @@ namespace Ryujinx.Memory.Range
{ {
Lock.EnterWriteLock(); Lock.EnterWriteLock();
Count = 0; Count = 0;
_quickAccess.Clear();
_fastQuickAccess.Clear();
Lock.ExitWriteLock(); Lock.ExitWriteLock();
// _quickAccess.Clear();
// _fastQuickAccess.Clear();
} }
/// <summary> /// <summary>
@ -344,7 +344,7 @@ namespace Ryujinx.Memory.Range
// So we need to return both the split 0-1 and 1-2 ranges. // So we need to return both the split 0-1 and 1-2 ranges.
Lock.EnterWriteLock(); Lock.EnterWriteLock();
(RangeItem<T> first, RangeItem<T> last) = FindOverlaps(address, size); (RangeItem<T> first, RangeItem<T> last) = FindOverlapsAsNodes(address, size);
list = new List<T>(); list = new List<T>();
if (first is null) if (first is null)
@ -436,10 +436,10 @@ namespace Ryujinx.Memory.Range
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public override RangeItem<T> FindOverlap(ulong address, ulong size) public override RangeItem<T> FindOverlap(ulong address, ulong size)
{ {
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap)) // if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap))
{ // {
return overlap; // return overlap;
} // }
int index = BinarySearchLeftEdge(address, address + size); int index = BinarySearchLeftEdge(address, address + size);
@ -448,11 +448,11 @@ namespace Ryujinx.Memory.Range
return null; return null;
} }
if (Items[index].Address < address) // if (Items[index].Address < address)
{ // {
_quickAccess.TryAdd(address, Items[index]); // _quickAccess.TryAdd(address, Items[index]);
Items[index].QuickAccessAddresses.Add(address); // Items[index].QuickAccessAddresses.Add(address);
} // }
return Items[index]; return Items[index];
} }
@ -466,10 +466,10 @@ namespace Ryujinx.Memory.Range
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public override RangeItem<T> FindOverlapFast(ulong address, ulong size) public override RangeItem<T> FindOverlapFast(ulong address, ulong size)
{ {
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap) || _fastQuickAccess.TryGetValue(address, out overlap)) // if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap) || _fastQuickAccess.TryGetValue(address, out overlap))
{ // {
return overlap; // return overlap;
} // }
int index = BinarySearch(address, address + size); int index = BinarySearch(address, address + size);
@ -478,16 +478,16 @@ namespace Ryujinx.Memory.Range
return null; return null;
} }
if (Items[index].Address < address) // if (Items[index].Address < address)
{ // {
_quickAccess.TryAdd(address, Items[index]); // _quickAccess.TryAdd(address, Items[index]);
} // }
else // else
{ // {
_fastQuickAccess.TryAdd(address, Items[index]); // _fastQuickAccess.TryAdd(address, Items[index]);
} // }
//
Items[index].QuickAccessAddresses.Add(address); // Items[index].QuickAccessAddresses.Add(address);
return Items[index]; return Items[index];
} }
@ -499,18 +499,8 @@ namespace Ryujinx.Memory.Range
/// <param name="size">Size in bytes of the range</param> /// <param name="size">Size in bytes of the range</param>
/// <returns>The first and last overlapping items, or null if none are found</returns> /// <returns>The first and last overlapping items, or null if none are found</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public (RangeItem<T>, RangeItem<T>) FindOverlaps(ulong address, ulong size) public (RangeItem<T>, RangeItem<T>) FindOverlapsAsNodes(ulong address, ulong size)
{ {
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap))
{
if (overlap.Next is null || overlap.Next.Address >= address + size)
{
return (overlap, overlap);
}
return (overlap, Items[BinarySearchRightEdge(address, address + size)]);
}
(int index, int endIndex) = BinarySearchEdges(address, address + size); (int index, int endIndex) = BinarySearchEdges(address, address + size);
if (index < 0) if (index < 0)
@ -518,13 +508,45 @@ namespace Ryujinx.Memory.Range
return (null, null); return (null, null);
} }
if (Items[index].Address < address) return (Items[index], Items[endIndex - 1]);
}
public RangeItem<T>[] FindOverlapsAsArray(ulong address, ulong size)
{
(int index, int endIndex) = BinarySearchEdges(address, address + size);
RangeItem<T>[] result;
if (index < 0)
{ {
_quickAccess.TryAdd(address, Items[index]); result = [];
Items[index].QuickAccessAddresses.Add(address); }
else
{
result = new RangeItem<T>[endIndex - index];
Array.Copy(Items, index, result, 0, endIndex - index);
} }
return (Items[index], Items[endIndex - 1]); return result;
}
public Span<RangeItem<T>> FindOverlapsAsSpan(ulong address, ulong size)
{
(int index, int endIndex) = BinarySearchEdges(address, address + size);
Span<RangeItem<T>> result;
if (index < 0)
{
result = [];
}
else
{
result = Items.AsSpan().Slice(index, endIndex - index);
}
return result;
} }
public override IEnumerator<T> GetEnumerator() public override IEnumerator<T> GetEnumerator()

34
src/Ryujinx.Memory/Tracking/MemoryTracking.cs
View file

@ -1,4 +1,5 @@
using Ryujinx.Memory.Range; using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic; using System.Collections.Generic;
namespace Ryujinx.Memory.Tracking namespace Ryujinx.Memory.Tracking
@ -79,12 +80,10 @@ namespace Ryujinx.Memory.Tracking
{ {
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions; NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock(); regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(va, size); Span<RangeItem<VirtualRegion>> overlaps = regions.FindOverlapsAsSpan(va, size);
for (int i = 0; i < overlaps.Length; i++)
RangeItem<VirtualRegion> current = first;
while (last != null && current != last.Next)
{ {
VirtualRegion region = current.Value; VirtualRegion region = overlaps[i].Value;
// If the region has been fully remapped, signal that it has been mapped again. // If the region has been fully remapped, signal that it has been mapped again.
bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size);
@ -94,7 +93,6 @@ namespace Ryujinx.Memory.Tracking
} }
region.UpdateProtection(); region.UpdateProtection();
current = current.Next;
} }
regions.Lock.ExitReadLock(); regions.Lock.ExitReadLock();
} }
@ -118,15 +116,11 @@ namespace Ryujinx.Memory.Tracking
{ {
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions; NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock(); regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(va, size); Span<RangeItem<VirtualRegion>> overlaps = regions.FindOverlapsAsSpan(va, size);
RangeItem<VirtualRegion> current = first; for (int i = 0; i < overlaps.Length; i++)
while (last != null && current != last.Next)
{ {
VirtualRegion region = current.Value; overlaps[i].Value.SignalMappingChanged(false);
region.SignalMappingChanged(false);
current = current.Next;
} }
regions.Lock.ExitReadLock(); regions.Lock.ExitReadLock();
} }
@ -303,21 +297,13 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock) lock (TrackingLock)
{ {
NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions; NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions;
List<RangeItem<VirtualRegion>> overlaps = [];
// We use the non-span method here because keeping the lock will cause a deadlock. // We use the non-span method here because keeping the lock will cause a deadlock.
regions.Lock.EnterReadLock(); regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(address, size); RangeItem<VirtualRegion>[] overlaps = regions.FindOverlapsAsArray(address, size);
RangeItem<VirtualRegion> current = first;
while (last != null && current != last.Next)
{
overlaps.Add(current);
current = current.Next;
}
regions.Lock.ExitReadLock(); regions.Lock.ExitReadLock();
if (first is null && !precise) if (overlaps.Length == 0 && !precise)
{ {
if (_memoryManager.IsRangeMapped(address, size)) if (_memoryManager.IsRangeMapped(address, size))
{ {
@ -338,7 +324,7 @@ namespace Ryujinx.Memory.Tracking
size += (ulong)_pageSize; size += (ulong)_pageSize;
} }
for (int i = 0; i < overlaps.Count; i++) for (int i = 0; i < overlaps.Length; i++)
{ {
VirtualRegion region = overlaps[i].Value; VirtualRegion region = overlaps[i].Value;