mirrors/kenji-nx
1
0
Fork 0
mirror of https://git.ryujinx.app/kenji-nx/ryujinx.git synced 2025-12-13 13:37:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Memory changes 2.2

Browse source 
A few more internal changes to the RangeList systems.

* No longer using a QuickAccess dictionary.

  * The performance of the dictionary wasn't much faster than just doing binary searches.

  * Using just binary searches allows us to take advantage of span and array returns as they're are faster than linked lists when iterating or copying the overlaps.

Small code optimizations.

Fixes a few leftover crashes.
This commit is contained in:
LotP 2025-09-06 11:10:55 -05:00 • committed by KeatonTheBot
parent c2f54d0a5c
commit dae7ae7eaf
14 changed files with 242 additions and 239 deletions
Download patch file Download diff file Expand all files Collapse all files

25
src/Ryujinx.Graphics.Device/DeviceState.cs
View file

@ -96,7 +96,7 @@ namespace Ryujinx.Graphics.Device
uint alignedOffset = index * RegisterSize;
DebugWrite(alignedOffset, data);
GetRefIntAlignedUncheck(index) = data;
SetIntAlignedUncheck(index, data);
_writeCallbacks[index]?.Invoke(data);
}
@ -111,9 +111,7 @@ namespace Ryujinx.Graphics.Device
uint alignedOffset = index * RegisterSize;
DebugWrite(alignedOffset, data);
ref var storage = ref GetRefIntAlignedUncheck(index);
changed = storage != data;
storage = data;
changed = SetIntAlignedUncheckChanged(index, data);
_writeCallbacks[index]?.Invoke(data);
}
@ -153,5 +151,24 @@ namespace Ryujinx.Graphics.Device
{
return ref Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (IntPtr)index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void SetIntAlignedUncheck(ulong index, int data)
{
Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (nint)index) = data;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool SetIntAlignedUncheckChanged(ulong index, int data)
{
ref int val = ref Unsafe.Add(ref Unsafe.As<TState, int>(ref State), (nint)index);
if (val == data)
{
return false;
}
val = data;
return true;
}
}
}

12
src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
View file

@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong size,
BufferStage stage,
bool sparseCompatible,
List<Buffer> baseBuffers)
RangeItem<Buffer>[] baseBuffers)
{
_context = context;
_physicalMemory = physicalMemory;
@ -128,18 +128,18 @@ namespace Ryujinx.Graphics.Gpu.Memory
List<IRegionHandle> baseHandles = null;
if (baseBuffers.Count != 0)
if (baseBuffers.Length != 0)
{
baseHandles = new List<IRegionHandle>();
foreach (Buffer buffer in baseBuffers)
foreach (RangeItem<Buffer> item in baseBuffers)
{
if (buffer._useGranular)
if (item.Value._useGranular)
{
baseHandles.AddRange((buffer._memoryTrackingGranular.GetHandles()));
baseHandles.AddRange((item.Value._memoryTrackingGranular.GetHandles()));
}
else
{
baseHandles.Add(buffer._memoryTracking);
baseHandles.Add(item.Value._memoryTracking);
}
}
}

11
src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs
View file

@ -1,4 +1,5 @@
using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="parent">Parent buffer</param>
/// <param name="stage">Initial buffer stage</param>
/// <param name="baseBuffers">Buffers to inherit state from</param>
public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, List<Buffer> baseBuffers)
public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, RangeItem<Buffer>[] baseBuffers)
{
_size = (int)parent.Size;
_systemMemoryType = context.Capabilities.MemoryType;
@ -72,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferStage storageFlags = stage & BufferStage.StorageMask;
if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Count == 0)
if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Length == 0)
{
_desiredType = BufferBackingType.DeviceMemory;
}
@ -100,11 +101,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
// TODO: Might be nice to force atomic access to be device local for any stage.
}
if (baseBuffers.Count != 0)
if (baseBuffers.Length != 0)
{
foreach (Buffer buffer in baseBuffers)
foreach (RangeItem<Buffer> item in baseBuffers)
{
CombineState(buffer.BackingState);
CombineState(item.Value.BackingState);
}
}
}

95
src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
View file

@ -81,13 +81,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
MemoryRange subRange = range.GetSubRange(index);
_buffers.Lock.EnterReadLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(subRange.Address, subRange.Size);
Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(subRange.Address, subRange.Size);
RangeItem<Buffer> current = first;
while (last != null && current != last.Next)
for (int i = 0; i < overlaps.Length; i++)
{
current.Value.Unmapped(subRange.Address, subRange.Size);
current = current.Next;
overlaps[i].Value.Unmapped(subRange.Address, subRange.Size);
}
_buffers.Lock.ExitReadLock();
@ -490,9 +488,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage)
{
_buffers.Lock.EnterWriteLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(address, size);
Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(address, size);
if (first is not null)
if (overlaps.Length > 0)
{
// The buffer already exists. We can just return the existing buffer
// if the buffer we need is fully contained inside the overlapping buffer.
@ -502,7 +500,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size;
if (first.Address > address || first.EndAddress < endAddress)
if (overlaps[0].Address > address || overlaps[0].EndAddress < endAddress)
{
bool anySparseCompatible = false;
@ -515,39 +513,40 @@ namespace Ryujinx.Graphics.Gpu.Memory
// sequential memory.
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the
// range crosses a page, and after alignment, ends having a size of 2 pages.
if (first == last &&
address >= first.Address &&
endAddress - first.EndAddress <= BufferAlignmentSize * 2)
if (overlaps.Length == 1 &&
address >= overlaps[0].Address &&
endAddress - overlaps[0].EndAddress <= BufferAlignmentSize * 2)
{
// Try to grow the buffer by 1.5x of its current size.
// This improves performance in the cases where the buffer is resized often by small amounts.
ulong existingSize = first.Value.Size;
ulong existingSize = overlaps[0].Value.Size;
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
size = Math.Max(size, growthSize);
endAddress = address + size;
(first, last) = _buffers.FindOverlaps(address, size);
overlaps = _buffers.FindOverlapsAsSpan(address, size);
}
address = Math.Min(address, first.Address);
endAddress = Math.Max(endAddress, last.EndAddress);
address = Math.Min(address, overlaps[0].Address);
endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
List<Buffer> overlaps = [];
RangeItem<Buffer>[] overlapsArray = overlaps.ToArray();
RangeItem<Buffer> current = first;
while (current != last.Next)
for (int i = 0; i < overlaps.Length; i++)
{
anySparseCompatible |= current.Value.SparseCompatible;
overlaps.Add(current.Value);
_buffers.Remove(current.Value);
current = current.Next;
anySparseCompatible |= overlaps[i].Value.SparseCompatible;
}
_buffers.RemoveRange(overlaps[0], overlaps[^1]);
_buffers.Lock.ExitWriteLock();
ulong newSize = endAddress - address;
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps);
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlapsArray);
_buffers.Lock.EnterWriteLock();
_buffers.Add(newBuffer);
}
@ -577,19 +576,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
bool sparseAligned = alignment >= SparseBufferAlignmentSize;
_buffers.Lock.EnterWriteLock();
(RangeItem<Buffer> first, RangeItem<Buffer> last) = _buffers.FindOverlaps(address, size);
Span<RangeItem<Buffer>> overlaps = _buffers.FindOverlapsAsSpan(address, size);
if (first is not null)
if (overlaps.Length > 0)
{
// If the buffer already exists, make sure if covers the entire range,
// and make sure it is properly aligned, otherwise sparse mapping may fail.
ulong endAddress = address + size;
if (first.Address > address ||
first.EndAddress < endAddress ||
(first.Address & (alignment - 1)) != 0 ||
(!first.Value.SparseCompatible && sparseAligned))
if (overlaps[0].Address > address ||
overlaps[0].EndAddress < endAddress ||
(overlaps[0].Address & (alignment - 1)) != 0 ||
(!overlaps[0].Value.SparseCompatible && sparseAligned))
{
// We need to make sure the new buffer is properly aligned.
// However, after the range is aligned, it is possible that it
@ -597,33 +596,30 @@ namespace Ryujinx.Graphics.Gpu.Memory
// and ensure we cover all overlaps.
RangeItem<Buffer> oldFirst;
endAddress = Math.Max(endAddress, last.EndAddress);
endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
do
{
address = Math.Min(address, first.Address);
address = Math.Min(address, overlaps[0].Address);
address &= ~(alignment - 1);
oldFirst = first;
(first, last) = _buffers.FindOverlaps(address, endAddress - address);
oldFirst = overlaps[0];
overlaps = _buffers.FindOverlapsAsSpan(address, endAddress - address);
}
while (oldFirst != first);
while (oldFirst != overlaps[0]);
ulong newSize = endAddress - address;
List<Buffer> overlaps = [];
RangeItem<Buffer>[] overlapsArray = overlaps.ToArray();
RangeItem<Buffer> current = first;
while (current != last.Next)
{
overlaps.Add(current.Value);
_buffers.Remove(current.Value);
current = current.Next;
}
_buffers.RemoveRange(overlaps[0], overlaps[^1]);
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps);
_buffers.Lock.ExitWriteLock();
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlapsArray);
_buffers.Lock.EnterWriteLock();
_buffers.Add(newBuffer);
}
@ -635,6 +631,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_buffers.Add(buffer);
}
_buffers.Lock.ExitWriteLock();
}
@ -648,13 +645,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="stage">The type of usage that created the buffer</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="overlaps">Buffers overlapping the range</param>
private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, List<Buffer> overlaps)
private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, RangeItem<Buffer>[] overlaps)
{
Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps);
for (int index = 0; index < overlaps.Count; index++)
for (int index = 0; index < overlaps.Length; index++)
{
Buffer buffer = overlaps[index];
Buffer buffer = overlaps[index].Value;
int dstOffset = (int)(buffer.Address - newBuffer.Address);
@ -930,7 +927,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (size != 0)
{
buffer = _buffers.FindOverlapFast(address, size).Value;
buffer = _buffers.FindOverlap(address, size).Value;
buffer.CopyFromDependantVirtualBuffers();
buffer.SynchronizeMemory(address, size);

60
src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
View file

@ -80,8 +80,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
private BufferMigration _source;
private BufferModifiedRangeList _migrationTarget;
private List<RangeItem<BufferModifiedRange>> _overlaps;
/// <summary>
/// Whether the modified range list has any entries or not.
@ -108,7 +106,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
_context = context;
_parent = parent;
_flushAction = flushAction;
_overlaps = [];
}
/// <summary>
@ -122,12 +119,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Slices a given region using the modified regions in the list. Calls the action for the new slices.
Lock.EnterReadLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
Span<RangeItem<BufferModifiedRange>> overlaps = FindOverlapsAsSpan(address, size);
RangeItem<BufferModifiedRange> current = first;
while (last != null && current != last.Next)
for (int i = 0; i < overlaps.Length; i++)
{
BufferModifiedRange overlap = current.Value;
BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.Address > address)
{
@ -138,7 +134,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Remaining region is after this overlap.
size -= overlap.EndAddress - address;
address = overlap.EndAddress;
current = current.Next;
}
Lock.ExitReadLock();
@ -158,12 +153,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size of the modified region in bytes</param>
public void SignalModified(ulong address, ulong size)
{
// We may overlap with some existing modified regions. They must be cut into by the new entry.
Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
ulong endAddress = address + size;
ulong syncNumber = _context.SyncNumber;
// We may overlap with some existing modified regions. They must be cut into by the new entry.
Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlapsAsNodes(address, size);
if (first is null)
{
@ -172,8 +166,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
return;
}
if (first == last)
{
if (first.Address == address && first.EndAddress == endAddress)
@ -260,19 +252,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action<ulong, ulong> rangeAction)
{
Lock.EnterReadLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
Span<RangeItem<BufferModifiedRange>> overlaps = FindOverlapsAsSpan(address, size);
RangeItem<BufferModifiedRange> current = first;
while (last != null && current != last.Next)
for (int i = 0; i < overlaps.Length; i++)
{
BufferModifiedRange overlap = current.Value;
BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.SyncNumber == syncNumber)
{
rangeAction(overlap.Address, overlap.Size);
}
current = current.Next;
}
Lock.ExitReadLock();
@ -288,22 +277,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
// We use the non-span method here because keeping the lock will cause a deadlock.
Lock.EnterReadLock();
_overlaps.Clear();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
RangeItem<BufferModifiedRange> current = first;
while (last != null && current != last.Next)
{
_overlaps.Add(current);
current = current.Next;
}
RangeItem<BufferModifiedRange>[] overlaps = FindOverlapsAsArray(address, size);
Lock.ExitReadLock();
for (int i = 0; i < _overlaps.Count; i++)
for (int i = 0; i < overlaps.Length; i++)
{
BufferModifiedRange overlap = _overlaps[i].Value;
BufferModifiedRange overlap = overlaps[i].Value;
rangeAction(overlap.Address, overlap.Size);
}
}
@ -404,8 +383,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size;
ulong currentSync = _context.SyncNumber;
List<RangeItem<BufferModifiedRange>> overlaps = [];
// Range list must be consistent for this operation
if (_migrationTarget != null)
{
@ -416,16 +393,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
Lock.EnterWriteLock();
// We use the non-span method here because the array is partially modified by the code, which would invalidate a span.
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
RangeItem<BufferModifiedRange> current = first;
while (last != null && current != last.Next)
{
overlaps.Add(current);
current = current.Next;
}
RangeItem<BufferModifiedRange>[] overlaps = FindOverlapsAsArray(address, size);
int rangeCount = overlaps.Count;
int rangeCount = overlaps.Length;
if (rangeCount == 0)
{
@ -582,7 +552,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
ulong endAddress = address + size;
Lock.EnterWriteLock();
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlaps(address, size);
(RangeItem<BufferModifiedRange> first, RangeItem<BufferModifiedRange> last) = FindOverlapsAsNodes(address, size);
if (first is null)
{

2
src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs
View file

@ -122,7 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong originalVa = gpuVa;
_virtualRanges.Lock.EnterWriteLock();
(RangeItem<VirtualRange> first, RangeItem<VirtualRange> last) = _virtualRanges.FindOverlaps(gpuVa, size);
(RangeItem<VirtualRange> first, RangeItem<VirtualRange> last) = _virtualRanges.FindOverlapsAsNodes(gpuVa, size);
if (first is not null)
{

4
src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs
View file

@ -873,7 +873,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
{
lock (_threadingLock)
{
thread.ProcessListNode = _threads.AddLast(thread);
_threads.AddLast(thread.ProcessListNode);
}
}
@ -1235,7 +1235,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
{
thread.Suspend(ThreadSchedState.ThreadPauseFlag);
thread.Context.RequestInterrupt();
if (!thread.DebugHalt.WaitOne(TimeSpan.FromMilliseconds(50)))
if (!thread.DebugHalt.Wait(TimeSpan.FromMilliseconds(50)))
{
Logger.Warning?.Print(LogClass.Kernel, $"Failed to suspend thread {thread.ThreadUid} in time.");
}

14
src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs
View file

@ -13,16 +13,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
Monitor.Exit(mutex);
currentThread.Withholder = threadList;
currentThread.Reschedule(ThreadSchedState.Paused);
currentThread.WithholderNode = threadList.AddLast(currentThread);
if (currentThread.TerminationRequested)
{
threadList.Remove(currentThread.WithholderNode);
currentThread.Reschedule(ThreadSchedState.Running);
currentThread.Withholder = null;
@ -31,6 +23,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
}
else
{
currentThread.Withholder = threadList;
currentThread.Reschedule(ThreadSchedState.Paused);
threadList.AddLast(currentThread.WithholderNode);
if (timeout > 0)
{
context.TimeManager.ScheduleFutureInvocation(currentThread, timeout);

2
src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs
View file

@ -50,7 +50,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
// even if they are not scheduled on guest cores.
if (currentThread != null && !currentThread.IsSchedulable && currentThread.Context.Running)
{
currentThread.SchedulerWaitEvent.WaitOne();
currentThread.SchedulerWaitEvent.Wait();
}
}
}

8
src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs
View file

@ -194,7 +194,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return;
}
thread.SiblingsPerCore[core] = SuggestedQueue(prio, core).AddFirst(thread);
SuggestedQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]);
_suggestedPrioritiesPerCore[core] |= 1L << prio;
}
@ -223,7 +223,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return;
}
thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddLast(thread);
ScheduledQueue(prio, core).AddLast(thread.SiblingsPerCore[core]);
_scheduledPrioritiesPerCore[core] |= 1L << prio;
}
@ -235,7 +235,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return;
}
thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddFirst(thread);
ScheduledQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]);
_scheduledPrioritiesPerCore[core] |= 1L << prio;
}
@ -251,7 +251,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
queue.Remove(thread.SiblingsPerCore[core]);
thread.SiblingsPerCore[core] = queue.AddLast(thread);
queue.AddLast(thread.SiblingsPerCore[core]);
return queue.First?.Value;
}

5
src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs
View file

@ -321,11 +321,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
if (nextThread == null)
{
ActivateIdleThread();
currentThread.SchedulerWaitEvent.WaitOne();
currentThread.SchedulerWaitEvent.Wait();
}
else
{
WaitHandle.SignalAndWait(nextThread.SchedulerWaitEvent, currentThread.SchedulerWaitEvent);
nextThread.SchedulerWaitEvent.Set();
currentThread.SchedulerWaitEvent.Wait();
}
}
else

25
src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs
View file

@ -39,9 +39,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
public const int MaxWaitSyncObjects = 64;
private ManualResetEvent _schedulerWaitEvent;
private ManualResetEventSlim _schedulerWaitEvent;
public ManualResetEvent SchedulerWaitEvent => _schedulerWaitEvent;
public ManualResetEventSlim SchedulerWaitEvent => _schedulerWaitEvent;
public Thread HostThread { get; private set; }
@ -93,6 +93,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
private LinkedListNode<KThread> _mutexWaiterNode;
private readonly LinkedList<KThread> _pinnedWaiters;
private LinkedListNode<KThread> _pinnedWaiterNode;
public KThread MutexOwner { get; private set; }
@ -135,7 +136,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
private readonly Lock _activityOperationLock = new();
internal readonly ManualResetEvent DebugHalt = new(false);
internal readonly ManualResetEventSlim DebugHalt = new(false);
public KThread(KernelContext context) : base(context)
{
@ -144,8 +145,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
for (int i = 0; i < SiblingsPerCore.Length; i++)
{
SiblingsPerCore[i] = new LinkedListNode<KThread>(this);
}
_mutexWaiters = [];
_pinnedWaiters = [];
WithholderNode = new LinkedListNode<KThread>(this);
ProcessListNode = new LinkedListNode<KThread>(this);
_mutexWaiterNode = new LinkedListNode<KThread>(this);
_pinnedWaiterNode = new LinkedListNode<KThread>(this);
}
public Result Initialize(
@ -631,7 +642,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
break;
}
_pinnedWaiters.AddLast(currentThread);
_pinnedWaiters.AddLast(_pinnedWaiterNode);
currentThread.Reschedule(ThreadSchedState.Paused);
}
@ -848,7 +859,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
return KernelResult.ThreadTerminating;
}
_pinnedWaiters.AddLast(currentThread);
_pinnedWaiters.AddLast(_pinnedWaiterNode);
currentThread.Reschedule(ThreadSchedState.Paused);
}
@ -1262,7 +1273,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
{
if (_schedulerWaitEvent == null)
{
var schedulerWaitEvent = new ManualResetEvent(false);
var schedulerWaitEvent = new ManualResetEventSlim(false);
if (Interlocked.Exchange(ref _schedulerWaitEvent, schedulerWaitEvent) == null)
{
@ -1277,7 +1288,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
private void ThreadStart()
{
_schedulerWaitEvent.WaitOne();
_schedulerWaitEvent.Wait();
DebugHalt.Reset();
KernelStatic.SetKernelContext(KernelContext, this);

184
src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs
View file

@ -12,8 +12,8 @@ namespace Ryujinx.Memory.Range
/// <typeparam name="T">Type of the range.</typeparam>
public unsafe class NonOverlappingRangeList<T> : RangeListBase<T> where T : class, INonOverlappingRange
{
private readonly Dictionary<ulong, RangeItem<T>> _quickAccess = new(AddressEqualityComparer.Comparer);
private readonly Dictionary<ulong, RangeItem<T>> _fastQuickAccess = new(AddressEqualityComparer.Comparer);
// private readonly Dictionary<ulong, RangeItem<T>> _quickAccess = new(AddressEqualityComparer.Comparer);
// private readonly Dictionary<ulong, RangeItem<T>> _fastQuickAccess = new(AddressEqualityComparer.Comparer);
public readonly ReaderWriterLockSlim Lock = new();
@ -45,7 +45,7 @@ namespace Ryujinx.Memory.Range
Insert(index, rangeItem);
_quickAccess.Add(item.Address, rangeItem);
// _quickAccess.Add(item.Address, rangeItem);
}
/// <summary>
@ -71,15 +71,15 @@ namespace Ryujinx.Memory.Range
Items[index + 1].Previous = rangeItem;
}
foreach (ulong addr in Items[index].QuickAccessAddresses)
{
_quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr);
}
// foreach (ulong addr in Items[index].QuickAccessAddresses)
// {
// _quickAccess.Remove(addr);
// _fastQuickAccess.Remove(addr);
// }
Items[index] = rangeItem;
_quickAccess[item.Address] = rangeItem;
// _quickAccess[item.Address] = rangeItem;
return true;
}
@ -108,18 +108,18 @@ namespace Ryujinx.Memory.Range
Items[index + 1].Previous = rangeItem;
}
foreach (ulong addr in item.QuickAccessAddresses)
{
_quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr);
}
// foreach (ulong addr in item.QuickAccessAddresses)
// {
// _quickAccess.Remove(addr);
// _fastQuickAccess.Remove(addr);
// }
Items[index] = rangeItem;
if (item.Address != rangeItem.Address)
_quickAccess.Remove(item.Address);
_quickAccess[rangeItem.Address] = rangeItem;
// if (item.Address != rangeItem.Address)
// _quickAccess.Remove(item.Address);
//
// _quickAccess[rangeItem.Address] = rangeItem;
return true;
}
@ -196,13 +196,13 @@ namespace Ryujinx.Memory.Range
if (index >= 0 && Items[index].Value.Equals(item))
{
_quickAccess.Remove(item.Address);
foreach (ulong addr in Items[index].QuickAccessAddresses)
{
_quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr);
}
// _quickAccess.Remove(item.Address);
//
// foreach (ulong addr in Items[index].QuickAccessAddresses)
// {
// _quickAccess.Remove(addr);
// _fastQuickAccess.Remove(addr);
// }
RemoveAt(index);
@ -232,15 +232,15 @@ namespace Ryujinx.Memory.Range
(int startIndex, int endIndex) = BinarySearchEdges(startItem.Address, endItem.EndAddress);
for (int i = startIndex; i < endIndex; i++)
{
_quickAccess.Remove(Items[i].Address);
foreach (ulong addr in Items[i].QuickAccessAddresses)
{
_quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr);
}
}
// for (int i = startIndex; i < endIndex; i++)
// {
// _quickAccess.Remove(Items[i].Address);
// foreach (ulong addr in Items[i].QuickAccessAddresses)
// {
// _quickAccess.Remove(addr);
// _fastQuickAccess.Remove(addr);
// }
// }
if (endIndex < Count)
{
@ -279,12 +279,12 @@ namespace Ryujinx.Memory.Range
while (Items[endIndex] is not null && Items[endIndex].Address < address + size)
{
_quickAccess.Remove(Items[endIndex].Address);
foreach (ulong addr in Items[endIndex].QuickAccessAddresses)
{
_quickAccess.Remove(addr);
_fastQuickAccess.Remove(addr);
}
// _quickAccess.Remove(Items[endIndex].Address);
// foreach (ulong addr in Items[endIndex].QuickAccessAddresses)
// {
// _quickAccess.Remove(addr);
// _fastQuickAccess.Remove(addr);
// }
if (endIndex == Count - 1)
{
@ -321,9 +321,9 @@ namespace Ryujinx.Memory.Range
{
Lock.EnterWriteLock();
Count = 0;
_quickAccess.Clear();
_fastQuickAccess.Clear();
Lock.ExitWriteLock();
// _quickAccess.Clear();
// _fastQuickAccess.Clear();
}
/// <summary>
@ -344,7 +344,7 @@ namespace Ryujinx.Memory.Range
// So we need to return both the split 0-1 and 1-2 ranges.
Lock.EnterWriteLock();
(RangeItem<T> first, RangeItem<T> last) = FindOverlaps(address, size);
(RangeItem<T> first, RangeItem<T> last) = FindOverlapsAsNodes(address, size);
list = new List<T>();
if (first is null)
@ -436,10 +436,10 @@ namespace Ryujinx.Memory.Range
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override RangeItem<T> FindOverlap(ulong address, ulong size)
{
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap))
{
return overlap;
}
// if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap))
// {
// return overlap;
// }
int index = BinarySearchLeftEdge(address, address + size);
@ -448,11 +448,11 @@ namespace Ryujinx.Memory.Range
return null;
}
if (Items[index].Address < address)
{
_quickAccess.TryAdd(address, Items[index]);
Items[index].QuickAccessAddresses.Add(address);
}
// if (Items[index].Address < address)
// {
// _quickAccess.TryAdd(address, Items[index]);
// Items[index].QuickAccessAddresses.Add(address);
// }
return Items[index];
}
@ -466,10 +466,10 @@ namespace Ryujinx.Memory.Range
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override RangeItem<T> FindOverlapFast(ulong address, ulong size)
{
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap) || _fastQuickAccess.TryGetValue(address, out overlap))
{
return overlap;
}
// if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap) || _fastQuickAccess.TryGetValue(address, out overlap))
// {
// return overlap;
// }
int index = BinarySearch(address, address + size);
@ -478,16 +478,16 @@ namespace Ryujinx.Memory.Range
return null;
}
if (Items[index].Address < address)
{
_quickAccess.TryAdd(address, Items[index]);
}
else
{
_fastQuickAccess.TryAdd(address, Items[index]);
}
Items[index].QuickAccessAddresses.Add(address);
// if (Items[index].Address < address)
// {
// _quickAccess.TryAdd(address, Items[index]);
// }
// else
// {
// _fastQuickAccess.TryAdd(address, Items[index]);
// }
//
// Items[index].QuickAccessAddresses.Add(address);
return Items[index];
}
@ -499,18 +499,8 @@ namespace Ryujinx.Memory.Range
/// <param name="size">Size in bytes of the range</param>
/// <returns>The first and last overlapping items, or null if none are found</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public (RangeItem<T>, RangeItem<T>) FindOverlaps(ulong address, ulong size)
public (RangeItem<T>, RangeItem<T>) FindOverlapsAsNodes(ulong address, ulong size)
{
if (_quickAccess.TryGetValue(address, out RangeItem<T> overlap))
{
if (overlap.Next is null || overlap.Next.Address >= address + size)
{
return (overlap, overlap);
}
return (overlap, Items[BinarySearchRightEdge(address, address + size)]);
}
(int index, int endIndex) = BinarySearchEdges(address, address + size);
if (index < 0)
@ -518,13 +508,45 @@ namespace Ryujinx.Memory.Range
return (null, null);
}
if (Items[index].Address < address)
return (Items[index], Items[endIndex - 1]);
}
public RangeItem<T>[] FindOverlapsAsArray(ulong address, ulong size)
{
(int index, int endIndex) = BinarySearchEdges(address, address + size);
RangeItem<T>[] result;
if (index < 0)
{
_quickAccess.TryAdd(address, Items[index]);
Items[index].QuickAccessAddresses.Add(address);
result = [];
}
else
{
result = new RangeItem<T>[endIndex - index];
Array.Copy(Items, index, result, 0, endIndex - index);
}
return (Items[index], Items[endIndex - 1]);
return result;
}
public Span<RangeItem<T>> FindOverlapsAsSpan(ulong address, ulong size)
{
(int index, int endIndex) = BinarySearchEdges(address, address + size);
Span<RangeItem<T>> result;
if (index < 0)
{
result = [];
}
else
{
result = Items.AsSpan().Slice(index, endIndex - index);
}
return result;
}
public override IEnumerator<T> GetEnumerator()

34
src/Ryujinx.Memory/Tracking/MemoryTracking.cs
View file

@ -1,4 +1,5 @@
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
namespace Ryujinx.Memory.Tracking
@ -79,12 +80,10 @@ namespace Ryujinx.Memory.Tracking
{
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(va, size);
RangeItem<VirtualRegion> current = first;
while (last != null && current != last.Next)
Span<RangeItem<VirtualRegion>> overlaps = regions.FindOverlapsAsSpan(va, size);
for (int i = 0; i < overlaps.Length; i++)
{
VirtualRegion region = current.Value;
VirtualRegion region = overlaps[i].Value;
// If the region has been fully remapped, signal that it has been mapped again.
bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size);
@ -94,7 +93,6 @@ namespace Ryujinx.Memory.Tracking
}
region.UpdateProtection();
current = current.Next;
}
regions.Lock.ExitReadLock();
}
@ -118,15 +116,11 @@ namespace Ryujinx.Memory.Tracking
{
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(va, size);
Span<RangeItem<VirtualRegion>> overlaps = regions.FindOverlapsAsSpan(va, size);
RangeItem<VirtualRegion> current = first;
while (last != null && current != last.Next)
for (int i = 0; i < overlaps.Length; i++)
{
VirtualRegion region = current.Value;
region.SignalMappingChanged(false);
current = current.Next;
overlaps[i].Value.SignalMappingChanged(false);
}
regions.Lock.ExitReadLock();
}
@ -303,21 +297,13 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock)
{
NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions;
List<RangeItem<VirtualRegion>> overlaps = [];
// We use the non-span method here because keeping the lock will cause a deadlock.
regions.Lock.EnterReadLock();
(RangeItem<VirtualRegion> first, RangeItem<VirtualRegion> last) = regions.FindOverlaps(address, size);
RangeItem<VirtualRegion> current = first;
while (last != null && current != last.Next)
{
overlaps.Add(current);
current = current.Next;
}
RangeItem<VirtualRegion>[] overlaps = regions.FindOverlapsAsArray(address, size);
regions.Lock.ExitReadLock();
if (first is null && !precise)
if (overlaps.Length == 0 && !precise)
{
if (_memoryManager.IsRangeMapped(address, size))
{
@ -338,7 +324,7 @@ namespace Ryujinx.Memory.Tracking
size += (ulong)_pageSize;
}
for (int i = 0; i < overlaps.Count; i++)
for (int i = 0; i < overlaps.Length; i++)
{
VirtualRegion region = overlaps[i].Value;