From 65caa1e3f27ad45fbe75fccc395a09f25258b45f Mon Sep 17 00:00:00 2001 From: LotP <22-lotp@users.noreply.git.ryujinx.app> Date: Sat, 6 Sep 2025 11:10:55 -0500 Subject: [PATCH] Memory changes 2.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A few more internal changes to the RangeList systems. * No longer using a QuickAccess dictionary.   * The performance of the dictionary wasn't much faster than just doing binary searches.   * Using just binary searches allows us to take advantage of span and array returns as they're are faster than linked lists when iterating or copying the overlaps. Small code optimizations. Fixes a few leftover crashes. --- src/Ryujinx.Graphics.Device/DeviceState.cs | 25 ++- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 12 +- .../Memory/BufferBackingState.cs | 11 +- .../Memory/BufferCache.cs | 95 +++++---- .../Memory/BufferModifiedRangeList.cs | 60 ++---- .../Memory/VirtualRangeCache.cs | 2 +- .../HOS/Kernel/Process/KProcess.cs | 2 +- .../Kernel/Threading/KConditionVariable.cs | 14 +- .../HOS/Kernel/Threading/KCriticalSection.cs | 2 +- .../HOS/Kernel/Threading/KPriorityQueue.cs | 8 +- .../HOS/Kernel/Threading/KScheduler.cs | 5 +- .../HOS/Kernel/Threading/KThread.cs | 23 ++- .../Range/NonOverlappingRangeList.cs | 184 ++++++++++-------- src/Ryujinx.Memory/Tracking/MemoryTracking.cs | 34 +--- 14 files changed, 240 insertions(+), 237 deletions(-) diff --git a/src/Ryujinx.Graphics.Device/DeviceState.cs b/src/Ryujinx.Graphics.Device/DeviceState.cs index a8777cc5a..aaefc7494 100644 --- a/src/Ryujinx.Graphics.Device/DeviceState.cs +++ b/src/Ryujinx.Graphics.Device/DeviceState.cs @@ -96,7 +96,7 @@ namespace Ryujinx.Graphics.Device uint alignedOffset = index * RegisterSize; DebugWrite(alignedOffset, data); - GetRefIntAlignedUncheck(index) = data; + SetIntAlignedUncheck(index, data); _writeCallbacks[index]?.Invoke(data); } @@ -111,9 +111,7 @@ namespace Ryujinx.Graphics.Device uint alignedOffset = index * RegisterSize; DebugWrite(alignedOffset, data); - ref var storage = ref GetRefIntAlignedUncheck(index); - changed = storage != data; - storage = data; + changed = SetIntAlignedUncheckChanged(index, data); _writeCallbacks[index]?.Invoke(data); } @@ -153,5 +151,24 @@ namespace Ryujinx.Graphics.Device { return ref Unsafe.Add(ref Unsafe.As(ref State), (IntPtr)index); } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetIntAlignedUncheck(ulong index, int data) + { + Unsafe.Add(ref Unsafe.As(ref State), (nint)index) = data; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool SetIntAlignedUncheckChanged(ulong index, int data) + { + ref int val = ref Unsafe.Add(ref Unsafe.As(ref State), (nint)index); + if (val == data) + { + return false; + } + val = data; + + return true; + } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index d16889633..1d9cff1c4 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong size, BufferStage stage, bool sparseCompatible, - List baseBuffers) + RangeItem[] baseBuffers) { _context = context; _physicalMemory = physicalMemory; @@ -128,18 +128,18 @@ namespace Ryujinx.Graphics.Gpu.Memory List baseHandles = null; - if (baseBuffers.Count != 0) + if (baseBuffers.Length != 0) { baseHandles = new List(); - foreach (Buffer buffer in baseBuffers) + foreach (RangeItem item in baseBuffers) { - if (buffer._useGranular) + if (item.Value._useGranular) { - baseHandles.AddRange((buffer._memoryTrackingGranular.GetHandles())); + baseHandles.AddRange((item.Value._memoryTrackingGranular.GetHandles())); } else { - baseHandles.Add(buffer._memoryTracking); + baseHandles.Add(item.Value._memoryTracking); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs index 00abf0405..f5a5c13d5 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs @@ -1,4 +1,5 @@ using Ryujinx.Graphics.GAL; +using Ryujinx.Memory.Range; using System; using System.Collections.Generic; @@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Parent buffer /// Initial buffer stage /// Buffers to inherit state from - public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, List baseBuffers) + public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, RangeItem[] baseBuffers) { _size = (int)parent.Size; _systemMemoryType = context.Capabilities.MemoryType; @@ -72,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory BufferStage storageFlags = stage & BufferStage.StorageMask; - if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Count == 0) + if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Length == 0) { _desiredType = BufferBackingType.DeviceMemory; } @@ -100,11 +101,11 @@ namespace Ryujinx.Graphics.Gpu.Memory // TODO: Might be nice to force atomic access to be device local for any stage. } - if (baseBuffers.Count != 0) + if (baseBuffers.Length != 0) { - foreach (Buffer buffer in baseBuffers) + foreach (RangeItem item in baseBuffers) { - CombineState(buffer.BackingState); + CombineState(item.Value.BackingState); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index fe0ed4b4a..f5f870283 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -81,13 +81,11 @@ namespace Ryujinx.Graphics.Gpu.Memory MemoryRange subRange = range.GetSubRange(index); _buffers.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(subRange.Address, subRange.Size); + Span> overlaps = _buffers.FindOverlapsAsSpan(subRange.Address, subRange.Size); - RangeItem current = first; - while (last != null && current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - current.Value.Unmapped(subRange.Address, subRange.Size); - current = current.Next; + overlaps[i].Value.Unmapped(subRange.Address, subRange.Size); } _buffers.Lock.ExitReadLock(); @@ -490,9 +488,9 @@ namespace Ryujinx.Graphics.Gpu.Memory private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) { _buffers.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(address, size); + Span> overlaps = _buffers.FindOverlapsAsSpan(address, size); - if (first is not null) + if (overlaps.Length > 0) { // The buffer already exists. We can just return the existing buffer // if the buffer we need is fully contained inside the overlapping buffer. @@ -502,7 +500,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong endAddress = address + size; - if (first.Address > address || first.EndAddress < endAddress) + if (overlaps[0].Address > address || overlaps[0].EndAddress < endAddress) { bool anySparseCompatible = false; @@ -515,39 +513,40 @@ namespace Ryujinx.Graphics.Gpu.Memory // sequential memory. // Allowing for 2 pages (rather than just one) is necessary to catch cases where the // range crosses a page, and after alignment, ends having a size of 2 pages. - if (first == last && - address >= first.Address && - endAddress - first.EndAddress <= BufferAlignmentSize * 2) + if (overlaps.Length == 1 && + address >= overlaps[0].Address && + endAddress - overlaps[0].EndAddress <= BufferAlignmentSize * 2) { // Try to grow the buffer by 1.5x of its current size. // This improves performance in the cases where the buffer is resized often by small amounts. - ulong existingSize = first.Value.Size; + ulong existingSize = overlaps[0].Value.Size; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; size = Math.Max(size, growthSize); endAddress = address + size; - (first, last) = _buffers.FindOverlaps(address, size); + overlaps = _buffers.FindOverlapsAsSpan(address, size); } - address = Math.Min(address, first.Address); - endAddress = Math.Max(endAddress, last.EndAddress); + address = Math.Min(address, overlaps[0].Address); + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); - List overlaps = []; + RangeItem[] overlapsArray = overlaps.ToArray(); - RangeItem current = first; - while (current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - anySparseCompatible |= current.Value.SparseCompatible; - overlaps.Add(current.Value); - _buffers.Remove(current.Value); - - current = current.Next; + anySparseCompatible |= overlaps[i].Value.SparseCompatible; } + _buffers.RemoveRange(overlaps[0], overlaps[^1]); + + _buffers.Lock.ExitWriteLock(); + ulong newSize = endAddress - address; - Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps); + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlapsArray); + + _buffers.Lock.EnterWriteLock(); _buffers.Add(newBuffer); } @@ -577,19 +576,19 @@ namespace Ryujinx.Graphics.Gpu.Memory bool sparseAligned = alignment >= SparseBufferAlignmentSize; _buffers.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(address, size); + Span> overlaps = _buffers.FindOverlapsAsSpan(address, size); - if (first is not null) + if (overlaps.Length > 0) { // If the buffer already exists, make sure if covers the entire range, // and make sure it is properly aligned, otherwise sparse mapping may fail. ulong endAddress = address + size; - if (first.Address > address || - first.EndAddress < endAddress || - (first.Address & (alignment - 1)) != 0 || - (!first.Value.SparseCompatible && sparseAligned)) + if (overlaps[0].Address > address || + overlaps[0].EndAddress < endAddress || + (overlaps[0].Address & (alignment - 1)) != 0 || + (!overlaps[0].Value.SparseCompatible && sparseAligned)) { // We need to make sure the new buffer is properly aligned. // However, after the range is aligned, it is possible that it @@ -597,33 +596,30 @@ namespace Ryujinx.Graphics.Gpu.Memory // and ensure we cover all overlaps. RangeItem oldFirst; - endAddress = Math.Max(endAddress, last.EndAddress); + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); do { - address = Math.Min(address, first.Address); + address = Math.Min(address, overlaps[0].Address); address &= ~(alignment - 1); - oldFirst = first; - (first, last) = _buffers.FindOverlaps(address, endAddress - address); + oldFirst = overlaps[0]; + overlaps = _buffers.FindOverlapsAsSpan(address, endAddress - address); } - while (oldFirst != first); + while (oldFirst != overlaps[0]); ulong newSize = endAddress - address; - List overlaps = []; + RangeItem[] overlapsArray = overlaps.ToArray(); - RangeItem current = first; - while (current != last.Next) - { - overlaps.Add(current.Value); - _buffers.Remove(current.Value); - - current = current.Next; - } + _buffers.RemoveRange(overlaps[0], overlaps[^1]); - Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps); + _buffers.Lock.ExitWriteLock(); + + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlapsArray); + + _buffers.Lock.EnterWriteLock(); _buffers.Add(newBuffer); } @@ -635,6 +631,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _buffers.Add(buffer); } + _buffers.Lock.ExitWriteLock(); } @@ -648,13 +645,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer /// Indicates if the buffer can be used in a sparse buffer mapping /// Buffers overlapping the range - private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, List overlaps) + private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, RangeItem[] overlaps) { Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); - for (int index = 0; index < overlaps.Count; index++) + for (int index = 0; index < overlaps.Length; index++) { - Buffer buffer = overlaps[index]; + Buffer buffer = overlaps[index].Value; int dstOffset = (int)(buffer.Address - newBuffer.Address); @@ -930,7 +927,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (size != 0) { - buffer = _buffers.FindOverlapFast(address, size).Value; + buffer = _buffers.FindOverlap(address, size).Value; buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index 1c7adf2fc..46a76ed7c 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -80,8 +80,6 @@ namespace Ryujinx.Graphics.Gpu.Memory private BufferMigration _source; private BufferModifiedRangeList _migrationTarget; - - private List> _overlaps; /// /// Whether the modified range list has any entries or not. @@ -108,7 +106,6 @@ namespace Ryujinx.Graphics.Gpu.Memory _context = context; _parent = parent; _flushAction = flushAction; - _overlaps = []; } /// @@ -122,12 +119,11 @@ namespace Ryujinx.Graphics.Gpu.Memory // Slices a given region using the modified regions in the list. Calls the action for the new slices. Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + Span> overlaps = FindOverlapsAsSpan(address, size); - RangeItem current = first; - while (last != null && current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - BufferModifiedRange overlap = current.Value; + BufferModifiedRange overlap = overlaps[i].Value; if (overlap.Address > address) { @@ -138,7 +134,6 @@ namespace Ryujinx.Graphics.Gpu.Memory // Remaining region is after this overlap. size -= overlap.EndAddress - address; address = overlap.EndAddress; - current = current.Next; } Lock.ExitReadLock(); @@ -158,12 +153,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size of the modified region in bytes public void SignalModified(ulong address, ulong size) { - // We may overlap with some existing modified regions. They must be cut into by the new entry. - Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - ulong endAddress = address + size; ulong syncNumber = _context.SyncNumber; + // We may overlap with some existing modified regions. They must be cut into by the new entry. + Lock.EnterWriteLock(); + (RangeItem first, RangeItem last) = FindOverlapsAsNodes(address, size); if (first is null) { @@ -172,8 +166,6 @@ namespace Ryujinx.Graphics.Gpu.Memory return; } - - if (first == last) { if (first.Address == address && first.EndAddress == endAddress) @@ -260,19 +252,16 @@ namespace Ryujinx.Graphics.Gpu.Memory public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action rangeAction) { Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + Span> overlaps = FindOverlapsAsSpan(address, size); - RangeItem current = first; - while (last != null && current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - BufferModifiedRange overlap = current.Value; + BufferModifiedRange overlap = overlaps[i].Value; if (overlap.SyncNumber == syncNumber) { rangeAction(overlap.Address, overlap.Size); } - - current = current.Next; } Lock.ExitReadLock(); @@ -288,22 +277,12 @@ namespace Ryujinx.Graphics.Gpu.Memory { // We use the non-span method here because keeping the lock will cause a deadlock. Lock.EnterReadLock(); - - _overlaps.Clear(); - - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - - RangeItem current = first; - while (last != null && current != last.Next) - { - _overlaps.Add(current); - current = current.Next; - } + RangeItem[] overlaps = FindOverlapsAsArray(address, size); Lock.ExitReadLock(); - for (int i = 0; i < _overlaps.Count; i++) + for (int i = 0; i < overlaps.Length; i++) { - BufferModifiedRange overlap = _overlaps[i].Value; + BufferModifiedRange overlap = overlaps[i].Value; rangeAction(overlap.Address, overlap.Size); } } @@ -404,8 +383,6 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong endAddress = address + size; ulong currentSync = _context.SyncNumber; - List> overlaps = []; - // Range list must be consistent for this operation if (_migrationTarget != null) { @@ -416,16 +393,9 @@ namespace Ryujinx.Graphics.Gpu.Memory Lock.EnterWriteLock(); // We use the non-span method here because the array is partially modified by the code, which would invalidate a span. - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - - RangeItem current = first; - while (last != null && current != last.Next) - { - overlaps.Add(current); - current = current.Next; - } + RangeItem[] overlaps = FindOverlapsAsArray(address, size); - int rangeCount = overlaps.Count; + int rangeCount = overlaps.Length; if (rangeCount == 0) { @@ -582,7 +552,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { ulong endAddress = address + size; Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + (RangeItem first, RangeItem last) = FindOverlapsAsNodes(address, size); if (first is null) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs index 06253cb30..1d44ee65f 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs @@ -122,7 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong originalVa = gpuVa; _virtualRanges.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _virtualRanges.FindOverlaps(gpuVa, size); + (RangeItem first, RangeItem last) = _virtualRanges.FindOverlapsAsNodes(gpuVa, size); if (first is not null) { diff --git a/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs b/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs index 8e93cb4c8..62d9d310e 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs @@ -850,7 +850,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process { lock (_threadingLock) { - thread.ProcessListNode = _threads.AddLast(thread); + _threads.AddLast(thread.ProcessListNode); } } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs index c6aa984c2..cd2e9b543 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KConditionVariable.cs @@ -13,16 +13,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading Monitor.Exit(mutex); - currentThread.Withholder = threadList; - - currentThread.Reschedule(ThreadSchedState.Paused); - - currentThread.WithholderNode = threadList.AddLast(currentThread); - if (currentThread.TerminationRequested) { - threadList.Remove(currentThread.WithholderNode); - currentThread.Reschedule(ThreadSchedState.Running); currentThread.Withholder = null; @@ -31,6 +23,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading } else { + currentThread.Withholder = threadList; + + currentThread.Reschedule(ThreadSchedState.Paused); + + threadList.AddLast(currentThread.WithholderNode); + if (timeout > 0) { context.TimeManager.ScheduleFutureInvocation(currentThread, timeout); diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs index bfa6b68f6..d8887fbad 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KCriticalSection.cs @@ -50,7 +50,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading // even if they are not scheduled on guest cores. if (currentThread != null && !currentThread.IsSchedulable && currentThread.Context.Running) { - currentThread.SchedulerWaitEvent.WaitOne(); + currentThread.SchedulerWaitEvent.Wait(); } } } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs index 08f18be92..7e3d29851 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KPriorityQueue.cs @@ -194,7 +194,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading return; } - thread.SiblingsPerCore[core] = SuggestedQueue(prio, core).AddFirst(thread); + SuggestedQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]); _suggestedPrioritiesPerCore[core] |= 1L << prio; } @@ -223,7 +223,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading return; } - thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddLast(thread); + ScheduledQueue(prio, core).AddLast(thread.SiblingsPerCore[core]); _scheduledPrioritiesPerCore[core] |= 1L << prio; } @@ -235,7 +235,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading return; } - thread.SiblingsPerCore[core] = ScheduledQueue(prio, core).AddFirst(thread); + ScheduledQueue(prio, core).AddFirst(thread.SiblingsPerCore[core]); _scheduledPrioritiesPerCore[core] |= 1L << prio; } @@ -251,7 +251,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading queue.Remove(thread.SiblingsPerCore[core]); - thread.SiblingsPerCore[core] = queue.AddLast(thread); + queue.AddLast(thread.SiblingsPerCore[core]); return queue.First?.Value; } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs index b63b9dd41..86242e110 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs @@ -320,11 +320,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading if (nextThread == null) { ActivateIdleThread(); - currentThread.SchedulerWaitEvent.WaitOne(); + currentThread.SchedulerWaitEvent.Wait(); } else { - WaitHandle.SignalAndWait(nextThread.SchedulerWaitEvent, currentThread.SchedulerWaitEvent); + nextThread.SchedulerWaitEvent.Set(); + currentThread.SchedulerWaitEvent.Wait(); } } else diff --git a/src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs b/src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs index 0bc891036..437de7959 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs @@ -19,9 +19,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading public const int MaxWaitSyncObjects = 64; - private ManualResetEvent _schedulerWaitEvent; + private ManualResetEventSlim _schedulerWaitEvent; - public ManualResetEvent SchedulerWaitEvent => _schedulerWaitEvent; + public ManualResetEventSlim SchedulerWaitEvent => _schedulerWaitEvent; public Thread HostThread { get; private set; } @@ -73,6 +73,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading private LinkedListNode _mutexWaiterNode; private readonly LinkedList _pinnedWaiters; + private LinkedListNode _pinnedWaiterNode; public KThread MutexOwner { get; private set; } @@ -122,8 +123,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading SiblingsPerCore = new LinkedListNode[KScheduler.CpuCoresCount]; + for (int i = 0; i < SiblingsPerCore.Length; i++) + { + SiblingsPerCore[i] = new LinkedListNode(this); + } + _mutexWaiters = []; _pinnedWaiters = []; + + WithholderNode = new LinkedListNode(this); + ProcessListNode = new LinkedListNode(this); + _mutexWaiterNode = new LinkedListNode(this); + _pinnedWaiterNode = new LinkedListNode(this); } public Result Initialize( @@ -605,7 +616,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading break; } - _pinnedWaiters.AddLast(currentThread); + _pinnedWaiters.AddLast(_pinnedWaiterNode); currentThread.Reschedule(ThreadSchedState.Paused); } @@ -822,7 +833,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading return KernelResult.ThreadTerminating; } - _pinnedWaiters.AddLast(currentThread); + _pinnedWaiters.AddLast(_pinnedWaiterNode); currentThread.Reschedule(ThreadSchedState.Paused); } @@ -1236,7 +1247,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading { if (_schedulerWaitEvent == null) { - var schedulerWaitEvent = new ManualResetEvent(false); + var schedulerWaitEvent = new ManualResetEventSlim(false); if (Interlocked.Exchange(ref _schedulerWaitEvent, schedulerWaitEvent) == null) { @@ -1251,7 +1262,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading private void ThreadStart() { - _schedulerWaitEvent.WaitOne(); + _schedulerWaitEvent.Wait(); KernelStatic.SetKernelContext(KernelContext, this); if (_customThreadStart != null) diff --git a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs index 76513c9ef..2d0f36c72 100644 --- a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs +++ b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs @@ -12,8 +12,8 @@ namespace Ryujinx.Memory.Range /// Type of the range. public unsafe class NonOverlappingRangeList : RangeListBase where T : class, INonOverlappingRange { - private readonly Dictionary> _quickAccess = new(AddressEqualityComparer.Comparer); - private readonly Dictionary> _fastQuickAccess = new(AddressEqualityComparer.Comparer); + // private readonly Dictionary> _quickAccess = new(AddressEqualityComparer.Comparer); + // private readonly Dictionary> _fastQuickAccess = new(AddressEqualityComparer.Comparer); public readonly ReaderWriterLockSlim Lock = new(); @@ -45,7 +45,7 @@ namespace Ryujinx.Memory.Range Insert(index, rangeItem); - _quickAccess.Add(item.Address, rangeItem); + // _quickAccess.Add(item.Address, rangeItem); } /// @@ -71,15 +71,15 @@ namespace Ryujinx.Memory.Range Items[index + 1].Previous = rangeItem; } - foreach (ulong addr in Items[index].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } + // foreach (ulong addr in Items[index].QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } Items[index] = rangeItem; - _quickAccess[item.Address] = rangeItem; + // _quickAccess[item.Address] = rangeItem; return true; } @@ -108,18 +108,18 @@ namespace Ryujinx.Memory.Range Items[index + 1].Previous = rangeItem; } - foreach (ulong addr in item.QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } + // foreach (ulong addr in item.QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } Items[index] = rangeItem; - if (item.Address != rangeItem.Address) - _quickAccess.Remove(item.Address); - - _quickAccess[rangeItem.Address] = rangeItem; + // if (item.Address != rangeItem.Address) + // _quickAccess.Remove(item.Address); + // + // _quickAccess[rangeItem.Address] = rangeItem; return true; } @@ -196,13 +196,13 @@ namespace Ryujinx.Memory.Range if (index >= 0 && Items[index].Value.Equals(item)) { - _quickAccess.Remove(item.Address); - - foreach (ulong addr in Items[index].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } + // _quickAccess.Remove(item.Address); + // + // foreach (ulong addr in Items[index].QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } RemoveAt(index); @@ -232,15 +232,15 @@ namespace Ryujinx.Memory.Range (int startIndex, int endIndex) = BinarySearchEdges(startItem.Address, endItem.EndAddress); - for (int i = startIndex; i < endIndex; i++) - { - _quickAccess.Remove(Items[i].Address); - foreach (ulong addr in Items[i].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } - } + // for (int i = startIndex; i < endIndex; i++) + // { + // _quickAccess.Remove(Items[i].Address); + // foreach (ulong addr in Items[i].QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } + // } if (endIndex < Count) { @@ -279,12 +279,12 @@ namespace Ryujinx.Memory.Range while (Items[endIndex] is not null && Items[endIndex].Address < address + size) { - _quickAccess.Remove(Items[endIndex].Address); - foreach (ulong addr in Items[endIndex].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } + // _quickAccess.Remove(Items[endIndex].Address); + // foreach (ulong addr in Items[endIndex].QuickAccessAddresses) + // { + // _quickAccess.Remove(addr); + // _fastQuickAccess.Remove(addr); + // } if (endIndex == Count - 1) { @@ -321,9 +321,9 @@ namespace Ryujinx.Memory.Range { Lock.EnterWriteLock(); Count = 0; - _quickAccess.Clear(); - _fastQuickAccess.Clear(); Lock.ExitWriteLock(); + // _quickAccess.Clear(); + // _fastQuickAccess.Clear(); } /// @@ -344,7 +344,7 @@ namespace Ryujinx.Memory.Range // So we need to return both the split 0-1 and 1-2 ranges. Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + (RangeItem first, RangeItem last) = FindOverlapsAsNodes(address, size); list = new List(); if (first is null) @@ -436,10 +436,10 @@ namespace Ryujinx.Memory.Range [MethodImpl(MethodImplOptions.AggressiveInlining)] public override RangeItem FindOverlap(ulong address, ulong size) { - if (_quickAccess.TryGetValue(address, out RangeItem overlap)) - { - return overlap; - } + // if (_quickAccess.TryGetValue(address, out RangeItem overlap)) + // { + // return overlap; + // } int index = BinarySearchLeftEdge(address, address + size); @@ -448,11 +448,11 @@ namespace Ryujinx.Memory.Range return null; } - if (Items[index].Address < address) - { - _quickAccess.TryAdd(address, Items[index]); - Items[index].QuickAccessAddresses.Add(address); - } + // if (Items[index].Address < address) + // { + // _quickAccess.TryAdd(address, Items[index]); + // Items[index].QuickAccessAddresses.Add(address); + // } return Items[index]; } @@ -466,10 +466,10 @@ namespace Ryujinx.Memory.Range [MethodImpl(MethodImplOptions.AggressiveInlining)] public override RangeItem FindOverlapFast(ulong address, ulong size) { - if (_quickAccess.TryGetValue(address, out RangeItem overlap) || _fastQuickAccess.TryGetValue(address, out overlap)) - { - return overlap; - } + // if (_quickAccess.TryGetValue(address, out RangeItem overlap) || _fastQuickAccess.TryGetValue(address, out overlap)) + // { + // return overlap; + // } int index = BinarySearch(address, address + size); @@ -478,16 +478,16 @@ namespace Ryujinx.Memory.Range return null; } - if (Items[index].Address < address) - { - _quickAccess.TryAdd(address, Items[index]); - } - else - { - _fastQuickAccess.TryAdd(address, Items[index]); - } - - Items[index].QuickAccessAddresses.Add(address); + // if (Items[index].Address < address) + // { + // _quickAccess.TryAdd(address, Items[index]); + // } + // else + // { + // _fastQuickAccess.TryAdd(address, Items[index]); + // } + // + // Items[index].QuickAccessAddresses.Add(address); return Items[index]; } @@ -499,18 +499,8 @@ namespace Ryujinx.Memory.Range /// Size in bytes of the range /// The first and last overlapping items, or null if none are found [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (RangeItem, RangeItem) FindOverlaps(ulong address, ulong size) + public (RangeItem, RangeItem) FindOverlapsAsNodes(ulong address, ulong size) { - if (_quickAccess.TryGetValue(address, out RangeItem overlap)) - { - if (overlap.Next is null || overlap.Next.Address >= address + size) - { - return (overlap, overlap); - } - - return (overlap, Items[BinarySearchRightEdge(address, address + size)]); - } - (int index, int endIndex) = BinarySearchEdges(address, address + size); if (index < 0) @@ -518,13 +508,45 @@ namespace Ryujinx.Memory.Range return (null, null); } - if (Items[index].Address < address) + return (Items[index], Items[endIndex - 1]); + } + + public RangeItem[] FindOverlapsAsArray(ulong address, ulong size) + { + (int index, int endIndex) = BinarySearchEdges(address, address + size); + + RangeItem[] result; + + if (index < 0) { - _quickAccess.TryAdd(address, Items[index]); - Items[index].QuickAccessAddresses.Add(address); + result = []; + } + else + { + result = new RangeItem[endIndex - index]; + + Array.Copy(Items, index, result, 0, endIndex - index); } - return (Items[index], Items[endIndex - 1]); + return result; + } + + public Span> FindOverlapsAsSpan(ulong address, ulong size) + { + (int index, int endIndex) = BinarySearchEdges(address, address + size); + + Span> result; + + if (index < 0) + { + result = []; + } + else + { + result = Items.AsSpan().Slice(index, endIndex - index); + } + + return result; } public override IEnumerator GetEnumerator() diff --git a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs index 3b9b9fb3a..3511e7213 100644 --- a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -1,4 +1,5 @@ using Ryujinx.Memory.Range; +using System; using System.Collections.Generic; namespace Ryujinx.Memory.Tracking @@ -79,12 +80,10 @@ namespace Ryujinx.Memory.Tracking { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(va, size); - - RangeItem current = first; - while (last != null && current != last.Next) + Span> overlaps = regions.FindOverlapsAsSpan(va, size); + for (int i = 0; i < overlaps.Length; i++) { - VirtualRegion region = current.Value; + VirtualRegion region = overlaps[i].Value; // If the region has been fully remapped, signal that it has been mapped again. bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); @@ -94,7 +93,6 @@ namespace Ryujinx.Memory.Tracking } region.UpdateProtection(); - current = current.Next; } regions.Lock.ExitReadLock(); } @@ -118,15 +116,11 @@ namespace Ryujinx.Memory.Tracking { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(va, size); + Span> overlaps = regions.FindOverlapsAsSpan(va, size); - RangeItem current = first; - while (last != null && current != last.Next) + for (int i = 0; i < overlaps.Length; i++) { - VirtualRegion region = current.Value; - - region.SignalMappingChanged(false); - current = current.Next; + overlaps[i].Value.SignalMappingChanged(false); } regions.Lock.ExitReadLock(); } @@ -303,21 +297,13 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; - List> overlaps = []; // We use the non-span method here because keeping the lock will cause a deadlock. regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(address, size); - - RangeItem current = first; - while (last != null && current != last.Next) - { - overlaps.Add(current); - current = current.Next; - } + RangeItem[] overlaps = regions.FindOverlapsAsArray(address, size); regions.Lock.ExitReadLock(); - if (first is null && !precise) + if (overlaps.Length == 0 && !precise) { if (_memoryManager.IsRangeMapped(address, size)) { @@ -338,7 +324,7 @@ namespace Ryujinx.Memory.Tracking size += (ulong)_pageSize; } - for (int i = 0; i < overlaps.Count; i++) + for (int i = 0; i < overlaps.Length; i++) { VirtualRegion region = overlaps[i].Value;