From 48c809a80d1019ed420b283e25660b613188ff34 Mon Sep 17 00:00:00 2001 From: KeatonTheBot Date: Sat, 16 Aug 2025 23:15:55 -0500 Subject: [PATCH] Revert "Memory Changes" This reverts commit d5c9bc662c40b5e329781e106aa87c1eda26dbcf. Solves crash in Kirby Star Allies after a few mins of gameplay, possibly other titles. --- .../Collections/IntrusiveRedBlackTree.cs | 89 +-- .../Collections/IntrusiveRedBlackTreeNode.cs | 5 +- .../Collections/TreeDictionary.cs | 11 +- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 49 +- .../Memory/BufferBackingState.cs | 6 +- .../Memory/BufferCache.cs | 183 +++--- .../Memory/BufferManager.cs | 12 +- .../Memory/BufferModifiedRangeList.cs | 485 ++++++---------- .../Memory/BufferStage.cs | 2 - .../Memory/MemoryManager.cs | 7 +- .../Memory/VirtualRangeCache.cs | 62 +- .../HOS/Kernel/Memory/KMemoryBlockManager.cs | 30 +- .../Range/INonOverlappingRange.cs | 2 +- .../Range/NonOverlappingRangeList.cs | 409 +------------ src/Ryujinx.Memory/Range/RangeList.cs | 546 ++++++++++-------- src/Ryujinx.Memory/Range/RangeListBase.cs | 359 ------------ src/Ryujinx.Memory/Tracking/MemoryTracking.cs | 62 +- 17 files changed, 736 insertions(+), 1583 deletions(-) delete mode 100644 src/Ryujinx.Memory/Range/RangeListBase.cs diff --git a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs index 8cb0ae7e9..9e56f707b 100644 --- a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs +++ b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs @@ -14,13 +14,12 @@ namespace Ryujinx.Common.Collections /// Adds a new node into the tree. /// /// Node to be added - /// Node to be added under /// is null - public void Add(T node, T parent = null) + public void Add(T node) { ArgumentNullException.ThrowIfNull(node); - Insert(node, parent); + Insert(node); } /// @@ -76,11 +75,9 @@ namespace Ryujinx.Common.Collections /// Inserts a new node into the tree. /// /// Node to be inserted - /// Node to be inserted under - private void Insert(T node, T parent = null) + private void Insert(T node) { - T newNode = parent != null ? InsertWithParent(node, parent) : BSTInsert(node); - + T newNode = BSTInsert(node); RestoreBalanceAfterInsertion(newNode); } @@ -123,78 +120,10 @@ namespace Ryujinx.Common.Collections else if (newNode.CompareTo(parent) < 0) { parent.Left = newNode; - - newNode.Successor = parent; - - if (parent.Predecessor != null) - { - newNode.Predecessor = parent.Predecessor; - parent.Predecessor = newNode; - newNode.Predecessor.Successor = newNode; - } - - parent.Predecessor = newNode; } else { parent.Right = newNode; - - newNode.Predecessor = parent; - - if (parent.Successor != null) - { - newNode.Successor = parent.Successor; - newNode.Successor.Predecessor = newNode; - } - - parent.Successor = newNode; - } - Count++; - return newNode; - } - - /// - /// Insertion Mechanism for a Binary Search Tree (BST). - ///

- /// Inserts a new node directly under a parent node - /// where all children in the left subtree are less than , - /// and all children in the right subtree are greater than . - /// - /// Node to be inserted - /// Node to be inserted under - /// The inserted Node - private T InsertWithParent(T newNode, T parent) - { - newNode.Parent = parent; - - if (newNode.CompareTo(parent) < 0) - { - parent.Left = newNode; - - newNode.Successor = parent; - - if (parent.Predecessor != null) - { - newNode.Predecessor = parent.Predecessor; - parent.Predecessor = newNode; - newNode.Predecessor.Successor = newNode; - } - - parent.Predecessor = newNode; - } - else - { - parent.Right = newNode; - - newNode.Predecessor = parent; - - if (parent.Successor != null) - { - newNode.Successor = parent.Successor; - newNode.Successor.Predecessor = newNode; - } - - parent.Successor = newNode; } Count++; return newNode; @@ -227,7 +156,7 @@ namespace Ryujinx.Common.Collections } else { - T element = nodeToDelete.Successor; + T element = Minimum(RightOf(nodeToDelete)); child = RightOf(element); parent = ParentOf(element); @@ -255,9 +184,6 @@ namespace Ryujinx.Common.Collections element.Left = old.Left; element.Right = old.Right; element.Parent = old.Parent; - element.Predecessor = old.Predecessor; - if (element.Predecessor != null) - element.Predecessor.Successor = element; if (ParentOf(old) == null) { @@ -312,11 +238,6 @@ namespace Ryujinx.Common.Collections { RestoreBalanceAfterRemoval(child); } - - if (old.Successor != null) - old.Successor.Predecessor = old.Predecessor; - if (old.Predecessor != null) - old.Predecessor.Successor = old.Successor; return old; } diff --git a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs index 57e0b27c8..29d2d0c9a 100644 --- a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs +++ b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs @@ -9,7 +9,8 @@ namespace Ryujinx.Common.Collections public T Left; public T Right; public T Parent; - public T Predecessor; - public T Successor; + + public T Predecessor => IntrusiveRedBlackTreeImpl.PredecessorOf((T)this); + public T Successor => IntrusiveRedBlackTreeImpl.SuccessorOf((T)this); } } diff --git a/src/Ryujinx.Common/Collections/TreeDictionary.cs b/src/Ryujinx.Common/Collections/TreeDictionary.cs index 4bb871b93..9d15ba367 100644 --- a/src/Ryujinx.Common/Collections/TreeDictionary.cs +++ b/src/Ryujinx.Common/Collections/TreeDictionary.cs @@ -107,7 +107,7 @@ namespace Ryujinx.Common.Collections Node node = GetNode(key); if (node != null) { - Node successor = node.Successor; + Node successor = SuccessorOf(node); return successor != null ? successor.Key : default; } @@ -124,7 +124,7 @@ namespace Ryujinx.Common.Collections Node node = GetNode(key); if (node != null) { - Node predecessor = node.Predecessor; + Node predecessor = PredecessorOf(node); return predecessor != null ? predecessor.Key : default; } @@ -132,10 +132,11 @@ namespace Ryujinx.Common.Collections } /// - /// Adds all the nodes in the dictionary as key/value pairs into a list. + /// Adds all the nodes in the dictionary as key/value pairs into . ///

/// The key/value pairs will be added in Level Order. /// + /// List to add the tree pairs into public List> AsLevelOrderList() { List> list = []; @@ -162,7 +163,7 @@ namespace Ryujinx.Common.Collections } /// - /// Adds all the nodes in the dictionary into a list. + /// Adds all the nodes in the dictionary into . /// /// A list of all KeyValuePairs sorted by Key Order public List> AsList() @@ -274,7 +275,7 @@ namespace Ryujinx.Common.Collections } } Node newNode = new(key, value, parent); - if (parent == null) + if (newNode.Parent == null) { Root = newNode; } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index d16889633..a9444daa4 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -15,7 +15,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Buffer, used to store vertex and index data, uniform and storage buffers, and others. /// - class Buffer : INonOverlappingRange, ISyncActionHandler, IDisposable + class Buffer : IRange, ISyncActionHandler, IDisposable { private const ulong GranularBufferThreshold = 4096; @@ -35,7 +35,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Size of the buffer in bytes. /// - public ulong Size { get; private set; } + public ulong Size { get; } /// /// End address of the buffer in guest memory. @@ -60,13 +60,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// This is null until at least one modification occurs. /// - private BufferModifiedRangeList _modifiedRanges; + private BufferModifiedRangeList _modifiedRanges = null; /// /// A structure that is used to flush buffer data back to a host mapped buffer for cached readback. /// Only used if the buffer data is explicitly owned by device local memory. /// - private BufferPreFlush _preFlush; + private BufferPreFlush _preFlush = null; /// /// Usage tracking state that determines what type of backing the buffer should use. @@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong size, BufferStage stage, bool sparseCompatible, - List baseBuffers) + IEnumerable baseBuffers = null) { _context = context; _physicalMemory = physicalMemory; @@ -126,22 +126,21 @@ namespace Ryujinx.Graphics.Gpu.Memory _useGranular = size > GranularBufferThreshold; - List baseHandles = null; + IEnumerable baseHandles = null; - if (baseBuffers.Count != 0) + if (baseBuffers != null) { - baseHandles = new List(); - foreach (Buffer buffer in baseBuffers) + baseHandles = baseBuffers.SelectMany(buffer => { if (buffer._useGranular) { - baseHandles.AddRange((buffer._memoryTrackingGranular.GetHandles())); + return buffer._memoryTrackingGranular.GetHandles(); } else { - baseHandles.Add(buffer._memoryTracking); + return Enumerable.Repeat(buffer._memoryTracking, 1); } - } + }); } if (_useGranular) @@ -172,9 +171,9 @@ namespace Ryujinx.Graphics.Gpu.Memory _memoryTracking.RegisterPreciseAction(PreciseAction); } - _externalFlushDelegate = ExternalFlush; - _loadDelegate = LoadRegion; - _modifiedDelegate = RegionModified; + _externalFlushDelegate = new RegionSignal(ExternalFlush); + _loadDelegate = new Action(LoadRegion); + _modifiedDelegate = new Action(RegionModified); _virtualDependenciesLock = new ReaderWriterLockSlim(); } @@ -248,11 +247,6 @@ namespace Ryujinx.Graphics.Gpu.Memory return Address < address + size && address < EndAddress; } - public INonOverlappingRange Split(ulong splitAddress) - { - throw new NotImplementedException(); - } - /// /// Checks if a given range is fully contained in the buffer. /// @@ -441,7 +435,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The buffer to inherit from public void InheritModifiedRanges(Buffer from) { - if (from._modifiedRanges is { HasRanges: true }) + if (from._modifiedRanges != null && from._modifiedRanges.HasRanges) { if (from._syncActionRegistered && !_syncActionRegistered) { @@ -449,7 +443,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _syncActionRegistered = true; } - void RegisterRangeAction(ulong address, ulong size) + void registerRangeAction(ulong address, ulong size) { if (_useGranular) { @@ -463,7 +457,7 @@ namespace Ryujinx.Graphics.Gpu.Memory EnsureRangeList(); - _modifiedRanges.InheritRanges(from._modifiedRanges, RegisterRangeAction); + _modifiedRanges.InheritRanges(from._modifiedRanges, registerRangeAction); } if (from._dirtyStart != ulong.MaxValue) @@ -505,7 +499,14 @@ namespace Ryujinx.Graphics.Gpu.Memory { // Cut off the start. - _dirtyStart = end < _dirtyEnd ? end : ulong.MaxValue; + if (end < _dirtyEnd) + { + _dirtyStart = end; + } + else + { + _dirtyStart = ulong.MaxValue; + } } else if (end >= _dirtyEnd) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs index 00abf0405..3f65131e6 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs @@ -56,7 +56,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Parent buffer /// Initial buffer stage /// Buffers to inherit state from - public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, List baseBuffers) + public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable baseBuffers = null) { _size = (int)parent.Size; _systemMemoryType = context.Capabilities.MemoryType; @@ -72,7 +72,7 @@ namespace Ryujinx.Graphics.Gpu.Memory BufferStage storageFlags = stage & BufferStage.StorageMask; - if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Count == 0) + if (parent.Size > DeviceLocalSizeThreshold && baseBuffers == null) { _desiredType = BufferBackingType.DeviceMemory; } @@ -100,7 +100,7 @@ namespace Ryujinx.Graphics.Gpu.Memory // TODO: Might be nice to force atomic access to be device local for any stage. } - if (baseBuffers.Count != 0) + if (baseBuffers != null) { foreach (Buffer buffer in baseBuffers) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index f88381cd8..d19ba4779 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -2,6 +2,7 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Memory.Range; using System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory @@ -38,9 +39,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Only modified from the GPU thread. Must lock for add/remove. /// Must lock for any access from other threads. /// - private readonly NonOverlappingRangeList _buffers; + private readonly RangeList _buffers; private readonly MultiRangeList _multiRangeBuffers; + private Buffer[] _bufferOverlaps; + private readonly Dictionary _dirtyCache; private readonly Dictionary _modifiedCache; private bool _pruneCaches; @@ -61,6 +64,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _buffers = []; _multiRangeBuffers = []; + _bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity]; + _dirtyCache = new Dictionary(); // There are a lot more entries on the modified cache, so it is separate from the one for ForceDirty. @@ -74,23 +79,24 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Event arguments public void MemoryUnmappedHandler(object sender, UnmapEventArgs e) { + Buffer[] overlaps = new Buffer[10]; + int overlapCount; + MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size); for (int index = 0; index < range.Count; index++) { MemoryRange subRange = range.GetSubRange(index); - _buffers.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(subRange.Address, subRange.Size); - - RangeItem current = first; - while (last != null && current != last.Next) + lock (_buffers) { - current.Value.Unmapped(subRange.Address, subRange.Size); - current = current.Next; + overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps); } - _buffers.Lock.ExitReadLock(); + for (int i = 0; i < overlapCount; i++) + { + overlaps[i].Unmapped(subRange.Address, subRange.Size); + } } } @@ -131,7 +137,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Physical ranges of the buffer, after address translation public MultiRange TranslateAndCreateMultiBuffers(MemoryManager memoryManager, ulong gpuVa, ulong size, BufferStage stage) { - if (gpuVa == 0 || size == 0) + if (gpuVa == 0) { return new MultiRange(MemoryManager.PteUnmapped, size); } @@ -330,7 +336,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; ulong alignedSize = alignedEndAddress - alignedAddress; - Buffer buffer = _buffers.FindOverlap(alignedAddress, alignedSize).Value; + Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); @@ -397,7 +403,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (subRange.Address != MemoryManager.PteUnmapped) { - Buffer buffer = _buffers.FindOverlap(subRange.Address, subRange.Size).Value; + Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size); physicalBuffers.Add(buffer); @@ -489,10 +495,10 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) { - _buffers.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(address, size); + Buffer[] overlaps = _bufferOverlaps; + int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); - if (first is not null) + if (overlapsCount != 0) { // The buffer already exists. We can just return the existing buffer // if the buffer we need is fully contained inside the overlapping buffer. @@ -501,8 +507,9 @@ namespace Ryujinx.Graphics.Gpu.Memory // old buffer(s) to the new buffer. ulong endAddress = address + size; + Buffer overlap0 = overlaps[0]; - if (first.Address > address || first.EndAddress < endAddress) + if (overlap0.Address > address || overlap0.EndAddress < endAddress) { bool anySparseCompatible = false; @@ -515,52 +522,53 @@ namespace Ryujinx.Graphics.Gpu.Memory // sequential memory. // Allowing for 2 pages (rather than just one) is necessary to catch cases where the // range crosses a page, and after alignment, ends having a size of 2 pages. - if (first == last && - address >= first.Address && - endAddress - first.EndAddress <= BufferAlignmentSize * 2) + if (overlapsCount == 1 && + address >= overlap0.Address && + endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2) { // Try to grow the buffer by 1.5x of its current size. // This improves performance in the cases where the buffer is resized often by small amounts. - ulong existingSize = first.Value.Size; + ulong existingSize = overlap0.Size; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; size = Math.Max(size, growthSize); endAddress = address + size; - (first, last) = _buffers.FindOverlaps(address, size); + overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); } - address = Math.Min(address, first.Address); - endAddress = Math.Max(endAddress, last.EndAddress); - - List overlaps = []; - - RangeItem current = first; - while (current != last.Next) + for (int index = 0; index < overlapsCount; index++) { - anySparseCompatible |= current.Value.SparseCompatible; - overlaps.Add(current.Value); - _buffers.Remove(current.Value); + Buffer buffer = overlaps[index]; - current = current.Next; + anySparseCompatible |= buffer.SparseCompatible; + + address = Math.Min(address, buffer.Address); + endAddress = Math.Max(endAddress, buffer.EndAddress); + + lock (_buffers) + { + _buffers.Remove(buffer); + } } ulong newSize = endAddress - address; - Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps); - - _buffers.Add(newBuffer); + CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps, overlapsCount); } } else { // No overlap, just create a new buffer. - Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false, []); + Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false); - _buffers.Add(buffer); + lock (_buffers) + { + _buffers.Add(buffer); + } } - _buffers.Lock.ExitWriteLock(); + ShrinkOverlapsBufferIfNeeded(); } /// @@ -574,68 +582,72 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Alignment of the start address of the buffer private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, ulong alignment) { + Buffer[] overlaps = _bufferOverlaps; + int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); bool sparseAligned = alignment >= SparseBufferAlignmentSize; - _buffers.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _buffers.FindOverlaps(address, size); - - if (first is not null) + if (overlapsCount != 0) { // If the buffer already exists, make sure if covers the entire range, // and make sure it is properly aligned, otherwise sparse mapping may fail. ulong endAddress = address + size; + Buffer overlap0 = overlaps[0]; - if (first.Address > address || - first.EndAddress < endAddress || - (first.Address & (alignment - 1)) != 0 || - (!first.Value.SparseCompatible && sparseAligned)) + if (overlap0.Address > address || + overlap0.EndAddress < endAddress || + (overlap0.Address & (alignment - 1)) != 0 || + (!overlap0.SparseCompatible && sparseAligned)) { // We need to make sure the new buffer is properly aligned. // However, after the range is aligned, it is possible that it // overlaps more buffers, so try again after each extension // and ensure we cover all overlaps. - RangeItem oldFirst; - endAddress = Math.Max(endAddress, last.EndAddress); + int oldOverlapsCount; do { - address = Math.Min(address, first.Address); + for (int index = 0; index < overlapsCount; index++) + { + Buffer buffer = overlaps[index]; + + address = Math.Min(address, buffer.Address); + endAddress = Math.Max(endAddress, buffer.EndAddress); + } address &= ~(alignment - 1); - oldFirst = first; - (first, last) = _buffers.FindOverlaps(address, endAddress - address); + oldOverlapsCount = overlapsCount; + overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps); + } + while (oldOverlapsCount != overlapsCount); + + lock (_buffers) + { + for (int index = 0; index < overlapsCount; index++) + { + _buffers.Remove(overlaps[index]); + } } - while (oldFirst != first); ulong newSize = endAddress - address; - List overlaps = []; - - RangeItem current = first; - while (current != last.Next) - { - overlaps.Add(current.Value); - _buffers.Remove(current.Value); - - current = current.Next; - } - - Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps); - - _buffers.Add(newBuffer); + CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps, overlapsCount); } } else { // No overlap, just create a new buffer. - Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned, []); + Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned); - _buffers.Add(buffer); + lock (_buffers) + { + _buffers.Add(buffer); + } } - _buffers.Lock.ExitWriteLock(); + + ShrinkOverlapsBufferIfNeeded(); } /// @@ -648,11 +660,17 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer /// Indicates if the buffer can be used in a sparse buffer mapping /// Buffers overlapping the range - private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, List overlaps) + /// Total of overlaps + private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, Buffer[] overlaps, int overlapsCount) { - Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); + Buffer newBuffer = new Buffer(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps.Take(overlapsCount)); - for (int index = 0; index < overlaps.Count; index++) + lock (_buffers) + { + _buffers.Add(newBuffer); + } + + for (int index = 0; index < overlapsCount; index++) { Buffer buffer = overlaps[index]; @@ -670,8 +688,6 @@ namespace Ryujinx.Graphics.Gpu.Memory NotifyBuffersModified?.Invoke(); RecreateMultiRangeBuffers(address, size); - - return newBuffer; } /// @@ -702,6 +718,17 @@ namespace Ryujinx.Graphics.Gpu.Memory } } + /// + /// Resizes the temporary buffer used for range list intersection results, if it has grown too much. + /// + private void ShrinkOverlapsBufferIfNeeded() + { + if (_bufferOverlaps.Length > OverlapsBufferMaxCapacity) + { + Array.Resize(ref _bufferOverlaps, OverlapsBufferMaxCapacity); + } + } + /// /// Copy a buffer data from a given address to another. /// @@ -882,7 +909,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { MemoryRange subRange = range.GetSubRange(i); - Buffer subBuffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size).Value; + Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); subBuffer.SynchronizeMemory(subRange.Address, subRange.Size); @@ -930,7 +957,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (size != 0) { - buffer = _buffers.FindOverlapFast(address, size).Value; + buffer = _buffers.FindFirstOverlap(address, size); buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); @@ -942,7 +969,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { - buffer = _buffers.FindOverlapFast(address, 1).Value; + buffer = _buffers.FindFirstOverlap(address, 1); } return buffer; @@ -980,7 +1007,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { if (size != 0) { - Buffer buffer = _buffers.FindOverlapFast(address, size).Value; + Buffer buffer = _buffers.FindFirstOverlap(address, size); if (copyBackVirtual) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index d51f96783..88b240e6b 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -258,7 +258,7 @@ namespace Ryujinx.Graphics.Gpu.Memory RecordStorageAlignment(_cpStorageBuffers, index, gpuVa); - gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); + gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size, BufferStageUtils.ComputeStorage(flags)); @@ -282,7 +282,7 @@ namespace Ryujinx.Graphics.Gpu.Memory RecordStorageAlignment(buffers, index, gpuVa); - gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); + gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size, BufferStageUtils.GraphicsStorage(stage, flags)); @@ -761,7 +761,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!bounds.IsUnmapped) { - var isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write; + var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var range = isStorage ? bufferCache.GetBufferRangeAligned(bounds.Range, bufferStage | BufferStageUtils.FromUsage(bounds.Flags), isWrite) : bufferCache.GetBufferRange(bounds.Range, bufferStage); @@ -798,7 +798,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!bounds.IsUnmapped) { - var isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write; + var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var range = isStorage ? bufferCache.GetBufferRangeAligned(bounds.Range, BufferStageUtils.ComputeStorage(bounds.Flags), isWrite) : bufferCache.GetBufferRange(bounds.Range, BufferStage.Compute); @@ -817,6 +817,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Bind respective buffer bindings on the host API. /// /// Host buffers to bind, with their offsets and sizes + /// First binding point /// Number of bindings /// Indicates if the buffers are storage or uniform buffers [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -865,6 +866,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Buffer texture /// Physical ranges of memory where the buffer texture data is located /// Binding info for the buffer texture + /// Format of the buffer texture /// Whether the binding is for an image or a sampler public void SetBufferTextureStorage( ShaderStage stage, @@ -887,6 +889,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Physical ranges of memory where the buffer texture data is located /// Binding info for the buffer texture /// Index of the binding on the array + /// Format of the buffer texture public void SetBufferTextureStorage( ShaderStage stage, ITextureArray array, @@ -909,6 +912,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Physical ranges of memory where the buffer texture data is located /// Binding info for the buffer texture /// Index of the binding on the array + /// Format of the buffer texture public void SetBufferTextureStorage( ShaderStage stage, IImageArray array, diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index 29cc0e209..7a125ca9c 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -1,24 +1,25 @@ +using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; using System; -using System.Collections.Generic; using System.Linq; +using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { /// /// A range within a buffer that has been modified by the GPU. /// - class BufferModifiedRange : INonOverlappingRange + class BufferModifiedRange : IRange { /// /// Start address of the range in guest memory. /// - public ulong Address { get; internal set; } + public ulong Address { get; } /// /// Size of the range in bytes. /// - public ulong Size { get; internal set; } + public ulong Size { get; } /// /// End address of the range in guest memory. @@ -60,19 +61,14 @@ namespace Ryujinx.Graphics.Gpu.Memory { return Address < address + size && address < EndAddress; } - - public INonOverlappingRange Split(ulong splitAddress) - { - throw new NotImplementedException(); - } } /// /// A structure used to track GPU modified ranges within a buffer. /// - class BufferModifiedRangeList : NonOverlappingRangeList + class BufferModifiedRangeList : RangeList { - private new const int BackingInitialSize = 8; + private const int BackingInitialSize = 8; private readonly GpuContext _context; private readonly Buffer _parent; @@ -81,6 +77,8 @@ namespace Ryujinx.Graphics.Gpu.Memory private BufferMigration _source; private BufferModifiedRangeList _migrationTarget; + private readonly Lock _lock = new(); + /// /// Whether the modified range list has any entries or not. /// @@ -88,10 +86,10 @@ namespace Ryujinx.Graphics.Gpu.Memory { get { - Lock.EnterReadLock(); - bool result = Count > 0; - Lock.ExitReadLock(); - return result; + lock (_lock) + { + return Count > 0; + } } } @@ -116,41 +114,33 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Action to perform for each remaining sub-range of the input range public void ExcludeModifiedRegions(ulong address, ulong size, Action action) { - // Slices a given region using the modified regions in the list. Calls the action for the new slices. - bool lockOwner = Lock.IsReadLockHeld; - if (!lockOwner) + lock (_lock) { - Lock.EnterReadLock(); - } + // Slices a given region using the modified regions in the list. Calls the action for the new slices. + ref var overlaps = ref ThreadStaticArray.Get(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + int count = FindOverlapsNonOverlapping(address, size, ref overlaps); - RangeItem current = first; - while (last != null && current != last.Next) - { - BufferModifiedRange overlap = current.Value; - - if (overlap.Address > address) + for (int i = 0; i < count; i++) { - // The start of the remaining region is uncovered by this overlap. Call the action for it. - action(address, overlap.Address - address); + BufferModifiedRange overlap = overlaps[i]; + + if (overlap.Address > address) + { + // The start of the remaining region is uncovered by this overlap. Call the action for it. + action(address, overlap.Address - address); + } + + // Remaining region is after this overlap. + size -= overlap.EndAddress - address; + address = overlap.EndAddress; } - // Remaining region is after this overlap. - size -= overlap.EndAddress - address; - address = overlap.EndAddress; - current = current.Next; - } - - if (!lockOwner) - { - Lock.ExitReadLock(); - } - - if ((long)size > 0) - { - // If there is any region left after removing the overlaps, signal it. - action(address, size); + if ((long)size > 0) + { + // If there is any region left after removing the overlaps, signal it. + action(address, size); + } } } @@ -162,101 +152,51 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size of the modified region in bytes public void SignalModified(ulong address, ulong size) { - // We may overlap with some existing modified regions. They must be cut into by the new entry. - Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - - ulong endAddress = address + size; - ulong syncNumber = _context.SyncNumber; - - if (first is null) + // Must lock, as this can affect flushes from the background thread. + lock (_lock) { - Add(new BufferModifiedRange(address, size, syncNumber, this)); - Lock.ExitWriteLock(); - return; - } + // We may overlap with some existing modified regions. They must be cut into by the new entry. + ref var overlaps = ref ThreadStaticArray.Get(); - BufferModifiedRange buffPost = null; - bool extendsPost = false; - bool extendsPre = false; + int count = FindOverlapsNonOverlapping(address, size, ref overlaps); - if (first == last) - { - if (first.Address == address && first.EndAddress == endAddress) + ulong endAddress = address + size; + ulong syncNumber = _context.SyncNumber; + + for (int i = 0; i < count; i++) { - first.Value.SyncNumber = syncNumber; - first.Value.Parent = this; - Lock.ExitWriteLock(); - return; - } + // The overlaps must be removed or split. - if (first.Address < address) - { - first.Value.Size = address - first.Address; + BufferModifiedRange overlap = overlaps[i]; - extendsPre = true; - - if (first.EndAddress > endAddress) + if (overlap.Address == address && overlap.Size == size) { - buffPost = new BufferModifiedRange(endAddress, first.EndAddress - endAddress, - first.Value.SyncNumber, first.Value.Parent); - extendsPost = true; - } - } - else - { - if (first.EndAddress > endAddress) - { - first.Value.Size = first.EndAddress - endAddress; - first.Value.Address = endAddress; - } - else - { - Remove(first.Value); - } - } + // Region already exists. Just update the existing sync number. + overlap.SyncNumber = syncNumber; + overlap.Parent = this; - if (extendsPre && extendsPost) - { - Add(buffPost); + return; + } + + Remove(overlap); + + if (overlap.Address < address && overlap.EndAddress > address) + { + // A split item must be created behind this overlap. + + Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent)); + } + + if (overlap.Address < endAddress && overlap.EndAddress > endAddress) + { + // A split item must be created after this overlap. + + Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent)); + } } Add(new BufferModifiedRange(address, size, syncNumber, this)); - Lock.ExitWriteLock(); - - return; } - - BufferModifiedRange buffPre = null; - - if (first.Address < address) - { - buffPre = new BufferModifiedRange(first.Address, address - first.Address, - first.Value.SyncNumber, first.Value.Parent); - extendsPre = true; - } - - if (last.EndAddress > endAddress) - { - buffPost = new BufferModifiedRange(endAddress, last.EndAddress - endAddress, - last.Value.SyncNumber, last.Value.Parent); - extendsPost = true; - } - - RemoveRange(first, last); - - if (extendsPre) - { - Add(buffPre); - } - - if (extendsPost) - { - Add(buffPost); - } - - Add(new BufferModifiedRange(address, size, syncNumber, this)); - Lock.ExitWriteLock(); } /// @@ -268,23 +208,25 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action rangeAction) { - Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + int count = 0; - RangeItem current = first; - while (last != null && current != last.Next) + ref var overlaps = ref ThreadStaticArray.Get(); + + // Range list must be consistent for this operation. + lock (_lock) { - BufferModifiedRange overlap = current.Value; + count = FindOverlapsNonOverlapping(address, size, ref overlaps); + } + + for (int i = 0; i < count; i++) + { + BufferModifiedRange overlap = overlaps[i]; if (overlap.SyncNumber == syncNumber) { rangeAction(overlap.Address, overlap.Size); } - - current = current.Next; } - - Lock.ExitReadLock(); } /// @@ -295,23 +237,19 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void GetRanges(ulong address, ulong size, Action rangeAction) { - List> overlaps = []; + int count = 0; - // We use the non-span method here because keeping the lock will cause a deadlock. - Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); + ref var overlaps = ref ThreadStaticArray.Get(); - RangeItem current = first; - while (last != null && current != last.Next) + // Range list must be consistent for this operation. + lock (_lock) { - overlaps.Add(current); - current = current.Next; + count = FindOverlapsNonOverlapping(address, size, ref overlaps); } - Lock.ExitReadLock(); - for (int i = 0; i < overlaps.Count; i++) + for (int i = 0; i < count; i++) { - BufferModifiedRange overlap = overlaps[i].Value; + BufferModifiedRange overlap = overlaps[i]; rangeAction(overlap.Address, overlap.Size); } } @@ -324,11 +262,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// True if a range exists in the specified region, false otherwise public bool HasRange(ulong address, ulong size) { - Lock.EnterReadLock(); - (RangeItem first, RangeItem _) = FindOverlaps(address, size); - bool result = first is not null; - Lock.ExitReadLock(); - return result; + // Range list must be consistent for this operation. + lock (_lock) + { + return FindOverlapsNonOverlapping(address, size, ref ThreadStaticArray.Get()) > 0; + } } /// @@ -360,37 +298,38 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The start address of the flush range /// The end address of the flush range private void RemoveRangesAndFlush( - RangeItem[] overlaps, + BufferModifiedRange[] overlaps, int rangeCount, long highestDiff, ulong currentSync, ulong address, ulong endAddress) { - if (_migrationTarget == null) + lock (_lock) { - ulong waitSync = currentSync + (ulong)highestDiff; - - for (int i = 0; i < rangeCount; i++) + if (_migrationTarget == null) { - BufferModifiedRange overlap = overlaps[i].Value; + ulong waitSync = currentSync + (ulong)highestDiff; - long diff = (long)(overlap.SyncNumber - currentSync); - - if (diff <= highestDiff) + for (int i = 0; i < rangeCount; i++) { - ulong clampAddress = Math.Max(address, overlap.Address); - ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); + BufferModifiedRange overlap = overlaps[i]; - Lock.EnterWriteLock(); - ClearPart(overlap, clampAddress, clampEnd); - Lock.ExitWriteLock(); + long diff = (long)(overlap.SyncNumber - currentSync); - RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); + if (diff <= highestDiff) + { + ulong clampAddress = Math.Max(address, overlap.Address); + ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); + + ClearPart(overlap, clampAddress, clampEnd); + + RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); + } } - } - return; + return; + } } // There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine. @@ -416,37 +355,28 @@ namespace Ryujinx.Graphics.Gpu.Memory int rangeCount = 0; - List> overlaps = []; + ref var overlaps = ref ThreadStaticArray.Get(); // Range list must be consistent for this operation - Lock.EnterReadLock(); - if (_migrationTarget != null) + lock (_lock) { - rangeCount = -1; - } - else - { - // We use the non-span method here because the array is partially modified by the code, which would invalidate a span. - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - - RangeItem current = first; - while (last != null && current != last.Next) + if (_migrationTarget != null) { - rangeCount++; - overlaps.Add(current); - current = current.Next; + rangeCount = -1; + } + else + { + rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps); } } - Lock.ExitReadLock(); if (rangeCount == -1) { - _migrationTarget!.WaitForAndFlushRanges(address, size); + _migrationTarget?.WaitForAndFlushRanges(address, size); return; } - - if (rangeCount == 0) + else if (rangeCount == 0) { return; } @@ -458,7 +388,7 @@ namespace Ryujinx.Graphics.Gpu.Memory for (int i = 0; i < rangeCount; i++) { - BufferModifiedRange overlap = overlaps[i].Value; + BufferModifiedRange overlap = overlaps[i]; long diff = (long)(overlap.SyncNumber - currentSync); @@ -476,7 +406,7 @@ namespace Ryujinx.Graphics.Gpu.Memory // Wait for the syncpoint. _context.Renderer.WaitSync(currentSync + (ulong)highestDiff); - RemoveRangesAndFlush(overlaps.ToArray(), rangeCount, highestDiff, currentSync, address, endAddress); + RemoveRangesAndFlush(overlaps, rangeCount, highestDiff, currentSync, address, endAddress); } /// @@ -489,39 +419,42 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void InheritRanges(BufferModifiedRangeList ranges, Action registerRangeAction) { - ranges.Lock.EnterReadLock(); - BufferModifiedRange[] inheritRanges = ranges.ToArray(); - ranges.Lock.ExitReadLock(); + BufferModifiedRange[] inheritRanges; - // Copy over the migration from the previous range list - - BufferMigration oldMigration = ranges._source; - - BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration); - ranges._parent.IncrementReferenceCount(); - - if (_source == null) + lock (ranges._lock) { - // Create a new migration. - _source = new BufferMigration([span], this, _context.SyncNumber); + inheritRanges = ranges.ToArray(); - _context.RegisterBufferMigration(_source); + lock (_lock) + { + // Copy over the migration from the previous range list + + BufferMigration oldMigration = ranges._source; + + BufferMigrationSpan span = new BufferMigrationSpan(ranges._parent, ranges._flushAction, oldMigration); + ranges._parent.IncrementReferenceCount(); + + if (_source == null) + { + // Create a new migration. + _source = new BufferMigration([span], this, _context.SyncNumber); + + _context.RegisterBufferMigration(_source); + } + else + { + // Extend the migration + _source.AddSpanToEnd(span); + } + + ranges._migrationTarget = this; + + foreach (BufferModifiedRange range in inheritRanges) + { + Add(range); + } + } } - else - { - // Extend the migration - _source.AddSpanToEnd(span); - } - - ranges._migrationTarget = this; - - Lock.EnterWriteLock(); - foreach (BufferModifiedRange range in inheritRanges) - { - Add(range); - } - - Lock.ExitWriteLock(); ulong currentSync = _context.SyncNumber; foreach (BufferModifiedRange range in inheritRanges) @@ -540,18 +473,18 @@ namespace Ryujinx.Graphics.Gpu.Memory /// public void SelfMigration() { - BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), - _parent.GetSnapshotFlushAction(), _source); - BufferMigration migration = new([span], this, _context.SyncNumber); + lock (_lock) + { + BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), _parent.GetSnapshotFlushAction(), _source); + BufferMigration migration = new([span], this, _context.SyncNumber); - // Migration target is used to redirect flush actions to the latest range list, - // so we don't need to set it here. (this range list is still the latest) + // Migration target is used to redirect flush actions to the latest range list, + // so we don't need to set it here. (this range list is still the latest) - _context.RegisterBufferMigration(migration); + _context.RegisterBufferMigration(migration); - Lock.EnterWriteLock(); - _source = migration; - Lock.ExitWriteLock(); + _source = migration; + } } /// @@ -560,13 +493,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The migration to remove public void RemoveMigration(BufferMigration migration) { - Lock.EnterWriteLock(); - if (_source == migration) + lock (_lock) { - _source = null; + if (_source == migration) + { + _source = null; + } } - - Lock.ExitWriteLock(); } private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress) @@ -593,85 +526,33 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size to clear public void Clear(ulong address, ulong size) { - ulong endAddress = address + size; - Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - - if (first is null) + lock (_lock) { - Lock.ExitWriteLock(); - return; - } + // This function can be called from any thread, so it cannot use the arrays for background or foreground. + BufferModifiedRange[] toClear = new BufferModifiedRange[1]; - BufferModifiedRange buffPost = null; - bool extendsPost = false; - bool extendsPre = false; + int rangeCount = FindOverlapsNonOverlapping(address, size, ref toClear); - if (first == last) - { - if (first.Address < address) + ulong endAddress = address + size; + + for (int i = 0; i < rangeCount; i++) { - first.Value.Size = address - first.Address; - extendsPre = true; + BufferModifiedRange overlap = toClear[i]; - if (first.EndAddress > endAddress) - { - buffPost = new BufferModifiedRange(endAddress, first.EndAddress - endAddress, - first.Value.SyncNumber, first.Value.Parent); - extendsPost = true; - } + ClearPart(overlap, address, endAddress); } - else - { - if (first.EndAddress > endAddress) - { - first.Value.Size = first.EndAddress - endAddress; - first.Value.Address = endAddress; - } - else - { - Remove(first.Value); - } - } - - if (extendsPre && extendsPost) - { - Add(buffPost); - } - - Lock.ExitWriteLock(); - return; } + } - BufferModifiedRange buffPre = null; - - if (first.Address < address) + /// + /// Clear all modified ranges. + /// + public void Clear() + { + lock (_lock) { - buffPre = new BufferModifiedRange(first.Address, address - first.Address, - first.Value.SyncNumber, first.Value.Parent); - extendsPre = true; + Count = 0; } - - if (last.EndAddress > endAddress) - { - buffPost = new BufferModifiedRange(endAddress, last.EndAddress - endAddress, - last.Value.SyncNumber, last.Value.Parent); - extendsPost = true; - } - - RemoveRange(first, last); - - if (extendsPre) - { - Add(buffPre); - } - - if (extendsPost) - { - Add(buffPost); - } - - Lock.ExitWriteLock(); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferStage.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferStage.cs index 7cccee823..d56abda28 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferStage.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferStage.cs @@ -1,5 +1,4 @@ using Ryujinx.Graphics.Shader; -using System; using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory @@ -8,7 +7,6 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Pipeline stages that can modify buffer data, as well as flags indicating storage usage. /// Must match ShaderStage for the shader stages, though anything after that can be in any order. /// - [Flags] internal enum BufferStage : byte { Compute, diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs index 85bdb3cb3..192ebc223 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -690,8 +690,11 @@ namespace Ryujinx.Graphics.Gpu.Memory if (_pageTable[l0] == null) { _pageTable[l0] = new ulong[PtLvl1Size]; - - Array.Fill(_pageTable[l0], PteUnmapped); + + for (ulong index = 0; index < PtLvl1Size; index++) + { + _pageTable[l0][index] = PteUnmapped; + } } _pageTable[l0][l1] = pte; diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs index cb8a69d4a..ac25b3e5d 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs @@ -15,7 +15,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Represents a GPU virtual memory range. /// - private class VirtualRange : INonOverlappingRange + private readonly struct VirtualRange : IRange { /// /// GPU virtual address where the range starts. @@ -25,7 +25,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Size of the range in bytes. /// - public ulong Size { get; private set; } + public ulong Size { get; } /// /// GPU virtual address where the range ends. @@ -35,7 +35,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Physical regions where the GPU virtual region is mapped. /// - public MultiRange Range { get; private set; } + public MultiRange Range { get; } /// /// Creates a new virtual memory range. @@ -60,14 +60,10 @@ namespace Ryujinx.Graphics.Gpu.Memory { return Address < address + size && address < EndAddress; } - - public INonOverlappingRange Split(ulong splitAddress) - { - throw new NotImplementedException(); - } } - private readonly NonOverlappingRangeList _virtualRanges; + private readonly RangeList _virtualRanges; + private VirtualRange[] _virtualRangeOverlaps; private readonly ConcurrentQueue _deferredUnmaps; private int _hasDeferredUnmaps; @@ -79,6 +75,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { _memoryManager = memoryManager; _virtualRanges = []; + _virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity]; _deferredUnmaps = new ConcurrentQueue(); } @@ -109,11 +106,19 @@ namespace Ryujinx.Graphics.Gpu.Memory /// True if the range already existed, false if a new one was created and added public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range) { + VirtualRange[] overlaps = _virtualRangeOverlaps; + int overlapsCount; + if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0) { while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange)) { - _virtualRanges.RemoveRange(unmappedRange.Address, unmappedRange.Size); + overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps); + + for (int index = 0; index < overlapsCount; index++) + { + _virtualRanges.Remove(overlaps[index]); + } } } @@ -121,22 +126,27 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong originalVa = gpuVa; - _virtualRanges.Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = _virtualRanges.FindOverlaps(gpuVa, size); + overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps); - if (first is not null) + if (overlapsCount != 0) { // The virtual range already exists. We just need to check if our range fits inside // the existing one, and if not, we must extend the existing one. ulong endAddress = gpuVa + size; + VirtualRange overlap0 = overlaps[0]; - if (first.Address > gpuVa || first.EndAddress < endAddress) + if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress) { - gpuVa = Math.Min(gpuVa, first.Address); - endAddress = Math.Max(endAddress, last.EndAddress); + for (int index = 0; index < overlapsCount; index++) + { + VirtualRange virtualRange = overlaps[index]; - _virtualRanges.RemoveRange(first, last); + gpuVa = Math.Min(gpuVa, virtualRange.Address); + endAddress = Math.Max(endAddress, virtualRange.EndAddress); + + _virtualRanges.Remove(virtualRange); + } ulong newSize = endAddress - gpuVa; MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize); @@ -147,8 +157,8 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { - found = first.Value.Range.Count == 1 || IsSparseAligned(first.Value.Range); - range = first.Value.Range.Slice(gpuVa - first.Address, size); + found = overlap0.Range.Count == 1 || IsSparseAligned(overlap0.Range); + range = overlap0.Range.Slice(gpuVa - overlap0.Address, size); } } else @@ -160,7 +170,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _virtualRanges.Add(virtualRange); } - _virtualRanges.Lock.ExitWriteLock(); + + ShrinkOverlapsBufferIfNeeded(); // If the range is not properly aligned for sparse mapping, // let's just force it to a single range. @@ -210,5 +221,16 @@ namespace Ryujinx.Graphics.Gpu.Memory return true; } + + /// + /// Resizes the temporary buffer used for range list intersection results, if it has grown too much. + /// + private void ShrinkOverlapsBufferIfNeeded() + { + if (_virtualRangeOverlaps.Length > BufferCache.OverlapsBufferMaxCapacity) + { + Array.Resize(ref _virtualRangeOverlaps, BufferCache.OverlapsBufferMaxCapacity); + } + } } } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs b/src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs index 6a57aa3d1..f13a3554a 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs @@ -89,19 +89,13 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory if (baseAddress > currBaseAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - if (currBlock.Left == null) - _blockTree.Add(newBlock, currBlock); - else - _blockTree.Add(newBlock, currBlock.Predecessor); + _blockTree.Add(newBlock); } if (endAddr < currEndAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - if (currBlock.Left == null) - _blockTree.Add(newBlock, currBlock); - else - _blockTree.Add(newBlock, currBlock.Predecessor); + _blockTree.Add(newBlock); currBlock = newBlock; } @@ -149,19 +143,13 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory if (baseAddress > currBaseAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - if (currBlock.Left == null) - _blockTree.Add(newBlock, currBlock); - else - _blockTree.Add(newBlock, currBlock.Predecessor); + _blockTree.Add(newBlock); } if (endAddr < currEndAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - if (currBlock.Left == null) - _blockTree.Add(newBlock, currBlock); - else - _blockTree.Add(newBlock, currBlock.Predecessor); + _blockTree.Add(newBlock); currBlock = newBlock; } @@ -211,19 +199,13 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory if (baseAddress > currBaseAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - if (currBlock.Left == null) - _blockTree.Add(newBlock, currBlock); - else - _blockTree.Add(newBlock, currBlock.Predecessor); + _blockTree.Add(newBlock); } if (endAddr < currEndAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - if (currBlock.Left == null) - _blockTree.Add(newBlock, currBlock); - else - _blockTree.Add(newBlock, currBlock.Predecessor); + _blockTree.Add(newBlock); currBlock = newBlock; } diff --git a/src/Ryujinx.Memory/Range/INonOverlappingRange.cs b/src/Ryujinx.Memory/Range/INonOverlappingRange.cs index c6a0197d4..23194e0f2 100644 --- a/src/Ryujinx.Memory/Range/INonOverlappingRange.cs +++ b/src/Ryujinx.Memory/Range/INonOverlappingRange.cs @@ -3,7 +3,7 @@ namespace Ryujinx.Memory.Range /// /// Range of memory that can be split in two. /// - public interface INonOverlappingRange : IRange + interface INonOverlappingRange : IRange { /// /// Split this region into two, around the specified address. diff --git a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs index 26f3d81e3..511589176 100644 --- a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs +++ b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs @@ -1,8 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Threading; namespace Ryujinx.Memory.Range { @@ -10,283 +7,8 @@ namespace Ryujinx.Memory.Range /// A range list that assumes ranges are non-overlapping, with list items that can be split in two to avoid overlaps. /// /// Type of the range. - public class NonOverlappingRangeList : RangeListBase where T : INonOverlappingRange + class NonOverlappingRangeList : RangeList where T : INonOverlappingRange { - private readonly Dictionary> _quickAccess = new(AddressEqualityComparer.Comparer); - private readonly Dictionary> _fastQuickAccess = new(AddressEqualityComparer.Comparer); - - public readonly ReaderWriterLockSlim Lock = new(); - - /// - /// Creates a new non-overlapping range list. - /// - public NonOverlappingRangeList() { } - - /// - /// Creates a new non-overlapping range list. - /// - /// The initial size of the backing array - public NonOverlappingRangeList(int backingInitialSize) : base(backingInitialSize) { } - - /// - /// Adds a new item to the list. - /// - /// The item to be added - public override void Add(T item) - { - int index = BinarySearch(item.Address); - - if (index < 0) - { - index = ~index; - } - - RangeItem rangeItem = new(item); - - Insert(index, rangeItem); - - _quickAccess.Add(item.Address, rangeItem); - } - - /// - /// Updates an item's end address on the list. Address must be the same. - /// - /// The item to be updated - /// True if the item was located and updated, false otherwise - protected override bool Update(T item) - { - int index = BinarySearch(item.Address); - - if (index >= 0 && Items[index].Value.Equals(item)) - { - RangeItem rangeItem = new(item) { Previous = Items[index].Previous, Next = Items[index].Next }; - - if (index > 0) - { - Items[index - 1].Next = rangeItem; - } - - if (index < Count - 1) - { - Items[index + 1].Previous = rangeItem; - } - - foreach (ulong addr in Items[index].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } - - Items[index] = rangeItem; - - _quickAccess[item.Address] = rangeItem; - - return true; - } - - return false; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void Insert(int index, RangeItem item) - { - Debug.Assert(item.Address != item.EndAddress); - - if (Count + 1 > Items.Length) - { - Array.Resize(ref Items, Items.Length + BackingGrowthSize); - } - - if (index >= Count) - { - if (index == Count) - { - if (index != 0) - { - item.Previous = Items[index - 1]; - Items[index - 1].Next = item; - } - Items[index] = item; - Count++; - } - } - else - { - Array.Copy(Items, index, Items, index + 1, Count - index); - - Items[index] = item; - if (index != 0) - { - item.Previous = Items[index - 1]; - Items[index - 1].Next = item; - } - - item.Next = Items[index + 1]; - Items[index + 1].Previous = item; - - Count++; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void RemoveAt(int index) - { - if (index < Count - 1) - { - Items[index + 1].Previous = index > 0 ? Items[index - 1] : null; - } - - if (index > 0) - { - Items[index - 1].Next = index < Count - 1 ? Items[index + 1] : null; - } - - if (index < --Count) - { - Array.Copy(Items, index + 1, Items, index, Count - index); - } - } - - /// - /// Removes an item from the list. - /// - /// The item to be removed - /// True if the item was removed, or false if it was not found - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override bool Remove(T item) - { - int index = BinarySearch(item.Address); - - if (index >= 0 && Items[index].Value.Equals(item)) - { - _quickAccess.Remove(item.Address); - - foreach (ulong addr in Items[index].QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } - - RemoveAt(index); - - return true; - } - - return false; - } - - /// - /// Removes a range of items from the item list - /// - /// The first item in the range of items to be removed - /// The last item in the range of items to be removed - public override void RemoveRange(RangeItem startItem, RangeItem endItem) - { - if (startItem is null) - { - return; - } - - if (startItem == endItem) - { - Remove(startItem.Value); - return; - } - - int startIndex = BinarySearch(startItem.Address); - int endIndex = BinarySearch(endItem.Address); - - if (endIndex < Count - 1) - { - Items[endIndex + 1].Previous = startIndex > 0 ? Items[startIndex - 1] : null; - } - - if (startIndex > 0) - { - Items[startIndex - 1].Next = endIndex < Count - 1 ? Items[endIndex + 1] : null; - } - - if (endIndex < Count - 1) - { - Array.Copy(Items, endIndex + 1, Items, startIndex, Count - endIndex - 1); - } - - Count -= endIndex - startIndex + 1; - - while (startItem != endItem.Next) - { - _quickAccess.Remove(startItem.Address); - foreach (ulong addr in startItem.QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } - startItem = startItem.Next; - } - } - - /// - /// Removes a range of items from the item list - /// - /// Start address of the range - /// Size of the range - public void RemoveRange(ulong address, ulong size) - { - int startIndex = BinarySearchLeftEdge(address, address + size); - - if (startIndex < 0) - { - return; - } - - RangeItem startItem = Items[startIndex]; - - int endIndex = startIndex; - - while (startItem is not null && startItem.Address < address + size) - { - _quickAccess.Remove(startItem.Address); - foreach (ulong addr in startItem.QuickAccessAddresses) - { - _quickAccess.Remove(addr); - _fastQuickAccess.Remove(addr); - } - startItem = startItem.Next; - endIndex++; - } - - if (endIndex < Count - 1) - { - Items[endIndex + 1].Previous = startIndex > 0 ? Items[startIndex - 1] : null; - } - - if (startIndex > 0) - { - Items[startIndex - 1].Next = endIndex < Count - 1 ? Items[endIndex + 1] : null; - } - - - if (endIndex < Count - 1) - { - Array.Copy(Items, endIndex + 1, Items, startIndex, Count - endIndex - 1); - } - - Count -= endIndex - startIndex + 1; - } - - /// - /// Clear all ranges. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Clear() - { - Lock.EnterWriteLock(); - Count = 0; - _quickAccess.Clear(); - _fastQuickAccess.Clear(); - Lock.ExitWriteLock(); - } - /// /// Finds a list of regions that cover the desired (address, size) range. /// If this range starts or ends in the middle of an existing region, it is split and only the relevant part is added. @@ -297,18 +19,17 @@ namespace Ryujinx.Memory.Range /// Start address of the search region /// Size of the search region /// Factory for creating new ranges - public void GetOrAddRegions(out List list, ulong address, ulong size, Func factory) + public void GetOrAddRegions(List list, ulong address, ulong size, Func factory) { // (regarding the specific case this generalized function is used for) // A new region may be split into multiple parts if multiple virtual regions have mapped to it. // For instance, while a virtual mapping could cover 0-2 in physical space, the space 0-1 may have already been reserved... // So we need to return both the split 0-1 and 1-2 ranges. - Lock.EnterWriteLock(); - (RangeItem first, RangeItem last) = FindOverlaps(address, size); - list = []; + var results = new T[1]; + int count = FindOverlapsNonOverlapping(address, size, ref results); - if (first is null) + if (count == 0) { // The region is fully unmapped. Create and add it to the range list. T region = factory(address, size); @@ -320,15 +41,13 @@ namespace Ryujinx.Memory.Range ulong lastAddress = address; ulong endAddress = address + size; - RangeItem current = first; - while (last is not null && current is not null && current.Address < endAddress) + for (int i = 0; i < count; i++) { - T region = current.Value; - if (first == last && region.Address == address && region.Size == size) + T region = results[i]; + if (count == 1 && region.Address == address && region.Size == size) { // Exact match, no splitting required. list.Add(region); - Lock.ExitWriteLock(); return; } @@ -356,7 +75,6 @@ namespace Ryujinx.Memory.Range list.Add(region); lastAddress = region.EndAddress; - current = current.Next; } if (lastAddress < endAddress) @@ -367,8 +85,6 @@ namespace Ryujinx.Memory.Range Add(fillRegion); } } - - Lock.ExitWriteLock(); } /// @@ -379,7 +95,6 @@ namespace Ryujinx.Memory.Range /// The region to split /// The address to split with /// The new region (high part) - [MethodImpl(MethodImplOptions.AggressiveInlining)] private T Split(T region, ulong splitAddress) { T newRegion = (T)region.Split(splitAddress); @@ -387,113 +102,5 @@ namespace Ryujinx.Memory.Range Add(newRegion); return newRegion; } - - /// - /// Gets an item on the list overlapping the specified memory range. - /// - /// Start address of the range - /// Size in bytes of the range - /// The leftmost overlapping item, or null if none is found - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override RangeItem FindOverlap(ulong address, ulong size) - { - if (_quickAccess.TryGetValue(address, out RangeItem overlap)) - { - return overlap; - } - - int index = BinarySearchLeftEdge(address, address + size); - - if (index < 0) - { - return null; - } - - if (Items[index].Address < address) - { - _quickAccess.TryAdd(address, Items[index]); - Items[index].QuickAccessAddresses.Add(address); - } - - return Items[index]; - } - - /// - /// Gets an item on the list overlapping the specified memory range. - /// - /// Start address of the range - /// Size in bytes of the range - /// The overlapping item, or null if none is found - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override RangeItem FindOverlapFast(ulong address, ulong size) - { - if (_quickAccess.TryGetValue(address, out RangeItem overlap) || _fastQuickAccess.TryGetValue(address, out overlap)) - { - return overlap; - } - - int index = BinarySearch(address, address + size); - - if (index < 0) - { - return null; - } - - if (Items[index].Address < address) - { - _quickAccess.TryAdd(address, Items[index]); - } - else - { - _fastQuickAccess.TryAdd(address, Items[index]); - } - - Items[index].QuickAccessAddresses.Add(address); - - return Items[index]; - } - - /// - /// Gets all items on the list overlapping the specified memory range. - /// - /// Start address of the range - /// Size in bytes of the range - /// The first and last overlapping items, or null if none are found - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (RangeItem, RangeItem) FindOverlaps(ulong address, ulong size) - { - if (_quickAccess.TryGetValue(address, out RangeItem overlap)) - { - if (overlap.Next is null || overlap.Next.Address >= address + size) - { - return (overlap, overlap); - } - - return (overlap, Items[BinarySearchRightEdge(address, address + size)]); - } - - (int index, int endIndex) = BinarySearchEdges(address, address + size); - - if (index < 0) - { - return (null, null); - } - - if (Items[index].Address < address) - { - _quickAccess.TryAdd(address, Items[index]); - Items[index].QuickAccessAddresses.Add(address); - } - - return (Items[index], Items[endIndex - 1]); - } - - public override IEnumerator GetEnumerator() - { - for (int i = 0; i < Count; i++) - { - yield return Items[i].Value; - } - } } } diff --git a/src/Ryujinx.Memory/Range/RangeList.cs b/src/Ryujinx.Memory/Range/RangeList.cs index 7460dcc68..72cef1de0 100644 --- a/src/Ryujinx.Memory/Range/RangeList.cs +++ b/src/Ryujinx.Memory/Range/RangeList.cs @@ -1,91 +1,61 @@ using System; +using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Runtime.CompilerServices; -using System.Threading; namespace Ryujinx.Memory.Range { - public class RangeItem(TValue value) where TValue : IRange - { - public RangeItem Next; - public RangeItem Previous; - - public readonly ulong Address = value.Address; - public readonly ulong EndAddress = value.Address + value.Size; - - public readonly TValue Value = value; - - public readonly List QuickAccessAddresses = []; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool OverlapsWith(ulong address, ulong endAddress) - { - return Address < endAddress && address < EndAddress; - } - } - - class AddressEqualityComparer : IEqualityComparer - { - public bool Equals(ulong u1, ulong u2) - { - return u1 == u2; - } - - public int GetHashCode(ulong value) => (int)(value >> 5); - - public static readonly AddressEqualityComparer Comparer = new(); - } - - /// - /// Result of an Overlaps Finder function. WARNING: if the result is from the optimized - /// Overlaps Finder, the StartIndex will be -1 even when the result isn't empty - /// - /// - /// startIndex is inclusive. - /// endIndex is exclusive. - /// - public readonly struct OverlapResult where T : IRange - { - public readonly int StartIndex = -1; - public readonly int EndIndex = -1; - public readonly RangeItem QuickResult; - public int Count => EndIndex - StartIndex; - - public OverlapResult(int startIndex, int endIndex, RangeItem quickResult = null) - { - this.StartIndex = startIndex; - this.EndIndex = endIndex; - this.QuickResult = quickResult; - } - } - /// /// Sorted list of ranges that supports binary search. /// /// Type of the range. - public class RangeList : RangeListBase where T : IRange + public class RangeList : IEnumerable where T : IRange { - public readonly ReaderWriterLockSlim Lock = new(); - - private readonly Dictionary> _quickAccess = new(AddressEqualityComparer.Comparer); + private readonly struct RangeItem where TValue : IRange + { + public readonly ulong Address; + public readonly ulong EndAddress; - /// - /// Creates a new range list. - /// - public RangeList() { } + public readonly TValue Value; + + public RangeItem(TValue value) + { + Value = value; + + Address = value.Address; + EndAddress = value.Address + value.Size; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool OverlapsWith(ulong address, ulong endAddress) + { + return Address < endAddress && address < EndAddress; + } + } + + private const int BackingInitialSize = 1024; + private const int ArrayGrowthSize = 32; + + private RangeItem[] _items; + private readonly int _backingGrowthSize; + + public int Count { get; protected set; } /// /// Creates a new range list. /// /// The initial size of the backing array - public RangeList(int backingInitialSize) : base(backingInitialSize) { } + public RangeList(int backingInitialSize = BackingInitialSize) + { + _backingGrowthSize = backingInitialSize; + _items = new RangeItem[backingInitialSize]; + } /// /// Adds a new item to the list. /// /// The item to be added - public override void Add(T item) + public void Add(T item) { int index = BinarySearch(item.Address); @@ -102,27 +72,27 @@ namespace Ryujinx.Memory.Range /// /// The item to be updated /// True if the item was located and updated, false otherwise - protected override bool Update(T item) + public bool Update(T item) { int index = BinarySearch(item.Address); if (index >= 0) { + while (index > 0 && _items[index - 1].Address == item.Address) + { + index--; + } + while (index < Count) { - if (Items[index].Value.Equals(item)) + if (_items[index].Value.Equals(item)) { - foreach (ulong address in Items[index].QuickAccessAddresses) - { - _quickAccess.Remove(address); - } - - Items[index] = new RangeItem(item); + _items[index] = new RangeItem(item); return true; } - if (Items[index].Address > item.Address) + if (_items[index].Address > item.Address) { break; } @@ -137,42 +107,23 @@ namespace Ryujinx.Memory.Range [MethodImpl(MethodImplOptions.AggressiveInlining)] private void Insert(int index, RangeItem item) { - Debug.Assert(item.Address != item.EndAddress); - - Debug.Assert(item.Address % 32 == 0); - - if (Count + 1 > Items.Length) + if (Count + 1 > _items.Length) { - Array.Resize(ref Items, Items.Length + BackingGrowthSize); + Array.Resize(ref _items, _items.Length + _backingGrowthSize); } if (index >= Count) { if (index == Count) { - if (index != 0) - { - item.Previous = Items[index - 1]; - Items[index - 1].Next = item; - } - Items[index] = item; - Count++; + _items[Count++] = item; } } else { - Array.Copy(Items, index, Items, index + 1, Count - index); + Array.Copy(_items, index, _items, index + 1, Count - index); - Items[index] = item; - if (index != 0) - { - item.Previous = Items[index - 1]; - Items[index - 1].Next = item; - } - - item.Next = Items[index + 1]; - Items[index + 1].Previous = item; - + _items[index] = item; Count++; } } @@ -180,71 +131,9 @@ namespace Ryujinx.Memory.Range [MethodImpl(MethodImplOptions.AggressiveInlining)] private void RemoveAt(int index) { - foreach (ulong address in Items[index].QuickAccessAddresses) - { - _quickAccess.Remove(address); - } - - if (index < Count - 1) - { - Items[index + 1].Previous = index > 0 ? Items[index - 1] : null; - } - - if (index > 0) - { - Items[index - 1].Next = index < Count - 1 ? Items[index + 1] : null; - } - if (index < --Count) { - Array.Copy(Items, index + 1, Items, index, Count - index); - } - } - - /// - /// Removes a range of items from the item list - /// - /// The first item in the range of items to be removed - /// The last item in the range of items to be removed - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override void RemoveRange(RangeItem startItem, RangeItem endItem) - { - if (endItem.Next is not null) - { - endItem.Next.Previous = startItem.Previous; - } - - if (startItem.Previous is not null) - { - startItem.Previous.Next = endItem.Next; - } - - RangeItem current = startItem; - while (current != endItem.Next) - { - foreach (ulong address in current.QuickAccessAddresses) - { - _quickAccess.Remove(address); - } - - current = current.Next; - } - - RangeItem[] array = []; - OverlapResult overlapResult = FindOverlaps(startItem.Address, endItem.EndAddress, ref array); - - if (overlapResult.EndIndex < Count) - { - Array.Copy(Items, overlapResult.EndIndex, Items, overlapResult.StartIndex, Count - overlapResult.EndIndex); - Count -= overlapResult.Count; - } - else if (overlapResult.EndIndex == Count) - { - Count = overlapResult.StartIndex; - } - else - { - Debug.Assert(false); + Array.Copy(_items, index + 1, _items, index, Count - index); } } @@ -253,22 +142,27 @@ namespace Ryujinx.Memory.Range /// /// The item to be removed /// True if the item was removed, or false if it was not found - public override bool Remove(T item) + public bool Remove(T item) { int index = BinarySearch(item.Address); if (index >= 0) { + while (index > 0 && _items[index - 1].Address == item.Address) + { + index--; + } + while (index < Count) { - if (Items[index].Value.Equals(item)) + if (_items[index].Value.Equals(item)) { RemoveAt(index); return true; } - if (Items[index].Address > item.Address) + if (_items[index].Address > item.Address) { break; } @@ -279,62 +173,86 @@ namespace Ryujinx.Memory.Range return false; } - + /// - /// Gets an item on the list overlapping the specified memory range. + /// Updates an item's end address. /// - /// - /// This has no ordering guarantees of the returned item. - /// It only ensures that the item returned overlaps the specified memory range. - /// - /// Start address of the range - /// Size in bytes of the range - /// The overlapping item, or the default value for the type if none found - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override RangeItem FindOverlap(ulong address, ulong size) + /// The item to be updated + public void UpdateEndAddress(T item) { - int index = BinarySearchLeftEdge(address, address + size); + int index = BinarySearch(item.Address); - if (index < 0) + if (index >= 0) { - return null; - } + while (index > 0 && _items[index - 1].Address == item.Address) + { + index--; + } - return Items[index]; + while (index < Count) + { + if (_items[index].Value.Equals(item)) + { + _items[index] = new RangeItem(item); + + return; + } + + if (_items[index].Address > item.Address) + { + break; + } + + index++; + } + } } /// - /// Gets an item on the list overlapping the specified memory range. + /// Gets the first item on the list overlapping in memory with the specified item. /// /// - /// This has no ordering guarantees of the returned item. + /// Despite the name, this has no ordering guarantees of the returned item. + /// It only ensures that the item returned overlaps the specified item. + /// + /// Item to check for overlaps + /// The overlapping item, or the default value for the type if none found + public T FindFirstOverlap(T item) + { + return FindFirstOverlap(item.Address, item.Size); + } + + /// + /// Gets the first item on the list overlapping the specified memory range. + /// + /// + /// Despite the name, this has no ordering guarantees of the returned item. /// It only ensures that the item returned overlaps the specified memory range. /// /// Start address of the range /// Size in bytes of the range /// The overlapping item, or the default value for the type if none found - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public override RangeItem FindOverlapFast(ulong address, ulong size) + public T FindFirstOverlap(ulong address, ulong size) { - if (_quickAccess.TryGetValue(address, out RangeItem quickResult)) - { - return quickResult; - } - int index = BinarySearch(address, address + size); if (index < 0) { - return null; + return default; } - if (Items[index].OverlapsWith(address, address + 1)) - { - _quickAccess.Add(address, Items[index]); - Items[index].QuickAccessAddresses.Add(address); - } + return _items[index].Value; + } - return Items[index]; + /// + /// Gets all items overlapping with the specified item in memory. + /// + /// Item to check for overlaps + /// Output array where matches will be written. It is automatically resized to fit the results + /// The number of overlapping items found + public int FindOverlaps(T item, ref T[] output) + { + return FindOverlaps(item.Address, item.Size, ref output); } /// @@ -343,66 +261,222 @@ namespace Ryujinx.Memory.Range /// Start address of the range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results - /// Range information of overlapping items found - private OverlapResult FindOverlaps(ulong address, ulong size, ref RangeItem[] output) + /// The number of overlapping items found + public int FindOverlaps(ulong address, ulong size, ref T[] output) { - int outputCount = 0; + int outputIndex = 0; ulong endAddress = address + size; - int startIndex = BinarySearch(address, endAddress); - if (startIndex < 0) - startIndex = ~startIndex; - int endIndex = -1; - - for (int i = startIndex; i < Count; i++) + for (int i = 0; i < Count; i++) { - ref RangeItem item = ref Items[i]; + ref RangeItem item = ref _items[i]; if (item.Address >= endAddress) { - endIndex = i; break; } if (item.OverlapsWith(address, endAddress)) { - outputCount++; + if (outputIndex == output.Length) + { + Array.Resize(ref output, outputIndex + ArrayGrowthSize); + } + + output[outputIndex++] = item.Value; } } - if (endIndex == -1 && outputCount > 0) - { - endIndex = Count; - } - - if (outputCount > 0 && outputCount == endIndex - startIndex) - { - Array.Resize(ref output, outputCount); - Array.Copy(Items, endIndex - outputCount, output, 0, outputCount); - - return new OverlapResult(startIndex, endIndex); - } - else if (outputCount > 0) - { - Array.Resize(ref output, outputCount); - int arrIndex = 0; - for (int i = startIndex; i < endIndex; i++) - { - output[arrIndex++] = Items[i]; - } - - return new OverlapResult(endIndex - outputCount, endIndex); - } - - return new OverlapResult(); + return outputIndex; } - public override IEnumerator GetEnumerator() + /// + /// Gets all items overlapping with the specified item in memory. + /// + /// + /// This method only returns correct results if none of the items on the list overlaps with + /// each other. If that is not the case, this method should not be used. + /// This method is faster than the regular method to find all overlaps. + /// + /// Item to check for overlaps + /// Output array where matches will be written. It is automatically resized to fit the results + /// The number of overlapping items found + public int FindOverlapsNonOverlapping(T item, ref T[] output) + { + return FindOverlapsNonOverlapping(item.Address, item.Size, ref output); + } + + /// + /// Gets all items on the list overlapping the specified memory range. + /// + /// + /// This method only returns correct results if none of the items on the list overlaps with + /// each other. If that is not the case, this method should not be used. + /// This method is faster than the regular method to find all overlaps. + /// + /// Start address of the range + /// Size in bytes of the range + /// Output array where matches will be written. It is automatically resized to fit the results + /// The number of overlapping items found + public int FindOverlapsNonOverlapping(ulong address, ulong size, ref T[] output) + { + // This is a bit faster than FindOverlaps, but only works + // when none of the items on the list overlaps with each other. + int outputIndex = 0; + + ulong endAddress = address + size; + + int index = BinarySearch(address, endAddress); + + if (index >= 0) + { + while (index > 0 && _items[index - 1].OverlapsWith(address, endAddress)) + { + index--; + } + + do + { + if (outputIndex == output.Length) + { + Array.Resize(ref output, outputIndex + ArrayGrowthSize); + } + + output[outputIndex++] = _items[index++].Value; + } + while (index < Count && _items[index].OverlapsWith(address, endAddress)); + } + + return outputIndex; + } + + /// + /// Gets all items on the list with the specified memory address. + /// + /// Address to find + /// Output array where matches will be written. It is automatically resized to fit the results + /// The number of matches found + public int FindOverlaps(ulong address, ref T[] output) + { + int index = BinarySearch(address); + + int outputIndex = 0; + + if (index >= 0) + { + while (index > 0 && _items[index - 1].Address == address) + { + index--; + } + + while (index < Count) + { + ref RangeItem overlap = ref _items[index++]; + + if (overlap.Address != address) + { + break; + } + + if (outputIndex == output.Length) + { + Array.Resize(ref output, outputIndex + ArrayGrowthSize); + } + + output[outputIndex++] = overlap.Value; + } + } + + return outputIndex; + } + + /// + /// Performs binary search on the internal list of items. + /// + /// Address to find + /// List index of the item, or complement index of nearest item with lower value on the list + private int BinarySearch(ulong address) + { + int left = 0; + int right = Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref _items[middle]; + + if (item.Address == address) + { + return middle; + } + + if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + /// + /// Performs binary search for items overlapping a given memory range. + /// + /// Start address of the range + /// End address of the range + /// List index of the item, or complement index of nearest item with lower value on the list + private int BinarySearch(ulong address, ulong endAddress) + { + int left = 0; + int right = Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref _items[middle]; + + if (item.OverlapsWith(address, endAddress)) + { + return middle; + } + + if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + public IEnumerator GetEnumerator() { for (int i = 0; i < Count; i++) { - yield return Items[i].Value; + yield return _items[i].Value; + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + for (int i = 0; i < Count; i++) + { + yield return _items[i].Value; } } } diff --git a/src/Ryujinx.Memory/Range/RangeListBase.cs b/src/Ryujinx.Memory/Range/RangeListBase.cs deleted file mode 100644 index 7e26442f0..000000000 --- a/src/Ryujinx.Memory/Range/RangeListBase.cs +++ /dev/null @@ -1,359 +0,0 @@ -using System.Collections; -using System.Collections.Generic; -using System.Runtime.CompilerServices; - -namespace Ryujinx.Memory.Range -{ - public abstract class RangeListBase : IEnumerable where T : IRange - { - protected const int BackingInitialSize = 1024; - - protected RangeItem[] Items; - protected readonly int BackingGrowthSize; - - public int Count { get; protected set; } - - /// - /// Creates a new range list. - /// - /// The initial size of the backing array - protected RangeListBase(int backingInitialSize = BackingInitialSize) - { - BackingGrowthSize = backingInitialSize; - Items = new RangeItem[backingInitialSize]; - } - - public abstract void Add(T item); - - /// - /// Updates an item's end address on the list. Address must be the same. - /// - /// The item to be updated - /// True if the item was located and updated, false otherwise - protected abstract bool Update(T item); - - public abstract bool Remove(T item); - - public abstract void RemoveRange(RangeItem startItem, RangeItem endItem); - - public abstract RangeItem FindOverlap(ulong address, ulong size); - - public abstract RangeItem FindOverlapFast(ulong address, ulong size); - - /// - /// Performs binary search on the internal list of items. - /// - /// Address to find - /// List index of the item, or complement index of nearest item with lower value on the list - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected int BinarySearch(ulong address) - { - int left = 0; - int right = Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref Items[middle]; - - if (item.Address == address) - { - return middle; - } - - if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return ~left; - } - - /// - /// Performs binary search for items overlapping a given memory range. - /// - /// Start address of the range - /// End address of the range - /// List index of the item, or complement index of nearest item with lower value on the list - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected int BinarySearch(ulong address, ulong endAddress) - { - int left = 0; - int right = Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref Items[middle]; - - if (item.OverlapsWith(address, endAddress)) - { - return middle; - } - - if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return ~left; - } - - /// - /// Performs binary search for items overlapping a given memory range. - /// - /// Start address of the range - /// End address of the range - /// List index of the item, or complement index of nearest item with lower value on the list - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected int BinarySearchLeftEdge(ulong address, ulong endAddress) - { - if (Count == 0) - return ~0; - - int left = 0; - int right = Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref Items[middle]; - - bool match = item.OverlapsWith(address, endAddress); - - if (range == 0) - { - if (match) - return middle; - else if (address < item.Address) - return ~(right); - else - return ~(right + 1); - } - - if (match) - { - right = middle; - } - else if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return ~left; - } - - /// - /// Performs binary search for items overlapping a given memory range. - /// - /// Start address of the range - /// End address of the range - /// List index of the item, or complement index of nearest item with lower value on the list - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected int BinarySearchRightEdge(ulong address, ulong endAddress) - { - if (Count == 0) - return ~0; - - int left = 0; - int right = Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = right - (range >> 1); - - ref RangeItem item = ref Items[middle]; - - bool match = item.OverlapsWith(address, endAddress); - - if (range == 0) - { - if (match) - return middle; - else if (endAddress > item.EndAddress) - return ~(left + 1); - else - return ~(left); - } - - if (match) - { - left = middle; - } - else if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return ~left; - } - - /// - /// Performs binary search for items overlapping a given memory range. - /// - /// Start address of the range - /// End address of the range - /// Range information (inclusive, exclusive) of items that overlaps, or complement index of nearest item with lower value on the list - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected (int, int) BinarySearchEdges(ulong address, ulong endAddress) - { - if (Count == 0) - return (~0, ~0); - - if (Count == 1) - { - ref RangeItem item = ref Items[0]; - - if (item.OverlapsWith(address, endAddress)) - { - return (0, 1); - } - - if (address < item.Address) - { - return (~0, ~0); - } - else - { - return (~1, ~1); - } - } - - int left = 0; - int right = Count - 1; - - int leftEdge = -1; - int rightEdgeMatch = -1; - int rightEdgeNoMatch = -1; - - while (left <= right) - { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref Items[middle]; - - bool match = item.OverlapsWith(address, endAddress); - - if (range == 0) - { - if (match) - { - leftEdge = middle; - break; - } - else if (address < item.Address) - { - return (~right, ~right); - } - else - { - return (~(right + 1), ~(right + 1)); - } - } - - if (match) - { - right = middle; - if (rightEdgeMatch == -1) - rightEdgeMatch = middle; - } - else if (address < item.Address) - { - right = middle - 1; - rightEdgeNoMatch = middle; - } - else - { - left = middle + 1; - } - } - - if (left > right) - { - return (~left, ~left); - } - - if (rightEdgeMatch == -1) - { - return (leftEdge, leftEdge + 1); - } - - left = rightEdgeMatch; - right = rightEdgeNoMatch > 0 ? rightEdgeNoMatch : Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = right - (range >> 1); - - ref RangeItem item = ref Items[middle]; - - bool match = item.OverlapsWith(address, endAddress); - - if (range == 0) - { - if (match) - return (leftEdge, middle + 1); - else - return (leftEdge, middle); - } - - if (match) - { - left = middle; - } - else if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return (leftEdge, right + 1); - } - - public abstract IEnumerator GetEnumerator(); - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - } -} diff --git a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs index 2bad3c850..fb5659d5f 100644 --- a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -1,3 +1,4 @@ +using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; using System.Collections.Generic; @@ -75,16 +76,17 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { + ref var overlaps = ref ThreadStaticArray.Get(); + for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; - regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(va, size); - RangeItem current = first; - while (last != null && current != last.Next) + int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); + + for (int i = 0; i < count; i++) { - VirtualRegion region = current.Value; + VirtualRegion region = overlaps[i]; // If the region has been fully remapped, signal that it has been mapped again. bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); @@ -94,9 +96,7 @@ namespace Ryujinx.Memory.Tracking } region.UpdateProtection(); - current = current.Next; } - regions.Lock.ExitReadLock(); } } } @@ -114,21 +114,20 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { + ref var overlaps = ref ThreadStaticArray.Get(); + for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; - regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(va, size); - RangeItem current = first; - while (last != null && current != last.Next) + int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); + + for (int i = 0; i < count; i++) { - VirtualRegion region = current.Value; + VirtualRegion region = overlaps[i]; region.SignalMappingChanged(false); - current = current.Next; } - regions.Lock.ExitReadLock(); } } } @@ -166,11 +165,10 @@ namespace Ryujinx.Memory.Tracking /// A list of virtual regions within the given range internal List GetVirtualRegionsForHandle(ulong va, ulong size, bool guest) { + List result = []; NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; - regions.Lock.EnterUpgradeableReadLock(); - regions.GetOrAddRegions(out List result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); - regions.Lock.ExitUpgradeableReadLock(); - + regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); + return result; } @@ -298,33 +296,25 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { + ref var overlaps = ref ThreadStaticArray.Get(); + NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; - List> overlaps = []; - // We use the non-span method here because keeping the lock will cause a deadlock. - regions.Lock.EnterReadLock(); - (RangeItem first, RangeItem last) = regions.FindOverlaps(address, size); + int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); - RangeItem current = first; - while (last != null && current != last.Next) - { - overlaps.Add(current); - current = current.Next; - } - regions.Lock.ExitReadLock(); - - if (first is null && !precise) + if (count == 0 && !precise) { if (_memoryManager.IsRangeMapped(address, size)) { // TODO: There is currently the possibility that a page can be protected after its virtual region is removed. // This code handles that case when it happens, but it would be better to find out how this happens. _memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest); - return true; // This memory _should_ be mapped, so we need to try again. } - - shouldThrow = true; + else + { + shouldThrow = true; + } } else { @@ -334,9 +324,9 @@ namespace Ryujinx.Memory.Tracking size += (ulong)_pageSize; } - for (int i = 0; i < overlaps.Count; i++) + for (int i = 0; i < count; i++) { - VirtualRegion region = overlaps[i].Value; + VirtualRegion region = overlaps[i]; if (precise) {