Android: NCE support

This commit is contained in:
KeatonTheBot 2025-05-26 11:12:50 -05:00
parent 19dd23c288
commit cd3221abdc
35 changed files with 3783 additions and 45 deletions

View file

@ -5,6 +5,8 @@ namespace Ryujinx.Cpu
{
public class AddressSpace : IDisposable
{
private const MemoryAllocationFlags AsFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible;
private readonly MemoryBlock _backingMemory;
public MemoryBlock Base { get; }
@ -25,29 +27,43 @@ namespace Ryujinx.Cpu
{
addressSpace = null;
const MemoryAllocationFlags AsFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible;
ulong minAddressSpaceSize = Math.Min(asSize, 1UL << 36);
// Attempt to create the address space with expected size or try to reduce it until it succeed.
for (ulong addressSpaceSize = asSize; addressSpaceSize >= minAddressSpaceSize; addressSpaceSize >>= 1)
{
MemoryBlock baseMemory = null;
MemoryBlock mirrorMemory = null;
try
{
baseMemory = new MemoryBlock(addressSpaceSize, AsFlags);
mirrorMemory = new MemoryBlock(addressSpaceSize, AsFlags);
addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, addressSpaceSize);
break;
baseMemory = new MemoryBlock(asSize, AsFlags);
mirrorMemory = new MemoryBlock(asSize, AsFlags);
addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, asSize);
}
catch (SystemException)
{
baseMemory?.Dispose();
mirrorMemory?.Dispose();
}
return addressSpace != null;
}
public static bool TryCreateWithoutMirror(ulong asSize, out MemoryBlock addressSpace)
{
addressSpace = null;
ulong minAddressSpaceSize = Math.Min(asSize, 1UL << 36);
// Attempt to create the address space with expected size or try to reduce it until it succeed.
for (ulong addressSpaceSize = asSize; addressSpaceSize >= minAddressSpaceSize; addressSpaceSize -= 0x100000000UL)
{
try
{
MemoryBlock baseMemory = new MemoryBlock(addressSpaceSize, AsFlags);
addressSpace = baseMemory;
break;
}
catch (SystemException)
{
}
}
return addressSpace != null;

View file

@ -0,0 +1,330 @@
using ARMeilleure.Memory;
using Ryujinx.Memory;
using Ryujinx.Memory.Range;
using Ryujinx.Memory.Tracking;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
namespace Ryujinx.Cpu.Jit
{
/// <summary>
/// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region.
/// </summary>
public sealed class MemoryManagerHostNoMirror : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
{
private readonly InvalidAccessHandler _invalidAccessHandler;
private readonly bool _unsafeMode;
private readonly MemoryBlock _addressSpace;
private readonly MemoryBlock _backingMemory;
private readonly PageTable<ulong> _pageTable;
public int AddressSpaceBits { get; }
protected override ulong AddressSpaceSize { get; }
private readonly MemoryEhMeilleure _memoryEh;
private readonly ManagedPageFlags _pages;
/// <inheritdoc/>
public bool UsesPrivateAllocations => false;
public IntPtr PageTablePointer => _addressSpace.Pointer;
public MemoryManagerType Type => _unsafeMode ? MemoryManagerType.HostMappedUnsafe : MemoryManagerType.HostMapped;
public MemoryTracking Tracking { get; }
public event Action<ulong, ulong> UnmapEvent;
/// <summary>
/// Creates a new instance of the host mapped memory manager.
/// </summary>
/// <param name="addressSpace">Address space instance to use</param>
/// <param name="unsafeMode">True if unmanaged access should not be masked (unsafe), false otherwise.</param>
/// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
public MemoryManagerHostNoMirror(
MemoryBlock addressSpace,
MemoryBlock backingMemory,
bool unsafeMode,
InvalidAccessHandler invalidAccessHandler)
{
_addressSpace = addressSpace;
_backingMemory = backingMemory;
_pageTable = new PageTable<ulong>();
_invalidAccessHandler = invalidAccessHandler;
_unsafeMode = unsafeMode;
AddressSpaceSize = addressSpace.Size;
ulong asSize = PageSize;
int asBits = PageBits;
while (asSize < addressSpace.Size)
{
asSize <<= 1;
asBits++;
}
AddressSpaceBits = asBits;
_pages = new ManagedPageFlags(asBits);
Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler);
_memoryEh = new MemoryEhMeilleure(addressSpace, null, Tracking);
}
/// <summary>
/// Ensures the combination of virtual address and size is part of the addressable space and fully mapped.
/// </summary>
/// <param name="va">Virtual address of the range</param>
/// <param name="size">Size of the range in bytes</param>
private void AssertMapped(ulong va, ulong size)
{
if (!ValidateAddressAndSize(va, size) || !_pages.IsRangeMapped(va, size))
{
throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
}
}
/// <inheritdoc/>
public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
{
AssertValidAddressAndSize(va, size);
_addressSpace.MapView(_backingMemory, pa, va, size);
_pages.AddMapping(va, size);
PtMap(va, pa, size);
Tracking.Map(va, size);
}
private void PtMap(ulong va, ulong pa, ulong size)
{
while (size != 0)
{
_pageTable.Map(va, pa);
va += PageSize;
pa += PageSize;
size -= PageSize;
}
}
/// <inheritdoc/>
public void Unmap(ulong va, ulong size)
{
AssertValidAddressAndSize(va, size);
UnmapEvent?.Invoke(va, size);
Tracking.Unmap(va, size);
_pages.RemoveMapping(va, size);
PtUnmap(va, size);
_addressSpace.UnmapView(_backingMemory, va, size);
}
private void PtUnmap(ulong va, ulong size)
{
while (size != 0)
{
_pageTable.Unmap(va);
va += PageSize;
size -= PageSize;
}
}
/// <inheritdoc/>
public void Reprotect(ulong va, ulong size, MemoryPermission permission)
{
}
public ref T GetRef<T>(ulong va) where T : unmanaged
{
if (!IsContiguous(va, Unsafe.SizeOf<T>()))
{
ThrowMemoryNotContiguous();
}
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
return ref _backingMemory.GetRef<T>(GetPhysicalAddressChecked(va));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool IsMapped(ulong va)
{
return ValidateAddress(va) && _pages.IsMapped(va);
}
/// <inheritdoc/>
public bool IsRangeMapped(ulong va, ulong size)
{
AssertValidAddressAndSize(va, size);
return _pages.IsRangeMapped(va, size);
}
/// <inheritdoc/>
public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
{
if (size == 0)
{
return Enumerable.Empty<HostMemoryRange>();
}
var guestRegions = GetPhysicalRegionsImpl(va, size);
if (guestRegions == null)
{
return null;
}
var regions = new HostMemoryRange[guestRegions.Count];
for (int i = 0; i < regions.Length; i++)
{
var guestRegion = guestRegions[i];
IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size);
regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size);
}
return regions;
}
/// <inheritdoc/>
public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
{
if (size == 0)
{
return Enumerable.Empty<MemoryRange>();
}
return GetPhysicalRegionsImpl(va, size);
}
private List<MemoryRange> GetPhysicalRegionsImpl(ulong va, ulong size)
{
if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
{
return null;
}
int pages = GetPagesCount(va, (uint)size, out va);
var regions = new List<MemoryRange>();
ulong regionStart = GetPhysicalAddressInternal(va);
ulong regionSize = PageSize;
for (int page = 0; page < pages - 1; page++)
{
if (!ValidateAddress(va + PageSize))
{
return null;
}
ulong newPa = GetPhysicalAddressInternal(va + PageSize);
if (GetPhysicalAddressInternal(va) + PageSize != newPa)
{
regions.Add(new MemoryRange(regionStart, regionSize));
regionStart = newPa;
regionSize = 0;
}
va += PageSize;
regionSize += PageSize;
}
regions.Add(new MemoryRange(regionStart, regionSize));
return regions;
}
private ulong GetPhysicalAddressChecked(ulong va)
{
if (!IsMapped(va))
{
ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}");
}
return GetPhysicalAddressInternal(va);
}
private ulong GetPhysicalAddressInternal(ulong va)
{
return _pageTable.Read(va) + (va & PageMask);
}
/// <inheritdoc/>
/// <remarks>
/// This function also validates that the given range is both valid and mapped, and will throw if it is not.
/// </remarks>
public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
{
AssertValidAddressAndSize(va, size);
if (precise)
{
Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
return;
}
_pages.SignalMemoryTracking(Tracking, va, size, write, exemptId);
}
/// <inheritdoc/>
public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest)
{
if (guest)
{
_addressSpace.Reprotect(va, size, protection, false);
}
else
{
_pages.TrackingReprotect(va, size, protection);
}
}
/// <inheritdoc/>
public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags)
{
return Tracking.BeginTracking(address, size, id, flags);
}
/// <inheritdoc/>
public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id, RegionFlags flags)
{
return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags);
}
/// <inheritdoc/>
public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
{
return Tracking.BeginSmartGranularTracking(address, size, granularity, id);
}
/// <summary>
/// Disposes of resources used by the memory manager.
/// </summary>
protected override void Destroy()
{
_addressSpace.Dispose();
_memoryEh.Dispose();
}
protected override Memory<byte> GetPhysicalAddressMemory(nuint pa, int size)
=> _backingMemory.GetMemory(pa, size);
protected override Span<byte> GetPhysicalAddressSpan(nuint pa, int size)
=> _backingMemory.GetSpan(pa, size);
protected override nuint TranslateVirtualAddressChecked(ulong va)
=> (nuint)GetPhysicalAddressChecked(va);
protected override nuint TranslateVirtualAddressUnchecked(ulong va)
=> (nuint)GetPhysicalAddressInternal(va);
}
}

View file

@ -45,6 +45,11 @@ namespace Ryujinx.Cpu.LightningJit
_translator.InvalidateJitCacheRegion(address, size);
}
/// <inheritdoc/>
public void PatchCodeForNce(ulong textAddress, ulong textSize, ulong patchRegionAddress, ulong patchRegionSize)
{
}
/// <inheritdoc/>
public IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled, string cacheSelector)
{

View file

@ -0,0 +1,22 @@
namespace Ryujinx.Cpu.Nce.Arm64
{
enum ArmCondition
{
Eq = 0,
Ne = 1,
GeUn = 2,
LtUn = 3,
Mi = 4,
Pl = 5,
Vs = 6,
Vc = 7,
GtUn = 8,
LeUn = 9,
Ge = 10,
Lt = 11,
Gt = 12,
Le = 13,
Al = 14,
Nv = 15,
}
}

View file

@ -0,0 +1,14 @@
namespace Ryujinx.Cpu.Nce.Arm64
{
enum ArmExtensionType
{
Uxtb = 0,
Uxth = 1,
Uxtw = 2,
Uxtx = 3,
Sxtb = 4,
Sxth = 5,
Sxtw = 6,
Sxtx = 7,
}
}

View file

@ -0,0 +1,11 @@
namespace Ryujinx.Cpu.Nce.Arm64
{
enum ArmShiftType
{
Lsl = 0,
Lsr = 1,
Asr = 2,
Ror = 3,
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,66 @@
using System.Numerics;
namespace Ryujinx.Cpu.Nce.Arm64
{
static class CodeGenCommon
{
public static bool TryEncodeBitMask(Operand operand, out int immN, out int immS, out int immR)
{
return TryEncodeBitMask(operand.Type, operand.Value, out immN, out immS, out immR);
}
public static bool TryEncodeBitMask(OperandType type, ulong value, out int immN, out int immS, out int immR)
{
if (type == OperandType.I32)
{
value |= value << 32;
}
return TryEncodeBitMask(value, out immN, out immS, out immR);
}
public static bool TryEncodeBitMask(ulong value, out int immN, out int immS, out int immR)
{
// Some special values also can't be encoded:
// 0 can't be encoded because we need to subtract 1 from onesCount (which would became negative if 0).
// A value with all bits set can't be encoded because it is reserved according to the spec, because:
// Any value AND all ones will be equal itself, so it's effectively a no-op.
// Any value OR all ones will be equal all ones, so one can just use MOV.
// Any value XOR all ones will be equal its inverse, so one can just use MVN.
if (value == 0 || value == ulong.MaxValue)
{
immN = 0;
immS = 0;
immR = 0;
return false;
}
// Normalize value, rotating it such that the LSB is 1: Ensures we get a complete element that has not
// been cut-in-half across the word boundary.
int rotation = BitOperations.TrailingZeroCount(value & (value + 1));
ulong rotatedValue = ulong.RotateRight(value, rotation);
// Now that we have a complete element in the LSB with the LSB = 1, determine size and number of ones
// in element.
int elementSize = BitOperations.TrailingZeroCount(rotatedValue & (rotatedValue + 1));
int onesInElement = BitOperations.TrailingZeroCount(~rotatedValue);
// Check the value is repeating; also ensures element size is a power of two.
if (ulong.RotateRight(value, elementSize) != value)
{
immN = 0;
immS = 0;
immR = 0;
return false;
}
immN = (elementSize >> 6) & 1;
immS = (((~elementSize + 1) << 1) | (onesInElement - 1)) & 0x3f;
immR = (elementSize - rotation) & (elementSize - 1);
return true;
}
}
}

View file

@ -0,0 +1,40 @@
using System.Diagnostics;
using System.Runtime.CompilerServices;
namespace Ryujinx.Cpu.Nce.Arm64
{
struct Operand
{
public readonly OperandKind Kind { get; }
public readonly OperandType Type { get; }
public readonly ulong Value { get; }
public Operand(OperandKind kind, OperandType type, ulong value)
{
Kind = kind;
Type = type;
Value = value;
}
public Operand(int index, RegisterType regType, OperandType type) : this(OperandKind.Register, type, (ulong)((int)regType << 24 | index))
{
}
public Operand(OperandType type, ulong value) : this(OperandKind.Constant, type, value)
{
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly Register GetRegister()
{
Debug.Assert(Kind == OperandKind.Register);
return new Register((int)Value & 0xffffff, (RegisterType)(Value >> 24));
}
public readonly int AsInt32()
{
return (int)Value;
}
}
}

View file

@ -0,0 +1,10 @@
namespace Ryujinx.Cpu.Nce.Arm64
{
enum OperandKind
{
None,
Constant,
Label,
Register,
}
}

View file

@ -0,0 +1,36 @@
using System;
namespace Ryujinx.Cpu.Nce.Arm64
{
enum OperandType
{
None,
I32,
I64,
FP32,
FP64,
V128,
}
static class OperandTypeExtensions
{
public static bool IsInteger(this OperandType type)
{
return type == OperandType.I32 ||
type == OperandType.I64;
}
public static int GetSizeInBytes(this OperandType type)
{
return type switch
{
OperandType.FP32 => 4,
OperandType.FP64 => 8,
OperandType.I32 => 4,
OperandType.I64 => 8,
OperandType.V128 => 16,
_ => throw new InvalidOperationException($"Invalid operand type \"{type}\"."),
};
}
}
}

View file

@ -0,0 +1,43 @@
using System;
namespace Ryujinx.Cpu.Nce.Arm64
{
readonly struct Register : IEquatable<Register>
{
public int Index { get; }
public RegisterType Type { get; }
public Register(int index, RegisterType type)
{
Index = index;
Type = type;
}
public override int GetHashCode()
{
return (ushort)Index | ((int)Type << 16);
}
public static bool operator ==(Register x, Register y)
{
return x.Equals(y);
}
public static bool operator !=(Register x, Register y)
{
return !x.Equals(y);
}
public override bool Equals(object obj)
{
return obj is Register reg && Equals(reg);
}
public bool Equals(Register other)
{
return other.Index == Index &&
other.Type == Type;
}
}
}

View file

@ -0,0 +1,220 @@
using System.Numerics;
namespace Ryujinx.Cpu.Nce.Arm64
{
readonly struct RegisterSaveRestore
{
private const int FpRegister = 29;
private const int LrRegister = 30;
private const int Encodable9BitsOffsetLimit = 0x200;
private readonly int _intMask;
private readonly int _vecMask;
private readonly OperandType _vecType;
private readonly bool _hasCall;
public RegisterSaveRestore(int intMask, int vecMask = 0, OperandType vecType = OperandType.FP64, bool hasCall = false)
{
_intMask = intMask;
_vecMask = vecMask;
_vecType = vecType;
_hasCall = hasCall;
}
public void WritePrologue(Assembler asm)
{
int intMask = _intMask;
int vecMask = _vecMask;
int intCalleeSavedRegsCount = BitOperations.PopCount((uint)intMask);
int vecCalleeSavedRegsCount = BitOperations.PopCount((uint)vecMask);
int calleeSaveRegionSize = Align16(intCalleeSavedRegsCount * 8 + vecCalleeSavedRegsCount * _vecType.GetSizeInBytes());
int offset = 0;
WritePrologueCalleeSavesPreIndexed(asm, ref intMask, ref offset, calleeSaveRegionSize, OperandType.I64);
if (_vecType == OperandType.V128 && (intCalleeSavedRegsCount & 1) != 0)
{
offset += 8;
}
WritePrologueCalleeSavesPreIndexed(asm, ref vecMask, ref offset, calleeSaveRegionSize, _vecType);
if (_hasCall)
{
Operand rsp = Register(Assembler.SpRegister);
asm.StpRiPre(Register(FpRegister), Register(LrRegister), rsp, -16);
asm.MovSp(Register(FpRegister), rsp);
}
}
private static void WritePrologueCalleeSavesPreIndexed(
Assembler asm,
ref int mask,
ref int offset,
int calleeSaveRegionSize,
OperandType type)
{
if ((BitOperations.PopCount((uint)mask) & 1) != 0)
{
int reg = BitOperations.TrailingZeroCount(mask);
mask &= ~(1 << reg);
if (offset != 0)
{
asm.StrRiUn(Register(reg, type), Register(Assembler.SpRegister), offset);
}
else if (calleeSaveRegionSize < Encodable9BitsOffsetLimit)
{
asm.StrRiPre(Register(reg, type), Register(Assembler.SpRegister), -calleeSaveRegionSize);
}
else
{
asm.Sub(Register(Assembler.SpRegister), Register(Assembler.SpRegister), new Operand(OperandType.I64, (ulong)calleeSaveRegionSize));
asm.StrRiUn(Register(reg, type), Register(Assembler.SpRegister), 0);
}
offset += type.GetSizeInBytes();
}
while (mask != 0)
{
int reg = BitOperations.TrailingZeroCount(mask);
mask &= ~(1 << reg);
int reg2 = BitOperations.TrailingZeroCount(mask);
mask &= ~(1 << reg2);
if (offset != 0)
{
asm.StpRiUn(Register(reg, type), Register(reg2, type), Register(Assembler.SpRegister), offset);
}
else if (calleeSaveRegionSize < Encodable9BitsOffsetLimit)
{
asm.StpRiPre(Register(reg, type), Register(reg2, type), Register(Assembler.SpRegister), -calleeSaveRegionSize);
}
else
{
asm.Sub(Register(Assembler.SpRegister), Register(Assembler.SpRegister), new Operand(OperandType.I64, (ulong)calleeSaveRegionSize));
asm.StpRiUn(Register(reg, type), Register(reg2, type), Register(Assembler.SpRegister), 0);
}
offset += type.GetSizeInBytes() * 2;
}
}
public void WriteEpilogue(Assembler asm)
{
int intMask = _intMask;
int vecMask = _vecMask;
int intCalleeSavedRegsCount = BitOperations.PopCount((uint)intMask);
int vecCalleeSavedRegsCount = BitOperations.PopCount((uint)vecMask);
bool misalignedVector = _vecType == OperandType.V128 && (intCalleeSavedRegsCount & 1) != 0;
int offset = intCalleeSavedRegsCount * 8 + vecCalleeSavedRegsCount * _vecType.GetSizeInBytes();
if (misalignedVector)
{
offset += 8;
}
int calleeSaveRegionSize = Align16(offset);
if (_hasCall)
{
Operand rsp = Register(Assembler.SpRegister);
asm.LdpRiPost(Register(FpRegister), Register(LrRegister), rsp, 16);
}
WriteEpilogueCalleeSavesPostIndexed(asm, ref vecMask, ref offset, calleeSaveRegionSize, _vecType);
if (misalignedVector)
{
offset -= 8;
}
WriteEpilogueCalleeSavesPostIndexed(asm, ref intMask, ref offset, calleeSaveRegionSize, OperandType.I64);
}
private static void WriteEpilogueCalleeSavesPostIndexed(
Assembler asm,
ref int mask,
ref int offset,
int calleeSaveRegionSize,
OperandType type)
{
while (mask != 0)
{
int reg = HighestBitSet(mask);
mask &= ~(1 << reg);
if (mask != 0)
{
int reg2 = HighestBitSet(mask);
mask &= ~(1 << reg2);
offset -= type.GetSizeInBytes() * 2;
if (offset != 0)
{
asm.LdpRiUn(Register(reg2, type), Register(reg, type), Register(Assembler.SpRegister), offset);
}
else if (calleeSaveRegionSize < Encodable9BitsOffsetLimit)
{
asm.LdpRiPost(Register(reg2, type), Register(reg, type), Register(Assembler.SpRegister), calleeSaveRegionSize);
}
else
{
asm.LdpRiUn(Register(reg2, type), Register(reg, type), Register(Assembler.SpRegister), 0);
asm.Add(Register(Assembler.SpRegister), Register(Assembler.SpRegister), new Operand(OperandType.I64, (ulong)calleeSaveRegionSize));
}
}
else
{
offset -= type.GetSizeInBytes();
if (offset != 0)
{
asm.LdrRiUn(Register(reg, type), Register(Assembler.SpRegister), offset);
}
else if (calleeSaveRegionSize < Encodable9BitsOffsetLimit)
{
asm.LdrRiPost(Register(reg, type), Register(Assembler.SpRegister), calleeSaveRegionSize);
}
else
{
asm.LdrRiUn(Register(reg, type), Register(Assembler.SpRegister), 0);
asm.Add(Register(Assembler.SpRegister), Register(Assembler.SpRegister), new Operand(OperandType.I64, (ulong)calleeSaveRegionSize));
}
}
}
}
private static int HighestBitSet(int value)
{
return 31 - BitOperations.LeadingZeroCount((uint)value);
}
private static Operand Register(int register, OperandType type = OperandType.I64)
{
return new Operand(register, RegisterType.Integer, type);
}
private static int Align16(int value)
{
return (value + 0xf) & ~0xf;
}
}
}

View file

@ -0,0 +1,8 @@
namespace Ryujinx.Cpu.Nce.Arm64
{
enum RegisterType
{
Integer,
Vector,
}
}

View file

@ -0,0 +1,328 @@
using ARMeilleure.Memory;
using Ryujinx.Memory;
using Ryujinx.Memory.Range;
using Ryujinx.Memory.Tracking;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
namespace Ryujinx.Cpu.Nce
{
/// <summary>
/// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region.
/// </summary>
public sealed class MemoryManagerNative : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
{
private readonly MemoryBlock _addressSpace;
private readonly MemoryBlock _backingMemory;
private readonly PageTable<ulong> _pageTable;
private readonly MemoryEhMeilleure _memoryEh;
private readonly ManagedPageFlags _pages;
/// <inheritdoc/>
public bool UsesPrivateAllocations => false;
public IntPtr PageTablePointer => IntPtr.Zero;
public ulong ReservedSize => (ulong)_addressSpace.Pointer.ToInt64();
public MemoryManagerType Type => MemoryManagerType.HostMappedUnsafe;
public MemoryTracking Tracking { get; }
public event Action<ulong, ulong> UnmapEvent;
public int AddressSpaceBits { get; }
protected override ulong AddressSpaceSize { get; }
/// <summary>
/// Creates a new instance of the host mapped memory manager.
/// </summary>
/// <param name="addressSpace">Address space memory block</param>
/// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
/// <param name="addressSpaceSize">Size of the address space</param>
/// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
public MemoryManagerNative(
MemoryBlock addressSpace,
MemoryBlock backingMemory,
ulong addressSpaceSize,
InvalidAccessHandler invalidAccessHandler = null)
{
_backingMemory = backingMemory;
_pageTable = new PageTable<ulong>();
AddressSpaceSize = addressSpaceSize;
ulong asSize = PageSize;
int asBits = PageBits;
while (asSize < addressSpaceSize)
{
asSize <<= 1;
asBits++;
}
AddressSpaceBits = asBits;
_pages = new ManagedPageFlags(asBits);
_addressSpace = addressSpace;
Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler);
_memoryEh = new MemoryEhMeilleure(addressSpaceSize, Tracking);
}
/// <inheritdoc/>
public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
{
AssertValidAddressAndSize(va, size);
_addressSpace.MapView(_backingMemory, pa, AddressToOffset(va), size);
_pages.AddMapping(va, size);
PtMap(va, pa, size);
Tracking.Map(va, size);
}
private void PtMap(ulong va, ulong pa, ulong size)
{
while (size != 0)
{
_pageTable.Map(va, pa);
va += PageSize;
pa += PageSize;
size -= PageSize;
}
}
/// <inheritdoc/>
public void Unmap(ulong va, ulong size)
{
AssertValidAddressAndSize(va, size);
UnmapEvent?.Invoke(va, size);
Tracking.Unmap(va, size);
_pages.RemoveMapping(va, size);
PtUnmap(va, size);
_addressSpace.UnmapView(_backingMemory, AddressToOffset(va), size);
}
private void PtUnmap(ulong va, ulong size)
{
while (size != 0)
{
_pageTable.Unmap(va);
va += PageSize;
size -= PageSize;
}
}
/// <inheritdoc/>
public void Reprotect(ulong va, ulong size, MemoryPermission protection)
{
_addressSpace.Reprotect(AddressToOffset(va), size, protection);
}
public ref T GetRef<T>(ulong va) where T : unmanaged
{
if (!IsContiguous(va, Unsafe.SizeOf<T>()))
{
ThrowMemoryNotContiguous();
}
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
return ref _backingMemory.GetRef<T>(GetPhysicalAddressChecked(va));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool IsMapped(ulong va)
{
return ValidateAddress(va) && _pages.IsMapped(va);
}
/// <inheritdoc/>
public bool IsRangeMapped(ulong va, ulong size)
{
AssertValidAddressAndSize(va, size);
return _pages.IsRangeMapped(va, size);
}
/// <inheritdoc/>
public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
{
if (size == 0)
{
return Enumerable.Empty<HostMemoryRange>();
}
var guestRegions = GetPhysicalRegionsImpl(va, size);
if (guestRegions == null)
{
return null;
}
var regions = new HostMemoryRange[guestRegions.Count];
for (int i = 0; i < regions.Length; i++)
{
var guestRegion = guestRegions[i];
IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size);
regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size);
}
return regions;
}
/// <inheritdoc/>
public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
{
if (size == 0)
{
return Enumerable.Empty<MemoryRange>();
}
return GetPhysicalRegionsImpl(va, size);
}
private List<MemoryRange> GetPhysicalRegionsImpl(ulong va, ulong size)
{
if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
{
return null;
}
int pages = GetPagesCount(va, (uint)size, out va);
var regions = new List<MemoryRange>();
ulong regionStart = GetPhysicalAddressInternal(va);
ulong regionSize = PageSize;
for (int page = 0; page < pages - 1; page++)
{
if (!ValidateAddress(va + PageSize))
{
return null;
}
ulong newPa = GetPhysicalAddressInternal(va + PageSize);
if (GetPhysicalAddressInternal(va) + PageSize != newPa)
{
regions.Add(new MemoryRange(regionStart, regionSize));
regionStart = newPa;
regionSize = 0;
}
va += PageSize;
regionSize += PageSize;
}
regions.Add(new MemoryRange(regionStart, regionSize));
return regions;
}
private ulong GetPhysicalAddressChecked(ulong va)
{
if (!IsMapped(va))
{
ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}");
}
return GetPhysicalAddressInternal(va);
}
private ulong GetPhysicalAddressInternal(ulong va)
{
return _pageTable.Read(va) + (va & PageMask);
}
/// <inheritdoc/>
/// <remarks>
/// This function also validates that the given range is both valid and mapped, and will throw if it is not.
/// </remarks>
public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
{
AssertValidAddressAndSize(va, size);
if (precise)
{
Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
return;
}
_pages.SignalMemoryTracking(Tracking, va, size, write, exemptId);
}
/// <inheritdoc/>
public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest)
{
if (guest)
{
_addressSpace.Reprotect(AddressToOffset(va), size, protection, false);
}
else
{
_pages.TrackingReprotect(va, size, protection);
}
}
/// <inheritdoc/>
public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags)
{
return Tracking.BeginTracking(address, size, id, flags);
}
/// <inheritdoc/>
public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id, RegionFlags flags)
{
return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags);
}
/// <inheritdoc/>
public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
{
return Tracking.BeginSmartGranularTracking(address, size, granularity, id);
}
private ulong AddressToOffset(ulong address)
{
if (address < ReservedSize)
{
throw new ArgumentException($"Invalid address 0x{address:x16}");
}
return address - ReservedSize;
}
/// <summary>
/// Disposes of resources used by the memory manager.
/// </summary>
protected override void Destroy()
{
_addressSpace.Dispose();
_memoryEh.Dispose();
}
protected override Memory<byte> GetPhysicalAddressMemory(nuint pa, int size)
=> _backingMemory.GetMemory(pa, size);
protected override Span<byte> GetPhysicalAddressSpan(nuint pa, int size)
=> _backingMemory.GetSpan(pa, size);
protected override nuint TranslateVirtualAddressChecked(ulong va)
=> (nuint)GetPhysicalAddressChecked(va);
protected override nuint TranslateVirtualAddressUnchecked(ulong va)
=> (nuint)GetPhysicalAddressInternal(va);
}
}

View file

@ -0,0 +1,81 @@
using Ryujinx.Common;
using Ryujinx.Memory;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.InteropServices;
namespace Ryujinx.Cpu.Nce
{
/// <summary>
/// Native Code Execution CPU code patch.
/// </summary>
public class NceCpuCodePatch
{
private readonly List<uint> _code;
private readonly struct PatchTarget
{
public readonly int TextIndex;
public readonly int PatchStartIndex;
public readonly int PatchBranchIndex;
public PatchTarget(int textIndex, int patchStartIndex, int patchBranchIndex)
{
TextIndex = textIndex;
PatchStartIndex = patchStartIndex;
PatchBranchIndex = patchBranchIndex;
}
}
private readonly List<PatchTarget> _patchTargets;
/// <inheritdoc/>
public ulong Size => BitUtils.AlignUp((ulong)_code.Count * sizeof(uint), 0x1000UL);
public NceCpuCodePatch()
{
_code = new();
_patchTargets = new();
}
internal void AddCode(int textIndex, IEnumerable<uint> code)
{
int patchStartIndex = _code.Count;
_code.AddRange(code);
_patchTargets.Add(new PatchTarget(textIndex, patchStartIndex, _code.Count - 1));
}
/// <inheritdoc/>
public void Write(IVirtualMemoryManager memoryManager, ulong patchAddress, ulong textAddress)
{
uint[] code = _code.ToArray();
foreach (var patchTarget in _patchTargets)
{
ulong instPatchStartAddress = patchAddress + (ulong)patchTarget.PatchStartIndex * sizeof(uint);
ulong instPatchBranchAddress = patchAddress + (ulong)patchTarget.PatchBranchIndex * sizeof(uint);
ulong instTextAddress = textAddress + (ulong)patchTarget.TextIndex * sizeof(uint);
uint prevInst = memoryManager.Read<uint>(instTextAddress);
code[patchTarget.PatchBranchIndex] |= EncodeSImm26_2(checked((int)((long)instTextAddress - (long)instPatchBranchAddress + sizeof(uint))));
memoryManager.Write(instTextAddress, 0x14000000u | EncodeSImm26_2(checked((int)((long)instPatchStartAddress - (long)instTextAddress))));
uint newInst = memoryManager.Read<uint>(instTextAddress);
}
if (Size != 0)
{
memoryManager.Write(patchAddress, MemoryMarshal.Cast<uint, byte>(code));
memoryManager.Reprotect(patchAddress, Size, MemoryPermission.ReadAndExecute);
}
}
private static uint EncodeSImm26_2(int value)
{
uint imm = (uint)(value >> 2) & 0x3ffffff;
Debug.Assert(((int)imm << 6) >> 4 == value, $"Failed to encode constant 0x{value:X}.");
return imm;
}
}
}

View file

@ -0,0 +1,144 @@
using ARMeilleure.Memory;
using Ryujinx.Cpu.Signal;
using Ryujinx.Common;
using Ryujinx.Memory;
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
namespace Ryujinx.Cpu.Nce
{
class NceCpuContext : ICpuContext
{
private static uint[] _getTpidrEl0Code = new uint[]
{
GetMrsTpidrEl0(0), // mrs x0, tpidr_el0
0xd65f03c0u, // ret
};
private static uint GetMrsTpidrEl0(uint rd)
{
if (OperatingSystem.IsMacOS())
{
return 0xd53bd060u | rd; // TPIDRRO
}
else
{
return 0xd53bd040u | rd; // TPIDR
}
}
readonly struct CodeWriter
{
private readonly List<uint> _fullCode;
public CodeWriter()
{
_fullCode = new List<uint>();
}
public ulong Write(uint[] code)
{
ulong offset = (ulong)_fullCode.Count * sizeof(uint);
_fullCode.AddRange(code);
return offset;
}
public MemoryBlock CreateMemoryBlock()
{
ReadOnlySpan<byte> codeBytes = MemoryMarshal.Cast<uint, byte>(_fullCode.ToArray());
MemoryBlock codeBlock = new(BitUtils.AlignUp((ulong)codeBytes.Length, 0x1000UL));
codeBlock.Write(0, codeBytes);
codeBlock.Reprotect(0, (ulong)codeBytes.Length, MemoryPermission.ReadAndExecute, true);
return codeBlock;
}
}
private delegate void ThreadStart(IntPtr nativeContextPtr);
private delegate IntPtr GetTpidrEl0();
private static MemoryBlock _codeBlock;
private static ThreadStart _threadStart;
private static GetTpidrEl0 _getTpidrEl0;
private readonly ITickSource _tickSource;
private readonly IMemoryManager _memoryManager;
static NceCpuContext()
{
CodeWriter codeWriter = new();
uint[] threadStartCode = NcePatcher.GenerateThreadStartCode();
uint[] ehSuspendCode = NcePatcher.GenerateSuspendExceptionHandler();
ulong threadStartCodeOffset = codeWriter.Write(threadStartCode);
ulong getTpidrEl0CodeOffset = codeWriter.Write(_getTpidrEl0Code);
ulong ehSuspendCodeOffset = codeWriter.Write(ehSuspendCode);
MemoryBlock codeBlock = null;
NativeSignalHandler.InitializeSignalHandler((IntPtr oldSignalHandlerSegfaultPtr, IntPtr signalHandlerPtr) =>
{
uint[] ehWrapperCode = NcePatcher.GenerateWrapperExceptionHandler(oldSignalHandlerSegfaultPtr, signalHandlerPtr);
ulong ehWrapperCodeOffset = codeWriter.Write(ehWrapperCode);
codeBlock = codeWriter.CreateMemoryBlock();
return codeBlock.GetPointer(ehWrapperCodeOffset, (ulong)ehWrapperCode.Length * sizeof(uint));
});
NativeSignalHandler.InstallUnixSignalHandler(NceThreadPal.UnixSuspendSignal, codeBlock.GetPointer(ehSuspendCodeOffset, (ulong)ehSuspendCode.Length * sizeof(uint)));
_threadStart = Marshal.GetDelegateForFunctionPointer<ThreadStart>(codeBlock.GetPointer(threadStartCodeOffset, (ulong)threadStartCode.Length * sizeof(uint)));
_getTpidrEl0 = Marshal.GetDelegateForFunctionPointer<GetTpidrEl0>(codeBlock.GetPointer(getTpidrEl0CodeOffset, (ulong)_getTpidrEl0Code.Length * sizeof(uint)));
_codeBlock = codeBlock;
}
public NceCpuContext(ITickSource tickSource, IMemoryManager memory, bool for64Bit)
{
_tickSource = tickSource;
_memoryManager = memory;
}
/// <inheritdoc/>
public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks)
{
return new NceExecutionContext(exceptionCallbacks);
}
/// <inheritdoc/>
public void Execute(IExecutionContext context, ulong address)
{
NceExecutionContext nec = (NceExecutionContext)context;
NceNativeInterface.RegisterThread(nec, _tickSource);
int tableIndex = NceThreadTable.Register(_getTpidrEl0(), nec.NativeContextPtr);
nec.SetStartAddress(address);
_threadStart(nec.NativeContextPtr);
nec.Exit();
NceThreadTable.Unregister(tableIndex);
}
/// <inheritdoc/>
public void InvalidateCacheRegion(ulong address, ulong size)
{
}
/// <inheritdoc/>
public IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled, string cacheSelector)
{
return new DummyDiskCacheLoadState();
}
/// <inheritdoc/>
public void PrepareCodeRange(ulong address, ulong size)
{
}
public void Dispose()
{
}
}
}

View file

@ -0,0 +1,19 @@
using ARMeilleure.Memory;
namespace Ryujinx.Cpu.Nce
{
public class NceEngine : ICpuEngine
{
public ITickSource TickSource{ get; }
public NceEngine(ITickSource tickSource)
{
TickSource = tickSource;
}
public ICpuContext CreateCpuContext(IMemoryManager memoryManager, bool for64Bit)
{
return new NceCpuContext(TickSource, memoryManager, for64Bit);
}
}
}

View file

@ -0,0 +1,174 @@
using ARMeilleure.State;
using Ryujinx.Cpu.Signal;
using Ryujinx.Memory;
using System;
using System.Runtime.InteropServices;
using System.Threading;
namespace Ryujinx.Cpu.Nce
{
class NceExecutionContext : IExecutionContext
{
private const ulong AlternateStackSize = 0x4000;
private readonly NceNativeContext _context;
private readonly ExceptionCallbacks _exceptionCallbacks;
internal IntPtr NativeContextPtr => _context.BasePtr;
public ulong Pc => 0UL;
public long TpidrEl0
{
get => (long)_context.GetStorage().TpidrEl0;
set => _context.GetStorage().TpidrEl0 = (ulong)value;
}
public long TpidrroEl0
{
get => (long)_context.GetStorage().TpidrroEl0;
set => _context.GetStorage().TpidrroEl0 = (ulong)value;
}
public uint Pstate
{
get => _context.GetStorage().Pstate;
set => _context.GetStorage().Pstate = value;
}
public uint Fpcr
{
get => _context.GetStorage().Fpcr;
set => _context.GetStorage().Fpcr = value;
}
public uint Fpsr
{
get => _context.GetStorage().Fpsr;
set => _context.GetStorage().Fpsr = value;
}
public bool IsAarch32
{
get => false;
set
{
if (value)
{
throw new NotSupportedException();
}
}
}
public bool Running { get; private set; }
private delegate bool SupervisorCallHandler(int imm);
private SupervisorCallHandler _svcHandler;
private MemoryBlock _alternateStackMemory;
public NceExecutionContext(ExceptionCallbacks exceptionCallbacks)
{
_svcHandler = OnSupervisorCall;
IntPtr svcHandlerPtr = Marshal.GetFunctionPointerForDelegate(_svcHandler);
_context = new NceNativeContext();
ref var storage = ref _context.GetStorage();
storage.SvcCallHandler = svcHandlerPtr;
storage.InManaged = 1u;
storage.CtrEl0 = 0x8444c004; // TODO: Get value from host CPU instead of using guest one?
Running = true;
_exceptionCallbacks = exceptionCallbacks;
}
public ulong GetX(int index) => _context.GetStorage().X[index];
public void SetX(int index, ulong value) => _context.GetStorage().X[index] = value;
public V128 GetV(int index) => _context.GetStorage().V[index];
public void SetV(int index, V128 value) => _context.GetStorage().V[index] = value;
// TODO
public bool GetPstateFlag(PState flag) => false;
public void SetPstateFlag(PState flag, bool value) { }
// TODO
public bool GetFPstateFlag(FPState flag) => false;
public void SetFPstateFlag(FPState flag, bool value) { }
public void SetStartAddress(ulong address)
{
ref var storage = ref _context.GetStorage();
storage.X[30] = address;
storage.HostThreadHandle = NceThreadPal.GetCurrentThreadHandle();
RegisterAlternateStack();
}
public void Exit()
{
_context.GetStorage().HostThreadHandle = IntPtr.Zero;
UnregisterAlternateStack();
}
private void RegisterAlternateStack()
{
// We need to use an alternate stack to handle the suspend signal,
// as the guest stack may be in a state that is not suitable for the signal handlers.
_alternateStackMemory = new MemoryBlock(AlternateStackSize);
NativeSignalHandler.InstallUnixAlternateStackForCurrentThread(_alternateStackMemory.GetPointer(0UL, AlternateStackSize), AlternateStackSize);
}
private void UnregisterAlternateStack()
{
NativeSignalHandler.UninstallUnixAlternateStackForCurrentThread();
_alternateStackMemory.Dispose();
_alternateStackMemory = null;
}
public bool OnSupervisorCall(int imm)
{
_exceptionCallbacks.SupervisorCallback?.Invoke(this, 0UL, imm);
return Running;
}
public bool OnInterrupt()
{
_exceptionCallbacks.InterruptCallback?.Invoke(this);
return Running;
}
public void RequestInterrupt()
{
IntPtr threadHandle = _context.GetStorage().HostThreadHandle;
if (threadHandle != IntPtr.Zero)
{
// Bit 0 set means that the thread is currently running managed code.
// Bit 1 set means that an interrupt was requested for the thread.
// This, we only need to send the suspend signal if the value was 0 (not running managed code,
// and no interrupt was requested before).
ref uint inManaged = ref _context.GetStorage().InManaged;
uint oldValue = Interlocked.Or(ref inManaged, 2);
if (oldValue == 0)
{
NceThreadPal.SuspendThread(threadHandle);
}
}
}
public void StopRunning()
{
Running = false;
}
public void Dispose()
{
_context.Dispose();
}
}
}

View file

@ -0,0 +1,13 @@
using ARMeilleure.Memory;
using Ryujinx.Memory;
namespace Ryujinx.Cpu.Nce
{
class NceMemoryAllocator : IJitMemoryAllocator
{
public IJitMemoryBlock Allocate(ulong size) => new NceMemoryBlock(size, MemoryAllocationFlags.None);
public IJitMemoryBlock Reserve(ulong size) => new NceMemoryBlock(size, MemoryAllocationFlags.Reserve);
public ulong GetPageSize() => MemoryBlock.GetPageSize();
}
}

View file

@ -0,0 +1,25 @@
using ARMeilleure.Memory;
using Ryujinx.Memory;
using System;
namespace Ryujinx.Cpu.Nce
{
class NceMemoryBlock : IJitMemoryBlock
{
private readonly MemoryBlock _impl;
public IntPtr Pointer => _impl.Pointer;
public NceMemoryBlock(ulong size, MemoryAllocationFlags flags)
{
_impl = new MemoryBlock(size, flags);
}
public void Commit(ulong offset, ulong size) => _impl.Commit(offset, size);
public void MapAsRw(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadAndWrite);
public void MapAsRx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadAndExecute);
public void MapAsRwx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadWriteExecute);
public void Dispose() => _impl.Dispose();
}
}

View file

@ -0,0 +1,98 @@
using ARMeilleure.State;
using Ryujinx.Common.Memory;
using Ryujinx.Memory;
using System;
using System.Runtime.CompilerServices;
namespace Ryujinx.Cpu.Nce
{
class NceNativeContext : IDisposable
{
public struct NativeCtxStorage
{
public Array32<ulong> X;
public Array32<V128> V;
public ulong TpidrEl0;
public ulong TpidrroEl0;
public ulong CtrEl0;
public uint Pstate;
public uint Fpcr;
public uint Fpsr;
public uint InManaged;
public ulong HostSp;
public IntPtr HostThreadHandle;
public ulong TempStorage;
public IntPtr SvcCallHandler;
}
private static NativeCtxStorage _dummyStorage = new();
private readonly MemoryBlock _block;
public IntPtr BasePtr => _block.Pointer;
public NceNativeContext()
{
_block = new MemoryBlock((ulong)Unsafe.SizeOf<NativeCtxStorage>());
}
public static int GetXOffset(int index)
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.X[index]);
}
public static int GetGuestSPOffset()
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.X[31]);
}
public static int GetVOffset(int index)
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.V[index]);
}
public static int GetTpidrEl0Offset()
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.TpidrEl0);
}
public static int GetTpidrroEl0Offset()
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.TpidrroEl0);
}
public static int GetInManagedOffset()
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.InManaged);
}
public static int GetHostSPOffset()
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.HostSp);
}
public static int GetCtrEl0Offset()
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.CtrEl0);
}
public static int GetTempStorageOffset()
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.TempStorage);
}
public static int GetSvcCallHandlerOffset()
{
return StorageOffset(ref _dummyStorage, ref _dummyStorage.SvcCallHandler);
}
private static int StorageOffset<T>(ref NativeCtxStorage storage, ref T target)
{
return (int)Unsafe.ByteOffset(ref Unsafe.As<NativeCtxStorage, T>(ref storage), ref target);
}
public unsafe ref NativeCtxStorage GetStorage() => ref Unsafe.AsRef<NativeCtxStorage>((void*)_block.Pointer);
public void Dispose() => _block.Dispose();
}
}

View file

@ -0,0 +1,55 @@
using System;
using System.Runtime.InteropServices;
namespace Ryujinx.Cpu.Nce
{
static class NceNativeInterface
{
private delegate ulong GetTickCounterDelegate();
private delegate bool SuspendThreadHandlerDelegate();
private static GetTickCounterDelegate _getTickCounter;
private static SuspendThreadHandlerDelegate _suspendThreadHandler;
private static IntPtr _getTickCounterPtr;
private static IntPtr _suspendThreadHandlerPtr;
[ThreadStatic]
private static NceExecutionContext _context;
[ThreadStatic]
private static ITickSource _tickSource;
static NceNativeInterface()
{
_getTickCounter = GetTickCounter;
_suspendThreadHandler = SuspendThreadHandler;
_getTickCounterPtr = Marshal.GetFunctionPointerForDelegate(_getTickCounter);
_suspendThreadHandlerPtr = Marshal.GetFunctionPointerForDelegate(_suspendThreadHandler);
}
public static void RegisterThread(NceExecutionContext context, ITickSource tickSource)
{
_context = context;
_tickSource = tickSource;
}
public static ulong GetTickCounter()
{
return _tickSource.Counter;
}
public static bool SuspendThreadHandler()
{
return _context.OnInterrupt();
}
public static IntPtr GetTickCounterAccessFunctionPointer()
{
return _getTickCounterPtr;
}
public static IntPtr GetSuspendThreadHandlerFunctionPointer()
{
return _suspendThreadHandlerPtr;
}
}
}

View file

@ -0,0 +1,603 @@
using Ryujinx.Cpu.Nce.Arm64;
using Ryujinx.Common.Logging;
using System;
using System.Runtime.InteropServices;
namespace Ryujinx.Cpu.Nce
{
public static class NcePatcher
{
private const int ScratchBaseReg = 19;
private const uint IntCalleeSavedRegsMask = 0x1ff80000; // X19 to X28
private const uint FpCalleeSavedRegsMask = 0xff00; // D8 to D15
public static NceCpuCodePatch CreatePatch(ReadOnlySpan<byte> textSection)
{
NceCpuCodePatch codePatch = new();
var textUint = MemoryMarshal.Cast<byte, uint>(textSection);
for (int i = 0; i < textUint.Length; i++)
{
uint inst = textUint[i];
ulong address = (ulong)i * sizeof(uint);
if ((inst & ~(0xffffu << 5)) == 0xd4000001u) // svc #0
{
uint svcId = (ushort)(inst >> 5);
codePatch.AddCode(i, WriteSvcPatch(svcId));
Logger.Debug?.Print(LogClass.Cpu, $"Patched SVC #{svcId} at 0x{address:X}.");
}
else if ((inst & ~0x1f) == 0xd53bd060) // mrs x0, tpidrro_el0
{
uint rd = inst & 0x1f;
codePatch.AddCode(i, WriteMrsTpidrroEl0Patch(rd));
Logger.Debug?.Print(LogClass.Cpu, $"Patched MRS x{rd}, tpidrro_el0 at 0x{address:X}.");
}
else if ((inst & ~0x1f) == 0xd53bd040) // mrs x0, tpidr_el0
{
uint rd = inst & 0x1f;
codePatch.AddCode(i, WriteMrsTpidrEl0Patch(rd));
Logger.Debug?.Print(LogClass.Cpu, $"Patched MRS x{rd}, tpidr_el0 at 0x{address:X}.");
}
else if ((inst & ~0x1f) == 0xd53b0020 && OperatingSystem.IsMacOS()) // mrs x0, ctr_el0
{
uint rd = inst & 0x1f;
codePatch.AddCode(i, WriteMrsCtrEl0Patch(rd));
Logger.Debug?.Print(LogClass.Cpu, $"Patched MRS x{rd}, ctr_el0 at 0x{address:X}.");
}
else if ((inst & ~0x1f) == 0xd53be020) // mrs x0, cntpct_el0
{
uint rd = inst & 0x1f;
codePatch.AddCode(i, WriteMrsCntpctEl0Patch(rd));
Logger.Debug?.Print(LogClass.Cpu, $"Patched MRS x{rd}, cntpct_el0 at 0x{address:X}.");
}
else if ((inst & ~0x1f) == 0xd51bd040) // msr tpidr_el0, x0
{
uint rd = inst & 0x1f;
codePatch.AddCode(i, WriteMsrTpidrEl0Patch(rd));
Logger.Debug?.Print(LogClass.Cpu, $"Patched MSR tpidr_el0, x{rd} at 0x{address:X}.");
}
}
return codePatch;
}
private static uint[] WriteSvcPatch(uint svcId)
{
Assembler asm = new();
WriteManagedCall(asm, (asm, ctx, tmp, tmp2) =>
{
for (int i = 0; i < 8; i++)
{
asm.StrRiUn(Gpr(i), ctx, NceNativeContext.GetXOffset(i));
}
WriteInManagedLockAcquire(asm, ctx, tmp, tmp2);
asm.Mov(Gpr(0, OperandType.I32), svcId);
asm.LdrRiUn(tmp, ctx, NceNativeContext.GetSvcCallHandlerOffset());
asm.Blr(tmp);
Operand lblContinue = asm.CreateLabel();
Operand lblQuit = asm.CreateLabel();
asm.Cbnz(Gpr(0, OperandType.I32), lblContinue);
asm.MarkLabel(lblQuit);
CreateRegisterSaveRestoreForManaged().WriteEpilogue(asm);
asm.Ret(Gpr(30));
asm.MarkLabel(lblContinue);
WriteInManagedLockRelease(asm, ctx, tmp, tmp2, ThreadExitMethod.Label, lblQuit);
for (int i = 0; i < 8; i++)
{
asm.LdrRiUn(Gpr(i), ctx, NceNativeContext.GetXOffset(i));
}
}, 0xff);
asm.B(0);
return asm.GetCode();
}
private static uint[] WriteMrsTpidrroEl0Patch(uint rd)
{
return WriteMrsContextRead(rd, NceNativeContext.GetTpidrroEl0Offset());
}
private static uint[] WriteMrsTpidrEl0Patch(uint rd)
{
return WriteMrsContextRead(rd, NceNativeContext.GetTpidrEl0Offset());
}
private static uint[] WriteMrsCtrEl0Patch(uint rd)
{
return WriteMrsContextRead(rd, NceNativeContext.GetCtrEl0Offset());
}
private static uint[] WriteMrsCntpctEl0Patch(uint rd)
{
Assembler asm = new();
WriteManagedCall(asm, (asm, ctx, tmp, tmp2) =>
{
WriteInManagedLockAcquire(asm, ctx, tmp, tmp2);
asm.Mov(tmp, (ulong)NceNativeInterface.GetTickCounterAccessFunctionPointer());
asm.Blr(tmp);
asm.StrRiUn(Gpr(0), ctx, NceNativeContext.GetTempStorageOffset());
WriteInManagedLockRelease(asm, ctx, tmp, tmp2, ThreadExitMethod.GenerateReturn);
asm.LdrRiUn(Gpr((int)rd), ctx, NceNativeContext.GetTempStorageOffset());
}, 1u << (int)rd);
asm.B(0);
return asm.GetCode();
}
private static uint[] WriteMsrTpidrEl0Patch(uint rd)
{
Assembler asm = new();
Span<int> scratchRegs = stackalloc int[3];
PickScratchRegs(scratchRegs, 1u << (int)rd);
RegisterSaveRestore rsr = new((1 << scratchRegs[0]) | (1 << scratchRegs[1]) | (1 << scratchRegs[2]));
rsr.WritePrologue(asm);
WriteLoadContext(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2]));
asm.StrRiUn(Gpr((int)rd), Gpr(scratchRegs[0]),NceNativeContext.GetTpidrEl0Offset());
rsr.WriteEpilogue(asm);
asm.B(0);
return asm.GetCode();
}
private static uint[] WriteMrsContextRead(uint rd, int contextOffset)
{
Assembler asm = new();
Span<int> scratchRegs = stackalloc int[3];
PickScratchRegs(scratchRegs, 1u << (int)rd);
RegisterSaveRestore rsr = new((1 << scratchRegs[0]) | (1 << scratchRegs[1]) | (1 << scratchRegs[2]));
rsr.WritePrologue(asm);
WriteLoadContext(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2]));
asm.Add(Gpr((int)rd), Gpr(scratchRegs[0]), Const((ulong)contextOffset));
rsr.WriteEpilogue(asm);
asm.LdrRiUn(Gpr((int)rd), Gpr((int)rd), 0);
asm.B(0);
return asm.GetCode();
}
private static void WriteLoadContext(Assembler asm, Operand tmp0, Operand tmp1, Operand tmp2)
{
asm.Mov(tmp0, (ulong)NceThreadTable.EntriesPointer);
if (OperatingSystem.IsMacOS())
{
asm.MrsTpidrroEl0(tmp1);
}
else
{
asm.MrsTpidrEl0(tmp1);
}
Operand lblFound = asm.CreateLabel();
Operand lblLoop = asm.CreateLabel();
asm.MarkLabel(lblLoop);
asm.LdrRiPost(tmp2, tmp0, 16);
asm.Cmp(tmp1, tmp2);
asm.B(lblFound, ArmCondition.Eq);
asm.B(lblLoop);
asm.MarkLabel(lblFound);
asm.Ldur(tmp0, tmp0, -8);
}
private static void WriteLoadContextSafe(Assembler asm, Operand lblFail, Operand tmp0, Operand tmp1, Operand tmp2, Operand tmp3)
{
asm.Mov(tmp0, (ulong)NceThreadTable.EntriesPointer);
asm.Ldur(tmp3, tmp0, -8);
asm.Add(tmp3, tmp0, tmp3, ArmShiftType.Lsl, 4);
if (OperatingSystem.IsMacOS())
{
asm.MrsTpidrroEl0(tmp1);
}
else
{
asm.MrsTpidrEl0(tmp1);
}
Operand lblFound = asm.CreateLabel();
Operand lblLoop = asm.CreateLabel();
asm.MarkLabel(lblLoop);
asm.Cmp(tmp0, tmp3);
asm.B(lblFail, ArmCondition.GeUn);
asm.LdrRiPost(tmp2, tmp0, 16);
asm.Cmp(tmp1, tmp2);
asm.B(lblFound, ArmCondition.Eq);
asm.B(lblLoop);
asm.MarkLabel(lblFound);
asm.Ldur(tmp0, tmp0, -8);
}
private static void PickScratchRegs(Span<int> scratchRegs, uint blacklistedRegMask)
{
int scratchReg = ScratchBaseReg;
for (int i = 0; i < scratchRegs.Length; i++)
{
while ((blacklistedRegMask & (1u << scratchReg)) != 0)
{
scratchReg++;
}
if (scratchReg >= 29)
{
throw new ArgumentException($"No enough register for {scratchRegs.Length} scratch register, started from {ScratchBaseReg}");
}
scratchRegs[i] = scratchReg++;
}
}
private static Operand Gpr(int register, OperandType type = OperandType.I64)
{
return new Operand(register, RegisterType.Integer, type);
}
private static Operand Vec(int register, OperandType type = OperandType.V128)
{
return new Operand(register, RegisterType.Vector, type);
}
private static Operand Const(ulong value)
{
return new Operand(OperandType.I64, value);
}
private static Operand Const(OperandType type, ulong value)
{
return new Operand(type, value);
}
private static uint GetImm26(ulong sourceAddress, ulong targetAddress)
{
long offset = (long)(targetAddress - sourceAddress);
long offsetTrunc = (offset >> 2) & 0x3FFFFFF;
if ((offsetTrunc << 38) >> 36 != offset)
{
throw new Exception($"Offset out of range: 0x{sourceAddress:X} -> 0x{targetAddress:X} (0x{offset:X})");
}
return (uint)offsetTrunc;
}
private static int GetOffset(ulong sourceAddress, ulong targetAddress)
{
long offset = (long)(targetAddress - sourceAddress);
return checked((int)offset);
}
private static uint[] GetCopy(uint[] code)
{
uint[] codeCopy = new uint[code.Length];
code.CopyTo(codeCopy, 0);
return codeCopy;
}
private static void WriteManagedCall(Assembler asm, Action<Assembler, Operand, Operand, Operand> writeCall, uint blacklistedRegMask)
{
int intMask = 0x7fffffff & (int)~blacklistedRegMask;
int vecMask = unchecked((int)0xffffffff);
Span<int> scratchRegs = stackalloc int[3];
PickScratchRegs(scratchRegs, blacklistedRegMask);
RegisterSaveRestore rsr = new(intMask, vecMask, OperandType.V128);
rsr.WritePrologue(asm);
WriteLoadContext(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2]));
asm.MovSp(Gpr(scratchRegs[1]), Gpr(Assembler.SpRegister));
asm.StrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetGuestSPOffset());
asm.LdrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetHostSPOffset());
asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[1]));
writeCall(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2]));
asm.LdrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetGuestSPOffset());
asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[1]));
rsr.WriteEpilogue(asm);
}
internal static uint[] GenerateThreadStartCode()
{
Assembler asm = new();
CreateRegisterSaveRestoreForManaged().WritePrologue(asm);
asm.MovSp(Gpr(1), Gpr(Assembler.SpRegister));
asm.StrRiUn(Gpr(1), Gpr(0), NceNativeContext.GetHostSPOffset());
for (int i = 2; i < 30; i += 2)
{
asm.LdpRiUn(Gpr(i), Gpr(i + 1), Gpr(0), NceNativeContext.GetXOffset(i));
}
for (int i = 0; i < 32; i += 2)
{
asm.LdpRiUn(Vec(i), Vec(i + 1), Gpr(0), NceNativeContext.GetVOffset(i));
}
asm.LdpRiUn(Gpr(30), Gpr(1), Gpr(0), NceNativeContext.GetXOffset(30));
asm.MovSp(Gpr(Assembler.SpRegister), Gpr(1));
asm.StrRiUn(Gpr(Assembler.ZrRegister, OperandType.I32), Gpr(0), NceNativeContext.GetInManagedOffset());
asm.LdpRiUn(Gpr(0), Gpr(1), Gpr(0), NceNativeContext.GetXOffset(0));
asm.Br(Gpr(30));
return asm.GetCode();
}
internal static uint[] GenerateSuspendExceptionHandler()
{
Assembler asm = new();
Span<int> scratchRegs = stackalloc int[4];
PickScratchRegs(scratchRegs, 0u);
RegisterSaveRestore rsr = new((1 << scratchRegs[0]) | (1 << scratchRegs[1]) | (1 << scratchRegs[2]) | (1 << scratchRegs[3]), hasCall: true);
rsr.WritePrologue(asm);
Operand lblAgain = asm.CreateLabel();
Operand lblFail = asm.CreateLabel();
WriteLoadContextSafe(asm, lblFail, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2]), Gpr(scratchRegs[3]));
asm.LdrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetHostSPOffset());
asm.MovSp(Gpr(scratchRegs[2]), Gpr(Assembler.SpRegister));
asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[1]));
asm.Cmp(Gpr(0, OperandType.I32), Const((ulong)NceThreadPal.UnixSuspendSignal));
asm.B(lblFail, ArmCondition.Ne);
// SigUsr2
asm.Mov(Gpr(scratchRegs[1], OperandType.I32), Const(OperandType.I32, 1));
asm.StrRiUn(Gpr(scratchRegs[1], OperandType.I32), Gpr(scratchRegs[0]), NceNativeContext.GetInManagedOffset());
asm.MarkLabel(lblAgain);
asm.Mov(Gpr(scratchRegs[3]), (ulong)NceNativeInterface.GetSuspendThreadHandlerFunctionPointer());
asm.Blr(Gpr(scratchRegs[3]));
// TODO: Check return value, exit if we must.
WriteInManagedLockReleaseForSuspendHandler(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[3]), lblAgain);
asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[2]));
rsr.WriteEpilogue(asm);
asm.Ret(Gpr(30));
asm.MarkLabel(lblFail);
rsr.WriteEpilogue(asm);
asm.Ret(Gpr(30));
return asm.GetCode();
}
internal static uint[] GenerateWrapperExceptionHandler(IntPtr oldSignalHandlerSegfaultPtr, IntPtr signalHandlerPtr)
{
Assembler asm = new();
Span<int> scratchRegs = stackalloc int[4];
PickScratchRegs(scratchRegs, 0u);
RegisterSaveRestore rsr = new((1 << scratchRegs[0]) | (1 << scratchRegs[1]) | (1 << scratchRegs[2]) | (1 << scratchRegs[3]), hasCall: true);
rsr.WritePrologue(asm);
Operand lblFail = asm.CreateLabel();
WriteLoadContextSafe(asm, lblFail, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2]), Gpr(scratchRegs[3]));
asm.LdrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetHostSPOffset());
asm.MovSp(Gpr(scratchRegs[2]), Gpr(Assembler.SpRegister));
asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[1]));
// SigSegv
WriteInManagedLockAcquire(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[3]));
asm.Mov(Gpr(scratchRegs[3]), (ulong)signalHandlerPtr);
asm.Blr(Gpr(scratchRegs[3]));
WriteInManagedLockRelease(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[3]), ThreadExitMethod.None);
asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[2]));
rsr.WriteEpilogue(asm);
asm.Ret(Gpr(30));
asm.MarkLabel(lblFail);
rsr.WriteEpilogue(asm);
asm.Mov(Gpr(3), (ulong)oldSignalHandlerSegfaultPtr);
asm.Br(Gpr(3));
return asm.GetCode();
}
private static void WriteInManagedLockAcquire(Assembler asm, Operand ctx, Operand tmp, Operand tmp2)
{
Operand tmpUint = new Operand(tmp.GetRegister().Index, RegisterType.Integer, OperandType.I32);
Operand tmp2Uint = new Operand(tmp2.GetRegister().Index, RegisterType.Integer, OperandType.I32);
Operand lblLoop = asm.CreateLabel();
// Bit 0 set means that the thread is currently executing managed code (that case should be impossible here).
// Bit 1 being set means there is a signal pending, we should wait for the signal, otherwise it could trigger
// while running managed code.
asm.MarkLabel(lblLoop);
asm.Add(tmp, ctx, Const((ulong)NceNativeContext.GetInManagedOffset()));
asm.Ldaxr(tmp2Uint, tmp);
asm.Cbnz(tmp2Uint, lblLoop);
asm.Mov(tmp2Uint, Const(OperandType.I32, 1));
asm.Stlxr(tmp2Uint, tmp, tmpUint);
asm.Cbnz(tmpUint, lblLoop); // Retry if store failed.
}
private enum ThreadExitMethod
{
None,
GenerateReturn,
Label
}
private static void WriteInManagedLockRelease(Assembler asm, Operand ctx, Operand tmp, Operand tmp2, ThreadExitMethod exitMethod, Operand lblQuit = default)
{
Operand tmpUint = new Operand(tmp.GetRegister().Index, RegisterType.Integer, OperandType.I32);
Operand tmp2Uint = new Operand(tmp2.GetRegister().Index, RegisterType.Integer, OperandType.I32);
Operand lblLoop = asm.CreateLabel();
Operand lblInterrupt = asm.CreateLabel();
Operand lblDone = asm.CreateLabel();
// Bit 0 set means that the thread is currently executing managed code (it should be always set here, as we just returned from managed code).
// Bit 1 being set means a interrupt was requested while it was in managed, we should service it.
asm.MarkLabel(lblLoop);
asm.Add(tmp, ctx, Const((ulong)NceNativeContext.GetInManagedOffset()));
asm.Ldaxr(tmp2Uint, tmp);
asm.Cmp(tmp2Uint, Const(OperandType.I32, 3));
asm.B(lblInterrupt, ArmCondition.Eq);
asm.Stlxr(Gpr(Assembler.ZrRegister, OperandType.I32), tmp, tmpUint);
asm.Cbnz(tmpUint, lblLoop); // Retry if store failed.
asm.B(lblDone);
asm.MarkLabel(lblInterrupt);
// If we got here, a interrupt was requested while it was in managed code.
// Let's service the interrupt and check what we should do next.
asm.Mov(tmp2Uint, Const(OperandType.I32, 1));
asm.Stlxr(tmp2Uint, tmp, tmpUint);
asm.Cbnz(tmpUint, lblLoop); // Retry if store failed.
asm.Mov(tmp, (ulong)NceNativeInterface.GetSuspendThreadHandlerFunctionPointer());
asm.Blr(tmp);
// The return value from the interrupt handler indicates if we should continue running.
// From here, we either try to release the lock again. We might have received another interrupt
// request in the meantime, in which case we should service it again.
// If we were requested to exit, then we exit if we can.
// TODO: We should also exit while on a signal handler. To do that we need to modify the PC value on the
// context. It's a bit more tricky to do, so for now we ignore that case with "ThreadExitMethod.None".
if (exitMethod == ThreadExitMethod.None)
{
asm.B(lblLoop);
}
else
{
asm.Cbnz(Gpr(0, OperandType.I32), lblLoop);
if (exitMethod == ThreadExitMethod.Label)
{
asm.B(lblQuit);
}
else if (exitMethod == ThreadExitMethod.GenerateReturn)
{
CreateRegisterSaveRestoreForManaged().WriteEpilogue(asm);
asm.Ret(Gpr(30));
}
}
asm.MarkLabel(lblDone);
}
private static void WriteInManagedLockReleaseForSuspendHandler(Assembler asm, Operand ctx, Operand tmp, Operand tmp2, Operand lblAgain)
{
Operand tmpUint = new Operand(tmp.GetRegister().Index, RegisterType.Integer, OperandType.I32);
Operand tmp2Uint = new Operand(tmp2.GetRegister().Index, RegisterType.Integer, OperandType.I32);
Operand lblLoop = asm.CreateLabel();
Operand lblInterrupt = asm.CreateLabel();
Operand lblDone = asm.CreateLabel();
// Bit 0 set means that the thread is currently executing managed code (it should be always set here, as we just returned from managed code).
// Bit 1 being set means a interrupt was requested while it was in managed, we should service it.
asm.MarkLabel(lblLoop);
asm.Add(tmp, ctx, Const((ulong)NceNativeContext.GetInManagedOffset()));
asm.Ldaxr(tmp2Uint, tmp);
asm.Cmp(tmp2Uint, Const(OperandType.I32, 3));
asm.B(lblInterrupt, ArmCondition.Eq);
asm.Stlxr(Gpr(Assembler.ZrRegister, OperandType.I32), tmp, tmpUint);
asm.Cbnz(tmpUint, lblLoop); // Retry if store failed.
asm.B(lblDone);
asm.MarkLabel(lblInterrupt);
// If we got here, a interrupt was requested while it was in managed code.
// Let's service the interrupt and check what we should do next.
asm.Mov(tmp2Uint, Const(OperandType.I32, 1));
asm.Stlxr(tmp2Uint, tmp, tmpUint);
asm.Cbnz(tmpUint, lblLoop); // Retry if store failed.
asm.B(lblAgain);
asm.MarkLabel(lblDone);
}
private static RegisterSaveRestore CreateRegisterSaveRestoreForManaged()
{
return new RegisterSaveRestore((int)IntCalleeSavedRegsMask, unchecked((int)FpCalleeSavedRegsMask), OperandType.FP64, hasCall: true);
}
}
}

View file

@ -0,0 +1,41 @@
using Ryujinx.Common;
using System;
namespace Ryujinx.Cpu.Nce
{
static class NceThreadPal
{
private const int SigUsr2Linux = 12;
private const int SigUsr2MacOS = 31;
public static int UnixSuspendSignal => OperatingSystem.IsMacOS() ? SigUsr2MacOS : SigUsr2Linux;
public static IntPtr GetCurrentThreadHandle()
{
if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS() || PlatformInfo.IsBionic)
{
return NceThreadPalUnix.GetCurrentThreadHandle();
}
else
{
throw new PlatformNotSupportedException();
}
}
public static void SuspendThread(IntPtr handle)
{
if (PlatformInfo.IsBionic)
{
NceThreadPalAndroid.SuspendThread(handle);
}
else if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
{
NceThreadPalUnix.SuspendThread(handle);
}
else
{
throw new PlatformNotSupportedException();
}
}
}
}

View file

@ -0,0 +1,20 @@
using System;
using System.Runtime.InteropServices;
namespace Ryujinx.Cpu.Nce
{
static class NceThreadPalAndroid
{
[DllImport("libc", SetLastError = true)]
private static extern int pthread_kill(IntPtr thread, int sig);
public static void SuspendThread(IntPtr handle)
{
int result = pthread_kill(handle, NceThreadPal.UnixSuspendSignal);
if (result != 0)
{
throw new Exception($"Thread kill returned error 0x{result:X}.");
}
}
}
}

View file

@ -0,0 +1,37 @@
using System;
using System.Runtime.InteropServices;
namespace Ryujinx.Cpu.Nce
{
static class NceThreadPalUnix
{
[DllImport("libc", SetLastError = true)]
private static extern IntPtr pthread_self();
[DllImport("libc", SetLastError = true)]
private static extern int pthread_threadid_np(IntPtr arg0, out ulong tid);
[DllImport("libpthread", SetLastError = true)]
private static extern int pthread_kill(IntPtr thread, int sig);
public static IntPtr GetCurrentThreadHandle()
{
return pthread_self();
}
public static ulong GetCurrentThreadId()
{
pthread_threadid_np(IntPtr.Zero, out ulong tid);
return tid;
}
public static void SuspendThread(IntPtr handle)
{
int result = pthread_kill(handle, NceThreadPal.UnixSuspendSignal);
if (result != 0)
{
throw new Exception($"Thread kill returned error 0x{result:X}.");
}
}
}
}

View file

@ -0,0 +1,97 @@
using Ryujinx.Memory;
using System;
using System.Runtime.CompilerServices;
namespace Ryujinx.Cpu.Nce
{
static class NceThreadTable
{
private const int MaxThreads = 4096;
private struct Entry
{
public IntPtr ThreadId;
public IntPtr NativeContextPtr;
public Entry(IntPtr threadId, IntPtr nativeContextPtr)
{
ThreadId = threadId;
NativeContextPtr = nativeContextPtr;
}
}
private static MemoryBlock _block;
public static IntPtr EntriesPointer => _block.Pointer + 8;
static NceThreadTable()
{
_block = new MemoryBlock((ulong)Unsafe.SizeOf<Entry>() * MaxThreads + 8UL);
_block.Write(0UL, 0UL);
}
public static int Register(IntPtr threadId, IntPtr nativeContextPtr)
{
Span<Entry> entries = GetStorage();
lock (_block)
{
ref ulong currentThreadCount = ref GetThreadsCount();
for (int i = 0; i < MaxThreads; i++)
{
if (entries[i].ThreadId == IntPtr.Zero)
{
entries[i] = new Entry(threadId, nativeContextPtr);
if (currentThreadCount < (ulong)i + 1)
{
currentThreadCount = (ulong)i + 1;
}
return i;
}
}
}
throw new Exception($"Number of active threads exceeds limit of {MaxThreads}.");
}
public static void Unregister(int tableIndex)
{
Span<Entry> entries = GetStorage();
lock (_block)
{
if (entries[tableIndex].ThreadId != IntPtr.Zero)
{
entries[tableIndex] = default;
ulong currentThreadCount = GetThreadsCount();
for (int i = (int)currentThreadCount - 1; i >= 0; i--)
{
if (entries[i].ThreadId != IntPtr.Zero)
{
break;
}
currentThreadCount = (ulong)i;
}
GetThreadsCount() = currentThreadCount;
}
}
}
private static ref ulong GetThreadsCount()
{
return ref _block.GetRef<ulong>(0UL);
}
private static unsafe Span<Entry> GetStorage()
{
return new Span<Entry>((void*)_block.GetPointer(8UL, (ulong)Unsafe.SizeOf<Entry>() * MaxThreads), MaxThreads);
}
}
}

View file

@ -24,6 +24,8 @@ namespace Ryujinx.HLE.HOS
private readonly ICpuContext _cpuContext;
private T _memoryManager;
public ulong ReservedSize { get; }
public IVirtualMemoryManager AddressSpace => _memoryManager;
public ulong AddressSpaceSize { get; }
@ -34,7 +36,8 @@ namespace Ryujinx.HLE.HOS
GpuContext gpuContext,
T memoryManager,
ulong addressSpaceSize,
bool for64Bit)
bool for64Bit,
ulong reservedSize = 0UL)
{
if (memoryManager is IRefCounted rc)
{
@ -47,8 +50,8 @@ namespace Ryujinx.HLE.HOS
_gpuContext = gpuContext;
_cpuContext = cpuEngine.CreateCpuContext(memoryManager, for64Bit);
_memoryManager = memoryManager;
AddressSpaceSize = addressSpaceSize;
ReservedSize = reservedSize;
}
public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks)

View file

@ -4,6 +4,7 @@ using Ryujinx.Cpu;
using Ryujinx.Cpu.AppleHv;
using Ryujinx.Cpu.Jit;
using Ryujinx.Cpu.LightningJit;
using Ryujinx.Cpu.Nce;
using Ryujinx.Graphics.Gpu;
using Ryujinx.HLE.HOS.Kernel;
using Ryujinx.HLE.HOS.Kernel.Process;
@ -46,19 +47,45 @@ namespace Ryujinx.HLE.HOS
_codeSize = codeSize;
}
public static NceCpuCodePatch CreateCodePatchForNce(KernelContext context, bool for64Bit, ReadOnlySpan<byte> textSection)
{
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64 && for64Bit && context.Device.Configuration.UseHypervisor && !OperatingSystem.IsMacOS())
{
return NcePatcher.CreatePatch(textSection);
}
return null;
}
public IProcessContext Create(KernelContext context, ulong pid, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit)
{
IArmProcessContext processContext;
bool isArm64Host = RuntimeInformation.ProcessArchitecture == Architecture.Arm64;
if (OperatingSystem.IsMacOS() && isArm64Host && for64Bit && context.Device.Configuration.UseHypervisor)
if (isArm64Host && for64Bit && context.Device.Configuration.UseHypervisor)
{
if (OperatingSystem.IsMacOS())
{
var cpuEngine = new HvEngine(_tickSource);
var memoryManager = new HvMemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler);
processContext = new ArmProcessContext<HvMemoryManager>(pid, cpuEngine, _gpu, memoryManager, addressSpaceSize, for64Bit);
}
else
{
if (!AddressSpace.TryCreateWithoutMirror(addressSpaceSize, out var addressSpace))
{
throw new Exception("Address space creation failed");
}
Logger.Info?.Print(LogClass.Cpu, $"NCE Base AS Address: 0x{addressSpace.Pointer.ToInt64():X} Size: 0x{addressSpace.Size:X}");
var cpuEngine = new NceEngine(_tickSource);
var memoryManager = new MemoryManagerNative(addressSpace, context.Memory, addressSpaceSize, invalidAccessHandler);
processContext = new ArmProcessContext<MemoryManagerNative>(pid, cpuEngine, _gpu, memoryManager, addressSpace.Size, for64Bit, memoryManager.ReservedSize);
}
}
else
{
MemoryManagerMode mode = context.Device.Configuration.MemoryManagerMode;
@ -74,11 +101,13 @@ namespace Ryujinx.HLE.HOS
: new JitEngine(_tickSource);
AddressSpace addressSpace = null;
MemoryBlock asNoMirror = null;
// We want to use host tracked mode if the host page size is > 4KB.
if ((mode == MemoryManagerMode.HostMapped || mode == MemoryManagerMode.HostMappedUnsafe) && MemoryBlock.GetPageSize() <= 0x1000)
{
if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, out addressSpace))
if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, out addressSpace) &&
!AddressSpace.TryCreateWithoutMirror(addressSpaceSize, out asNoMirror))
{
Logger.Warning?.Print(LogClass.Cpu, "Address space creation failed, falling back to software page table");
@ -89,35 +118,47 @@ namespace Ryujinx.HLE.HOS
switch (mode)
{
case MemoryManagerMode.SoftwarePageTable:
var memoryManager = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler);
processContext = new ArmProcessContext<MemoryManager>(pid, cpuEngine, _gpu, memoryManager, addressSpaceSize, for64Bit);
{
var mm = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler);
processContext = new ArmProcessContext<MemoryManager>(pid, cpuEngine, _gpu, mm, addressSpaceSize, for64Bit);
}
break;
case MemoryManagerMode.HostMapped:
case MemoryManagerMode.HostMappedUnsafe:
if (addressSpace == null)
if (addressSpace == null && asNoMirror == null)
{
var memoryManagerHostTracked = new MemoryManagerHostTracked(context.Memory, addressSpaceSize, mode == MemoryManagerMode.HostMappedUnsafe, invalidAccessHandler);
processContext = new ArmProcessContext<MemoryManagerHostTracked>(pid, cpuEngine, _gpu, memoryManagerHostTracked, addressSpaceSize, for64Bit);
}
else
{
if (addressSpaceSize != addressSpace.AddressSpaceSize)
{
Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{addressSpace.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})");
}
bool unsafeMode = mode == MemoryManagerMode.HostMappedUnsafe;
var memoryManagerHostMapped = new MemoryManagerHostMapped(addressSpace, mode == MemoryManagerMode.HostMappedUnsafe, invalidAccessHandler);
processContext = new ArmProcessContext<MemoryManagerHostMapped>(pid, cpuEngine, _gpu, memoryManagerHostMapped, addressSpace.AddressSpaceSize, for64Bit);
if (addressSpace != null)
{
var mm = new MemoryManagerHostMapped(addressSpace, unsafeMode, invalidAccessHandler);
processContext = new ArmProcessContext<MemoryManagerHostMapped>(pid, cpuEngine, _gpu, mm, addressSpace.AddressSpaceSize, for64Bit);
}
else
{
var mm = new MemoryManagerHostNoMirror(asNoMirror, context.Memory, unsafeMode, invalidAccessHandler);
processContext = new ArmProcessContext<MemoryManagerHostNoMirror>(pid, cpuEngine, _gpu, mm, asNoMirror.Size, for64Bit);
}
}
break;
default:
throw new InvalidOperationException($"{nameof(mode)} contains an invalid value: {mode}");
}
if (addressSpaceSize != processContext.AddressSpaceSize)
{
Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{processContext.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})");
}
}
DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, _diskCacheSelector ?? "default");
DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, _diskCacheSelector);
return processContext;
}

View file

@ -104,6 +104,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
MemoryRegion memRegion,
ulong address,
ulong size,
ulong reservedSize,
KMemoryBlockSlabManager slabManager)
{
_contextId = Context.ContextIdManager.GetId();
@ -119,6 +120,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
memRegion,
address,
size,
reservedSize,
slabManager);
if (result != Result.Success)
@ -161,6 +163,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
MemoryRegion memRegion,
ulong address,
ulong size,
ulong reservedSize,
KMemoryBlockSlabManager slabManager)
{
ulong endAddr = address + size;
@ -218,11 +221,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
break;
case ProcessCreationFlags.AddressSpace64Bit:
ulong reservedAddressSpaceSize = _reservedAddressSpaceSize;
if (_reservedAddressSpaceSize < addrSpaceEnd)
{
int addressSpaceWidth = (int)ulong.Log2(_reservedAddressSpaceSize);
int addressSpaceWidth = (int)ulong.Log2(reservedAddressSpaceSize);
aliasRegion.Size = 1UL << (addressSpaceWidth - 3);
aliasRegion.Size = reservedAddressSpaceSize >= 0x1800000000 ? 0x1000000000 : 1UL << (addressSpaceWidth - 3);
heapRegion.Size = 0x180000000;
stackRegion.Size = 1UL << (addressSpaceWidth - 8);
tlsIoRegion.Size = 1UL << (addressSpaceWidth - 3);
@ -230,8 +234,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
codeRegionSize = BitUtils.AlignUp(endAddr, RegionAlignment) - CodeRegionStart;
stackAndTlsIoStart = 0;
stackAndTlsIoEnd = 0;
AslrRegionStart = 0x8000000;
addrSpaceEnd = 1UL << addressSpaceWidth;
AslrRegionStart = Math.Max(reservedSize, 0x8000000);
addrSpaceEnd = reservedSize + reservedAddressSpaceSize;
AslrRegionEnd = addrSpaceEnd;
}
else
@ -242,7 +246,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
tlsIoRegion.Size = 0x1000000000;
CodeRegionStart = BitUtils.AlignDown(address, RegionAlignment);
codeRegionSize = BitUtils.AlignUp(endAddr, RegionAlignment) - CodeRegionStart;
AslrRegionStart = 0x8000000;
AslrRegionStart = Math.Max(reservedSize, 0x8000000);
AslrRegionEnd = AslrRegionStart + 0x7ff8000000;
stackAndTlsIoStart = 0;
stackAndTlsIoEnd = 0;

View file

@ -7,6 +7,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
interface IProcessContext : IDisposable
{
IVirtualMemoryManager AddressSpace { get; }
ulong ReservedSize { get; }
ulong AddressSpaceSize { get; }

View file

@ -135,7 +135,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
InitializeMemoryManager(creationInfo.Flags);
ulong codeAddress = creationInfo.CodeAddress;
ulong codeAddress = creationInfo.CodeAddress + Context.ReservedSize;
ulong codeSize = (ulong)creationInfo.CodePagesCount * KPageTableBase.PageSize;
@ -149,6 +149,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
memRegion,
codeAddress,
codeSize,
Context.ReservedSize,
slabManager);
if (result != Result.Success)
@ -184,7 +185,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
KResourceLimit resourceLimit,
MemoryRegion memRegion,
IProcessContextFactory contextFactory,
ThreadStart customThreadStart = null)
ThreadStart customThreadStart = null,
ulong entrypointOffset = 0UL)
{
ResourceLimit = resourceLimit;
_memRegion = memRegion;
@ -238,7 +240,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
InitializeMemoryManager(creationInfo.Flags);
ulong codeAddress = creationInfo.CodeAddress;
ulong codeAddress = creationInfo.CodeAddress + Context.ReservedSize;
ulong codeSize = codePagesCount * KPageTableBase.PageSize;
@ -248,6 +250,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
memRegion,
codeAddress,
codeSize,
Context.ReservedSize,
slabManager);
if (result != Result.Success)
@ -293,6 +296,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
CleanUpForError();
}
_entrypoint += entrypointOffset;
return result;
}
@ -337,10 +342,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
Flags = creationInfo.Flags;
TitleId = creationInfo.TitleId;
_entrypoint = creationInfo.CodeAddress;
_entrypoint = creationInfo.CodeAddress + Context.ReservedSize;
_imageSize = (ulong)creationInfo.CodePagesCount * KPageTableBase.PageSize;
switch (Flags & ProcessCreationFlags.AddressSpaceMask)
// 19.0.0+ sets all regions to same size
_memoryUsageCapacity = MemoryManager.HeapRegionEnd - MemoryManager.HeapRegionStart;
/*switch (Flags & ProcessCreationFlags.AddressSpaceMask)
{
case ProcessCreationFlags.AddressSpace32Bit:
case ProcessCreationFlags.AddressSpace64BitDeprecated:
@ -357,7 +364,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
break;
default:
throw new InvalidOperationException($"Invalid MMU flags value 0x{Flags:x2}.");
}
}*/
GenerateRandomEntropy();
@ -535,7 +542,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
{
throw new InvalidOperationException("Trying to start a process with an invalid state!");
}
// TODO: after 19.0.0+ alignment is not needed
ulong stackSizeRounded = BitUtils.AlignUp<ulong>(stackSize, KPageTableBase.PageSize);
ulong neededSize = stackSizeRounded + _imageSize;

View file

@ -7,6 +7,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
class ProcessContext : IProcessContext
{
public IVirtualMemoryManager AddressSpace { get; }
public ulong ReservedSize => 0UL;
public ulong AddressSpaceSize { get; }

View file

@ -1,4 +1,4 @@
using LibHac.Account;
using LibHac.Account;
using LibHac.Common;
using LibHac.Fs;
using LibHac.Fs.Fsa;
@ -11,6 +11,7 @@ using LibHac.Tools.FsSystem;
using LibHac.Tools.FsSystem.NcaUtils;
using Ryujinx.Common;
using Ryujinx.Common.Logging;
using Ryujinx.Cpu.Nce;
using Ryujinx.HLE.HOS;
using Ryujinx.HLE.HOS.Kernel;
using Ryujinx.HLE.HOS.Kernel.Common;
@ -21,6 +22,7 @@ using Ryujinx.HLE.Loaders.Processes.Extensions;
using Ryujinx.Horizon.Common;
using Ryujinx.Horizon.Sdk.Arp;
using System;
using System.Linq;
using System.Runtime.InteropServices;
using ApplicationId = LibHac.Ncm.ApplicationId;
@ -250,7 +252,7 @@ namespace Ryujinx.HLE.Loaders.Processes
ulong argsStart = 0;
uint argsSize = 0;
ulong codeStart = ((meta.Flags & 1) != 0 ? 0x8000000UL : 0x200000UL) + CodeStartOffset;
uint codeSize = 0;
ulong codeSize = 0;
string[] buildIds = new string[executables.Length];
@ -264,6 +266,7 @@ namespace Ryujinx.HLE.Loaders.Processes
}).ToUpper();
}
NceCpuCodePatch[] nsoPatch = new NceCpuCodePatch[executables.Length];
ulong[] nsoBase = new ulong[executables.Length];
for (int index = 0; index < executables.Length; index++)
@ -288,6 +291,16 @@ namespace Ryujinx.HLE.Loaders.Processes
nsoSize = BitUtils.AlignUp<uint>(nsoSize, KPageTableBase.PageSize);
bool for64Bit = ((ProcessCreationFlags)meta.Flags).HasFlag(ProcessCreationFlags.Is64Bit);
NceCpuCodePatch codePatch = ArmProcessContextFactory.CreateCodePatchForNce(context, for64Bit, nso.Text);
nsoPatch[index] = codePatch;
if (codePatch != null)
{
codeSize += codePatch.Size;
}
nsoBase[index] = codeStart + codeSize;
codeSize += nsoSize;
@ -389,7 +402,8 @@ namespace Ryujinx.HLE.Loaders.Processes
MemoryMarshal.Cast<byte, uint>(npdm.KernelCapabilityData),
resourceLimit,
memoryRegion,
processContextFactory);
processContextFactory,
entrypointOffset: nsoPatch[0]?.Size ?? 0UL);
if (result != Result.Success)
{
@ -400,9 +414,12 @@ namespace Ryujinx.HLE.Loaders.Processes
for (int index = 0; index < executables.Length; index++)
{
Logger.Info?.Print(LogClass.Loader, $"Loading image {index} at 0x{nsoBase[index]:x16}...");
ulong nsoBaseAddress = process.Context.ReservedSize + nsoBase[index];
Logger.Info?.Print(LogClass.Loader, $"Loading image {index} at 0x{nsoBaseAddress:x16}...");
result = LoadIntoMemory(process, executables[index], nsoBaseAddress, nsoPatch[index]);
result = LoadIntoMemory(process, executables[index], nsoBase[index]);
if (result != Result.Success)
{
Logger.Error?.Print(LogClass.Loader, $"Process initialization returned error \"{result}\".");
@ -460,7 +477,7 @@ namespace Ryujinx.HLE.Loaders.Processes
return processResult;
}
public static Result LoadIntoMemory(KProcess process, IExecutable image, ulong baseAddress)
private static Result LoadIntoMemory(KProcess process, IExecutable image, ulong baseAddress, NceCpuCodePatch codePatch = null)
{
ulong textStart = baseAddress + image.TextOffset;
ulong roStart = baseAddress + image.RoOffset;
@ -480,6 +497,11 @@ namespace Ryujinx.HLE.Loaders.Processes
process.CpuMemory.Fill(bssStart, image.BssSize, 0);
if (codePatch != null)
{
codePatch.Write(process.CpuMemory, baseAddress - codePatch.Size, textStart);
}
Result SetProcessMemoryPermission(ulong address, ulong size, KMemoryPermission permission)
{
if (size == 0)