diff --git a/src/Ryujinx.Cpu/AddressSpace.cs b/src/Ryujinx.Cpu/AddressSpace.cs index 6664ed134..08595864a 100644 --- a/src/Ryujinx.Cpu/AddressSpace.cs +++ b/src/Ryujinx.Cpu/AddressSpace.cs @@ -5,6 +5,8 @@ namespace Ryujinx.Cpu { public class AddressSpace : IDisposable { + private const MemoryAllocationFlags AsFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible; + private readonly MemoryBlock _backingMemory; public MemoryBlock Base { get; } @@ -25,28 +27,42 @@ namespace Ryujinx.Cpu { addressSpace = null; - const MemoryAllocationFlags AsFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible; + MemoryBlock baseMemory = null; + MemoryBlock mirrorMemory = null; + + try + { + baseMemory = new MemoryBlock(asSize, AsFlags); + mirrorMemory = new MemoryBlock(asSize, AsFlags); + addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, asSize); + } + catch (SystemException) + { + baseMemory?.Dispose(); + mirrorMemory?.Dispose(); + } + + return addressSpace != null; + } + + public static bool TryCreateWithoutMirror(ulong asSize, out MemoryBlock addressSpace) + { + addressSpace = null; ulong minAddressSpaceSize = Math.Min(asSize, 1UL << 36); // Attempt to create the address space with expected size or try to reduce it until it succeed. - for (ulong addressSpaceSize = asSize; addressSpaceSize >= minAddressSpaceSize; addressSpaceSize >>= 1) + for (ulong addressSpaceSize = asSize; addressSpaceSize >= minAddressSpaceSize; addressSpaceSize -= 0x100000000UL) { - MemoryBlock baseMemory = null; - MemoryBlock mirrorMemory = null; - try { - baseMemory = new MemoryBlock(addressSpaceSize, AsFlags); - mirrorMemory = new MemoryBlock(addressSpaceSize, AsFlags); - addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, addressSpaceSize); + MemoryBlock baseMemory = new MemoryBlock(addressSpaceSize, AsFlags); + addressSpace = baseMemory; break; } catch (SystemException) { - baseMemory?.Dispose(); - mirrorMemory?.Dispose(); } } diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostNoMirror.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostNoMirror.cs new file mode 100644 index 000000000..743956271 --- /dev/null +++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostNoMirror.cs @@ -0,0 +1,330 @@ +using ARMeilleure.Memory; +using Ryujinx.Memory; +using Ryujinx.Memory.Range; +using Ryujinx.Memory.Tracking; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace Ryujinx.Cpu.Jit +{ + /// + /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. + /// + public sealed class MemoryManagerHostNoMirror : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + { + private readonly InvalidAccessHandler _invalidAccessHandler; + private readonly bool _unsafeMode; + + private readonly MemoryBlock _addressSpace; + private readonly MemoryBlock _backingMemory; + private readonly PageTable _pageTable; + + public int AddressSpaceBits { get; } + protected override ulong AddressSpaceSize { get; } + + private readonly MemoryEhMeilleure _memoryEh; + + private readonly ManagedPageFlags _pages; + + /// + public bool UsesPrivateAllocations => false; + + public IntPtr PageTablePointer => _addressSpace.Pointer; + + public MemoryManagerType Type => _unsafeMode ? MemoryManagerType.HostMappedUnsafe : MemoryManagerType.HostMapped; + + public MemoryTracking Tracking { get; } + + public event Action UnmapEvent; + + /// + /// Creates a new instance of the host mapped memory manager. + /// + /// Address space instance to use + /// True if unmanaged access should not be masked (unsafe), false otherwise. + /// Optional function to handle invalid memory accesses + public MemoryManagerHostNoMirror( + MemoryBlock addressSpace, + MemoryBlock backingMemory, + bool unsafeMode, + InvalidAccessHandler invalidAccessHandler) + { + _addressSpace = addressSpace; + _backingMemory = backingMemory; + _pageTable = new PageTable(); + _invalidAccessHandler = invalidAccessHandler; + _unsafeMode = unsafeMode; + AddressSpaceSize = addressSpace.Size; + + ulong asSize = PageSize; + int asBits = PageBits; + + while (asSize < addressSpace.Size) + { + asSize <<= 1; + asBits++; + } + + AddressSpaceBits = asBits; + + _pages = new ManagedPageFlags(asBits); + + Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler); + _memoryEh = new MemoryEhMeilleure(addressSpace, null, Tracking); + } + + /// + /// Ensures the combination of virtual address and size is part of the addressable space and fully mapped. + /// + /// Virtual address of the range + /// Size of the range in bytes + private void AssertMapped(ulong va, ulong size) + { + if (!ValidateAddressAndSize(va, size) || !_pages.IsRangeMapped(va, size)) + { + throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + } + + /// + public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags) + { + AssertValidAddressAndSize(va, size); + + _addressSpace.MapView(_backingMemory, pa, va, size); + _pages.AddMapping(va, size); + PtMap(va, pa, size); + + Tracking.Map(va, size); + } + + private void PtMap(ulong va, ulong pa, ulong size) + { + while (size != 0) + { + _pageTable.Map(va, pa); + + va += PageSize; + pa += PageSize; + size -= PageSize; + } + } + + /// + public void Unmap(ulong va, ulong size) + { + AssertValidAddressAndSize(va, size); + + UnmapEvent?.Invoke(va, size); + Tracking.Unmap(va, size); + + _pages.RemoveMapping(va, size); + PtUnmap(va, size); + _addressSpace.UnmapView(_backingMemory, va, size); + } + + private void PtUnmap(ulong va, ulong size) + { + while (size != 0) + { + _pageTable.Unmap(va); + + va += PageSize; + size -= PageSize; + } + } + + /// + public void Reprotect(ulong va, ulong size, MemoryPermission permission) + { + } + + public ref T GetRef(ulong va) where T : unmanaged + { + if (!IsContiguous(va, Unsafe.SizeOf())) + { + ThrowMemoryNotContiguous(); + } + + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); + + return ref _backingMemory.GetRef(GetPhysicalAddressChecked(va)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override bool IsMapped(ulong va) + { + return ValidateAddress(va) && _pages.IsMapped(va); + } + + /// + public bool IsRangeMapped(ulong va, ulong size) + { + AssertValidAddressAndSize(va, size); + + return _pages.IsRangeMapped(va, size); + } + + /// + public IEnumerable GetHostRegions(ulong va, ulong size) + { + if (size == 0) + { + return Enumerable.Empty(); + } + + var guestRegions = GetPhysicalRegionsImpl(va, size); + if (guestRegions == null) + { + return null; + } + + var regions = new HostMemoryRange[guestRegions.Count]; + + for (int i = 0; i < regions.Length; i++) + { + var guestRegion = guestRegions[i]; + IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size); + regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size); + } + + return regions; + } + + /// + public IEnumerable GetPhysicalRegions(ulong va, ulong size) + { + if (size == 0) + { + return Enumerable.Empty(); + } + + return GetPhysicalRegionsImpl(va, size); + } + + private List GetPhysicalRegionsImpl(ulong va, ulong size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size)) + { + return null; + } + + int pages = GetPagesCount(va, (uint)size, out va); + + var regions = new List(); + + ulong regionStart = GetPhysicalAddressInternal(va); + ulong regionSize = PageSize; + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return null; + } + + ulong newPa = GetPhysicalAddressInternal(va + PageSize); + + if (GetPhysicalAddressInternal(va) + PageSize != newPa) + { + regions.Add(new MemoryRange(regionStart, regionSize)); + regionStart = newPa; + regionSize = 0; + } + + va += PageSize; + regionSize += PageSize; + } + + regions.Add(new MemoryRange(regionStart, regionSize)); + + return regions; + } + + private ulong GetPhysicalAddressChecked(ulong va) + { + if (!IsMapped(va)) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}"); + } + + return GetPhysicalAddressInternal(va); + } + + private ulong GetPhysicalAddressInternal(ulong va) + { + return _pageTable.Read(va) + (va & PageMask); + } + + /// + /// + /// This function also validates that the given range is both valid and mapped, and will throw if it is not. + /// + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + { + AssertValidAddressAndSize(va, size); + + if (precise) + { + Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId); + return; + } + + _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); + } + + /// + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) + { + if (guest) + { + _addressSpace.Reprotect(va, size, protection, false); + } + else + { + _pages.TrackingReprotect(va, size, protection); + } + } + + /// + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags) + { + return Tracking.BeginTracking(address, size, id, flags); + } + + /// + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags) + { + return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); + } + + /// + public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id) + { + return Tracking.BeginSmartGranularTracking(address, size, granularity, id); + } + + /// + /// Disposes of resources used by the memory manager. + /// + protected override void Destroy() + { + _addressSpace.Dispose(); + _memoryEh.Dispose(); + } + + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _backingMemory.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) + => _backingMemory.GetSpan(pa, size); + + protected override nuint TranslateVirtualAddressChecked(ulong va) + => (nuint)GetPhysicalAddressChecked(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) + => (nuint)GetPhysicalAddressInternal(va); + } +} diff --git a/src/Ryujinx.Cpu/LightningJit/LightningJitCpuContext.cs b/src/Ryujinx.Cpu/LightningJit/LightningJitCpuContext.cs index 0f47ffb15..cd38706a0 100644 --- a/src/Ryujinx.Cpu/LightningJit/LightningJitCpuContext.cs +++ b/src/Ryujinx.Cpu/LightningJit/LightningJitCpuContext.cs @@ -45,6 +45,11 @@ namespace Ryujinx.Cpu.LightningJit _translator.InvalidateJitCacheRegion(address, size); } + /// + public void PatchCodeForNce(ulong textAddress, ulong textSize, ulong patchRegionAddress, ulong patchRegionSize) + { + } + /// public IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled, string cacheSelector) { diff --git a/src/Ryujinx.Cpu/Nce/Arm64/ArmCondition.cs b/src/Ryujinx.Cpu/Nce/Arm64/ArmCondition.cs new file mode 100644 index 000000000..125fe2a10 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/ArmCondition.cs @@ -0,0 +1,22 @@ +namespace Ryujinx.Cpu.Nce.Arm64 +{ + enum ArmCondition + { + Eq = 0, + Ne = 1, + GeUn = 2, + LtUn = 3, + Mi = 4, + Pl = 5, + Vs = 6, + Vc = 7, + GtUn = 8, + LeUn = 9, + Ge = 10, + Lt = 11, + Gt = 12, + Le = 13, + Al = 14, + Nv = 15, + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/ArmExtensionType.cs b/src/Ryujinx.Cpu/Nce/Arm64/ArmExtensionType.cs new file mode 100644 index 000000000..48416da46 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/ArmExtensionType.cs @@ -0,0 +1,14 @@ +namespace Ryujinx.Cpu.Nce.Arm64 +{ + enum ArmExtensionType + { + Uxtb = 0, + Uxth = 1, + Uxtw = 2, + Uxtx = 3, + Sxtb = 4, + Sxth = 5, + Sxtw = 6, + Sxtx = 7, + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/ArmShiftType.cs b/src/Ryujinx.Cpu/Nce/Arm64/ArmShiftType.cs new file mode 100644 index 000000000..290e141b9 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/ArmShiftType.cs @@ -0,0 +1,11 @@ + +namespace Ryujinx.Cpu.Nce.Arm64 +{ + enum ArmShiftType + { + Lsl = 0, + Lsr = 1, + Asr = 2, + Ror = 3, + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/Assembler.cs b/src/Ryujinx.Cpu/Nce/Arm64/Assembler.cs new file mode 100644 index 000000000..8db1575f1 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/Assembler.cs @@ -0,0 +1,1103 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +namespace Ryujinx.Cpu.Nce.Arm64 +{ + class Assembler + { + private const uint SfFlag = 1u << 31; + + public const int SpRegister = 31; + public const int ZrRegister = 31; + + private readonly List _code; + + public int CodeWords => _code.Count; + + private class LabelState + { + public int BranchIndex; + public int TargetIndex; + public bool HasBranch; + public bool HasTarget; + } + + private List _labels; + + public Assembler() + { + _code = new List(); + _labels = new List(); + } + + public Operand CreateLabel() + { + int labelIndex = _labels.Count; + _labels.Add(new LabelState()); + + return new Operand(OperandKind.Label, OperandType.None, (ulong)labelIndex); + } + + public void MarkLabel(Operand label) + { + int targetIndex = _code.Count; + + var state = _labels[label.AsInt32()]; + + state.TargetIndex = targetIndex; + state.HasTarget = true; + + if (state.HasBranch) + { + int imm = (targetIndex - state.BranchIndex) * sizeof(uint); + uint code = _code[state.BranchIndex]; + + if ((code & 0xfc000000u) == 0x14000000u) + { + _code[state.BranchIndex] = code | EncodeSImm26_2(imm); + } + else + { + _code[state.BranchIndex] = code | (EncodeSImm19_2(imm) << 5); + } + } + } + + public void B(Operand label, ArmCondition condition = ArmCondition.Al) + { + int branchIndex = _code.Count; + + var state = _labels[label.AsInt32()]; + + state.BranchIndex = branchIndex; + state.HasBranch = true; + + int imm = 0; + + if (state.HasTarget) + { + imm = (state.TargetIndex - branchIndex) * sizeof(uint); + } + + if (condition == ArmCondition.Al) + { + B(imm); + } + else + { + B(condition, imm); + } + } + + public void Cbz(Operand rt, Operand label) + { + int branchIndex = _code.Count; + + var state = _labels[label.AsInt32()]; + + state.BranchIndex = branchIndex; + state.HasBranch = true; + + int imm = 0; + + if (state.HasTarget) + { + imm = (state.TargetIndex - branchIndex) * sizeof(uint); + } + + Cbz(rt, imm); + } + + public void Cbnz(Operand rt, Operand label) + { + int branchIndex = _code.Count; + + var state = _labels[label.AsInt32()]; + + state.BranchIndex = branchIndex; + state.HasBranch = true; + + int imm = 0; + + if (state.HasTarget) + { + imm = (state.TargetIndex - branchIndex) * sizeof(uint); + } + + Cbnz(rt, imm); + } + + public void Add(Operand rd, Operand rn, Operand rm, ArmExtensionType extensionType, int shiftAmount = 0) + { + WriteInstructionAuto(0x0b200000u, rd, rn, rm, extensionType, shiftAmount); + } + + public void Add(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0, bool immForm = false) + { + WriteInstructionAuto(0x11000000u, 0x0b000000u, rd, rn, rm, shiftType, shiftAmount, immForm); + } + + public void And(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + WriteInstructionBitwiseAuto(0x12000000u, 0x0a000000u, rd, rn, rm, shiftType, shiftAmount); + } + + public void Ands(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + WriteInstructionBitwiseAuto(0x72000000u, 0x6a000000u, rd, rn, rm, shiftType, shiftAmount); + } + + public void Asr(Operand rd, Operand rn, Operand rm) + { + if (rm.Kind == OperandKind.Constant) + { + int shift = rm.AsInt32(); + int mask = rd.Type == OperandType.I64 ? 63 : 31; + shift &= mask; + Sbfm(rd, rn, shift, mask); + } + else + { + Asrv(rd, rn, rm); + } + } + + public void Asrv(Operand rd, Operand rn, Operand rm) + { + WriteInstructionBitwiseAuto(0x1ac02800u, rd, rn, rm); + } + + public void B(int imm) + { + WriteUInt32(0x14000000u | EncodeSImm26_2(imm)); + } + + public void B(ArmCondition condition, int imm) + { + WriteUInt32(0x54000000u | (uint)condition | (EncodeSImm19_2(imm) << 5)); + } + + public void Blr(Operand rn) + { + WriteUInt32(0xd63f0000u | (EncodeReg(rn) << 5)); + } + + public void Br(Operand rn) + { + WriteUInt32(0xd61f0000u | (EncodeReg(rn) << 5)); + } + + public void Brk() + { + WriteUInt32(0xd4200000u); + } + + public void Cbz(Operand rt, int imm) + { + WriteInstructionAuto(0x34000000u | (EncodeSImm19_2(imm) << 5), rt); + } + + public void Cbnz(Operand rt, int imm) + { + WriteInstructionAuto(0x35000000u | (EncodeSImm19_2(imm) << 5), rt); + } + + public void Clrex(int crm = 15) + { + WriteUInt32(0xd503305fu | (EncodeUImm4(crm) << 8)); + } + + public void Clz(Operand rd, Operand rn) + { + WriteInstructionAuto(0x5ac01000u, rd, rn); + } + + public void Cmp(Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + Subs(new Operand(ZrRegister, RegisterType.Integer, rn.Type), rn, rm, shiftType, shiftAmount); + } + + public void Csel(Operand rd, Operand rn, Operand rm, ArmCondition condition) + { + WriteInstructionBitwiseAuto(0x1a800000u | ((uint)condition << 12), rd, rn, rm); + } + + public void Cset(Operand rd, ArmCondition condition) + { + var zr = new Operand(ZrRegister, RegisterType.Integer, rd.Type); + Csinc(rd, zr, zr, (ArmCondition)((int)condition ^ 1)); + } + + public void Csinc(Operand rd, Operand rn, Operand rm, ArmCondition condition) + { + WriteInstructionBitwiseAuto(0x1a800400u | ((uint)condition << 12), rd, rn, rm); + } + + public void Dmb(uint option) + { + WriteUInt32(0xd50330bfu | (option << 8)); + } + + public void Eor(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + WriteInstructionBitwiseAuto(0x52000000u, 0x4a000000u, rd, rn, rm, shiftType, shiftAmount); + } + + public void Extr(Operand rd, Operand rn, Operand rm, int imms) + { + uint n = rd.Type == OperandType.I64 ? 1u << 22 : 0u; + WriteInstructionBitwiseAuto(0x13800000u | n | (EncodeUImm6(imms) << 10), rd, rn, rm); + } + + public void Ldaxp(Operand rt, Operand rt2, Operand rn) + { + WriteInstruction(0x887f8000u | ((rt.Type == OperandType.I64 ? 3u : 2u) << 30), rt, rn, rt2); + } + + public void Ldaxr(Operand rt, Operand rn) + { + WriteInstruction(0x085ffc00u | ((rt.Type == OperandType.I64 ? 3u : 2u) << 30), rt, rn); + } + + public void Ldaxrb(Operand rt, Operand rn) + { + WriteInstruction(0x085ffc00u, rt, rn); + } + + public void Ldaxrh(Operand rt, Operand rn) + { + WriteInstruction(0x085ffc00u | (1u << 30), rt, rn); + } + + public void LdpRiPost(Operand rt, Operand rt2, Operand rn, int imm) + { + uint instruction = GetLdpStpInstruction(0x28c00000u, 0x2cc00000u, imm, rt.Type); + WriteInstruction(instruction, rt, rn, rt2); + } + + public void LdpRiPre(Operand rt, Operand rt2, Operand rn, int imm) + { + uint instruction = GetLdpStpInstruction(0x29c00000u, 0x2dc00000u, imm, rt.Type); + WriteInstruction(instruction, rt, rn, rt2); + } + + public void LdpRiUn(Operand rt, Operand rt2, Operand rn, int imm) + { + uint instruction = GetLdpStpInstruction(0x29400000u, 0x2d400000u, imm, rt.Type); + WriteInstruction(instruction, rt, rn, rt2); + } + + public void LdrLit(Operand rt, int offset) + { + uint instruction = 0x18000000u | (EncodeSImm19_2(offset) << 5); + + if (rt.Type == OperandType.I64) + { + instruction |= 1u << 30; + } + + WriteInstruction(instruction, rt); + } + + public void LdrRiPost(Operand rt, Operand rn, int imm) + { + uint instruction = GetLdrStrInstruction(0xb8400400u, 0x3c400400u, rt.Type) | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void LdrRiPre(Operand rt, Operand rn, int imm) + { + uint instruction = GetLdrStrInstruction(0xb8400c00u, 0x3c400c00u, rt.Type) | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void LdrRiUn(Operand rt, Operand rn, int imm) + { + uint instruction = GetLdrStrInstruction(0xb9400000u, 0x3d400000u, rt.Type) | (EncodeUImm12(imm, rt.Type) << 10); + WriteInstruction(instruction, rt, rn); + } + + public void LdrRr(Operand rt, Operand rn, Operand rm, ArmExtensionType extensionType, bool shift) + { + uint instruction = GetLdrStrInstruction(0xb8600800u, 0x3ce00800u, rt.Type); + WriteInstructionLdrStrAuto(instruction, rt, rn, rm, extensionType, shift); + } + + public void LdrbRiPost(Operand rt, Operand rn, int imm) + { + uint instruction = 0x38400400u | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void LdrbRiPre(Operand rt, Operand rn, int imm) + { + uint instruction = 0x38400c00u | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void LdrbRiUn(Operand rt, Operand rn, int imm) + { + uint instruction = 0x39400000u | (EncodeUImm12(imm, 0) << 10); + WriteInstruction(instruction, rt, rn); + } + + public void LdrhRiPost(Operand rt, Operand rn, int imm) + { + uint instruction = 0x78400400u | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void LdrhRiPre(Operand rt, Operand rn, int imm) + { + uint instruction = 0x78400c00u | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void LdrhRiUn(Operand rt, Operand rn, int imm) + { + uint instruction = 0x79400000u | (EncodeUImm12(imm, 1) << 10); + WriteInstruction(instruction, rt, rn); + } + + public void Ldur(Operand rt, Operand rn, int imm) + { + uint instruction = GetLdrStrInstruction(0xb8400000u, 0x3c400000u, rt.Type) | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void Lsl(Operand rd, Operand rn, Operand rm) + { + if (rm.Kind == OperandKind.Constant) + { + int shift = rm.AsInt32(); + int mask = rd.Type == OperandType.I64 ? 63 : 31; + shift &= mask; + Ubfm(rd, rn, -shift & mask, mask - shift); + } + else + { + Lslv(rd, rn, rm); + } + } + + public void Lslv(Operand rd, Operand rn, Operand rm) + { + WriteInstructionBitwiseAuto(0x1ac02000u, rd, rn, rm); + } + + public void Lsr(Operand rd, Operand rn, Operand rm) + { + if (rm.Kind == OperandKind.Constant) + { + int shift = rm.AsInt32(); + int mask = rd.Type == OperandType.I64 ? 63 : 31; + shift &= mask; + Ubfm(rd, rn, shift, mask); + } + else + { + Lsrv(rd, rn, rm); + } + } + + public void Lsrv(Operand rd, Operand rn, Operand rm) + { + WriteInstructionBitwiseAuto(0x1ac02400u, rd, rn, rm); + } + + public void Madd(Operand rd, Operand rn, Operand rm, Operand ra) + { + WriteInstructionAuto(0x1b000000u, rd, rn, rm, ra); + } + + public void Mul(Operand rd, Operand rn, Operand rm) + { + Madd(rd, rn, rm, new Operand(ZrRegister, RegisterType.Integer, rd.Type)); + } + + public void Mov(Operand rd, Operand rn) + { + Debug.Assert(rd.Type.IsInteger()); + Orr(rd, new Operand(ZrRegister, RegisterType.Integer, rd.Type), rn); + } + + public void MovSp(Operand rd, Operand rn) + { + if (rd.GetRegister().Index == SpRegister || + rn.GetRegister().Index == SpRegister) + { + Add(rd, rn, new Operand(rd.Type, 0), immForm: true); + } + else + { + Mov(rd, rn); + } + } + + public void Mov(Operand rd, ulong value) + { + if (value == 0) + { + Mov(rd, new Operand(ZrRegister, RegisterType.Integer, rd.Type)); + } + else if (CodeGenCommon.TryEncodeBitMask(rd.Type, value, out _, out _, out _)) + { + Orr(rd, new Operand(ZrRegister, RegisterType.Integer, rd.Type), new Operand(OperandKind.Constant, rd.Type, value)); + } + else + { + int hw = 0; + bool first = true; + + while (value != 0) + { + int valueLow = (ushort)value; + if (valueLow != 0) + { + if (first) + { + Movz(rd, valueLow, hw); + first = false; + } + else + { + Movk(rd, valueLow, hw); + } + } + + hw++; + value >>= 16; + } + } + } + + public void Mov(Operand rd, int imm) + { + Movz(rd, imm, 0); + } + + public void Movz(Operand rd, int imm, int hw) + { + Debug.Assert((hw & (rd.Type == OperandType.I64 ? 3 : 1)) == hw); + WriteInstructionAuto(0x52800000u | (EncodeUImm16(imm) << 5) | ((uint)hw << 21), rd); + } + + public void Movk(Operand rd, int imm, int hw) + { + Debug.Assert((hw & (rd.Type == OperandType.I64 ? 3 : 1)) == hw); + WriteInstructionAuto(0x72800000u | (EncodeUImm16(imm) << 5) | ((uint)hw << 21), rd); + } + + public void MrsTpidrEl0(Operand rt) + { + WriteInstruction(0xd53bd040u, rt); + } + + public void MrsTpidrroEl0(Operand rt) + { + WriteInstruction(0xd53bd060u, rt); + } + + public void Mrs(Operand rt, uint o0, uint op1, uint crn, uint crm, uint op2) + { + uint instruction = 0xd5300000u; + + instruction |= (op2 & 7) << 5; + instruction |= (crm & 15) << 8; + instruction |= (crn & 15) << 12; + instruction |= (op1 & 7) << 16; + instruction |= (o0 & 1) << 19; + + WriteInstruction(instruction, rt); + } + + public void Mvn(Operand rd, Operand rn, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + Orn(rd, new Operand(ZrRegister, RegisterType.Integer, rd.Type), rn, shiftType, shiftAmount); + } + + public void Neg(Operand rd, Operand rn, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + Sub(rd, new Operand(ZrRegister, RegisterType.Integer, rd.Type), rn, shiftType, shiftAmount); + } + + public void Orn(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + WriteInstructionBitwiseAuto(0x2a200000u, rd, rn, rm, shiftType, shiftAmount); + } + + public void Orr(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + WriteInstructionBitwiseAuto(0x32000000u, 0x2a000000u, rd, rn, rm, shiftType, shiftAmount); + } + + public void Ret(Operand rn) + { + WriteUInt32(0xd65f0000u | (EncodeReg(rn) << 5)); + } + + public void Rev(Operand rd, Operand rn) + { + uint opc0 = rd.Type == OperandType.I64 ? 1u << 10 : 0u; + WriteInstructionAuto(0x5ac00800u | opc0, rd, rn); + } + + public void Ror(Operand rd, Operand rn, Operand rm) + { + if (rm.Kind == OperandKind.Constant) + { + int shift = rm.AsInt32(); + int mask = rd.Type == OperandType.I64 ? 63 : 31; + shift &= mask; + Extr(rd, rn, rn, shift); + } + else + { + Rorv(rd, rn, rm); + } + } + + public void Rorv(Operand rd, Operand rn, Operand rm) + { + WriteInstructionBitwiseAuto(0x1ac02c00u, rd, rn, rm); + } + + public void Sbfm(Operand rd, Operand rn, int immr, int imms) + { + uint n = rd.Type == OperandType.I64 ? 1u << 22 : 0u; + WriteInstructionAuto(0x13000000u | n | (EncodeUImm6(imms) << 10) | (EncodeUImm6(immr) << 16), rd, rn); + } + + public void Sdiv(Operand rd, Operand rn, Operand rm) + { + WriteInstructionRm16Auto(0x1ac00c00u, rd, rn, rm); + } + + public void Smulh(Operand rd, Operand rn, Operand rm) + { + WriteInstructionRm16(0x9b407c00u, rd, rn, rm); + } + + public void Stlxp(Operand rt, Operand rt2, Operand rn, Operand rs) + { + WriteInstruction(0x88208000u | ((rt.Type == OperandType.I64 ? 3u : 2u) << 30), rt, rn, rs, rt2); + } + + public void Stlxr(Operand rt, Operand rn, Operand rs) + { + WriteInstructionRm16(0x0800fc00u | ((rt.Type == OperandType.I64 ? 3u : 2u) << 30), rt, rn, rs); + } + + public void Stlxrb(Operand rt, Operand rn, Operand rs) + { + WriteInstructionRm16(0x0800fc00u, rt, rn, rs); + } + + public void Stlxrh(Operand rt, Operand rn, Operand rs) + { + WriteInstructionRm16(0x0800fc00u | (1u << 30), rt, rn, rs); + } + + public void StpRiPost(Operand rt, Operand rt2, Operand rn, int imm) + { + uint instruction = GetLdpStpInstruction(0x28800000u, 0x2c800000u, imm, rt.Type); + WriteInstruction(instruction, rt, rn, rt2); + } + + public void StpRiPre(Operand rt, Operand rt2, Operand rn, int imm) + { + uint instruction = GetLdpStpInstruction(0x29800000u, 0x2d800000u, imm, rt.Type); + WriteInstruction(instruction, rt, rn, rt2); + } + + public void StpRiUn(Operand rt, Operand rt2, Operand rn, int imm) + { + uint instruction = GetLdpStpInstruction(0x29000000u, 0x2d000000u, imm, rt.Type); + WriteInstruction(instruction, rt, rn, rt2); + } + + public void StrRiPost(Operand rt, Operand rn, int imm) + { + uint instruction = GetLdrStrInstruction(0xb8000400u, 0x3c000400u, rt.Type) | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void StrRiPre(Operand rt, Operand rn, int imm) + { + uint instruction = GetLdrStrInstruction(0xb8000c00u, 0x3c000c00u, rt.Type) | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void StrRiUn(Operand rt, Operand rn, int imm) + { + uint instruction = GetLdrStrInstruction(0xb9000000u, 0x3d000000u, rt.Type) | (EncodeUImm12(imm, rt.Type) << 10); + WriteInstruction(instruction, rt, rn); + } + + public void StrRr(Operand rt, Operand rn, Operand rm, ArmExtensionType extensionType, bool shift) + { + uint instruction = GetLdrStrInstruction(0xb8200800u, 0x3ca00800u, rt.Type); + WriteInstructionLdrStrAuto(instruction, rt, rn, rm, extensionType, shift); + } + + public void StrbRiPost(Operand rt, Operand rn, int imm) + { + uint instruction = 0x38000400u | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void StrbRiPre(Operand rt, Operand rn, int imm) + { + uint instruction = 0x38000c00u | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void StrbRiUn(Operand rt, Operand rn, int imm) + { + uint instruction = 0x39000000u | (EncodeUImm12(imm, 0) << 10); + WriteInstruction(instruction, rt, rn); + } + + public void StrhRiPost(Operand rt, Operand rn, int imm) + { + uint instruction = 0x78000400u | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void StrhRiPre(Operand rt, Operand rn, int imm) + { + uint instruction = 0x78000c00u | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void StrhRiUn(Operand rt, Operand rn, int imm) + { + uint instruction = 0x79000000u | (EncodeUImm12(imm, 1) << 10); + WriteInstruction(instruction, rt, rn); + } + + public void Stur(Operand rt, Operand rn, int imm) + { + uint instruction = GetLdrStrInstruction(0xb8000000u, 0x3c000000u, rt.Type) | (EncodeSImm9(imm) << 12); + WriteInstruction(instruction, rt, rn); + } + + public void Sub(Operand rd, Operand rn, Operand rm, ArmExtensionType extensionType, int shiftAmount = 0) + { + WriteInstructionAuto(0x4b200000u, rd, rn, rm, extensionType, shiftAmount); + } + + public void Sub(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + WriteInstructionAuto(0x51000000u, 0x4b000000u, rd, rn, rm, shiftType, shiftAmount); + } + + public void Subs(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + WriteInstructionAuto(0x71000000u, 0x6b000000u, rd, rn, rm, shiftType, shiftAmount); + } + + public void Sxtb(Operand rd, Operand rn) + { + Sbfm(rd, rn, 0, 7); + } + + public void Sxth(Operand rd, Operand rn) + { + Sbfm(rd, rn, 0, 15); + } + + public void Sxtw(Operand rd, Operand rn) + { + Sbfm(rd, rn, 0, 31); + } + + public void Tst(Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0) + { + Ands(new Operand(ZrRegister, RegisterType.Integer, rn.Type), rn, rm, shiftType, shiftAmount); + } + + public void Ubfm(Operand rd, Operand rn, int immr, int imms) + { + uint n = rd.Type == OperandType.I64 ? 1u << 22 : 0u; + WriteInstructionAuto(0x53000000u | n | (EncodeUImm6(imms) << 10) | (EncodeUImm6(immr) << 16), rd, rn); + } + + public void Udiv(Operand rd, Operand rn, Operand rm) + { + WriteInstructionRm16Auto(0x1ac00800u, rd, rn, rm); + } + + public void Umov(Operand rd, Operand rn, int index, int size) + { + uint q = size == 3 ? 1u << 30 : 0u; + WriteInstruction(0x0e003c00u | (EncodeIndexSizeImm5(index, size) << 16) | q, rd, rn); + } + + public void Umulh(Operand rd, Operand rn, Operand rm) + { + WriteInstructionRm16(0x9bc07c00u, rd, rn, rm); + } + + public void Uxtb(Operand rd, Operand rn) + { + Ubfm(rd, rn, 0, 7); + } + + public void Uxth(Operand rd, Operand rn) + { + Ubfm(rd, rn, 0, 15); + } + + private void WriteInstructionAuto( + uint instI, + uint instR, + Operand rd, + Operand rn, + Operand rm, + ArmShiftType shiftType = ArmShiftType.Lsl, + int shiftAmount = 0, + bool immForm = false) + { + if (rm.Kind == OperandKind.Constant && (rm.Value != 0 || immForm)) + { + Debug.Assert(shiftAmount == 0); + int imm = rm.AsInt32(); + Debug.Assert((uint)imm == rm.Value); + if (imm != 0 && (imm & 0xfff) == 0) + { + instI |= 1 << 22; // sh flag + imm >>= 12; + } + WriteInstructionAuto(instI | (EncodeUImm12(imm, 0) << 10), rd, rn); + } + else + { + instR |= EncodeUImm6(shiftAmount) << 10; + instR |= (uint)shiftType << 22; + + WriteInstructionRm16Auto(instR, rd, rn, rm); + } + } + + private void WriteInstructionAuto( + uint instruction, + Operand rd, + Operand rn, + Operand rm, + ArmExtensionType extensionType, + int shiftAmount = 0) + { + Debug.Assert((uint)shiftAmount <= 4); + + instruction |= (uint)shiftAmount << 10; + instruction |= (uint)extensionType << 13; + + WriteInstructionRm16Auto(instruction, rd, rn, rm); + } + + private void WriteInstructionBitwiseAuto( + uint instI, + uint instR, + Operand rd, + Operand rn, + Operand rm, + ArmShiftType shiftType = ArmShiftType.Lsl, + int shiftAmount = 0) + { + if (rm.Kind == OperandKind.Constant && rm.Value != 0) + { + Debug.Assert(shiftAmount == 0); + bool canEncode = CodeGenCommon.TryEncodeBitMask(rm, out int immN, out int immS, out int immR); + Debug.Assert(canEncode); + uint instruction = instI | ((uint)immS << 10) | ((uint)immR << 16) | ((uint)immN << 22); + + WriteInstructionAuto(instruction, rd, rn); + } + else + { + WriteInstructionBitwiseAuto(instR, rd, rn, rm, shiftType, shiftAmount); + } + } + + private void WriteInstructionBitwiseAuto( + uint instruction, + Operand rd, + Operand rn, + Operand rm, + ArmShiftType shiftType = ArmShiftType.Lsl, + int shiftAmount = 0) + { + if (rd.Type == OperandType.I64) + { + instruction |= SfFlag; + } + + instruction |= EncodeUImm6(shiftAmount) << 10; + instruction |= (uint)shiftType << 22; + + WriteInstructionRm16(instruction, rd, rn, rm); + } + + private void WriteInstructionLdrStrAuto( + uint instruction, + Operand rd, + Operand rn, + Operand rm, + ArmExtensionType extensionType, + bool shift) + { + if (shift) + { + instruction |= 1u << 12; + } + + instruction |= (uint)extensionType << 13; + + if (rd.Type == OperandType.I64) + { + instruction |= 1u << 30; + } + + WriteInstructionRm16(instruction, rd, rn, rm); + } + + private void WriteInstructionAuto(uint instruction, Operand rd) + { + if (rd.Type == OperandType.I64) + { + instruction |= SfFlag; + } + + WriteInstruction(instruction, rd); + } + + public void WriteInstructionAuto(uint instruction, Operand rd, Operand rn) + { + if (rd.Type == OperandType.I64) + { + instruction |= SfFlag; + } + + WriteInstruction(instruction, rd, rn); + } + + private void WriteInstructionAuto(uint instruction, Operand rd, Operand rn, Operand rm, Operand ra) + { + if (rd.Type == OperandType.I64) + { + instruction |= SfFlag; + } + + WriteInstruction(instruction, rd, rn, rm, ra); + } + + public void WriteInstruction(uint instruction, Operand rd) + { + WriteUInt32(instruction | EncodeReg(rd)); + } + + public void WriteInstruction(uint instruction, Operand rd, Operand rn) + { + WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5)); + } + + public void WriteInstruction(uint instruction, Operand rd, Operand rn, Operand rm) + { + WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5) | (EncodeReg(rm) << 10)); + } + + public void WriteInstruction(uint instruction, Operand rd, Operand rn, Operand rm, Operand ra) + { + WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5) | (EncodeReg(ra) << 10) | (EncodeReg(rm) << 16)); + } + + private void WriteInstructionRm16Auto(uint instruction, Operand rd, Operand rn, Operand rm) + { + if (rd.Type == OperandType.I64) + { + instruction |= SfFlag; + } + + WriteInstructionRm16(instruction, rd, rn, rm); + } + + public void WriteInstructionRm16(uint instruction, Operand rd, Operand rn, Operand rm) + { + WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5) | (EncodeReg(rm) << 16)); + } + + private static uint GetLdpStpInstruction(uint intInst, uint vecInst, int imm, OperandType type) + { + uint instruction; + int scale; + + if (type.IsInteger()) + { + instruction = intInst; + + if (type == OperandType.I64) + { + instruction |= SfFlag; + scale = 3; + } + else + { + scale = 2; + } + } + else + { + int opc = type switch + { + OperandType.FP32 => 0, + OperandType.FP64 => 1, + _ => 2, + }; + + instruction = vecInst | ((uint)opc << 30); + scale = 2 + opc; + } + + instruction |= (EncodeSImm7(imm, scale) << 15); + + return instruction; + } + + private static uint GetLdrStrInstruction(uint intInst, uint vecInst, OperandType type) + { + uint instruction; + + if (type.IsInteger()) + { + instruction = intInst; + + if (type == OperandType.I64) + { + instruction |= 1 << 30; + } + } + else + { + instruction = vecInst; + + if (type == OperandType.V128) + { + instruction |= 1u << 23; + } + else + { + instruction |= type == OperandType.FP32 ? 2u << 30 : 3u << 30; + } + } + + return instruction; + } + + private static uint EncodeIndexSizeImm5(int index, int size) + { + Debug.Assert((uint)size < 4); + Debug.Assert((uint)index < (16u >> size), $"Invalid index {index} and size {size} combination."); + return ((uint)index << (size + 1)) | (1u << size); + } + + private static uint EncodeSImm7(int value, int scale) + { + uint imm = (uint)(value >> scale) & 0x7f; + Debug.Assert(((int)imm << 25) >> (25 - scale) == value, $"Failed to encode constant 0x{value:X} with scale {scale}."); + return imm; + } + + private static uint EncodeSImm9(int value) + { + uint imm = (uint)value & 0x1ff; + Debug.Assert(((int)imm << 23) >> 23 == value, $"Failed to encode constant 0x{value:X}."); + return imm; + } + + private static uint EncodeSImm19_2(int value) + { + uint imm = (uint)(value >> 2) & 0x7ffff; + Debug.Assert(((int)imm << 13) >> 11 == value, $"Failed to encode constant 0x{value:X}."); + return imm; + } + + private static uint EncodeSImm26_2(int value) + { + uint imm = (uint)(value >> 2) & 0x3ffffff; + Debug.Assert(((int)imm << 6) >> 4 == value, $"Failed to encode constant 0x{value:X}."); + return imm; + } + + private static uint EncodeUImm4(int value) + { + uint imm = (uint)value & 0xf; + Debug.Assert((int)imm == value, $"Failed to encode constant 0x{value:X}."); + return imm; + } + + private static uint EncodeUImm6(int value) + { + uint imm = (uint)value & 0x3f; + Debug.Assert((int)imm == value, $"Failed to encode constant 0x{value:X}."); + return imm; + } + + private static uint EncodeUImm12(int value, OperandType type) + { + return EncodeUImm12(value, GetScaleForType(type)); + } + + private static uint EncodeUImm12(int value, int scale) + { + uint imm = (uint)(value >> scale) & 0xfff; + Debug.Assert((int)imm << scale == value, $"Failed to encode constant 0x{value:X} with scale {scale}."); + return imm; + } + + private static uint EncodeUImm16(int value) + { + uint imm = (uint)value & 0xffff; + Debug.Assert((int)imm == value, $"Failed to encode constant 0x{value:X}."); + return imm; + } + + private static uint EncodeReg(Operand reg) + { + if (reg.Kind == OperandKind.Constant && reg.Value == 0) + { + return ZrRegister; + } + + uint regIndex = (uint)reg.GetRegister().Index; + Debug.Assert(reg.Kind == OperandKind.Register); + Debug.Assert(regIndex < 32); + return regIndex; + } + + private static int GetScaleForType(OperandType type) + { + return type switch + { + OperandType.I32 => 2, + OperandType.I64 => 3, + OperandType.FP32 => 2, + OperandType.FP64 => 3, + OperandType.V128 => 4, + _ => throw new ArgumentException($"Invalid type {type}."), + }; + } + + private void WriteUInt32(uint value) + { + _code.Add(value); + } + + public uint[] GetCode() + { + return _code.ToArray(); + } + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/CodeGenCommon.cs b/src/Ryujinx.Cpu/Nce/Arm64/CodeGenCommon.cs new file mode 100644 index 000000000..205929760 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/CodeGenCommon.cs @@ -0,0 +1,66 @@ +using System.Numerics; + +namespace Ryujinx.Cpu.Nce.Arm64 +{ + static class CodeGenCommon + { + public static bool TryEncodeBitMask(Operand operand, out int immN, out int immS, out int immR) + { + return TryEncodeBitMask(operand.Type, operand.Value, out immN, out immS, out immR); + } + + public static bool TryEncodeBitMask(OperandType type, ulong value, out int immN, out int immS, out int immR) + { + if (type == OperandType.I32) + { + value |= value << 32; + } + + return TryEncodeBitMask(value, out immN, out immS, out immR); + } + + public static bool TryEncodeBitMask(ulong value, out int immN, out int immS, out int immR) + { + // Some special values also can't be encoded: + // 0 can't be encoded because we need to subtract 1 from onesCount (which would became negative if 0). + // A value with all bits set can't be encoded because it is reserved according to the spec, because: + // Any value AND all ones will be equal itself, so it's effectively a no-op. + // Any value OR all ones will be equal all ones, so one can just use MOV. + // Any value XOR all ones will be equal its inverse, so one can just use MVN. + if (value == 0 || value == ulong.MaxValue) + { + immN = 0; + immS = 0; + immR = 0; + + return false; + } + + // Normalize value, rotating it such that the LSB is 1: Ensures we get a complete element that has not + // been cut-in-half across the word boundary. + int rotation = BitOperations.TrailingZeroCount(value & (value + 1)); + ulong rotatedValue = ulong.RotateRight(value, rotation); + + // Now that we have a complete element in the LSB with the LSB = 1, determine size and number of ones + // in element. + int elementSize = BitOperations.TrailingZeroCount(rotatedValue & (rotatedValue + 1)); + int onesInElement = BitOperations.TrailingZeroCount(~rotatedValue); + + // Check the value is repeating; also ensures element size is a power of two. + if (ulong.RotateRight(value, elementSize) != value) + { + immN = 0; + immS = 0; + immR = 0; + + return false; + } + + immN = (elementSize >> 6) & 1; + immS = (((~elementSize + 1) << 1) | (onesInElement - 1)) & 0x3f; + immR = (elementSize - rotation) & (elementSize - 1); + + return true; + } + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/Operand.cs b/src/Ryujinx.Cpu/Nce/Arm64/Operand.cs new file mode 100644 index 000000000..d5d36b70f --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/Operand.cs @@ -0,0 +1,40 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace Ryujinx.Cpu.Nce.Arm64 +{ + struct Operand + { + public readonly OperandKind Kind { get; } + public readonly OperandType Type { get; } + public readonly ulong Value { get; } + + public Operand(OperandKind kind, OperandType type, ulong value) + { + Kind = kind; + Type = type; + Value = value; + } + + public Operand(int index, RegisterType regType, OperandType type) : this(OperandKind.Register, type, (ulong)((int)regType << 24 | index)) + { + } + + public Operand(OperandType type, ulong value) : this(OperandKind.Constant, type, value) + { + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly Register GetRegister() + { + Debug.Assert(Kind == OperandKind.Register); + + return new Register((int)Value & 0xffffff, (RegisterType)(Value >> 24)); + } + + public readonly int AsInt32() + { + return (int)Value; + } + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/OperandKind.cs b/src/Ryujinx.Cpu/Nce/Arm64/OperandKind.cs new file mode 100644 index 000000000..4a822694b --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/OperandKind.cs @@ -0,0 +1,10 @@ +namespace Ryujinx.Cpu.Nce.Arm64 +{ + enum OperandKind + { + None, + Constant, + Label, + Register, + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/OperandType.cs b/src/Ryujinx.Cpu/Nce/Arm64/OperandType.cs new file mode 100644 index 000000000..27e72815d --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/OperandType.cs @@ -0,0 +1,36 @@ +using System; + +namespace Ryujinx.Cpu.Nce.Arm64 +{ + enum OperandType + { + None, + I32, + I64, + FP32, + FP64, + V128, + } + + static class OperandTypeExtensions + { + public static bool IsInteger(this OperandType type) + { + return type == OperandType.I32 || + type == OperandType.I64; + } + + public static int GetSizeInBytes(this OperandType type) + { + return type switch + { + OperandType.FP32 => 4, + OperandType.FP64 => 8, + OperandType.I32 => 4, + OperandType.I64 => 8, + OperandType.V128 => 16, + _ => throw new InvalidOperationException($"Invalid operand type \"{type}\"."), + }; + } + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/Register.cs b/src/Ryujinx.Cpu/Nce/Arm64/Register.cs new file mode 100644 index 000000000..8277fda7a --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/Register.cs @@ -0,0 +1,43 @@ +using System; + +namespace Ryujinx.Cpu.Nce.Arm64 +{ + readonly struct Register : IEquatable + { + public int Index { get; } + + public RegisterType Type { get; } + + public Register(int index, RegisterType type) + { + Index = index; + Type = type; + } + + public override int GetHashCode() + { + return (ushort)Index | ((int)Type << 16); + } + + public static bool operator ==(Register x, Register y) + { + return x.Equals(y); + } + + public static bool operator !=(Register x, Register y) + { + return !x.Equals(y); + } + + public override bool Equals(object obj) + { + return obj is Register reg && Equals(reg); + } + + public bool Equals(Register other) + { + return other.Index == Index && + other.Type == Type; + } + } +} diff --git a/src/Ryujinx.Cpu/Nce/Arm64/RegisterSaveRestore.cs b/src/Ryujinx.Cpu/Nce/Arm64/RegisterSaveRestore.cs new file mode 100644 index 000000000..aade05977 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/RegisterSaveRestore.cs @@ -0,0 +1,220 @@ +using System.Numerics; + +namespace Ryujinx.Cpu.Nce.Arm64 +{ + readonly struct RegisterSaveRestore + { + private const int FpRegister = 29; + private const int LrRegister = 30; + + private const int Encodable9BitsOffsetLimit = 0x200; + + private readonly int _intMask; + private readonly int _vecMask; + private readonly OperandType _vecType; + private readonly bool _hasCall; + + public RegisterSaveRestore(int intMask, int vecMask = 0, OperandType vecType = OperandType.FP64, bool hasCall = false) + { + _intMask = intMask; + _vecMask = vecMask; + _vecType = vecType; + _hasCall = hasCall; + } + + public void WritePrologue(Assembler asm) + { + int intMask = _intMask; + int vecMask = _vecMask; + + int intCalleeSavedRegsCount = BitOperations.PopCount((uint)intMask); + int vecCalleeSavedRegsCount = BitOperations.PopCount((uint)vecMask); + + int calleeSaveRegionSize = Align16(intCalleeSavedRegsCount * 8 + vecCalleeSavedRegsCount * _vecType.GetSizeInBytes()); + + int offset = 0; + + WritePrologueCalleeSavesPreIndexed(asm, ref intMask, ref offset, calleeSaveRegionSize, OperandType.I64); + + if (_vecType == OperandType.V128 && (intCalleeSavedRegsCount & 1) != 0) + { + offset += 8; + } + + WritePrologueCalleeSavesPreIndexed(asm, ref vecMask, ref offset, calleeSaveRegionSize, _vecType); + + if (_hasCall) + { + Operand rsp = Register(Assembler.SpRegister); + + asm.StpRiPre(Register(FpRegister), Register(LrRegister), rsp, -16); + asm.MovSp(Register(FpRegister), rsp); + } + } + + private static void WritePrologueCalleeSavesPreIndexed( + Assembler asm, + ref int mask, + ref int offset, + int calleeSaveRegionSize, + OperandType type) + { + if ((BitOperations.PopCount((uint)mask) & 1) != 0) + { + int reg = BitOperations.TrailingZeroCount(mask); + + mask &= ~(1 << reg); + + if (offset != 0) + { + asm.StrRiUn(Register(reg, type), Register(Assembler.SpRegister), offset); + } + else if (calleeSaveRegionSize < Encodable9BitsOffsetLimit) + { + asm.StrRiPre(Register(reg, type), Register(Assembler.SpRegister), -calleeSaveRegionSize); + } + else + { + asm.Sub(Register(Assembler.SpRegister), Register(Assembler.SpRegister), new Operand(OperandType.I64, (ulong)calleeSaveRegionSize)); + asm.StrRiUn(Register(reg, type), Register(Assembler.SpRegister), 0); + } + + offset += type.GetSizeInBytes(); + } + + while (mask != 0) + { + int reg = BitOperations.TrailingZeroCount(mask); + + mask &= ~(1 << reg); + + int reg2 = BitOperations.TrailingZeroCount(mask); + + mask &= ~(1 << reg2); + + if (offset != 0) + { + asm.StpRiUn(Register(reg, type), Register(reg2, type), Register(Assembler.SpRegister), offset); + } + else if (calleeSaveRegionSize < Encodable9BitsOffsetLimit) + { + asm.StpRiPre(Register(reg, type), Register(reg2, type), Register(Assembler.SpRegister), -calleeSaveRegionSize); + } + else + { + asm.Sub(Register(Assembler.SpRegister), Register(Assembler.SpRegister), new Operand(OperandType.I64, (ulong)calleeSaveRegionSize)); + asm.StpRiUn(Register(reg, type), Register(reg2, type), Register(Assembler.SpRegister), 0); + } + + offset += type.GetSizeInBytes() * 2; + } + } + + public void WriteEpilogue(Assembler asm) + { + int intMask = _intMask; + int vecMask = _vecMask; + + int intCalleeSavedRegsCount = BitOperations.PopCount((uint)intMask); + int vecCalleeSavedRegsCount = BitOperations.PopCount((uint)vecMask); + + bool misalignedVector = _vecType == OperandType.V128 && (intCalleeSavedRegsCount & 1) != 0; + + int offset = intCalleeSavedRegsCount * 8 + vecCalleeSavedRegsCount * _vecType.GetSizeInBytes(); + + if (misalignedVector) + { + offset += 8; + } + + int calleeSaveRegionSize = Align16(offset); + + if (_hasCall) + { + Operand rsp = Register(Assembler.SpRegister); + + asm.LdpRiPost(Register(FpRegister), Register(LrRegister), rsp, 16); + } + + WriteEpilogueCalleeSavesPostIndexed(asm, ref vecMask, ref offset, calleeSaveRegionSize, _vecType); + + if (misalignedVector) + { + offset -= 8; + } + + WriteEpilogueCalleeSavesPostIndexed(asm, ref intMask, ref offset, calleeSaveRegionSize, OperandType.I64); + } + + private static void WriteEpilogueCalleeSavesPostIndexed( + Assembler asm, + ref int mask, + ref int offset, + int calleeSaveRegionSize, + OperandType type) + { + while (mask != 0) + { + int reg = HighestBitSet(mask); + + mask &= ~(1 << reg); + + if (mask != 0) + { + int reg2 = HighestBitSet(mask); + + mask &= ~(1 << reg2); + + offset -= type.GetSizeInBytes() * 2; + + if (offset != 0) + { + asm.LdpRiUn(Register(reg2, type), Register(reg, type), Register(Assembler.SpRegister), offset); + } + else if (calleeSaveRegionSize < Encodable9BitsOffsetLimit) + { + asm.LdpRiPost(Register(reg2, type), Register(reg, type), Register(Assembler.SpRegister), calleeSaveRegionSize); + } + else + { + asm.LdpRiUn(Register(reg2, type), Register(reg, type), Register(Assembler.SpRegister), 0); + asm.Add(Register(Assembler.SpRegister), Register(Assembler.SpRegister), new Operand(OperandType.I64, (ulong)calleeSaveRegionSize)); + } + } + else + { + offset -= type.GetSizeInBytes(); + + if (offset != 0) + { + asm.LdrRiUn(Register(reg, type), Register(Assembler.SpRegister), offset); + } + else if (calleeSaveRegionSize < Encodable9BitsOffsetLimit) + { + asm.LdrRiPost(Register(reg, type), Register(Assembler.SpRegister), calleeSaveRegionSize); + } + else + { + asm.LdrRiUn(Register(reg, type), Register(Assembler.SpRegister), 0); + asm.Add(Register(Assembler.SpRegister), Register(Assembler.SpRegister), new Operand(OperandType.I64, (ulong)calleeSaveRegionSize)); + } + } + } + } + + private static int HighestBitSet(int value) + { + return 31 - BitOperations.LeadingZeroCount((uint)value); + } + + private static Operand Register(int register, OperandType type = OperandType.I64) + { + return new Operand(register, RegisterType.Integer, type); + } + + private static int Align16(int value) + { + return (value + 0xf) & ~0xf; + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/Arm64/RegisterType.cs b/src/Ryujinx.Cpu/Nce/Arm64/RegisterType.cs new file mode 100644 index 000000000..f53090e66 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/Arm64/RegisterType.cs @@ -0,0 +1,8 @@ +namespace Ryujinx.Cpu.Nce.Arm64 +{ + enum RegisterType + { + Integer, + Vector, + } +} diff --git a/src/Ryujinx.Cpu/Nce/MemoryManagerNative.cs b/src/Ryujinx.Cpu/Nce/MemoryManagerNative.cs new file mode 100644 index 000000000..cbb3fd2b4 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/MemoryManagerNative.cs @@ -0,0 +1,328 @@ +using ARMeilleure.Memory; +using Ryujinx.Memory; +using Ryujinx.Memory.Range; +using Ryujinx.Memory.Tracking; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace Ryujinx.Cpu.Nce +{ + /// + /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. + /// + public sealed class MemoryManagerNative : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + { + private readonly MemoryBlock _addressSpace; + + private readonly MemoryBlock _backingMemory; + private readonly PageTable _pageTable; + + private readonly MemoryEhMeilleure _memoryEh; + + private readonly ManagedPageFlags _pages; + + /// + public bool UsesPrivateAllocations => false; + + public IntPtr PageTablePointer => IntPtr.Zero; + + public ulong ReservedSize => (ulong)_addressSpace.Pointer.ToInt64(); + + public MemoryManagerType Type => MemoryManagerType.HostMappedUnsafe; + + public MemoryTracking Tracking { get; } + + public event Action UnmapEvent; + + public int AddressSpaceBits { get; } + protected override ulong AddressSpaceSize { get; } + + /// + /// Creates a new instance of the host mapped memory manager. + /// + /// Address space memory block + /// Physical backing memory where virtual memory will be mapped to + /// Size of the address space + /// Optional function to handle invalid memory accesses + public MemoryManagerNative( + MemoryBlock addressSpace, + MemoryBlock backingMemory, + ulong addressSpaceSize, + InvalidAccessHandler invalidAccessHandler = null) + { + _backingMemory = backingMemory; + _pageTable = new PageTable(); + AddressSpaceSize = addressSpaceSize; + + ulong asSize = PageSize; + int asBits = PageBits; + + while (asSize < addressSpaceSize) + { + asSize <<= 1; + asBits++; + } + + AddressSpaceBits = asBits; + + _pages = new ManagedPageFlags(asBits); + + _addressSpace = addressSpace; + + Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler); + _memoryEh = new MemoryEhMeilleure(addressSpaceSize, Tracking); + } + + /// + public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags) + { + AssertValidAddressAndSize(va, size); + + _addressSpace.MapView(_backingMemory, pa, AddressToOffset(va), size); + _pages.AddMapping(va, size); + PtMap(va, pa, size); + + Tracking.Map(va, size); + } + + private void PtMap(ulong va, ulong pa, ulong size) + { + while (size != 0) + { + _pageTable.Map(va, pa); + + va += PageSize; + pa += PageSize; + size -= PageSize; + } + } + + /// + public void Unmap(ulong va, ulong size) + { + AssertValidAddressAndSize(va, size); + + UnmapEvent?.Invoke(va, size); + Tracking.Unmap(va, size); + + _pages.RemoveMapping(va, size); + PtUnmap(va, size); + _addressSpace.UnmapView(_backingMemory, AddressToOffset(va), size); + } + + private void PtUnmap(ulong va, ulong size) + { + while (size != 0) + { + _pageTable.Unmap(va); + + va += PageSize; + size -= PageSize; + } + } + + /// + public void Reprotect(ulong va, ulong size, MemoryPermission protection) + { + _addressSpace.Reprotect(AddressToOffset(va), size, protection); + } + + public ref T GetRef(ulong va) where T : unmanaged + { + if (!IsContiguous(va, Unsafe.SizeOf())) + { + ThrowMemoryNotContiguous(); + } + + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); + + return ref _backingMemory.GetRef(GetPhysicalAddressChecked(va)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override bool IsMapped(ulong va) + { + return ValidateAddress(va) && _pages.IsMapped(va); + } + + /// + public bool IsRangeMapped(ulong va, ulong size) + { + AssertValidAddressAndSize(va, size); + + return _pages.IsRangeMapped(va, size); + } + + /// + public IEnumerable GetHostRegions(ulong va, ulong size) + { + if (size == 0) + { + return Enumerable.Empty(); + } + + var guestRegions = GetPhysicalRegionsImpl(va, size); + if (guestRegions == null) + { + return null; + } + + var regions = new HostMemoryRange[guestRegions.Count]; + + for (int i = 0; i < regions.Length; i++) + { + var guestRegion = guestRegions[i]; + IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size); + regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size); + } + + return regions; + } + + /// + public IEnumerable GetPhysicalRegions(ulong va, ulong size) + { + if (size == 0) + { + return Enumerable.Empty(); + } + + return GetPhysicalRegionsImpl(va, size); + } + + private List GetPhysicalRegionsImpl(ulong va, ulong size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size)) + { + return null; + } + + int pages = GetPagesCount(va, (uint)size, out va); + + var regions = new List(); + + ulong regionStart = GetPhysicalAddressInternal(va); + ulong regionSize = PageSize; + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return null; + } + + ulong newPa = GetPhysicalAddressInternal(va + PageSize); + + if (GetPhysicalAddressInternal(va) + PageSize != newPa) + { + regions.Add(new MemoryRange(regionStart, regionSize)); + regionStart = newPa; + regionSize = 0; + } + + va += PageSize; + regionSize += PageSize; + } + + regions.Add(new MemoryRange(regionStart, regionSize)); + + return regions; + } + + private ulong GetPhysicalAddressChecked(ulong va) + { + if (!IsMapped(va)) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}"); + } + + return GetPhysicalAddressInternal(va); + } + + private ulong GetPhysicalAddressInternal(ulong va) + { + return _pageTable.Read(va) + (va & PageMask); + } + + /// + /// + /// This function also validates that the given range is both valid and mapped, and will throw if it is not. + /// + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + { + AssertValidAddressAndSize(va, size); + + if (precise) + { + Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId); + return; + } + + _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); + } + + /// + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) + { + if (guest) + { + _addressSpace.Reprotect(AddressToOffset(va), size, protection, false); + } + else + { + _pages.TrackingReprotect(va, size, protection); + } + } + + /// + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags) + { + return Tracking.BeginTracking(address, size, id, flags); + } + + /// + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags) + { + return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); + } + + /// + public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id) + { + return Tracking.BeginSmartGranularTracking(address, size, granularity, id); + } + + private ulong AddressToOffset(ulong address) + { + if (address < ReservedSize) + { + throw new ArgumentException($"Invalid address 0x{address:x16}"); + } + + return address - ReservedSize; + } + + /// + /// Disposes of resources used by the memory manager. + /// + protected override void Destroy() + { + _addressSpace.Dispose(); + _memoryEh.Dispose(); + } + + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _backingMemory.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) + => _backingMemory.GetSpan(pa, size); + + protected override nuint TranslateVirtualAddressChecked(ulong va) + => (nuint)GetPhysicalAddressChecked(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) + => (nuint)GetPhysicalAddressInternal(va); + } +} diff --git a/src/Ryujinx.Cpu/Nce/NceCpuCodePatch.cs b/src/Ryujinx.Cpu/Nce/NceCpuCodePatch.cs new file mode 100644 index 000000000..bb1f65109 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceCpuCodePatch.cs @@ -0,0 +1,81 @@ +using Ryujinx.Common; +using Ryujinx.Memory; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace Ryujinx.Cpu.Nce +{ + /// + /// Native Code Execution CPU code patch. + /// + public class NceCpuCodePatch + { + private readonly List _code; + + private readonly struct PatchTarget + { + public readonly int TextIndex; + public readonly int PatchStartIndex; + public readonly int PatchBranchIndex; + + public PatchTarget(int textIndex, int patchStartIndex, int patchBranchIndex) + { + TextIndex = textIndex; + PatchStartIndex = patchStartIndex; + PatchBranchIndex = patchBranchIndex; + } + } + + private readonly List _patchTargets; + + /// + public ulong Size => BitUtils.AlignUp((ulong)_code.Count * sizeof(uint), 0x1000UL); + + public NceCpuCodePatch() + { + _code = new(); + _patchTargets = new(); + } + + internal void AddCode(int textIndex, IEnumerable code) + { + int patchStartIndex = _code.Count; + _code.AddRange(code); + _patchTargets.Add(new PatchTarget(textIndex, patchStartIndex, _code.Count - 1)); + } + + /// + public void Write(IVirtualMemoryManager memoryManager, ulong patchAddress, ulong textAddress) + { + uint[] code = _code.ToArray(); + + foreach (var patchTarget in _patchTargets) + { + ulong instPatchStartAddress = patchAddress + (ulong)patchTarget.PatchStartIndex * sizeof(uint); + ulong instPatchBranchAddress = patchAddress + (ulong)patchTarget.PatchBranchIndex * sizeof(uint); + ulong instTextAddress = textAddress + (ulong)patchTarget.TextIndex * sizeof(uint); + + uint prevInst = memoryManager.Read(instTextAddress); + + code[patchTarget.PatchBranchIndex] |= EncodeSImm26_2(checked((int)((long)instTextAddress - (long)instPatchBranchAddress + sizeof(uint)))); + memoryManager.Write(instTextAddress, 0x14000000u | EncodeSImm26_2(checked((int)((long)instPatchStartAddress - (long)instTextAddress)))); + + uint newInst = memoryManager.Read(instTextAddress); + } + + if (Size != 0) + { + memoryManager.Write(patchAddress, MemoryMarshal.Cast(code)); + memoryManager.Reprotect(patchAddress, Size, MemoryPermission.ReadAndExecute); + } + } + + private static uint EncodeSImm26_2(int value) + { + uint imm = (uint)(value >> 2) & 0x3ffffff; + Debug.Assert(((int)imm << 6) >> 4 == value, $"Failed to encode constant 0x{value:X}."); + return imm; + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/NceCpuContext.cs b/src/Ryujinx.Cpu/Nce/NceCpuContext.cs new file mode 100644 index 000000000..154ffd5bf --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceCpuContext.cs @@ -0,0 +1,144 @@ +using ARMeilleure.Memory; +using Ryujinx.Cpu.Signal; +using Ryujinx.Common; +using Ryujinx.Memory; +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Ryujinx.Cpu.Nce +{ + class NceCpuContext : ICpuContext + { + private static uint[] _getTpidrEl0Code = new uint[] + { + GetMrsTpidrEl0(0), // mrs x0, tpidr_el0 + 0xd65f03c0u, // ret + }; + + private static uint GetMrsTpidrEl0(uint rd) + { + if (OperatingSystem.IsMacOS()) + { + return 0xd53bd060u | rd; // TPIDRRO + } + else + { + return 0xd53bd040u | rd; // TPIDR + } + } + + readonly struct CodeWriter + { + private readonly List _fullCode; + + public CodeWriter() + { + _fullCode = new List(); + } + + public ulong Write(uint[] code) + { + ulong offset = (ulong)_fullCode.Count * sizeof(uint); + _fullCode.AddRange(code); + + return offset; + } + + public MemoryBlock CreateMemoryBlock() + { + ReadOnlySpan codeBytes = MemoryMarshal.Cast(_fullCode.ToArray()); + + MemoryBlock codeBlock = new(BitUtils.AlignUp((ulong)codeBytes.Length, 0x1000UL)); + + codeBlock.Write(0, codeBytes); + codeBlock.Reprotect(0, (ulong)codeBytes.Length, MemoryPermission.ReadAndExecute, true); + + return codeBlock; + } + } + + private delegate void ThreadStart(IntPtr nativeContextPtr); + private delegate IntPtr GetTpidrEl0(); + private static MemoryBlock _codeBlock; + private static ThreadStart _threadStart; + private static GetTpidrEl0 _getTpidrEl0; + + private readonly ITickSource _tickSource; + private readonly IMemoryManager _memoryManager; + + static NceCpuContext() + { + CodeWriter codeWriter = new(); + + uint[] threadStartCode = NcePatcher.GenerateThreadStartCode(); + uint[] ehSuspendCode = NcePatcher.GenerateSuspendExceptionHandler(); + + ulong threadStartCodeOffset = codeWriter.Write(threadStartCode); + ulong getTpidrEl0CodeOffset = codeWriter.Write(_getTpidrEl0Code); + ulong ehSuspendCodeOffset = codeWriter.Write(ehSuspendCode); + + MemoryBlock codeBlock = null; + + NativeSignalHandler.InitializeSignalHandler((IntPtr oldSignalHandlerSegfaultPtr, IntPtr signalHandlerPtr) => + { + uint[] ehWrapperCode = NcePatcher.GenerateWrapperExceptionHandler(oldSignalHandlerSegfaultPtr, signalHandlerPtr); + ulong ehWrapperCodeOffset = codeWriter.Write(ehWrapperCode); + codeBlock = codeWriter.CreateMemoryBlock(); + return codeBlock.GetPointer(ehWrapperCodeOffset, (ulong)ehWrapperCode.Length * sizeof(uint)); + }); + + NativeSignalHandler.InstallUnixSignalHandler(NceThreadPal.UnixSuspendSignal, codeBlock.GetPointer(ehSuspendCodeOffset, (ulong)ehSuspendCode.Length * sizeof(uint))); + + _threadStart = Marshal.GetDelegateForFunctionPointer(codeBlock.GetPointer(threadStartCodeOffset, (ulong)threadStartCode.Length * sizeof(uint))); + _getTpidrEl0 = Marshal.GetDelegateForFunctionPointer(codeBlock.GetPointer(getTpidrEl0CodeOffset, (ulong)_getTpidrEl0Code.Length * sizeof(uint))); + _codeBlock = codeBlock; + } + + public NceCpuContext(ITickSource tickSource, IMemoryManager memory, bool for64Bit) + { + _tickSource = tickSource; + _memoryManager = memory; + } + + /// + public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks) + { + return new NceExecutionContext(exceptionCallbacks); + } + + /// + public void Execute(IExecutionContext context, ulong address) + { + NceExecutionContext nec = (NceExecutionContext)context; + NceNativeInterface.RegisterThread(nec, _tickSource); + int tableIndex = NceThreadTable.Register(_getTpidrEl0(), nec.NativeContextPtr); + + nec.SetStartAddress(address); + _threadStart(nec.NativeContextPtr); + nec.Exit(); + + NceThreadTable.Unregister(tableIndex); + } + + /// + public void InvalidateCacheRegion(ulong address, ulong size) + { + } + + /// + public IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled, string cacheSelector) + { + return new DummyDiskCacheLoadState(); + } + + /// + public void PrepareCodeRange(ulong address, ulong size) + { + } + + public void Dispose() + { + } + } +} diff --git a/src/Ryujinx.Cpu/Nce/NceEngine.cs b/src/Ryujinx.Cpu/Nce/NceEngine.cs new file mode 100644 index 000000000..402616695 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceEngine.cs @@ -0,0 +1,19 @@ +using ARMeilleure.Memory; + +namespace Ryujinx.Cpu.Nce +{ + public class NceEngine : ICpuEngine + { + public ITickSource TickSource{ get; } + + public NceEngine(ITickSource tickSource) + { + TickSource = tickSource; + } + + public ICpuContext CreateCpuContext(IMemoryManager memoryManager, bool for64Bit) + { + return new NceCpuContext(TickSource, memoryManager, for64Bit); + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/NceExecutionContext.cs b/src/Ryujinx.Cpu/Nce/NceExecutionContext.cs new file mode 100644 index 000000000..37069939e --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceExecutionContext.cs @@ -0,0 +1,174 @@ +using ARMeilleure.State; +using Ryujinx.Cpu.Signal; +using Ryujinx.Memory; +using System; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Ryujinx.Cpu.Nce +{ + class NceExecutionContext : IExecutionContext + { + private const ulong AlternateStackSize = 0x4000; + + private readonly NceNativeContext _context; + private readonly ExceptionCallbacks _exceptionCallbacks; + + internal IntPtr NativeContextPtr => _context.BasePtr; + + public ulong Pc => 0UL; + + public long TpidrEl0 + { + get => (long)_context.GetStorage().TpidrEl0; + set => _context.GetStorage().TpidrEl0 = (ulong)value; + } + + public long TpidrroEl0 + { + get => (long)_context.GetStorage().TpidrroEl0; + set => _context.GetStorage().TpidrroEl0 = (ulong)value; + } + + public uint Pstate + { + get => _context.GetStorage().Pstate; + set => _context.GetStorage().Pstate = value; + } + + public uint Fpcr + { + get => _context.GetStorage().Fpcr; + set => _context.GetStorage().Fpcr = value; + } + + public uint Fpsr + { + get => _context.GetStorage().Fpsr; + set => _context.GetStorage().Fpsr = value; + } + + public bool IsAarch32 + { + get => false; + set + { + if (value) + { + throw new NotSupportedException(); + } + } + } + + public bool Running { get; private set; } + + private delegate bool SupervisorCallHandler(int imm); + private SupervisorCallHandler _svcHandler; + + private MemoryBlock _alternateStackMemory; + + public NceExecutionContext(ExceptionCallbacks exceptionCallbacks) + { + _svcHandler = OnSupervisorCall; + IntPtr svcHandlerPtr = Marshal.GetFunctionPointerForDelegate(_svcHandler); + + _context = new NceNativeContext(); + + ref var storage = ref _context.GetStorage(); + storage.SvcCallHandler = svcHandlerPtr; + storage.InManaged = 1u; + storage.CtrEl0 = 0x8444c004; // TODO: Get value from host CPU instead of using guest one? + + Running = true; + _exceptionCallbacks = exceptionCallbacks; + } + + public ulong GetX(int index) => _context.GetStorage().X[index]; + public void SetX(int index, ulong value) => _context.GetStorage().X[index] = value; + + public V128 GetV(int index) => _context.GetStorage().V[index]; + public void SetV(int index, V128 value) => _context.GetStorage().V[index] = value; + + // TODO + public bool GetPstateFlag(PState flag) => false; + public void SetPstateFlag(PState flag, bool value) { } + + // TODO + public bool GetFPstateFlag(FPState flag) => false; + public void SetFPstateFlag(FPState flag, bool value) { } + + public void SetStartAddress(ulong address) + { + ref var storage = ref _context.GetStorage(); + storage.X[30] = address; + storage.HostThreadHandle = NceThreadPal.GetCurrentThreadHandle(); + + RegisterAlternateStack(); + } + + public void Exit() + { + _context.GetStorage().HostThreadHandle = IntPtr.Zero; + + UnregisterAlternateStack(); + } + + private void RegisterAlternateStack() + { + // We need to use an alternate stack to handle the suspend signal, + // as the guest stack may be in a state that is not suitable for the signal handlers. + + _alternateStackMemory = new MemoryBlock(AlternateStackSize); + NativeSignalHandler.InstallUnixAlternateStackForCurrentThread(_alternateStackMemory.GetPointer(0UL, AlternateStackSize), AlternateStackSize); + } + + private void UnregisterAlternateStack() + { + NativeSignalHandler.UninstallUnixAlternateStackForCurrentThread(); + _alternateStackMemory.Dispose(); + _alternateStackMemory = null; + } + + public bool OnSupervisorCall(int imm) + { + _exceptionCallbacks.SupervisorCallback?.Invoke(this, 0UL, imm); + return Running; + } + + public bool OnInterrupt() + { + _exceptionCallbacks.InterruptCallback?.Invoke(this); + return Running; + } + + public void RequestInterrupt() + { + IntPtr threadHandle = _context.GetStorage().HostThreadHandle; + if (threadHandle != IntPtr.Zero) + { + // Bit 0 set means that the thread is currently running managed code. + // Bit 1 set means that an interrupt was requested for the thread. + // This, we only need to send the suspend signal if the value was 0 (not running managed code, + // and no interrupt was requested before). + + ref uint inManaged = ref _context.GetStorage().InManaged; + uint oldValue = Interlocked.Or(ref inManaged, 2); + + if (oldValue == 0) + { + NceThreadPal.SuspendThread(threadHandle); + } + } + } + + public void StopRunning() + { + Running = false; + } + + public void Dispose() + { + _context.Dispose(); + } + } +} diff --git a/src/Ryujinx.Cpu/Nce/NceMemoryAllocator.cs b/src/Ryujinx.Cpu/Nce/NceMemoryAllocator.cs new file mode 100644 index 000000000..34badd7ab --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceMemoryAllocator.cs @@ -0,0 +1,13 @@ +using ARMeilleure.Memory; +using Ryujinx.Memory; + +namespace Ryujinx.Cpu.Nce +{ + class NceMemoryAllocator : IJitMemoryAllocator + { + public IJitMemoryBlock Allocate(ulong size) => new NceMemoryBlock(size, MemoryAllocationFlags.None); + public IJitMemoryBlock Reserve(ulong size) => new NceMemoryBlock(size, MemoryAllocationFlags.Reserve); + + public ulong GetPageSize() => MemoryBlock.GetPageSize(); + } +} diff --git a/src/Ryujinx.Cpu/Nce/NceMemoryBlock.cs b/src/Ryujinx.Cpu/Nce/NceMemoryBlock.cs new file mode 100644 index 000000000..7bf180eb4 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceMemoryBlock.cs @@ -0,0 +1,25 @@ +using ARMeilleure.Memory; +using Ryujinx.Memory; +using System; + +namespace Ryujinx.Cpu.Nce +{ + class NceMemoryBlock : IJitMemoryBlock + { + private readonly MemoryBlock _impl; + + public IntPtr Pointer => _impl.Pointer; + + public NceMemoryBlock(ulong size, MemoryAllocationFlags flags) + { + _impl = new MemoryBlock(size, flags); + } + + public void Commit(ulong offset, ulong size) => _impl.Commit(offset, size); + public void MapAsRw(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadAndWrite); + public void MapAsRx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadAndExecute); + public void MapAsRwx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadWriteExecute); + + public void Dispose() => _impl.Dispose(); + } +} diff --git a/src/Ryujinx.Cpu/Nce/NceNativeContext.cs b/src/Ryujinx.Cpu/Nce/NceNativeContext.cs new file mode 100644 index 000000000..a7d5f4eec --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceNativeContext.cs @@ -0,0 +1,98 @@ +using ARMeilleure.State; +using Ryujinx.Common.Memory; +using Ryujinx.Memory; +using System; +using System.Runtime.CompilerServices; + +namespace Ryujinx.Cpu.Nce +{ + class NceNativeContext : IDisposable + { + public struct NativeCtxStorage + { + public Array32 X; + public Array32 V; + public ulong TpidrEl0; + public ulong TpidrroEl0; + public ulong CtrEl0; + public uint Pstate; + public uint Fpcr; + public uint Fpsr; + public uint InManaged; + public ulong HostSp; + public IntPtr HostThreadHandle; + public ulong TempStorage; + public IntPtr SvcCallHandler; + } + + private static NativeCtxStorage _dummyStorage = new(); + + private readonly MemoryBlock _block; + + public IntPtr BasePtr => _block.Pointer; + + public NceNativeContext() + { + _block = new MemoryBlock((ulong)Unsafe.SizeOf()); + } + + public static int GetXOffset(int index) + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.X[index]); + } + + public static int GetGuestSPOffset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.X[31]); + } + + public static int GetVOffset(int index) + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.V[index]); + } + + public static int GetTpidrEl0Offset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.TpidrEl0); + } + + public static int GetTpidrroEl0Offset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.TpidrroEl0); + } + + public static int GetInManagedOffset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.InManaged); + } + + public static int GetHostSPOffset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.HostSp); + } + + public static int GetCtrEl0Offset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.CtrEl0); + } + + public static int GetTempStorageOffset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.TempStorage); + } + + public static int GetSvcCallHandlerOffset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.SvcCallHandler); + } + + private static int StorageOffset(ref NativeCtxStorage storage, ref T target) + { + return (int)Unsafe.ByteOffset(ref Unsafe.As(ref storage), ref target); + } + + public unsafe ref NativeCtxStorage GetStorage() => ref Unsafe.AsRef((void*)_block.Pointer); + + public void Dispose() => _block.Dispose(); + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/NceNativeInterface.cs b/src/Ryujinx.Cpu/Nce/NceNativeInterface.cs new file mode 100644 index 000000000..843af8b5c --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceNativeInterface.cs @@ -0,0 +1,55 @@ +using System; +using System.Runtime.InteropServices; + +namespace Ryujinx.Cpu.Nce +{ + static class NceNativeInterface + { + private delegate ulong GetTickCounterDelegate(); + private delegate bool SuspendThreadHandlerDelegate(); + private static GetTickCounterDelegate _getTickCounter; + private static SuspendThreadHandlerDelegate _suspendThreadHandler; + private static IntPtr _getTickCounterPtr; + private static IntPtr _suspendThreadHandlerPtr; + + [ThreadStatic] + private static NceExecutionContext _context; + + [ThreadStatic] + private static ITickSource _tickSource; + + static NceNativeInterface() + { + _getTickCounter = GetTickCounter; + _suspendThreadHandler = SuspendThreadHandler; + _getTickCounterPtr = Marshal.GetFunctionPointerForDelegate(_getTickCounter); + _suspendThreadHandlerPtr = Marshal.GetFunctionPointerForDelegate(_suspendThreadHandler); + } + + public static void RegisterThread(NceExecutionContext context, ITickSource tickSource) + { + _context = context; + _tickSource = tickSource; + } + + public static ulong GetTickCounter() + { + return _tickSource.Counter; + } + + public static bool SuspendThreadHandler() + { + return _context.OnInterrupt(); + } + + public static IntPtr GetTickCounterAccessFunctionPointer() + { + return _getTickCounterPtr; + } + + public static IntPtr GetSuspendThreadHandlerFunctionPointer() + { + return _suspendThreadHandlerPtr; + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/NcePatcher.cs b/src/Ryujinx.Cpu/Nce/NcePatcher.cs new file mode 100644 index 000000000..04a1c16c7 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NcePatcher.cs @@ -0,0 +1,603 @@ +using Ryujinx.Cpu.Nce.Arm64; +using Ryujinx.Common.Logging; +using System; +using System.Runtime.InteropServices; + +namespace Ryujinx.Cpu.Nce +{ + public static class NcePatcher + { + private const int ScratchBaseReg = 19; + + private const uint IntCalleeSavedRegsMask = 0x1ff80000; // X19 to X28 + private const uint FpCalleeSavedRegsMask = 0xff00; // D8 to D15 + + public static NceCpuCodePatch CreatePatch(ReadOnlySpan textSection) + { + NceCpuCodePatch codePatch = new(); + + var textUint = MemoryMarshal.Cast(textSection); + + for (int i = 0; i < textUint.Length; i++) + { + uint inst = textUint[i]; + ulong address = (ulong)i * sizeof(uint); + + if ((inst & ~(0xffffu << 5)) == 0xd4000001u) // svc #0 + { + uint svcId = (ushort)(inst >> 5); + codePatch.AddCode(i, WriteSvcPatch(svcId)); + Logger.Debug?.Print(LogClass.Cpu, $"Patched SVC #{svcId} at 0x{address:X}."); + } + else if ((inst & ~0x1f) == 0xd53bd060) // mrs x0, tpidrro_el0 + { + uint rd = inst & 0x1f; + codePatch.AddCode(i, WriteMrsTpidrroEl0Patch(rd)); + Logger.Debug?.Print(LogClass.Cpu, $"Patched MRS x{rd}, tpidrro_el0 at 0x{address:X}."); + } + else if ((inst & ~0x1f) == 0xd53bd040) // mrs x0, tpidr_el0 + { + uint rd = inst & 0x1f; + codePatch.AddCode(i, WriteMrsTpidrEl0Patch(rd)); + Logger.Debug?.Print(LogClass.Cpu, $"Patched MRS x{rd}, tpidr_el0 at 0x{address:X}."); + } + else if ((inst & ~0x1f) == 0xd53b0020 && OperatingSystem.IsMacOS()) // mrs x0, ctr_el0 + { + uint rd = inst & 0x1f; + codePatch.AddCode(i, WriteMrsCtrEl0Patch(rd)); + Logger.Debug?.Print(LogClass.Cpu, $"Patched MRS x{rd}, ctr_el0 at 0x{address:X}."); + } + else if ((inst & ~0x1f) == 0xd53be020) // mrs x0, cntpct_el0 + { + uint rd = inst & 0x1f; + codePatch.AddCode(i, WriteMrsCntpctEl0Patch(rd)); + Logger.Debug?.Print(LogClass.Cpu, $"Patched MRS x{rd}, cntpct_el0 at 0x{address:X}."); + } + else if ((inst & ~0x1f) == 0xd51bd040) // msr tpidr_el0, x0 + { + uint rd = inst & 0x1f; + codePatch.AddCode(i, WriteMsrTpidrEl0Patch(rd)); + Logger.Debug?.Print(LogClass.Cpu, $"Patched MSR tpidr_el0, x{rd} at 0x{address:X}."); + } + } + + return codePatch; + } + + private static uint[] WriteSvcPatch(uint svcId) + { + Assembler asm = new(); + + WriteManagedCall(asm, (asm, ctx, tmp, tmp2) => + { + for (int i = 0; i < 8; i++) + { + asm.StrRiUn(Gpr(i), ctx, NceNativeContext.GetXOffset(i)); + } + + WriteInManagedLockAcquire(asm, ctx, tmp, tmp2); + + asm.Mov(Gpr(0, OperandType.I32), svcId); + asm.LdrRiUn(tmp, ctx, NceNativeContext.GetSvcCallHandlerOffset()); + asm.Blr(tmp); + + Operand lblContinue = asm.CreateLabel(); + Operand lblQuit = asm.CreateLabel(); + + asm.Cbnz(Gpr(0, OperandType.I32), lblContinue); + + asm.MarkLabel(lblQuit); + + CreateRegisterSaveRestoreForManaged().WriteEpilogue(asm); + + asm.Ret(Gpr(30)); + + asm.MarkLabel(lblContinue); + + WriteInManagedLockRelease(asm, ctx, tmp, tmp2, ThreadExitMethod.Label, lblQuit); + + for (int i = 0; i < 8; i++) + { + asm.LdrRiUn(Gpr(i), ctx, NceNativeContext.GetXOffset(i)); + } + }, 0xff); + + asm.B(0); + + return asm.GetCode(); + } + + private static uint[] WriteMrsTpidrroEl0Patch(uint rd) + { + return WriteMrsContextRead(rd, NceNativeContext.GetTpidrroEl0Offset()); + } + + private static uint[] WriteMrsTpidrEl0Patch(uint rd) + { + return WriteMrsContextRead(rd, NceNativeContext.GetTpidrEl0Offset()); + } + + private static uint[] WriteMrsCtrEl0Patch(uint rd) + { + return WriteMrsContextRead(rd, NceNativeContext.GetCtrEl0Offset()); + } + + private static uint[] WriteMrsCntpctEl0Patch(uint rd) + { + Assembler asm = new(); + + WriteManagedCall(asm, (asm, ctx, tmp, tmp2) => + { + WriteInManagedLockAcquire(asm, ctx, tmp, tmp2); + + asm.Mov(tmp, (ulong)NceNativeInterface.GetTickCounterAccessFunctionPointer()); + asm.Blr(tmp); + asm.StrRiUn(Gpr(0), ctx, NceNativeContext.GetTempStorageOffset()); + + WriteInManagedLockRelease(asm, ctx, tmp, tmp2, ThreadExitMethod.GenerateReturn); + + asm.LdrRiUn(Gpr((int)rd), ctx, NceNativeContext.GetTempStorageOffset()); + }, 1u << (int)rd); + + asm.B(0); + + return asm.GetCode(); + } + + private static uint[] WriteMsrTpidrEl0Patch(uint rd) + { + Assembler asm = new(); + + Span scratchRegs = stackalloc int[3]; + PickScratchRegs(scratchRegs, 1u << (int)rd); + + RegisterSaveRestore rsr = new((1 << scratchRegs[0]) | (1 << scratchRegs[1]) | (1 << scratchRegs[2])); + + rsr.WritePrologue(asm); + + WriteLoadContext(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2])); + asm.StrRiUn(Gpr((int)rd), Gpr(scratchRegs[0]),NceNativeContext.GetTpidrEl0Offset()); + + rsr.WriteEpilogue(asm); + + asm.B(0); + + return asm.GetCode(); + } + + private static uint[] WriteMrsContextRead(uint rd, int contextOffset) + { + Assembler asm = new(); + + Span scratchRegs = stackalloc int[3]; + PickScratchRegs(scratchRegs, 1u << (int)rd); + + RegisterSaveRestore rsr = new((1 << scratchRegs[0]) | (1 << scratchRegs[1]) | (1 << scratchRegs[2])); + + rsr.WritePrologue(asm); + + WriteLoadContext(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2])); + asm.Add(Gpr((int)rd), Gpr(scratchRegs[0]), Const((ulong)contextOffset)); + + rsr.WriteEpilogue(asm); + + asm.LdrRiUn(Gpr((int)rd), Gpr((int)rd), 0); + + asm.B(0); + + return asm.GetCode(); + } + + private static void WriteLoadContext(Assembler asm, Operand tmp0, Operand tmp1, Operand tmp2) + { + asm.Mov(tmp0, (ulong)NceThreadTable.EntriesPointer); + + if (OperatingSystem.IsMacOS()) + { + asm.MrsTpidrroEl0(tmp1); + } + else + { + asm.MrsTpidrEl0(tmp1); + } + + Operand lblFound = asm.CreateLabel(); + Operand lblLoop = asm.CreateLabel(); + + asm.MarkLabel(lblLoop); + + asm.LdrRiPost(tmp2, tmp0, 16); + asm.Cmp(tmp1, tmp2); + asm.B(lblFound, ArmCondition.Eq); + asm.B(lblLoop); + + asm.MarkLabel(lblFound); + + asm.Ldur(tmp0, tmp0, -8); + } + + private static void WriteLoadContextSafe(Assembler asm, Operand lblFail, Operand tmp0, Operand tmp1, Operand tmp2, Operand tmp3) + { + asm.Mov(tmp0, (ulong)NceThreadTable.EntriesPointer); + asm.Ldur(tmp3, tmp0, -8); + asm.Add(tmp3, tmp0, tmp3, ArmShiftType.Lsl, 4); + + if (OperatingSystem.IsMacOS()) + { + asm.MrsTpidrroEl0(tmp1); + } + else + { + asm.MrsTpidrEl0(tmp1); + } + + Operand lblFound = asm.CreateLabel(); + Operand lblLoop = asm.CreateLabel(); + + asm.MarkLabel(lblLoop); + + asm.Cmp(tmp0, tmp3); + asm.B(lblFail, ArmCondition.GeUn); + asm.LdrRiPost(tmp2, tmp0, 16); + asm.Cmp(tmp1, tmp2); + asm.B(lblFound, ArmCondition.Eq); + asm.B(lblLoop); + + asm.MarkLabel(lblFound); + + asm.Ldur(tmp0, tmp0, -8); + } + + private static void PickScratchRegs(Span scratchRegs, uint blacklistedRegMask) + { + int scratchReg = ScratchBaseReg; + + for (int i = 0; i < scratchRegs.Length; i++) + { + while ((blacklistedRegMask & (1u << scratchReg)) != 0) + { + scratchReg++; + } + + if (scratchReg >= 29) + { + throw new ArgumentException($"No enough register for {scratchRegs.Length} scratch register, started from {ScratchBaseReg}"); + } + + scratchRegs[i] = scratchReg++; + } + } + + private static Operand Gpr(int register, OperandType type = OperandType.I64) + { + return new Operand(register, RegisterType.Integer, type); + } + + private static Operand Vec(int register, OperandType type = OperandType.V128) + { + return new Operand(register, RegisterType.Vector, type); + } + + private static Operand Const(ulong value) + { + return new Operand(OperandType.I64, value); + } + + private static Operand Const(OperandType type, ulong value) + { + return new Operand(type, value); + } + + private static uint GetImm26(ulong sourceAddress, ulong targetAddress) + { + long offset = (long)(targetAddress - sourceAddress); + long offsetTrunc = (offset >> 2) & 0x3FFFFFF; + + if ((offsetTrunc << 38) >> 36 != offset) + { + throw new Exception($"Offset out of range: 0x{sourceAddress:X} -> 0x{targetAddress:X} (0x{offset:X})"); + } + + return (uint)offsetTrunc; + } + + private static int GetOffset(ulong sourceAddress, ulong targetAddress) + { + long offset = (long)(targetAddress - sourceAddress); + + return checked((int)offset); + } + + private static uint[] GetCopy(uint[] code) + { + uint[] codeCopy = new uint[code.Length]; + code.CopyTo(codeCopy, 0); + + return codeCopy; + } + + private static void WriteManagedCall(Assembler asm, Action writeCall, uint blacklistedRegMask) + { + int intMask = 0x7fffffff & (int)~blacklistedRegMask; + int vecMask = unchecked((int)0xffffffff); + + Span scratchRegs = stackalloc int[3]; + PickScratchRegs(scratchRegs, blacklistedRegMask); + + RegisterSaveRestore rsr = new(intMask, vecMask, OperandType.V128); + + rsr.WritePrologue(asm); + + WriteLoadContext(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2])); + + asm.MovSp(Gpr(scratchRegs[1]), Gpr(Assembler.SpRegister)); + asm.StrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetGuestSPOffset()); + asm.LdrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetHostSPOffset()); + asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[1])); + + writeCall(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2])); + + asm.LdrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetGuestSPOffset()); + asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[1])); + + rsr.WriteEpilogue(asm); + } + + internal static uint[] GenerateThreadStartCode() + { + Assembler asm = new(); + + CreateRegisterSaveRestoreForManaged().WritePrologue(asm); + + asm.MovSp(Gpr(1), Gpr(Assembler.SpRegister)); + asm.StrRiUn(Gpr(1), Gpr(0), NceNativeContext.GetHostSPOffset()); + + for (int i = 2; i < 30; i += 2) + { + asm.LdpRiUn(Gpr(i), Gpr(i + 1), Gpr(0), NceNativeContext.GetXOffset(i)); + } + + for (int i = 0; i < 32; i += 2) + { + asm.LdpRiUn(Vec(i), Vec(i + 1), Gpr(0), NceNativeContext.GetVOffset(i)); + } + + asm.LdpRiUn(Gpr(30), Gpr(1), Gpr(0), NceNativeContext.GetXOffset(30)); + asm.MovSp(Gpr(Assembler.SpRegister), Gpr(1)); + + asm.StrRiUn(Gpr(Assembler.ZrRegister, OperandType.I32), Gpr(0), NceNativeContext.GetInManagedOffset()); + + asm.LdpRiUn(Gpr(0), Gpr(1), Gpr(0), NceNativeContext.GetXOffset(0)); + asm.Br(Gpr(30)); + + return asm.GetCode(); + } + + internal static uint[] GenerateSuspendExceptionHandler() + { + Assembler asm = new(); + + Span scratchRegs = stackalloc int[4]; + PickScratchRegs(scratchRegs, 0u); + + RegisterSaveRestore rsr = new((1 << scratchRegs[0]) | (1 << scratchRegs[1]) | (1 << scratchRegs[2]) | (1 << scratchRegs[3]), hasCall: true); + + rsr.WritePrologue(asm); + + Operand lblAgain = asm.CreateLabel(); + Operand lblFail = asm.CreateLabel(); + + WriteLoadContextSafe(asm, lblFail, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2]), Gpr(scratchRegs[3])); + + asm.LdrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetHostSPOffset()); + asm.MovSp(Gpr(scratchRegs[2]), Gpr(Assembler.SpRegister)); + asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[1])); + + asm.Cmp(Gpr(0, OperandType.I32), Const((ulong)NceThreadPal.UnixSuspendSignal)); + asm.B(lblFail, ArmCondition.Ne); + + // SigUsr2 + + asm.Mov(Gpr(scratchRegs[1], OperandType.I32), Const(OperandType.I32, 1)); + asm.StrRiUn(Gpr(scratchRegs[1], OperandType.I32), Gpr(scratchRegs[0]), NceNativeContext.GetInManagedOffset()); + + asm.MarkLabel(lblAgain); + + asm.Mov(Gpr(scratchRegs[3]), (ulong)NceNativeInterface.GetSuspendThreadHandlerFunctionPointer()); + asm.Blr(Gpr(scratchRegs[3])); + + // TODO: Check return value, exit if we must. + + WriteInManagedLockReleaseForSuspendHandler(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[3]), lblAgain); + + asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[2])); + + rsr.WriteEpilogue(asm); + + asm.Ret(Gpr(30)); + + asm.MarkLabel(lblFail); + + rsr.WriteEpilogue(asm); + + asm.Ret(Gpr(30)); + + return asm.GetCode(); + } + + internal static uint[] GenerateWrapperExceptionHandler(IntPtr oldSignalHandlerSegfaultPtr, IntPtr signalHandlerPtr) + { + Assembler asm = new(); + + Span scratchRegs = stackalloc int[4]; + PickScratchRegs(scratchRegs, 0u); + + RegisterSaveRestore rsr = new((1 << scratchRegs[0]) | (1 << scratchRegs[1]) | (1 << scratchRegs[2]) | (1 << scratchRegs[3]), hasCall: true); + + rsr.WritePrologue(asm); + + Operand lblFail = asm.CreateLabel(); + + WriteLoadContextSafe(asm, lblFail, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[2]), Gpr(scratchRegs[3])); + + asm.LdrRiUn(Gpr(scratchRegs[1]), Gpr(scratchRegs[0]), NceNativeContext.GetHostSPOffset()); + asm.MovSp(Gpr(scratchRegs[2]), Gpr(Assembler.SpRegister)); + asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[1])); + + // SigSegv + + WriteInManagedLockAcquire(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[3])); + + asm.Mov(Gpr(scratchRegs[3]), (ulong)signalHandlerPtr); + asm.Blr(Gpr(scratchRegs[3])); + + WriteInManagedLockRelease(asm, Gpr(scratchRegs[0]), Gpr(scratchRegs[1]), Gpr(scratchRegs[3]), ThreadExitMethod.None); + + asm.MovSp(Gpr(Assembler.SpRegister), Gpr(scratchRegs[2])); + + rsr.WriteEpilogue(asm); + + asm.Ret(Gpr(30)); + + asm.MarkLabel(lblFail); + + rsr.WriteEpilogue(asm); + + asm.Mov(Gpr(3), (ulong)oldSignalHandlerSegfaultPtr); + asm.Br(Gpr(3)); + + return asm.GetCode(); + } + + private static void WriteInManagedLockAcquire(Assembler asm, Operand ctx, Operand tmp, Operand tmp2) + { + Operand tmpUint = new Operand(tmp.GetRegister().Index, RegisterType.Integer, OperandType.I32); + Operand tmp2Uint = new Operand(tmp2.GetRegister().Index, RegisterType.Integer, OperandType.I32); + + Operand lblLoop = asm.CreateLabel(); + + // Bit 0 set means that the thread is currently executing managed code (that case should be impossible here). + // Bit 1 being set means there is a signal pending, we should wait for the signal, otherwise it could trigger + // while running managed code. + + asm.MarkLabel(lblLoop); + + asm.Add(tmp, ctx, Const((ulong)NceNativeContext.GetInManagedOffset())); + asm.Ldaxr(tmp2Uint, tmp); + asm.Cbnz(tmp2Uint, lblLoop); + asm.Mov(tmp2Uint, Const(OperandType.I32, 1)); + asm.Stlxr(tmp2Uint, tmp, tmpUint); + asm.Cbnz(tmpUint, lblLoop); // Retry if store failed. + } + + private enum ThreadExitMethod + { + None, + GenerateReturn, + Label + } + + private static void WriteInManagedLockRelease(Assembler asm, Operand ctx, Operand tmp, Operand tmp2, ThreadExitMethod exitMethod, Operand lblQuit = default) + { + Operand tmpUint = new Operand(tmp.GetRegister().Index, RegisterType.Integer, OperandType.I32); + Operand tmp2Uint = new Operand(tmp2.GetRegister().Index, RegisterType.Integer, OperandType.I32); + + Operand lblLoop = asm.CreateLabel(); + Operand lblInterrupt = asm.CreateLabel(); + Operand lblDone = asm.CreateLabel(); + + // Bit 0 set means that the thread is currently executing managed code (it should be always set here, as we just returned from managed code). + // Bit 1 being set means a interrupt was requested while it was in managed, we should service it. + + asm.MarkLabel(lblLoop); + + asm.Add(tmp, ctx, Const((ulong)NceNativeContext.GetInManagedOffset())); + asm.Ldaxr(tmp2Uint, tmp); + asm.Cmp(tmp2Uint, Const(OperandType.I32, 3)); + asm.B(lblInterrupt, ArmCondition.Eq); + asm.Stlxr(Gpr(Assembler.ZrRegister, OperandType.I32), tmp, tmpUint); + asm.Cbnz(tmpUint, lblLoop); // Retry if store failed. + asm.B(lblDone); + + asm.MarkLabel(lblInterrupt); + + // If we got here, a interrupt was requested while it was in managed code. + // Let's service the interrupt and check what we should do next. + + asm.Mov(tmp2Uint, Const(OperandType.I32, 1)); + asm.Stlxr(tmp2Uint, tmp, tmpUint); + asm.Cbnz(tmpUint, lblLoop); // Retry if store failed. + asm.Mov(tmp, (ulong)NceNativeInterface.GetSuspendThreadHandlerFunctionPointer()); + asm.Blr(tmp); + + // The return value from the interrupt handler indicates if we should continue running. + // From here, we either try to release the lock again. We might have received another interrupt + // request in the meantime, in which case we should service it again. + // If we were requested to exit, then we exit if we can. + // TODO: We should also exit while on a signal handler. To do that we need to modify the PC value on the + // context. It's a bit more tricky to do, so for now we ignore that case with "ThreadExitMethod.None". + + if (exitMethod == ThreadExitMethod.None) + { + asm.B(lblLoop); + } + else + { + asm.Cbnz(Gpr(0, OperandType.I32), lblLoop); + + if (exitMethod == ThreadExitMethod.Label) + { + asm.B(lblQuit); + } + else if (exitMethod == ThreadExitMethod.GenerateReturn) + { + CreateRegisterSaveRestoreForManaged().WriteEpilogue(asm); + + asm.Ret(Gpr(30)); + } + } + + asm.MarkLabel(lblDone); + } + + private static void WriteInManagedLockReleaseForSuspendHandler(Assembler asm, Operand ctx, Operand tmp, Operand tmp2, Operand lblAgain) + { + Operand tmpUint = new Operand(tmp.GetRegister().Index, RegisterType.Integer, OperandType.I32); + Operand tmp2Uint = new Operand(tmp2.GetRegister().Index, RegisterType.Integer, OperandType.I32); + + Operand lblLoop = asm.CreateLabel(); + Operand lblInterrupt = asm.CreateLabel(); + Operand lblDone = asm.CreateLabel(); + + // Bit 0 set means that the thread is currently executing managed code (it should be always set here, as we just returned from managed code). + // Bit 1 being set means a interrupt was requested while it was in managed, we should service it. + + asm.MarkLabel(lblLoop); + + asm.Add(tmp, ctx, Const((ulong)NceNativeContext.GetInManagedOffset())); + asm.Ldaxr(tmp2Uint, tmp); + asm.Cmp(tmp2Uint, Const(OperandType.I32, 3)); + asm.B(lblInterrupt, ArmCondition.Eq); + asm.Stlxr(Gpr(Assembler.ZrRegister, OperandType.I32), tmp, tmpUint); + asm.Cbnz(tmpUint, lblLoop); // Retry if store failed. + asm.B(lblDone); + + asm.MarkLabel(lblInterrupt); + + // If we got here, a interrupt was requested while it was in managed code. + // Let's service the interrupt and check what we should do next. + + asm.Mov(tmp2Uint, Const(OperandType.I32, 1)); + asm.Stlxr(tmp2Uint, tmp, tmpUint); + asm.Cbnz(tmpUint, lblLoop); // Retry if store failed. + asm.B(lblAgain); + + asm.MarkLabel(lblDone); + } + + private static RegisterSaveRestore CreateRegisterSaveRestoreForManaged() + { + return new RegisterSaveRestore((int)IntCalleeSavedRegsMask, unchecked((int)FpCalleeSavedRegsMask), OperandType.FP64, hasCall: true); + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/NceThreadPal.cs b/src/Ryujinx.Cpu/Nce/NceThreadPal.cs new file mode 100644 index 000000000..89b486c93 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceThreadPal.cs @@ -0,0 +1,41 @@ +using Ryujinx.Common; +using System; + +namespace Ryujinx.Cpu.Nce +{ + static class NceThreadPal + { + private const int SigUsr2Linux = 12; + private const int SigUsr2MacOS = 31; + + public static int UnixSuspendSignal => OperatingSystem.IsMacOS() ? SigUsr2MacOS : SigUsr2Linux; + + public static IntPtr GetCurrentThreadHandle() + { + if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS() || PlatformInfo.IsBionic) + { + return NceThreadPalUnix.GetCurrentThreadHandle(); + } + else + { + throw new PlatformNotSupportedException(); + } + } + + public static void SuspendThread(IntPtr handle) + { + if (PlatformInfo.IsBionic) + { + NceThreadPalAndroid.SuspendThread(handle); + } + else if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS()) + { + NceThreadPalUnix.SuspendThread(handle); + } + else + { + throw new PlatformNotSupportedException(); + } + } + } +} diff --git a/src/Ryujinx.Cpu/Nce/NceThreadPalAndroid.cs b/src/Ryujinx.Cpu/Nce/NceThreadPalAndroid.cs new file mode 100644 index 000000000..65bfd1bfb --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceThreadPalAndroid.cs @@ -0,0 +1,20 @@ +using System; +using System.Runtime.InteropServices; + +namespace Ryujinx.Cpu.Nce +{ + static class NceThreadPalAndroid + { + [DllImport("libc", SetLastError = true)] + private static extern int pthread_kill(IntPtr thread, int sig); + + public static void SuspendThread(IntPtr handle) + { + int result = pthread_kill(handle, NceThreadPal.UnixSuspendSignal); + if (result != 0) + { + throw new Exception($"Thread kill returned error 0x{result:X}."); + } + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/NceThreadPalUnix.cs b/src/Ryujinx.Cpu/Nce/NceThreadPalUnix.cs new file mode 100644 index 000000000..9f4877615 --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceThreadPalUnix.cs @@ -0,0 +1,37 @@ +using System; +using System.Runtime.InteropServices; + +namespace Ryujinx.Cpu.Nce +{ + static class NceThreadPalUnix + { + [DllImport("libc", SetLastError = true)] + private static extern IntPtr pthread_self(); + + [DllImport("libc", SetLastError = true)] + private static extern int pthread_threadid_np(IntPtr arg0, out ulong tid); + + [DllImport("libpthread", SetLastError = true)] + private static extern int pthread_kill(IntPtr thread, int sig); + + public static IntPtr GetCurrentThreadHandle() + { + return pthread_self(); + } + + public static ulong GetCurrentThreadId() + { + pthread_threadid_np(IntPtr.Zero, out ulong tid); + return tid; + } + + public static void SuspendThread(IntPtr handle) + { + int result = pthread_kill(handle, NceThreadPal.UnixSuspendSignal); + if (result != 0) + { + throw new Exception($"Thread kill returned error 0x{result:X}."); + } + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/NceThreadTable.cs b/src/Ryujinx.Cpu/Nce/NceThreadTable.cs new file mode 100644 index 000000000..8fe39b7bd --- /dev/null +++ b/src/Ryujinx.Cpu/Nce/NceThreadTable.cs @@ -0,0 +1,97 @@ +using Ryujinx.Memory; +using System; +using System.Runtime.CompilerServices; + +namespace Ryujinx.Cpu.Nce +{ + static class NceThreadTable + { + private const int MaxThreads = 4096; + + private struct Entry + { + public IntPtr ThreadId; + public IntPtr NativeContextPtr; + + public Entry(IntPtr threadId, IntPtr nativeContextPtr) + { + ThreadId = threadId; + NativeContextPtr = nativeContextPtr; + } + } + + private static MemoryBlock _block; + + public static IntPtr EntriesPointer => _block.Pointer + 8; + + static NceThreadTable() + { + _block = new MemoryBlock((ulong)Unsafe.SizeOf() * MaxThreads + 8UL); + _block.Write(0UL, 0UL); + } + + public static int Register(IntPtr threadId, IntPtr nativeContextPtr) + { + Span entries = GetStorage(); + + lock (_block) + { + ref ulong currentThreadCount = ref GetThreadsCount(); + + for (int i = 0; i < MaxThreads; i++) + { + if (entries[i].ThreadId == IntPtr.Zero) + { + entries[i] = new Entry(threadId, nativeContextPtr); + + if (currentThreadCount < (ulong)i + 1) + { + currentThreadCount = (ulong)i + 1; + } + + return i; + } + } + } + + throw new Exception($"Number of active threads exceeds limit of {MaxThreads}."); + } + + public static void Unregister(int tableIndex) + { + Span entries = GetStorage(); + + lock (_block) + { + if (entries[tableIndex].ThreadId != IntPtr.Zero) + { + entries[tableIndex] = default; + + ulong currentThreadCount = GetThreadsCount(); + + for (int i = (int)currentThreadCount - 1; i >= 0; i--) + { + if (entries[i].ThreadId != IntPtr.Zero) + { + break; + } + + currentThreadCount = (ulong)i; + } + + GetThreadsCount() = currentThreadCount; + } + } + } + + private static ref ulong GetThreadsCount() + { + return ref _block.GetRef(0UL); + } + + private static unsafe Span GetStorage() + { + return new Span((void*)_block.GetPointer(8UL, (ulong)Unsafe.SizeOf() * MaxThreads), MaxThreads); + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.HLE/HOS/ArmProcessContext.cs b/src/Ryujinx.HLE/HOS/ArmProcessContext.cs index 09a721644..9ce57a2a7 100644 --- a/src/Ryujinx.HLE/HOS/ArmProcessContext.cs +++ b/src/Ryujinx.HLE/HOS/ArmProcessContext.cs @@ -24,6 +24,8 @@ namespace Ryujinx.HLE.HOS private readonly ICpuContext _cpuContext; private T _memoryManager; + public ulong ReservedSize { get; } + public IVirtualMemoryManager AddressSpace => _memoryManager; public ulong AddressSpaceSize { get; } @@ -34,7 +36,8 @@ namespace Ryujinx.HLE.HOS GpuContext gpuContext, T memoryManager, ulong addressSpaceSize, - bool for64Bit) + bool for64Bit, + ulong reservedSize = 0UL) { if (memoryManager is IRefCounted rc) { @@ -47,8 +50,8 @@ namespace Ryujinx.HLE.HOS _gpuContext = gpuContext; _cpuContext = cpuEngine.CreateCpuContext(memoryManager, for64Bit); _memoryManager = memoryManager; - AddressSpaceSize = addressSpaceSize; + ReservedSize = reservedSize; } public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks) diff --git a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs index 080971dee..a50dd67d0 100644 --- a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs +++ b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs @@ -4,6 +4,7 @@ using Ryujinx.Cpu; using Ryujinx.Cpu.AppleHv; using Ryujinx.Cpu.Jit; using Ryujinx.Cpu.LightningJit; +using Ryujinx.Cpu.Nce; using Ryujinx.Graphics.Gpu; using Ryujinx.HLE.HOS.Kernel; using Ryujinx.HLE.HOS.Kernel.Process; @@ -46,17 +47,43 @@ namespace Ryujinx.HLE.HOS _codeSize = codeSize; } + public static NceCpuCodePatch CreateCodePatchForNce(KernelContext context, bool for64Bit, ReadOnlySpan textSection) + { + if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64 && for64Bit && context.Device.Configuration.UseHypervisor && !OperatingSystem.IsMacOS()) + { + return NcePatcher.CreatePatch(textSection); + } + + return null; + } + public IProcessContext Create(KernelContext context, ulong pid, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit) { IArmProcessContext processContext; bool isArm64Host = RuntimeInformation.ProcessArchitecture == Architecture.Arm64; - if (OperatingSystem.IsMacOS() && isArm64Host && for64Bit && context.Device.Configuration.UseHypervisor) + if (isArm64Host && for64Bit && context.Device.Configuration.UseHypervisor) { - var cpuEngine = new HvEngine(_tickSource); - var memoryManager = new HvMemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); - processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, addressSpaceSize, for64Bit); + if (OperatingSystem.IsMacOS()) + { + var cpuEngine = new HvEngine(_tickSource); + var memoryManager = new HvMemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, addressSpaceSize, for64Bit); + } + else + { + if (!AddressSpace.TryCreateWithoutMirror(addressSpaceSize, out var addressSpace)) + { + throw new Exception("Address space creation failed"); + } + + Logger.Info?.Print(LogClass.Cpu, $"NCE Base AS Address: 0x{addressSpace.Pointer.ToInt64():X} Size: 0x{addressSpace.Size:X}"); + + var cpuEngine = new NceEngine(_tickSource); + var memoryManager = new MemoryManagerNative(addressSpace, context.Memory, addressSpaceSize, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, addressSpace.Size, for64Bit, memoryManager.ReservedSize); + } } else { @@ -74,11 +101,13 @@ namespace Ryujinx.HLE.HOS : new JitEngine(_tickSource); AddressSpace addressSpace = null; + MemoryBlock asNoMirror = null; // We want to use host tracked mode if the host page size is > 4KB. if ((mode == MemoryManagerMode.HostMapped || mode == MemoryManagerMode.HostMappedUnsafe) && MemoryBlock.GetPageSize() <= 0x1000) { - if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, out addressSpace)) + if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, out addressSpace) && + !AddressSpace.TryCreateWithoutMirror(addressSpaceSize, out asNoMirror)) { Logger.Warning?.Print(LogClass.Cpu, "Address space creation failed, falling back to software page table"); @@ -89,35 +118,47 @@ namespace Ryujinx.HLE.HOS switch (mode) { case MemoryManagerMode.SoftwarePageTable: - var memoryManager = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); - processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, addressSpaceSize, for64Bit); + { + var mm = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, mm, addressSpaceSize, for64Bit); + } break; case MemoryManagerMode.HostMapped: case MemoryManagerMode.HostMappedUnsafe: - if (addressSpace == null) + if (addressSpace == null && asNoMirror == null) { var memoryManagerHostTracked = new MemoryManagerHostTracked(context.Memory, addressSpaceSize, mode == MemoryManagerMode.HostMappedUnsafe, invalidAccessHandler); processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManagerHostTracked, addressSpaceSize, for64Bit); } else { - if (addressSpaceSize != addressSpace.AddressSpaceSize) - { - Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{addressSpace.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})"); - } + bool unsafeMode = mode == MemoryManagerMode.HostMappedUnsafe; - var memoryManagerHostMapped = new MemoryManagerHostMapped(addressSpace, mode == MemoryManagerMode.HostMappedUnsafe, invalidAccessHandler); - processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManagerHostMapped, addressSpace.AddressSpaceSize, for64Bit); + if (addressSpace != null) + { + var mm = new MemoryManagerHostMapped(addressSpace, unsafeMode, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, mm, addressSpace.AddressSpaceSize, for64Bit); + } + else + { + var mm = new MemoryManagerHostNoMirror(asNoMirror, context.Memory, unsafeMode, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, mm, asNoMirror.Size, for64Bit); + } } break; default: throw new InvalidOperationException($"{nameof(mode)} contains an invalid value: {mode}"); } + + if (addressSpaceSize != processContext.AddressSpaceSize) + { + Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{processContext.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})"); + } } - DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, _diskCacheSelector ?? "default"); + DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, _diskCacheSelector); return processContext; } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs b/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs index 8c648d06f..b4eafab11 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs @@ -104,6 +104,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory MemoryRegion memRegion, ulong address, ulong size, + ulong reservedSize, KMemoryBlockSlabManager slabManager) { _contextId = Context.ContextIdManager.GetId(); @@ -119,6 +120,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory memRegion, address, size, + reservedSize, slabManager); if (result != Result.Success) @@ -161,6 +163,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory MemoryRegion memRegion, ulong address, ulong size, + ulong reservedSize, KMemoryBlockSlabManager slabManager) { ulong endAddr = address + size; @@ -218,11 +221,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory break; case ProcessCreationFlags.AddressSpace64Bit: + ulong reservedAddressSpaceSize = _reservedAddressSpaceSize; if (_reservedAddressSpaceSize < addrSpaceEnd) { - int addressSpaceWidth = (int)ulong.Log2(_reservedAddressSpaceSize); + int addressSpaceWidth = (int)ulong.Log2(reservedAddressSpaceSize); - aliasRegion.Size = 1UL << (addressSpaceWidth - 3); + aliasRegion.Size = reservedAddressSpaceSize >= 0x1800000000 ? 0x1000000000 : 1UL << (addressSpaceWidth - 3); heapRegion.Size = 0x180000000; stackRegion.Size = 1UL << (addressSpaceWidth - 8); tlsIoRegion.Size = 1UL << (addressSpaceWidth - 3); @@ -230,8 +234,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory codeRegionSize = BitUtils.AlignUp(endAddr, RegionAlignment) - CodeRegionStart; stackAndTlsIoStart = 0; stackAndTlsIoEnd = 0; - AslrRegionStart = 0x8000000; - addrSpaceEnd = 1UL << addressSpaceWidth; + AslrRegionStart = Math.Max(reservedSize, 0x8000000); + addrSpaceEnd = reservedSize + reservedAddressSpaceSize; AslrRegionEnd = addrSpaceEnd; } else @@ -242,7 +246,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory tlsIoRegion.Size = 0x1000000000; CodeRegionStart = BitUtils.AlignDown(address, RegionAlignment); codeRegionSize = BitUtils.AlignUp(endAddr, RegionAlignment) - CodeRegionStart; - AslrRegionStart = 0x8000000; + AslrRegionStart = Math.Max(reservedSize, 0x8000000); AslrRegionEnd = AslrRegionStart + 0x7ff8000000; stackAndTlsIoStart = 0; stackAndTlsIoEnd = 0; diff --git a/src/Ryujinx.HLE/HOS/Kernel/Process/IProcessContext.cs b/src/Ryujinx.HLE/HOS/Kernel/Process/IProcessContext.cs index ac36b781b..42e6b6416 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Process/IProcessContext.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Process/IProcessContext.cs @@ -7,6 +7,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process interface IProcessContext : IDisposable { IVirtualMemoryManager AddressSpace { get; } + ulong ReservedSize { get; } ulong AddressSpaceSize { get; } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs b/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs index 32dca1512..8e93cb4c8 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs @@ -135,7 +135,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process InitializeMemoryManager(creationInfo.Flags); - ulong codeAddress = creationInfo.CodeAddress; + ulong codeAddress = creationInfo.CodeAddress + Context.ReservedSize; ulong codeSize = (ulong)creationInfo.CodePagesCount * KPageTableBase.PageSize; @@ -149,6 +149,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process memRegion, codeAddress, codeSize, + Context.ReservedSize, slabManager); if (result != Result.Success) @@ -184,7 +185,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Process KResourceLimit resourceLimit, MemoryRegion memRegion, IProcessContextFactory contextFactory, - ThreadStart customThreadStart = null) + ThreadStart customThreadStart = null, + ulong entrypointOffset = 0UL) { ResourceLimit = resourceLimit; _memRegion = memRegion; @@ -238,7 +240,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process InitializeMemoryManager(creationInfo.Flags); - ulong codeAddress = creationInfo.CodeAddress; + ulong codeAddress = creationInfo.CodeAddress + Context.ReservedSize; ulong codeSize = codePagesCount * KPageTableBase.PageSize; @@ -248,6 +250,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process memRegion, codeAddress, codeSize, + Context.ReservedSize, slabManager); if (result != Result.Success) @@ -293,6 +296,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Process CleanUpForError(); } + _entrypoint += entrypointOffset; + return result; } @@ -337,10 +342,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Process Flags = creationInfo.Flags; TitleId = creationInfo.TitleId; - _entrypoint = creationInfo.CodeAddress; + _entrypoint = creationInfo.CodeAddress + Context.ReservedSize; _imageSize = (ulong)creationInfo.CodePagesCount * KPageTableBase.PageSize; - switch (Flags & ProcessCreationFlags.AddressSpaceMask) + // 19.0.0+ sets all regions to same size + _memoryUsageCapacity = MemoryManager.HeapRegionEnd - MemoryManager.HeapRegionStart; + /*switch (Flags & ProcessCreationFlags.AddressSpaceMask) { case ProcessCreationFlags.AddressSpace32Bit: case ProcessCreationFlags.AddressSpace64BitDeprecated: @@ -357,7 +364,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process break; default: throw new InvalidOperationException($"Invalid MMU flags value 0x{Flags:x2}."); - } + }*/ GenerateRandomEntropy(); @@ -535,7 +542,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process { throw new InvalidOperationException("Trying to start a process with an invalid state!"); } - + // TODO: after 19.0.0+ alignment is not needed ulong stackSizeRounded = BitUtils.AlignUp(stackSize, KPageTableBase.PageSize); ulong neededSize = stackSizeRounded + _imageSize; diff --git a/src/Ryujinx.HLE/HOS/Kernel/Process/ProcessContext.cs b/src/Ryujinx.HLE/HOS/Kernel/Process/ProcessContext.cs index b4ae6ec4e..6107487d8 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Process/ProcessContext.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Process/ProcessContext.cs @@ -7,6 +7,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Process class ProcessContext : IProcessContext { public IVirtualMemoryManager AddressSpace { get; } + public ulong ReservedSize => 0UL; public ulong AddressSpaceSize { get; } diff --git a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs index 5729052e8..dbcd7e475 100644 --- a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs +++ b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs @@ -1,4 +1,4 @@ -using LibHac.Account; +using LibHac.Account; using LibHac.Common; using LibHac.Fs; using LibHac.Fs.Fsa; @@ -11,6 +11,7 @@ using LibHac.Tools.FsSystem; using LibHac.Tools.FsSystem.NcaUtils; using Ryujinx.Common; using Ryujinx.Common.Logging; +using Ryujinx.Cpu.Nce; using Ryujinx.HLE.HOS; using Ryujinx.HLE.HOS.Kernel; using Ryujinx.HLE.HOS.Kernel.Common; @@ -21,6 +22,7 @@ using Ryujinx.HLE.Loaders.Processes.Extensions; using Ryujinx.Horizon.Common; using Ryujinx.Horizon.Sdk.Arp; using System; +using System.Linq; using System.Runtime.InteropServices; using ApplicationId = LibHac.Ncm.ApplicationId; @@ -250,7 +252,7 @@ namespace Ryujinx.HLE.Loaders.Processes ulong argsStart = 0; uint argsSize = 0; ulong codeStart = ((meta.Flags & 1) != 0 ? 0x8000000UL : 0x200000UL) + CodeStartOffset; - uint codeSize = 0; + ulong codeSize = 0; string[] buildIds = new string[executables.Length]; @@ -264,6 +266,7 @@ namespace Ryujinx.HLE.Loaders.Processes }).ToUpper(); } + NceCpuCodePatch[] nsoPatch = new NceCpuCodePatch[executables.Length]; ulong[] nsoBase = new ulong[executables.Length]; for (int index = 0; index < executables.Length; index++) @@ -288,6 +291,16 @@ namespace Ryujinx.HLE.Loaders.Processes nsoSize = BitUtils.AlignUp(nsoSize, KPageTableBase.PageSize); + bool for64Bit = ((ProcessCreationFlags)meta.Flags).HasFlag(ProcessCreationFlags.Is64Bit); + + NceCpuCodePatch codePatch = ArmProcessContextFactory.CreateCodePatchForNce(context, for64Bit, nso.Text); + nsoPatch[index] = codePatch; + + if (codePatch != null) + { + codeSize += codePatch.Size; + } + nsoBase[index] = codeStart + codeSize; codeSize += nsoSize; @@ -389,7 +402,8 @@ namespace Ryujinx.HLE.Loaders.Processes MemoryMarshal.Cast(npdm.KernelCapabilityData), resourceLimit, memoryRegion, - processContextFactory); + processContextFactory, + entrypointOffset: nsoPatch[0]?.Size ?? 0UL); if (result != Result.Success) { @@ -400,9 +414,12 @@ namespace Ryujinx.HLE.Loaders.Processes for (int index = 0; index < executables.Length; index++) { - Logger.Info?.Print(LogClass.Loader, $"Loading image {index} at 0x{nsoBase[index]:x16}..."); + ulong nsoBaseAddress = process.Context.ReservedSize + nsoBase[index]; + + Logger.Info?.Print(LogClass.Loader, $"Loading image {index} at 0x{nsoBaseAddress:x16}..."); + + result = LoadIntoMemory(process, executables[index], nsoBaseAddress, nsoPatch[index]); - result = LoadIntoMemory(process, executables[index], nsoBase[index]); if (result != Result.Success) { Logger.Error?.Print(LogClass.Loader, $"Process initialization returned error \"{result}\"."); @@ -460,7 +477,7 @@ namespace Ryujinx.HLE.Loaders.Processes return processResult; } - public static Result LoadIntoMemory(KProcess process, IExecutable image, ulong baseAddress) + private static Result LoadIntoMemory(KProcess process, IExecutable image, ulong baseAddress, NceCpuCodePatch codePatch = null) { ulong textStart = baseAddress + image.TextOffset; ulong roStart = baseAddress + image.RoOffset; @@ -480,6 +497,11 @@ namespace Ryujinx.HLE.Loaders.Processes process.CpuMemory.Fill(bssStart, image.BssSize, 0); + if (codePatch != null) + { + codePatch.Write(process.CpuMemory, baseAddress - codePatch.Size, textStart); + } + Result SetProcessMemoryPermission(ulong address, ulong size, KMemoryPermission permission) { if (size == 0)