add stream based loaders

* extend stream loading support

* fix content manager rebase

* fix update searching
This commit is contained in:
Emmanuel Hansen 2023-07-02 16:49:14 +00:00 committed by KeatonTheBot
parent 6ecd6111d5
commit c16559f20e
6 changed files with 462 additions and 262 deletions

View file

@ -42,12 +42,14 @@ namespace Ryujinx.HLE.FileSystem
private readonly struct AocItem
{
public readonly string ContainerPath;
public readonly Stream ContainerStream;
public readonly string NcaPath;
public readonly string Extension;
public AocItem(string containerPath, string ncaPath)
public AocItem(Stream containerStream, string ncaPath, string extension)
{
ContainerPath = containerPath;
ContainerStream = containerStream;
Extension = extension;
NcaPath = ncaPath;
}
}
@ -187,10 +189,10 @@ namespace Ryujinx.HLE.FileSystem
}
}
public void AddAocItem(ulong titleId, string containerPath, string ncaPath, bool mergedToContainer = false)
public void AddAocItem(ulong titleId, Stream containerStream, string ncaPath, string extension, bool mergedToContainer = false)
{
// TODO: Check Aoc version.
if (!AocData.TryAdd(titleId, new AocItem(containerPath, ncaPath)))
if (!AocData.TryAdd(titleId, new AocItem(containerStream, ncaPath, extension)))
{
Logger.Warning?.Print(LogClass.Application, $"Duplicate AddOnContent detected. TitleId {titleId:X16}");
}
@ -200,12 +202,20 @@ namespace Ryujinx.HLE.FileSystem
if (!mergedToContainer)
{
using var pfs = PartitionFileSystemUtils.OpenApplicationFileSystem(containerPath, _virtualFileSystem);
using var pfs = PartitionFileSystemUtils.OpenApplicationFileSystem(containerStream, extension == ".xci", _virtualFileSystem);
}
}
}
public void ClearAocData() => AocData.Clear();
public void ClearAocData()
{
foreach (var aoc in AocData)
{
aoc.Value.ContainerStream?.Dispose();
}
AocData.Clear();
}
public int GetAocCount() => AocData.Count;
@ -217,18 +227,17 @@ namespace Ryujinx.HLE.FileSystem
if (AocData.TryGetValue(aocTitleId, out AocItem aoc))
{
var file = new FileStream(aoc.ContainerPath, FileMode.Open, FileAccess.Read);
using var ncaFile = new UniqueRef<IFile>();
switch (Path.GetExtension(aoc.ContainerPath))
switch (aoc.Extension)
{
case ".xci":
var xci = new Xci(_virtualFileSystem.KeySet, file.AsStorage()).OpenPartition(XciPartitionType.Secure);
xci.OpenFile(ref ncaFile.Ref, aoc.NcaPath.ToU8Span(), OpenMode.Read).ThrowIfFailure();
var xci = new Xci(_virtualFileSystem.KeySet, aoc.ContainerStream.AsStorage()).OpenPartition(XciPartitionType.Secure);
xci.OpenFile(ref ncaFile.Ref, aoc.NcaPath.ToU8Span(), OpenMode.Read);
break;
case ".nsp":
var pfs = new PartitionFileSystem();
pfs.Initialize(file.AsStorage());
pfs.Initialize(aoc.ContainerStream.AsStorage());
pfs.OpenFile(ref ncaFile.Ref, aoc.NcaPath.ToU8Span(), OpenMode.Read).ThrowIfFailure();
break;
default:
@ -476,6 +485,27 @@ namespace Ryujinx.HLE.FileSystem
FinishInstallation(temporaryDirectory, registeredDirectory);
}
public void InstallFirmware(Stream stream, bool isXci)
{
ContentPath.TryGetContentPath(StorageId.BuiltInSystem, out var contentPathString);
ContentPath.TryGetRealPath(contentPathString, out var contentDirectory);
string registeredDirectory = Path.Combine(contentDirectory, "registered");
string temporaryDirectory = Path.Combine(contentDirectory, "temp");
if (!isXci)
{
using ZipArchive archive = new ZipArchive(stream);
InstallFromZip(archive, temporaryDirectory);
}
else
{
Xci xci = new(_virtualFileSystem.KeySet, stream.AsStorage());
InstallFromCart(xci, temporaryDirectory);
}
FinishInstallation(temporaryDirectory, registeredDirectory);
}
public void InstallKeys(string keysSource, string installDirectory)
{
if (Directory.Exists(keysSource))
@ -662,13 +692,16 @@ namespace Ryujinx.HLE.FileSystem
throw new MissingKeyException("HeaderKey is empty. Cannot decrypt NCA headers.");
}
Dictionary<ulong, List<(NcaContentType type, string path)>> updateNcas = new();
if (Directory.Exists(firmwarePackage))
{
return VerifyAndGetVersionDirectory(firmwarePackage);
}
SystemVersion VerifyAndGetVersionDirectory(string firmwareDirectory)
{
return VerifyAndGetVersion(new LocalFileSystem(firmwareDirectory));
}
if (!File.Exists(firmwarePackage))
{
throw new FileNotFoundException("Firmware file does not exist.");
@ -676,16 +709,28 @@ namespace Ryujinx.HLE.FileSystem
FileInfo info = new(firmwarePackage);
if (info.Extension == ".zip" || info.Extension == ".xci")
{
using FileStream file = File.OpenRead(firmwarePackage);
switch (info.Extension)
var isXci = info.Extension == ".xci";
return VerifyFirmwarePackage(file, isXci);
}
return null;
}
public SystemVersion VerifyFirmwarePackage(Stream file, bool isXci)
{
case ".zip":
using (ZipArchive archive = ZipFile.OpenRead(firmwarePackage))
if (!isXci)
{
using ZipArchive archive = new ZipArchive(file, ZipArchiveMode.Read);
return VerifyAndGetVersionZip(archive);
}
case ".xci":
else
{
Xci xci = new(_virtualFileSystem.KeySet, file.AsStorage());
if (xci.HasPartition(XciPartitionType.Update))
@ -698,17 +743,13 @@ namespace Ryujinx.HLE.FileSystem
{
throw new InvalidFirmwarePackageException("Update not found in xci file.");
}
default:
break;
}
}
SystemVersion VerifyAndGetVersionDirectory(string firmwareDirectory)
private SystemVersion VerifyAndGetVersionZip(ZipArchive archive)
{
return VerifyAndGetVersion(new LocalFileSystem(firmwareDirectory));
}
Dictionary<ulong, List<(NcaContentType type, string path)>> updateNcas = new();
SystemVersion VerifyAndGetVersionZip(ZipArchive archive)
{
SystemVersion systemVersion = null;
foreach (var entry in archive.Entries)
@ -863,8 +904,10 @@ namespace Ryujinx.HLE.FileSystem
return systemVersion;
}
SystemVersion VerifyAndGetVersion(IFileSystem filesystem)
private SystemVersion VerifyAndGetVersion(IFileSystem filesystem)
{
Dictionary<ulong, List<(NcaContentType type, string path)>> updateNcas = new();
SystemVersion systemVersion = null;
CnmtContentMetaEntry[] metaEntries = null;
@ -994,9 +1037,6 @@ namespace Ryujinx.HLE.FileSystem
return systemVersion;
}
return null;
}
public SystemVersion GetCurrentFirmwareVersion()
{
LoadEntries();

View file

@ -127,7 +127,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
return nca.Header.ContentType == NcaContentType.Control;
}
public static (Nca, Nca) GetUpdateData(this Nca mainNca, VirtualFileSystem fileSystem, IntegrityCheckLevel checkLevel, int programIndex, out string updatePath)
public static (Nca, Nca) GetUpdateData(this Nca mainNca, VirtualFileSystem fileSystem, IntegrityCheckLevel checkLevel, int programIndex, out string updatePath, Stream updateStream = null)
{
updatePath = null;
@ -138,6 +138,10 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
// Clear the program index part.
ulong titleIdBase = mainNca.GetProgramIdBase();
IFileSystem updatePartitionFileSystem = null;
if (updateStream == null)
{
// Load update information if exists.
string titleUpdateMetadataPath = Path.Combine(AppDataManager.GamesDirPath, titleIdBase.ToString("x16"), "updates.json");
if (File.Exists(titleUpdateMetadataPath))
@ -145,8 +149,17 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
updatePath = JsonHelper.DeserializeFromFile(titleUpdateMetadataPath, _applicationSerializerContext.TitleUpdateMetadata).Selected;
if (File.Exists(updatePath))
{
IFileSystem updatePartitionFileSystem = PartitionFileSystemUtils.OpenApplicationFileSystem(updatePath, fileSystem);
updatePartitionFileSystem = PartitionFileSystemUtils.OpenApplicationFileSystem(updatePath, fileSystem);
}
}
}
else
{
updatePartitionFileSystem = PartitionFileSystemUtils.OpenApplicationFileSystem(updateStream, false, fileSystem);
}
if (updatePartitionFileSystem != null)
{
foreach ((ulong applicationTitleId, ContentMetaData content) in updatePartitionFileSystem.GetContentData(ContentMetaType.Patch, fileSystem, checkLevel))
{
if ((applicationTitleId & ~0x1FFFUL) != titleIdBase)
@ -159,7 +172,6 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
break;
}
}
}
return (updatePatchNca, updateControlNca);
}

View file

@ -52,7 +52,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
return programs;
}
internal static (bool, ProcessResult) TryLoad<TMetaData, TFormat, THeader, TEntry>(this PartitionFileSystemCore<TMetaData, TFormat, THeader, TEntry> partitionFileSystem, Switch device, string path, ulong applicationId, out string errorMessage)
internal static (bool, ProcessResult) TryLoad<TMetaData, TFormat, THeader, TEntry>(this PartitionFileSystemCore<TMetaData, TFormat, THeader, TEntry> partitionFileSystem, Switch device, Stream stream, ulong applicationId, out string errorMessage, string extension, Stream updateStream = null)
where TMetaData : PartitionFileSystemMetaCore<TFormat, THeader, TEntry>, new()
where TFormat : IPartitionFileSystemFormat
where THeader : unmanaged, IPartitionFileSystemHeader
@ -102,7 +102,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
return (false, ProcessResult.Failed);
}
(Nca updatePatchNca, Nca updateControlNca) = mainNca.GetUpdateData(device.FileSystem, device.System.FsIntegrityCheckLevel, device.Configuration.UserChannelPersistence.Index, out string _);
(Nca updatePatchNca, Nca updateControlNca) = mainNca.GetUpdateData(device.FileSystem, device.System.FsIntegrityCheckLevel, device.Configuration.UserChannelPersistence.Index, out string _, updateStream);
if (updatePatchNca != null)
{
@ -131,7 +131,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
{
if (downloadableContentNca.Enabled)
{
device.Configuration.ContentManager.AddAocItem(downloadableContentNca.TitleId, downloadableContentContainer.ContainerPath, downloadableContentNca.FullPath);
device.Configuration.ContentManager.AddAocItem(downloadableContentNca.TitleId, stream, downloadableContentNca.FullPath, System.IO.Path.GetExtension(downloadableContentContainer.ContainerPath));
}
}
else

View file

@ -35,6 +35,12 @@ namespace Ryujinx.HLE.Loaders.Processes
public bool LoadXci(string path, ulong applicationId)
{
FileStream stream = new(path, FileMode.Open, FileAccess.Read);
return LoadXci(stream, applicationId);
}
public bool LoadXci(Stream stream, ulong applicationId, Stream updateStream = null)
{
Xci xci = new(_device.Configuration.VirtualFileSystem.KeySet, stream.AsStorage());
if (!xci.HasPartition(XciPartitionType.Secure))
@ -44,7 +50,7 @@ namespace Ryujinx.HLE.Loaders.Processes
return false;
}
(bool success, ProcessResult processResult) = xci.OpenPartition(XciPartitionType.Secure).TryLoad(_device, path, applicationId, out string errorMessage);
(bool success, ProcessResult processResult) = xci.OpenPartition(XciPartitionType.Secure).TryLoad(_device, stream, applicationId, out string errorMessage, "xci", updateStream);
if (!success)
{
@ -69,10 +75,16 @@ namespace Ryujinx.HLE.Loaders.Processes
public bool LoadNsp(string path, ulong applicationId)
{
FileStream file = new(path, FileMode.Open, FileAccess.Read);
PartitionFileSystem partitionFileSystem = new();
partitionFileSystem.Initialize(file.AsStorage()).ThrowIfFailure();
(bool success, ProcessResult processResult) = partitionFileSystem.TryLoad(_device, path, applicationId, out string errorMessage);
return LoadNsp(file, applicationId);
}
public bool LoadNsp(Stream stream, ulong applicationId, Stream updateStream = null)
{
PartitionFileSystem partitionFileSystem = new();
partitionFileSystem.Initialize(stream.AsStorage()).ThrowIfFailure();
(bool success, ProcessResult processResult) = partitionFileSystem.TryLoad(_device, stream, applicationId, out string errorMessage, "nsp", updateStream);
if (processResult.ProcessId == 0)
{
@ -101,7 +113,13 @@ namespace Ryujinx.HLE.Loaders.Processes
public bool LoadNca(string path)
{
FileStream file = new(path, FileMode.Open, FileAccess.Read);
Nca nca = new(_device.Configuration.VirtualFileSystem.KeySet, file.AsStorage(false));
return LoadNca(file);
}
public bool LoadNca(Stream ncaStream)
{
Nca nca = new(_device.Configuration.VirtualFileSystem.KeySet, ncaStream.AsStorage(false));
ProcessResult processResult = nca.Load(_device, null, null);
@ -249,5 +267,109 @@ namespace Ryujinx.HLE.Loaders.Processes
return false;
}
public bool LoadNxo(Stream stream, bool isNro, string name)
{
var nacpData = new BlitStruct<ApplicationControlProperty>(1);
IFileSystem dummyExeFs = null;
Stream romfsStream = null;
string programName = "";
ulong programId = 0000000000000000;
// Load executable.
IExecutable executable;
if (isNro)
{
NroExecutable nro = new(stream.AsStorage());
executable = nro;
// Open RomFS if exists.
IStorage romFsStorage = nro.OpenNroAssetSection(LibHac.Tools.Ro.NroAssetType.RomFs, false);
romFsStorage.GetSize(out long romFsSize).ThrowIfFailure();
if (romFsSize != 0)
{
romfsStream = romFsStorage.AsStream();
}
// Load Nacp if exists.
IStorage nacpStorage = nro.OpenNroAssetSection(LibHac.Tools.Ro.NroAssetType.Nacp, false);
nacpStorage.GetSize(out long nacpSize).ThrowIfFailure();
if (nacpSize != 0)
{
nacpStorage.Read(0, nacpData.ByteSpan);
programName = nacpData.Value.Title[(int)_device.System.State.DesiredTitleLanguage].NameString.ToString();
if (string.IsNullOrWhiteSpace(programName))
{
programName = Array.Find(nacpData.Value.Title.AsReadOnlySpan().ToArray(), x => x.Name[0] != 0).NameString.ToString();
}
if (nacpData.Value.PresenceGroupId != 0)
{
programId = nacpData.Value.PresenceGroupId;
}
else if (nacpData.Value.SaveDataOwnerId != 0)
{
programId = nacpData.Value.SaveDataOwnerId;
}
else if (nacpData.Value.AddOnContentBaseId != 0)
{
programId = nacpData.Value.AddOnContentBaseId - 0x1000;
}
}
// TODO: Add icon maybe ?
}
else
{
executable = new NsoExecutable(new LocalStorage(name, FileAccess.Read), programName);
}
// Explicitly null TitleId to disable the shader cache.
Graphics.Gpu.GraphicsConfig.TitleId = null;
_device.Gpu.HostInitalized.Set();
ProcessResult processResult = ProcessLoaderHelper.LoadNsos(_device,
_device.System.KernelContext,
dummyExeFs.GetNpdm(),
nacpData,
diskCacheEnabled: false,
diskCacheSelector: null,
allowCodeMemoryForJit: true,
programName,
programId,
0,
null,
executable);
// Make sure the process id is valid.
if (processResult.ProcessId != 0)
{
// Load RomFS.
if (romfsStream != null)
{
_device.Configuration.VirtualFileSystem.SetRomFs(processResult.ProcessId, romfsStream);
}
// Start process.
if (_processesByPid.TryAdd(processResult.ProcessId, processResult))
{
if (processResult.Start(_device))
{
_latestPid = processResult.ProcessId;
return true;
}
}
}
return false;
}
}
}

View file

@ -10,6 +10,7 @@ using Ryujinx.HLE.Loaders.Processes;
using Ryujinx.HLE.UI;
using Ryujinx.Memory;
using System;
using System.IO;
namespace Ryujinx.HLE
{
@ -101,6 +102,26 @@ namespace Ryujinx.HLE
return Processes.LoadNxo(fileName);
}
public bool LoadXci(Stream xciStream, ulong applicationId = 0, Stream updateStream = null)
{
return Processes.LoadXci(xciStream, applicationId, updateStream);
}
public bool LoadNca(Stream ncaStream)
{
return Processes.LoadNca(ncaStream);
}
public bool LoadNsp(Stream nspStream, ulong applicationId = 0, Stream updateStream = null)
{
return Processes.LoadNsp(nspStream, applicationId, updateStream);
}
public bool LoadProgram(Stream stream, bool isNro, string name)
{
return Processes.LoadNxo(stream, isNro, name);
}
public bool WaitFifo()
{
return Gpu.GPFifo.WaitForCommands();

View file

@ -14,16 +14,21 @@ namespace Ryujinx.HLE.Utilities
{
FileStream file = File.OpenRead(path);
return OpenApplicationFileSystem(file, Path.GetExtension(path).ToLower() == ".xci", fileSystem, throwOnFailure);
}
public static IFileSystem OpenApplicationFileSystem(Stream stream, bool isXci, VirtualFileSystem fileSystem, bool throwOnFailure = true)
{
IFileSystem partitionFileSystem;
if (Path.GetExtension(path).ToLower() == ".xci")
if (isXci)
{
partitionFileSystem = new Xci(fileSystem.KeySet, file.AsStorage()).OpenPartition(XciPartitionType.Secure);
partitionFileSystem = new Xci(fileSystem.KeySet, stream.AsStorage()).OpenPartition(XciPartitionType.Secure);
}
else
{
var pfsTemp = new PartitionFileSystem();
Result initResult = pfsTemp.Initialize(file.AsStorage());
Result initResult = pfsTemp.Initialize(stream.AsStorage());
if (throwOnFailure)
{