add stream based loaders

* extend stream loading support

* fix content manager rebase

* fix update searching

src/Ryujinx.HLE/FileSystem/ContentManager.cs

@@ -42,12 +42,14 @@ namespace Ryujinx.HLE.FileSystem
 
         private readonly struct AocItem
         {
-            public readonly string ContainerPath;
+            public readonly Stream ContainerStream;
             public readonly string NcaPath;
+            public readonly string Extension;
 
-            public AocItem(string containerPath, string ncaPath)
+            public AocItem(Stream containerStream, string ncaPath, string extension)
             {
-                ContainerPath = containerPath;
+                ContainerStream = containerStream;
+                Extension = extension;
                 NcaPath = ncaPath;
             }
         }
@@ -187,10 +189,10 @@ namespace Ryujinx.HLE.FileSystem
             }
         }
 
-        public void AddAocItem(ulong titleId, string containerPath, string ncaPath, bool mergedToContainer = false)
+        public void AddAocItem(ulong titleId, Stream containerStream, string ncaPath, string extension, bool mergedToContainer = false)
         {
             // TODO: Check Aoc version.
-            if (!AocData.TryAdd(titleId, new AocItem(containerPath, ncaPath)))
+            if (!AocData.TryAdd(titleId, new AocItem(containerStream, ncaPath, extension)))
             {
                 Logger.Warning?.Print(LogClass.Application, $"Duplicate AddOnContent detected. TitleId {titleId:X16}");
             }
@@ -200,12 +202,20 @@ namespace Ryujinx.HLE.FileSystem
 
                 if (!mergedToContainer)
                 {
-                    using var pfs = PartitionFileSystemUtils.OpenApplicationFileSystem(containerPath, _virtualFileSystem);
+                    using var pfs = PartitionFileSystemUtils.OpenApplicationFileSystem(containerStream, extension == ".xci", _virtualFileSystem);
                 }
             }
         }
 
-        public void ClearAocData() => AocData.Clear();
+        public void ClearAocData()
+        {
+            foreach (var aoc in AocData)
+            {
+                aoc.Value.ContainerStream?.Dispose();
+            }
+
+            AocData.Clear();
+        }
 
         public int GetAocCount() => AocData.Count;
 
@@ -217,18 +227,17 @@ namespace Ryujinx.HLE.FileSystem
 
             if (AocData.TryGetValue(aocTitleId, out AocItem aoc))
             {
-                var file = new FileStream(aoc.ContainerPath, FileMode.Open, FileAccess.Read);
                 using var ncaFile = new UniqueRef<IFile>();
 
-                switch (Path.GetExtension(aoc.ContainerPath))
+                switch (aoc.Extension)
                 {
                     case ".xci":
-                        var xci = new Xci(_virtualFileSystem.KeySet, file.AsStorage()).OpenPartition(XciPartitionType.Secure);
+                        var xci = new Xci(_virtualFileSystem.KeySet, aoc.ContainerStream.AsStorage()).OpenPartition(XciPartitionType.Secure);
-                        xci.OpenFile(ref ncaFile.Ref, aoc.NcaPath.ToU8Span(), OpenMode.Read).ThrowIfFailure();
+                        xci.OpenFile(ref ncaFile.Ref, aoc.NcaPath.ToU8Span(), OpenMode.Read);
                         break;
                     case ".nsp":
                         var pfs = new PartitionFileSystem();
-                        pfs.Initialize(file.AsStorage());
+                        pfs.Initialize(aoc.ContainerStream.AsStorage());
                         pfs.OpenFile(ref ncaFile.Ref, aoc.NcaPath.ToU8Span(), OpenMode.Read).ThrowIfFailure();
                         break;
                     default:
@@ -476,6 +485,27 @@ namespace Ryujinx.HLE.FileSystem
             FinishInstallation(temporaryDirectory, registeredDirectory);
         }
 
+        public void InstallFirmware(Stream stream, bool isXci)
+        {
+            ContentPath.TryGetContentPath(StorageId.BuiltInSystem, out var contentPathString);
+            ContentPath.TryGetRealPath(contentPathString, out var contentDirectory);
+            string registeredDirectory = Path.Combine(contentDirectory, "registered");
+            string temporaryDirectory = Path.Combine(contentDirectory, "temp");
+
+            if (!isXci)
+            {
+                using ZipArchive archive = new ZipArchive(stream);
+                InstallFromZip(archive, temporaryDirectory);
+            }
+            else
+            {
+                Xci xci = new(_virtualFileSystem.KeySet, stream.AsStorage());
+                InstallFromCart(xci, temporaryDirectory);
+            }
+
+            FinishInstallation(temporaryDirectory, registeredDirectory);
+        }
+
         public void InstallKeys(string keysSource, string installDirectory)
         {
             if (Directory.Exists(keysSource))
@@ -662,13 +692,16 @@ namespace Ryujinx.HLE.FileSystem
                 throw new MissingKeyException("HeaderKey is empty. Cannot decrypt NCA headers.");
             }
 
-            Dictionary<ulong, List<(NcaContentType type, string path)>> updateNcas = new();
-
             if (Directory.Exists(firmwarePackage))
             {
                 return VerifyAndGetVersionDirectory(firmwarePackage);
             }
 
+            SystemVersion VerifyAndGetVersionDirectory(string firmwareDirectory)
+            {
+                return VerifyAndGetVersion(new LocalFileSystem(firmwareDirectory));
+            }
+
             if (!File.Exists(firmwarePackage))
             {
                 throw new FileNotFoundException("Firmware file does not exist.");
@@ -676,16 +709,28 @@ namespace Ryujinx.HLE.FileSystem
 
             FileInfo info = new(firmwarePackage);
 
+
+            if (info.Extension == ".zip" || info.Extension == ".xci")
+            {
                 using FileStream file = File.OpenRead(firmwarePackage);
 
-            switch (info.Extension)
+                var isXci = info.Extension == ".xci";
+
+                return VerifyFirmwarePackage(file, isXci);
+            }
+
+            return null;
+        }
+
+        public SystemVersion VerifyFirmwarePackage(Stream file, bool isXci)
         {
-                case ".zip":
+            if (!isXci)
-                    using (ZipArchive archive = ZipFile.OpenRead(firmwarePackage))
             {
+                using ZipArchive archive = new ZipArchive(file, ZipArchiveMode.Read);
                 return VerifyAndGetVersionZip(archive);
             }
-                case ".xci":
+            else
+            {
                 Xci xci = new(_virtualFileSystem.KeySet, file.AsStorage());
 
                 if (xci.HasPartition(XciPartitionType.Update))
@@ -698,17 +743,13 @@ namespace Ryujinx.HLE.FileSystem
                 {
                     throw new InvalidFirmwarePackageException("Update not found in xci file.");
                 }
-                default:
+            }
-                    break;
         }
 
-            SystemVersion VerifyAndGetVersionDirectory(string firmwareDirectory)
+        private SystemVersion VerifyAndGetVersionZip(ZipArchive archive)
         {
-                return VerifyAndGetVersion(new LocalFileSystem(firmwareDirectory));
+            Dictionary<ulong, List<(NcaContentType type, string path)>> updateNcas = new();
-            }
 
-            SystemVersion VerifyAndGetVersionZip(ZipArchive archive)
-            {
             SystemVersion systemVersion = null;
 
             foreach (var entry in archive.Entries)
@@ -863,8 +904,10 @@ namespace Ryujinx.HLE.FileSystem
             return systemVersion;
         }
 
-            SystemVersion VerifyAndGetVersion(IFileSystem filesystem)
+        private SystemVersion VerifyAndGetVersion(IFileSystem filesystem)
         {
+            Dictionary<ulong, List<(NcaContentType type, string path)>> updateNcas = new();
+
             SystemVersion systemVersion = null;
 
             CnmtContentMetaEntry[] metaEntries = null;
@@ -994,9 +1037,6 @@ namespace Ryujinx.HLE.FileSystem
             return systemVersion;
         }
 
-            return null;
-        }
-
         public SystemVersion GetCurrentFirmwareVersion()
         {
             LoadEntries();

src/Ryujinx.HLE/Loaders/Processes/Extensions/NcaExtensions.cs

@@ -127,7 +127,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
             return nca.Header.ContentType == NcaContentType.Control;
         }
 
-        public static (Nca, Nca) GetUpdateData(this Nca mainNca, VirtualFileSystem fileSystem, IntegrityCheckLevel checkLevel, int programIndex, out string updatePath)
+        public static (Nca, Nca) GetUpdateData(this Nca mainNca, VirtualFileSystem fileSystem, IntegrityCheckLevel checkLevel, int programIndex, out string updatePath, Stream updateStream = null)
         {
             updatePath = null;
 
@@ -138,6 +138,10 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
             // Clear the program index part.
             ulong titleIdBase = mainNca.GetProgramIdBase();
 
+            IFileSystem updatePartitionFileSystem = null;
+
+            if (updateStream == null)
+            {
                 // Load update information if exists.
                 string titleUpdateMetadataPath = Path.Combine(AppDataManager.GamesDirPath, titleIdBase.ToString("x16"), "updates.json");
                 if (File.Exists(titleUpdateMetadataPath))
@@ -145,8 +149,17 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
                     updatePath = JsonHelper.DeserializeFromFile(titleUpdateMetadataPath, _applicationSerializerContext.TitleUpdateMetadata).Selected;
                     if (File.Exists(updatePath))
                     {
-                    IFileSystem updatePartitionFileSystem = PartitionFileSystemUtils.OpenApplicationFileSystem(updatePath, fileSystem);
+                        updatePartitionFileSystem = PartitionFileSystemUtils.OpenApplicationFileSystem(updatePath, fileSystem);
+                    }
+                }
+            }
+            else
+            {
+                updatePartitionFileSystem = PartitionFileSystemUtils.OpenApplicationFileSystem(updateStream, false, fileSystem);
+            }
 
+            if (updatePartitionFileSystem != null)
+            {
                 foreach ((ulong applicationTitleId, ContentMetaData content) in updatePartitionFileSystem.GetContentData(ContentMetaType.Patch, fileSystem, checkLevel))
                 {
                     if ((applicationTitleId & ~0x1FFFUL) != titleIdBase)
@@ -159,7 +172,6 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
                     break;
                 }
             }
-            }
 
             return (updatePatchNca, updateControlNca);
         }

src/Ryujinx.HLE/Loaders/Processes/Extensions/PartitionFileSystemExtensions.cs

@@ -52,7 +52,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
             return programs;
         }
 
-        internal static (bool, ProcessResult) TryLoad<TMetaData, TFormat, THeader, TEntry>(this PartitionFileSystemCore<TMetaData, TFormat, THeader, TEntry> partitionFileSystem, Switch device, string path, ulong applicationId, out string errorMessage)
+        internal static (bool, ProcessResult) TryLoad<TMetaData, TFormat, THeader, TEntry>(this PartitionFileSystemCore<TMetaData, TFormat, THeader, TEntry> partitionFileSystem, Switch device, Stream stream, ulong applicationId, out string errorMessage, string extension, Stream updateStream = null)
             where TMetaData : PartitionFileSystemMetaCore<TFormat, THeader, TEntry>, new()
             where TFormat : IPartitionFileSystemFormat
             where THeader : unmanaged, IPartitionFileSystemHeader
@@ -102,7 +102,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
                     return (false, ProcessResult.Failed);
                 }
 
-                (Nca updatePatchNca, Nca updateControlNca) = mainNca.GetUpdateData(device.FileSystem, device.System.FsIntegrityCheckLevel, device.Configuration.UserChannelPersistence.Index, out string _);
+                (Nca updatePatchNca, Nca updateControlNca) = mainNca.GetUpdateData(device.FileSystem, device.System.FsIntegrityCheckLevel, device.Configuration.UserChannelPersistence.Index, out string _, updateStream);
 
                 if (updatePatchNca != null)
                 {
@@ -131,7 +131,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
                             {
                                 if (downloadableContentNca.Enabled)
                                 {
-                                    device.Configuration.ContentManager.AddAocItem(downloadableContentNca.TitleId, downloadableContentContainer.ContainerPath, downloadableContentNca.FullPath);
+                                    device.Configuration.ContentManager.AddAocItem(downloadableContentNca.TitleId, stream, downloadableContentNca.FullPath, System.IO.Path.GetExtension(downloadableContentContainer.ContainerPath));
                                 }
                             }
                             else

src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs

@@ -35,6 +35,12 @@ namespace Ryujinx.HLE.Loaders.Processes
         public bool LoadXci(string path, ulong applicationId)
         {
             FileStream stream = new(path, FileMode.Open, FileAccess.Read);
+
+            return LoadXci(stream, applicationId);
+        }
+
+        public bool LoadXci(Stream stream, ulong applicationId, Stream updateStream = null)
+        {
             Xci xci = new(_device.Configuration.VirtualFileSystem.KeySet, stream.AsStorage());
 
             if (!xci.HasPartition(XciPartitionType.Secure))
@@ -44,7 +50,7 @@ namespace Ryujinx.HLE.Loaders.Processes
                 return false;
             }
 
-            (bool success, ProcessResult processResult) = xci.OpenPartition(XciPartitionType.Secure).TryLoad(_device, path, applicationId, out string errorMessage);
+            (bool success, ProcessResult processResult) = xci.OpenPartition(XciPartitionType.Secure).TryLoad(_device, stream, applicationId, out string errorMessage, "xci", updateStream);
 
             if (!success)
             {
@@ -69,10 +75,16 @@ namespace Ryujinx.HLE.Loaders.Processes
         public bool LoadNsp(string path, ulong applicationId)
         {
             FileStream file = new(path, FileMode.Open, FileAccess.Read);
-            PartitionFileSystem partitionFileSystem = new();
-            partitionFileSystem.Initialize(file.AsStorage()).ThrowIfFailure();
 
-            (bool success, ProcessResult processResult) = partitionFileSystem.TryLoad(_device, path, applicationId, out string errorMessage);
+            return LoadNsp(file, applicationId);
+        }
+
+        public bool LoadNsp(Stream stream, ulong applicationId, Stream updateStream = null)
+        {
+            PartitionFileSystem partitionFileSystem = new();
+            partitionFileSystem.Initialize(stream.AsStorage()).ThrowIfFailure();
+
+            (bool success, ProcessResult processResult) = partitionFileSystem.TryLoad(_device, stream, applicationId, out string errorMessage, "nsp", updateStream);
 
             if (processResult.ProcessId == 0)
             {
@@ -101,7 +113,13 @@ namespace Ryujinx.HLE.Loaders.Processes
         public bool LoadNca(string path)
         {
             FileStream file = new(path, FileMode.Open, FileAccess.Read);
-            Nca nca = new(_device.Configuration.VirtualFileSystem.KeySet, file.AsStorage(false));
+
+            return LoadNca(file);
+        }
+
+        public bool LoadNca(Stream ncaStream)
+        {
+            Nca nca = new(_device.Configuration.VirtualFileSystem.KeySet, ncaStream.AsStorage(false));
 
             ProcessResult processResult = nca.Load(_device, null, null);
 
@@ -249,5 +267,109 @@ namespace Ryujinx.HLE.Loaders.Processes
 
             return false;
         }
+
+        public bool LoadNxo(Stream stream, bool isNro, string name)
+        {
+            var nacpData = new BlitStruct<ApplicationControlProperty>(1);
+            IFileSystem dummyExeFs = null;
+            Stream romfsStream = null;
+
+            string programName = "";
+            ulong programId = 0000000000000000;
+
+            // Load executable.
+            IExecutable executable;
+
+            if (isNro)
+            {
+                NroExecutable nro = new(stream.AsStorage());
+
+                executable = nro;
+
+                // Open RomFS if exists.
+                IStorage romFsStorage = nro.OpenNroAssetSection(LibHac.Tools.Ro.NroAssetType.RomFs, false);
+                romFsStorage.GetSize(out long romFsSize).ThrowIfFailure();
+                if (romFsSize != 0)
+                {
+                    romfsStream = romFsStorage.AsStream();
+                }
+
+                // Load Nacp if exists.
+                IStorage nacpStorage = nro.OpenNroAssetSection(LibHac.Tools.Ro.NroAssetType.Nacp, false);
+                nacpStorage.GetSize(out long nacpSize).ThrowIfFailure();
+                if (nacpSize != 0)
+                {
+                    nacpStorage.Read(0, nacpData.ByteSpan);
+
+                    programName = nacpData.Value.Title[(int)_device.System.State.DesiredTitleLanguage].NameString.ToString();
+
+                    if (string.IsNullOrWhiteSpace(programName))
+                    {
+                        programName = Array.Find(nacpData.Value.Title.AsReadOnlySpan().ToArray(), x => x.Name[0] != 0).NameString.ToString();
+                    }
+
+                    if (nacpData.Value.PresenceGroupId != 0)
+                    {
+                        programId = nacpData.Value.PresenceGroupId;
+                    }
+                    else if (nacpData.Value.SaveDataOwnerId != 0)
+                    {
+                        programId = nacpData.Value.SaveDataOwnerId;
+                    }
+                    else if (nacpData.Value.AddOnContentBaseId != 0)
+                    {
+                        programId = nacpData.Value.AddOnContentBaseId - 0x1000;
+                    }
+                }
+
+                // TODO: Add icon maybe ?
+            }
+            else
+            {
+                executable = new NsoExecutable(new LocalStorage(name, FileAccess.Read), programName);
+            }
+
+            // Explicitly null TitleId to disable the shader cache.
+            Graphics.Gpu.GraphicsConfig.TitleId = null;
+            _device.Gpu.HostInitalized.Set();
+
+            ProcessResult processResult = ProcessLoaderHelper.LoadNsos(_device,
+                                                                       _device.System.KernelContext,
+                                                                       dummyExeFs.GetNpdm(),
+                                                                       nacpData,
+                                                                       diskCacheEnabled: false,
+                                                                       diskCacheSelector: null,
+                                                                       allowCodeMemoryForJit: true,
+                                                                       programName,
+                                                                       programId,
+                                                                       0,
+                                                                       null,
+                                                                       executable);
+
+            // Make sure the process id is valid.
+            if (processResult.ProcessId != 0)
+            {
+                // Load RomFS.
+                if (romfsStream != null)
+                {
+                    _device.Configuration.VirtualFileSystem.SetRomFs(processResult.ProcessId, romfsStream);
+                }
+
+                // Start process.
+                if (_processesByPid.TryAdd(processResult.ProcessId, processResult))
+                {
+                    if (processResult.Start(_device))
+                    {
+                        _latestPid = processResult.ProcessId;
+
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
+
     }
 }

src/Ryujinx.HLE/Switch.cs

@@ -10,6 +10,7 @@ using Ryujinx.HLE.Loaders.Processes;
 using Ryujinx.HLE.UI;
 using Ryujinx.Memory;
 using System;
+using System.IO;
 
 namespace Ryujinx.HLE
 {
@@ -101,6 +102,26 @@ namespace Ryujinx.HLE
             return Processes.LoadNxo(fileName);
         }
 
+        public bool LoadXci(Stream xciStream, ulong applicationId = 0, Stream updateStream = null)
+        {
+            return Processes.LoadXci(xciStream, applicationId, updateStream);
+        }
+
+        public bool LoadNca(Stream ncaStream)
+        {
+            return Processes.LoadNca(ncaStream);
+        }
+
+        public bool LoadNsp(Stream nspStream, ulong applicationId = 0, Stream updateStream = null)
+        {
+            return Processes.LoadNsp(nspStream, applicationId, updateStream);
+        }
+
+        public bool LoadProgram(Stream stream, bool isNro, string name)
+        {
+            return Processes.LoadNxo(stream, isNro, name);
+        }
+
         public bool WaitFifo()
         {
             return Gpu.GPFifo.WaitForCommands();

src/Ryujinx.HLE/Utilities/PartitionFileSystemUtils.cs

@@ -14,16 +14,21 @@ namespace Ryujinx.HLE.Utilities
         {
             FileStream file = File.OpenRead(path);
 
+            return OpenApplicationFileSystem(file, Path.GetExtension(path).ToLower() == ".xci", fileSystem, throwOnFailure);
+        }
+
+        public static IFileSystem OpenApplicationFileSystem(Stream stream, bool isXci, VirtualFileSystem fileSystem, bool throwOnFailure = true)
+        {
             IFileSystem partitionFileSystem;
 
-            if (Path.GetExtension(path).ToLower() == ".xci")
+            if (isXci)
             {
-                partitionFileSystem = new Xci(fileSystem.KeySet, file.AsStorage()).OpenPartition(XciPartitionType.Secure);
+                partitionFileSystem = new Xci(fileSystem.KeySet, stream.AsStorage()).OpenPartition(XciPartitionType.Secure);
             }
             else
             {
                 var pfsTemp = new PartitionFileSystem();
-                Result initResult = pfsTemp.Initialize(file.AsStorage());
+                Result initResult = pfsTemp.Initialize(stream.AsStorage());
 
                 if (throwOnFailure)
                 {