From fd7eeef1f3b3fc89b4f8a2061e7b224ab134a69a Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 01:14:42 -0700 Subject: [PATCH 01/21] Initial texture loading overhaul prototype --- KSPCommunityFixes/Performance/FastLoader.cs | 1393 ++++++++++++------- 1 file changed, 893 insertions(+), 500 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index c744a53f..893257ac 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -22,6 +22,9 @@ using KSPCommunityFixes.Library; using TMPro; using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; +using Unity.IO.LowLevel.Unsafe; +using Unity.Profiling; using UnityEngine; using UnityEngine.Experimental.Rendering; using UnityEngine.Networking; @@ -151,6 +154,11 @@ internal class KSPCFFastLoader : MonoBehaviour // min amount of files to try to keep in memory, regardless of maxBufferSize private const int minFileRead = 10; + // max concurrent per-texture coroutines spawned by TextureDriverCoroutine. + // Each in-flight request holds at most one of {ReadHandle, UnityWebRequest, background Task}, + // so a single semaphore covers all resource bounds. + private const int MaxConcurrentTextures = 512; + private static Harmony persistentHarmony; private static string PersistentHarmonyID => typeof(KSPCFFastLoader).FullName; @@ -473,7 +481,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) // Files loaded by our custom loaders List audioFiles = new List(1000); - List textureAssets = new List(10000); + List textureRequests = new List(10000); List modelAssets = new List(5000); // Files loaded by mod-defined loaders (ex : Shabby *.shab files) @@ -519,23 +527,23 @@ static IEnumerator FastAssetLoader(List configFileTypes) switch (file.fileExtension) { case "dds": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureDDS)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureDDS)); break; case "jpg": case "jpeg": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureJPG)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureJPG)); break; case "mbm": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureMBM)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureMBM)); break; case "png": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TexturePNG)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TexturePNG)); break; case "tga": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureTGA)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTGA)); break; case "truecolor": - textureAssets.Add(new RawAsset(file, RawAsset.AssetType.TextureTRUECOLOR)); + textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTRUECOLOR)); break; default: unsupportedTextureFiles.Add(file); @@ -567,16 +575,8 @@ static IEnumerator FastAssetLoader(List configFileTypes) } } - Thread textureCacheReaderThread; - if (textureCacheEnabled) - { - textureCacheReaderThread = new Thread(() => SetupTextureCacheThread(textureAssets)); - textureCacheReaderThread.Start(); - } - else - { - textureCacheReaderThread = null; - } + // PNG/DDS cache temporarily disabled — opt-in popup and settings are still + // wired up, but no cache I/O is performed during loading. gdb.progressTitle = "Loading sound assets..."; KSPCFFastLoaderReport.wAudioLoading.Restart(); @@ -668,6 +668,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) // start texture loading gdb.progressFraction = 0.25f; KSPCFFastLoaderReport.wAudioLoading.Stop(); + SupportedFormatCache.Build(); KSPCFFastLoaderReport.wTextureLoading.Restart(); gdb.progressTitle = "Loading texture assets..."; yield return null; @@ -716,19 +717,9 @@ static IEnumerator FastAssetLoader(List configFileTypes) } } - if (textureCacheReaderThread != null) - { - while (textureCacheReaderThread.IsAlive) - yield return null; - } - // call our custom loader - yield return gdb.StartCoroutine(FilesLoader(textureAssets, allTextureFiles, "Loading texture asset")); - - // write texture cache json to disk - Thread writeTextureCacheThread = new Thread(() => loader.WriteTextureCache()); - writeTextureCacheThread.Start(); + yield return gdb.StartCoroutine(TextureDriverCoroutine(textureRequests, allTextureFiles)); // start model loading gdb.progressFraction = 0.75f; @@ -1057,13 +1048,10 @@ public enum Result } private UrlFile file; - private CachedTextureInfo cachedTextureInfo; private AssetType assetType; private bool useRentedBuffer; private byte[] buffer; private int dataLength; - private MemoryStream memoryStream; - private BinaryReader binaryReader; private Result result; private string resultMessage; @@ -1112,9 +1100,6 @@ public void ReadFromDiskWorkerThread() { switch (assetType) { - case AssetType.TextureDDS: - case AssetType.TextureMBM: - case AssetType.TextureTGA: case AssetType.ModelMU: case AssetType.ModelDAE: useRentedBuffer = true; @@ -1123,7 +1108,7 @@ public void ReadFromDiskWorkerThread() try { - string path = assetType == AssetType.TexturePNGCached ? cachedTextureInfo.FilePath : file.fullPath; + string path = file.fullPath; using (FileStream fileStream = System.IO.File.OpenRead(path)) { @@ -1184,54 +1169,7 @@ public void LoadAndDisposeMainThread() if (result == Result.Failed) return; - if (file.fileType == FileType.Texture) - { - TextureInfo textureInfo; - switch (assetType) - { - case AssetType.TextureDDS: - textureInfo = LoadDDS(); - break; - case AssetType.TextureJPG: - textureInfo = LoadJPG(); - break; - case AssetType.TextureMBM: - textureInfo = LoadMBM(); - break; - case AssetType.TexturePNG: - textureInfo = LoadPNG(); - break; - case AssetType.TexturePNGCached: - textureInfo = LoadPNGCached(); - break; - case AssetType.TextureTGA: - textureInfo = LoadTGA(); - break; - case AssetType.TextureTRUECOLOR: - textureInfo = LoadTRUECOLOR(); - break; - default: - SetError("Unknown texture format"); - return; - } - - if (result == Result.Failed || textureInfo == null || textureInfo.texture.IsNullOrDestroyed()) - { - result = Result.Failed; - if (string.IsNullOrEmpty(resultMessage)) - resultMessage = $"{TypeName} load error"; - } - else - { - textureInfo.name = file.url; - textureInfo.texture.name = file.url; - Instance.databaseTexture.Add(textureInfo); - texturesByUrl[file.url] = textureInfo; - KSPCFFastLoaderReport.texturesBytesLoaded += dataLength; - KSPCFFastLoaderReport.texturesLoaded++; - } - } - else if (file.fileType == FileType.Model) + if (file.fileType == FileType.Model) { GameObject model; switch (assetType) @@ -1284,516 +1222,969 @@ public void LoadAndDisposeMainThread() public void Dispose() { - if (binaryReader != null) - binaryReader.Dispose(); - - if (memoryStream != null) - memoryStream.Dispose(); - if (useRentedBuffer) arrayPool.Return(buffer); } - public void CheckTextureCache() + private GameObject LoadMU() { - CachedTextureInfo cachedTextureInfo = GetCachedTextureInfo(file); + return MuParser.Parse(file.parent.url, buffer, dataLength); + } - if (cachedTextureInfo == null) - return; + private GameObject LoadDAE() + { + // given that this is a quite obsolete thing and that it's mess to reimplement, just call the stock + // stuff and re-load the file - assetType = AssetType.TexturePNGCached; - this.cachedTextureInfo = cachedTextureInfo; - } - - // see https://learn.microsoft.com/en-us/windows/win32/direct3ddds/dx-graphics-dds-pguide - private enum DDSFourCC : uint - { - DXT1 = 0x31545844, // "DXT1" - DXT2 = 0x32545844, // "DXT2" - DXT3 = 0x33545844, // "DXT3" - DXT4 = 0x34545844, // "DXT4" - DXT5 = 0x35545844, // "DXT5" - BC4U_ATI = 0x31495441, // "ATI1" (actually BC4U) - BC4U = 0x55344342, // "BC4U" - BC4S = 0x53344342, // "BC4S" - BC5U_ATI = 0x32495441, // "ATI2" (actually BC5U) - BC5U = 0x55354342, // "BC5U" - BC5S = 0x53354342, // "BC5S" - RGBG = 0x47424752, // "RGBG" - GRGB = 0x42475247, // "GRGB" - UYVY = 0x59565955, // "UYVY" - YUY2 = 0x32595559, // "YUY2" - DX10 = 0x30315844, // "DX10", actual DXGI format specified in DX10 header - R16G16B16A16_UNORM = 36, - R16G16B16A16_SNORM = 110, - R16_FLOAT = 111, - R16G16_FLOAT = 112, - R16G16B16A16_FLOAT = 113, - R32_FLOAT = 114, - R32G32_FLOAT = 115, - R32G32B32A32_FLOAT = 116, - CxV8U8 = 117, - } - - private TextureInfo LoadDDS() - { - memoryStream = new MemoryStream(buffer, 0, dataLength); - binaryReader = new BinaryReader(memoryStream); - - if (binaryReader.ReadUInt32() != DDSValues.uintMagic) - { - SetError("DDS: File is not a DDS format file!"); - return null; - } - DDSHeader dDSHeader = new DDSHeader(binaryReader); - bool mipChain = (dDSHeader.dwCaps & DDSPixelFormatCaps.MIPMAP) != 0; - bool isNormalMap = (dDSHeader.ddspf.dwFlags & 0x80000u) != 0 || (dDSHeader.ddspf.dwFlags & 0x80000000u) != 0; - - DDSFourCC ddsFourCC = (DDSFourCC)dDSHeader.ddspf.dwFourCC; - Texture2D texture2D = null; - GraphicsFormat graphicsFormat = GraphicsFormat.None; - - switch (ddsFourCC) - { - case DDSFourCC.DXT1: - graphicsFormat = GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT1, true); - break; - case DDSFourCC.DXT5: - graphicsFormat = GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT5, true); - break; - case DDSFourCC.BC4U_ATI: - case DDSFourCC.BC4U: - graphicsFormat = GraphicsFormat.R_BC4_UNorm; - break; - case DDSFourCC.BC4S: - graphicsFormat = GraphicsFormat.R_BC4_SNorm; - break; - case DDSFourCC.BC5U_ATI: - case DDSFourCC.BC5U: - graphicsFormat = GraphicsFormat.RG_BC5_UNorm; - break; - case DDSFourCC.BC5S: - graphicsFormat = GraphicsFormat.RG_BC5_SNorm; - break; - case DDSFourCC.R16G16B16A16_UNORM: - graphicsFormat = GraphicsFormat.R16G16B16A16_UNorm; - break; - case DDSFourCC.R16G16B16A16_SNORM: - graphicsFormat = GraphicsFormat.R16G16B16A16_SNorm; - break; - case DDSFourCC.R16_FLOAT: - graphicsFormat = GraphicsFormat.R16_SFloat; - break; - case DDSFourCC.R16G16_FLOAT: - graphicsFormat = GraphicsFormat.R16G16_SFloat; - break; - case DDSFourCC.R16G16B16A16_FLOAT: - graphicsFormat = GraphicsFormat.R16G16B16A16_SFloat; - break; - case DDSFourCC.R32_FLOAT: - graphicsFormat = GraphicsFormat.R32_SFloat; - break; - case DDSFourCC.R32G32_FLOAT: - graphicsFormat = GraphicsFormat.R32G32_SFloat; - break; - case DDSFourCC.R32G32B32A32_FLOAT: - graphicsFormat = GraphicsFormat.R32G32B32A32_SFloat; - break; - case DDSFourCC.DX10: - DDSHeaderDX10 dx10Header = new DDSHeaderDX10(binaryReader); - switch (dx10Header.dxgiFormat) + GameObject gameObject = new DatabaseLoaderModel_DAE.DAE().Load(file, new FileInfo(file.fullPath)); + if (gameObject.IsNotNullOrDestroyed()) + { + MeshFilter[] componentsInChildren = gameObject.GetComponentsInChildren(); + foreach (MeshFilter meshFilter in componentsInChildren) + { + if (meshFilter.gameObject.name == "node_collider") { - case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM: - graphicsFormat = GraphicsFormat.RGBA_DXT1_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM_SRGB: - graphicsFormat = GraphicsFormat.RGBA_DXT1_SRGB; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM: - graphicsFormat = GraphicsFormat.RGBA_DXT5_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM_SRGB: - graphicsFormat = GraphicsFormat.RGBA_DXT5_SRGB; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC4_SNORM: - graphicsFormat = GraphicsFormat.R_BC4_SNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC4_UNORM: - graphicsFormat = GraphicsFormat.R_BC4_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC5_SNORM: - graphicsFormat = GraphicsFormat.RG_BC5_SNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC5_UNORM: - graphicsFormat = GraphicsFormat.RG_BC5_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM: - graphicsFormat = GraphicsFormat.RGBA_BC7_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM_SRGB: - graphicsFormat = GraphicsFormat.RGBA_BC7_SRGB; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC6H_SF16: - graphicsFormat = GraphicsFormat.RGB_BC6H_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_BC6H_UF16: - graphicsFormat = GraphicsFormat.RGB_BC6H_UFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_UNORM: - graphicsFormat = GraphicsFormat.R16G16B16A16_UNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_SNORM: - graphicsFormat = GraphicsFormat.R16G16B16A16_SNorm; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16_FLOAT: - graphicsFormat = GraphicsFormat.R16_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16G16_FLOAT: - graphicsFormat = GraphicsFormat.R16G16_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_FLOAT: - graphicsFormat = GraphicsFormat.R16G16B16A16_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R32_FLOAT: - graphicsFormat = GraphicsFormat.R32_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R32G32_FLOAT: - graphicsFormat = GraphicsFormat.R32G32_SFloat; - break; - case DXGI_FORMAT.DXGI_FORMAT_R32G32B32A32_FLOAT: - graphicsFormat = GraphicsFormat.R32G32B32A32_SFloat; - break; - default: - SetError($"DDS: The '{dx10Header.dxgiFormat}' DXT10 format isn't supported"); - break; + meshFilter.gameObject.AddComponent().sharedMesh = meshFilter.mesh; + MeshRenderer component = meshFilter.gameObject.GetComponent(); + UnityEngine.Object.Destroy(meshFilter); + UnityEngine.Object.Destroy(component); } - break; - case DDSFourCC.DXT2: - case DDSFourCC.DXT3: - case DDSFourCC.DXT4: - case DDSFourCC.RGBG: - case DDSFourCC.GRGB: - case DDSFourCC.UYVY: - case DDSFourCC.YUY2: - case DDSFourCC.CxV8U8: - SetError($"DDS: The '{ddsFourCC}' format isn't supported, use DXT1 for RGB textures or DXT5 for RGBA textures"); - break; - default: - SetError($"DDS: Unknown dwFourCC format '0x{ddsFourCC:X}'"); - break; + } } - if (graphicsFormat != GraphicsFormat.None) - { - if (!SystemInfo.IsFormatSupported(graphicsFormat, FormatUsage.Sample)) + return gameObject; + } + + } + + #endregion + + #region Per-texture coroutine loader + + // Profiling markers for the work scheduled on background threads via Task.Run. + // Each marker.Auto() scope is opened inside the Task lambda so the timing + // appears under that thread in the Unity profiler. + private static readonly ProfilerMarker s_pmParseDDSHeader = new ProfilerMarker("KSPCF.Tex.ParseDDSHeader"); + private static readonly ProfilerMarker s_pmSwizzleNormalMap = new ProfilerMarker("KSPCF.Tex.SwizzleNormalMap"); + private static readonly ProfilerMarker s_pmCopyLevel0 = new ProfilerMarker("KSPCF.Tex.CopyLevel0"); + private static readonly ProfilerMarker s_pmFileSize = new ProfilerMarker("KSPCF.Tex.FileSize"); + private static readonly ProfilerMarker s_pmReadAllBytes = new ProfilerMarker("KSPCF.Tex.ReadAllBytes"); + + // Result/error carrier for each texture file. Replaces RawAsset for textures. + private sealed class TextureLoadRequest + { + public enum State : byte { Pending, Ready, Failed } + + public UrlFile File; + public RawAsset.AssetType AssetType; + public long FileLength; + public CachedTextureInfo CachedInfo; + public bool CameFromCache; + public volatile State Status; + public TextureInfo Result; + public string ErrorMessage; + public Exception Exception; + + public TextureLoadRequest(UrlFile file, RawAsset.AssetType assetType) + { + File = file; + AssetType = assetType; + Status = State.Pending; + } + + // Called from the texture cache reader thread before the driver runs. + public void CheckTextureCache() + { + CachedTextureInfo info = GetCachedTextureInfo(File); + if (info == null) + return; + AssetType = RawAsset.AssetType.TexturePNGCached; + CachedInfo = info; + CameFromCache = true; + } + } + + // Result of background DDS header parsing. + private struct DDSPreparedHeader + { + public int Width; + public int Height; + public bool MipChain; + public bool IsNormalMap; + public GraphicsFormat Format; + public long DataOffset; + public long FileLength; + } + + // Probes which GraphicsFormats are actually usable on the running GPU. + // Built once on the main thread before texture loading starts so that the + // background DDS header parser can produce a format and we can verify it + // against this set without needing main-thread access. + private static class SupportedFormatCache + { + private static HashSet supported; + + public static void Build() + { + supported = new HashSet(); + GraphicsFormat[] candidates = new[] + { + GraphicsFormat.RGBA_DXT1_UNorm, + GraphicsFormat.RGBA_DXT1_SRGB, + GraphicsFormat.RGBA_DXT5_UNorm, + GraphicsFormat.RGBA_DXT5_SRGB, + GraphicsFormat.R_BC4_UNorm, + GraphicsFormat.R_BC4_SNorm, + GraphicsFormat.RG_BC5_UNorm, + GraphicsFormat.RG_BC5_SNorm, + GraphicsFormat.RGBA_BC7_UNorm, + GraphicsFormat.RGBA_BC7_SRGB, + GraphicsFormat.RGB_BC6H_SFloat, + GraphicsFormat.RGB_BC6H_UFloat, + GraphicsFormat.R16G16B16A16_UNorm, + GraphicsFormat.R16G16B16A16_SNorm, + GraphicsFormat.R16G16B16A16_SFloat, + GraphicsFormat.R16_SFloat, + GraphicsFormat.R16G16_SFloat, + GraphicsFormat.R32_SFloat, + GraphicsFormat.R32G32_SFloat, + GraphicsFormat.R32G32B32A32_SFloat, + }; + foreach (GraphicsFormat fmt in candidates) + if (SystemInfo.IsFormatSupported(fmt, FormatUsage.Sample)) + supported.Add(fmt); + } + + public static bool IsSupported(GraphicsFormat fmt) => supported != null && supported.Contains(fmt); + } + + // Block-compressed format check used to ignore mipChain on NPOT textures + // (Unity rejects DXT5 + mipmaps on non-power-of-two sources). + private static bool IsBlockCompressed(GraphicsFormat fmt) + { + switch (fmt) + { + case GraphicsFormat.RGBA_DXT1_UNorm: + case GraphicsFormat.RGBA_DXT1_SRGB: + case GraphicsFormat.RGBA_DXT5_UNorm: + case GraphicsFormat.RGBA_DXT5_SRGB: + case GraphicsFormat.R_BC4_UNorm: + case GraphicsFormat.R_BC4_SNorm: + case GraphicsFormat.RG_BC5_UNorm: + case GraphicsFormat.RG_BC5_SNorm: + case GraphicsFormat.RGBA_BC7_UNorm: + case GraphicsFormat.RGBA_BC7_SRGB: + case GraphicsFormat.RGB_BC6H_SFloat: + case GraphicsFormat.RGB_BC6H_UFloat: + return true; + default: + return false; + } + } + + // Background DDS header reader. Throws on bad magic or unsupported format. + // Does not call any Unity API (so it is safe on a worker thread). + private static DDSPreparedHeader ParseDDSHeader(string path) + { + FileInfo fi = new FileInfo(path); + long fileLength = fi.Length; + if (fileLength < 128) + throw new IOException($"DDS file '{path}' is too small ({fileLength} bytes)"); + + using FileStream fs = File.OpenRead(path); + using BinaryReader br = new BinaryReader(fs); + + if (br.ReadUInt32() != DDSValues.uintMagic) + throw new IOException($"DDS: '{path}' is not a DDS format file"); + + DDSHeader hdr = new DDSHeader(br); + bool mipChain = (hdr.dwCaps & DDSPixelFormatCaps.MIPMAP) != 0; + bool isNormalMap = (hdr.ddspf.dwFlags & 0x80000u) != 0 || (hdr.ddspf.dwFlags & 0x80000000u) != 0; + + DDSHeaderDX10 dx10Header = default; + bool hasDx10 = (DDSFourCCBg)hdr.ddspf.dwFourCC == DDSFourCCBg.DX10; + if (hasDx10) + { + if (fileLength < 148) + throw new IOException($"DDS file '{path}' has DX10 marker but is too small for DX10 header"); + dx10Header = new DDSHeaderDX10(br); + } + + GraphicsFormat fmt = MapDDSFormat(hdr, hasDx10, dx10Header, out string error); + if (fmt == GraphicsFormat.None || error != null) + throw new IOException($"DDS: {error ?? "unknown format"}"); + + long dataOffset = hasDx10 ? 148 : 128; + return new DDSPreparedHeader + { + Width = (int)hdr.dwWidth, + Height = (int)hdr.dwHeight, + MipChain = mipChain, + IsNormalMap = isNormalMap, + Format = fmt, + DataOffset = dataOffset, + FileLength = fileLength, + }; + } + + // Background-thread-safe FourCC enum (mirrors RawAsset.DDSFourCC, which is private to RawAsset). + private enum DDSFourCCBg : uint + { + DXT1 = 0x31545844, + DXT2 = 0x32545844, + DXT3 = 0x33545844, + DXT4 = 0x34545844, + DXT5 = 0x35545844, + BC4U_ATI = 0x31495441, + BC4U = 0x55344342, + BC4S = 0x53344342, + BC5U_ATI = 0x32495441, + BC5U = 0x55354342, + BC5S = 0x53354342, + RGBG = 0x47424752, + GRGB = 0x42475247, + UYVY = 0x59565955, + YUY2 = 0x32595559, + DX10 = 0x30315844, + R16G16B16A16_UNORM = 36, + R16G16B16A16_SNORM = 110, + R16_FLOAT = 111, + R16G16_FLOAT = 112, + R16G16B16A16_FLOAT = 113, + R32_FLOAT = 114, + R32G32_FLOAT = 115, + R32G32B32A32_FLOAT = 116, + CxV8U8 = 117, + } + + // Returns GraphicsFormat.None and sets error on failure. + private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeaderDX10 dx10, out string error) + { + error = null; + DDSFourCCBg fourCC = (DDSFourCCBg)hdr.ddspf.dwFourCC; + switch (fourCC) + { + case DDSFourCCBg.DXT1: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT1, true); + case DDSFourCCBg.DXT5: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT5, true); + case DDSFourCCBg.BC4U_ATI: + case DDSFourCCBg.BC4U: return GraphicsFormat.R_BC4_UNorm; + case DDSFourCCBg.BC4S: return GraphicsFormat.R_BC4_SNorm; + case DDSFourCCBg.BC5U_ATI: + case DDSFourCCBg.BC5U: return GraphicsFormat.RG_BC5_UNorm; + case DDSFourCCBg.BC5S: return GraphicsFormat.RG_BC5_SNorm; + case DDSFourCCBg.R16G16B16A16_UNORM: return GraphicsFormat.R16G16B16A16_UNorm; + case DDSFourCCBg.R16G16B16A16_SNORM: return GraphicsFormat.R16G16B16A16_SNorm; + case DDSFourCCBg.R16_FLOAT: return GraphicsFormat.R16_SFloat; + case DDSFourCCBg.R16G16_FLOAT: return GraphicsFormat.R16G16_SFloat; + case DDSFourCCBg.R16G16B16A16_FLOAT: return GraphicsFormat.R16G16B16A16_SFloat; + case DDSFourCCBg.R32_FLOAT: return GraphicsFormat.R32_SFloat; + case DDSFourCCBg.R32G32_FLOAT: return GraphicsFormat.R32G32_SFloat; + case DDSFourCCBg.R32G32B32A32_FLOAT: return GraphicsFormat.R32G32B32A32_SFloat; + case DDSFourCCBg.DX10: + if (!hasDx10) { - if (SystemInfo.operatingSystemFamily == OperatingSystemFamily.MacOSX && - (graphicsFormat == GraphicsFormat.RGBA_BC7_UNorm - || graphicsFormat == GraphicsFormat.RGBA_BC7_SRGB - || graphicsFormat == GraphicsFormat.RGB_BC6H_SFloat - || graphicsFormat == GraphicsFormat.RGB_BC6H_UFloat)) - { - SetError($"DDS: The '{graphicsFormat}' format is not supported on MacOS"); - } - else - { - SetError($"DDS: The '{graphicsFormat}' format is not supported by your GPU or OS"); - } + error = "DX10 marker without DX10 header"; + return GraphicsFormat.None; } - else + switch (dx10.dxgiFormat) { - texture2D = new Texture2D((int)dDSHeader.dwWidth, (int)dDSHeader.dwHeight, graphicsFormat, mipChain ? TextureCreationFlags.MipChain : TextureCreationFlags.None); - if (texture2D.IsNullOrDestroyed()) + case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM: return GraphicsFormat.RGBA_DXT1_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM_SRGB: return GraphicsFormat.RGBA_DXT1_SRGB; + case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM: return GraphicsFormat.RGBA_DXT5_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM_SRGB: return GraphicsFormat.RGBA_DXT5_SRGB; + case DXGI_FORMAT.DXGI_FORMAT_BC4_SNORM: return GraphicsFormat.R_BC4_SNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC4_UNORM: return GraphicsFormat.R_BC4_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC5_SNORM: return GraphicsFormat.RG_BC5_SNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC5_UNORM: return GraphicsFormat.RG_BC5_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM: return GraphicsFormat.RGBA_BC7_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM_SRGB: return GraphicsFormat.RGBA_BC7_SRGB; + case DXGI_FORMAT.DXGI_FORMAT_BC6H_SF16: return GraphicsFormat.RGB_BC6H_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_BC6H_UF16: return GraphicsFormat.RGB_BC6H_UFloat; + case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_UNORM: return GraphicsFormat.R16G16B16A16_UNorm; + case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_SNORM: return GraphicsFormat.R16G16B16A16_SNorm; + case DXGI_FORMAT.DXGI_FORMAT_R16_FLOAT: return GraphicsFormat.R16_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R16G16_FLOAT: return GraphicsFormat.R16G16_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R16G16B16A16_FLOAT: return GraphicsFormat.R16G16B16A16_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R32_FLOAT: return GraphicsFormat.R32_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R32G32_FLOAT: return GraphicsFormat.R32G32_SFloat; + case DXGI_FORMAT.DXGI_FORMAT_R32G32B32A32_FLOAT: return GraphicsFormat.R32G32B32A32_SFloat; + default: + error = $"DXT10 format '{dx10.dxgiFormat}' is not supported"; + return GraphicsFormat.None; + } + case DDSFourCCBg.DXT2: + case DDSFourCCBg.DXT3: + case DDSFourCCBg.DXT4: + case DDSFourCCBg.RGBG: + case DDSFourCCBg.GRGB: + case DDSFourCCBg.UYVY: + case DDSFourCCBg.YUY2: + case DDSFourCCBg.CxV8U8: + error = $"format '{fourCC}' is not supported, use DXT1 for RGB textures or DXT5 for RGBA textures"; + return GraphicsFormat.None; + default: + error = $"unknown dwFourCC format '0x{(uint)fourCC:X}'"; + return GraphicsFormat.None; + } + } + + // Channel-swizzle for normal maps (extracted from BitmapToCompressedNormalMapFast). + // src must hold pixel data in srcFormat; dst is written as RGBA32 (level 0 only — + // if dst has a mip chain, higher mip levels are left untouched and are populated + // by the caller's Apply(updateMipmaps: true)). + private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) + { + byte* s = (byte*)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(src); + byte* d = (byte*)NativeArrayUnsafeUtility.GetUnsafePtr(dst); + int srcLen = src.Length; + + switch (srcFormat) + { + case TextureFormat.RGBA32: + // (r, g, b, a) -> (g, g, g, r) + for (int i = 0; i < srcLen; i += 4) + { + byte r = s[i]; + byte g = s[i + 1]; + d[i] = g; d[i + 1] = g; d[i + 2] = g; d[i + 3] = r; + } + break; + case TextureFormat.ARGB32: + // (a, r, g, b) -> (g, g, g, r) + for (int i = 0; i < srcLen; i += 4) + { + byte r = s[i + 1]; + byte g = s[i + 2]; + d[i] = g; d[i + 1] = g; d[i + 2] = g; d[i + 3] = r; + } + break; + case TextureFormat.RGB24: + // (r, g, b) -> (g, g, g, r); 3-byte in, 4-byte out + { + int j = 0; + for (int i = 0; i < srcLen; i += 3) { - SetError($"DDS: Failed to load texture, unknown error"); + byte r = s[i]; + byte g = s[i + 1]; + d[j] = g; d[j + 1] = g; d[j + 2] = g; d[j + 3] = r; + j += 4; } - else - { - int position = (int)binaryReader.BaseStream.Position; - GCHandle pinnedHandle = GCHandle.Alloc(buffer, GCHandleType.Pinned); - try - { - IntPtr ptr = Marshal.UnsafeAddrOfPinnedArrayElement(buffer, position); - texture2D.LoadRawTextureData(ptr, dataLength - position); - } - finally - { - pinnedHandle.Free(); - } + } + break; + default: + throw new InvalidOperationException($"SwizzleNormalMap: unsupported source format {srcFormat}"); + } + } - texture2D.Apply(updateMipmaps: false, makeNoLongerReadable: true); - } + // Returns the most informative exception from a faulted Task without using the + // null-coalescing operator (which the type checker rejects when mixing + // AggregateException with concrete subtypes of Exception). + private static Exception UnwrapFaultedTask(Task task, string fallbackMessage) + { + AggregateException ae = task.Exception; + if (ae != null && ae.InnerException != null) + return ae.InnerException; + if (ae != null) + return ae; + return new IOException(fallbackMessage); + } + + // Iterator methods can't contain unsafe blocks in C# 8, so the AsyncReadManager + // pointer setup goes through this static helper. + private static unsafe ReadHandle BeginAsyncRead(string path, NativeArray dst, long offset, long size) + { + ReadCommand cmd = new ReadCommand + { + Buffer = NativeArrayUnsafeUtility.GetUnsafePtr(dst), + Offset = offset, + Size = size, + }; + return AsyncReadManager.Read(path, &cmd, 1); + } + + // Wraps an inner format-specific coroutine with exception capture and semaphore release. + // C# does not allow yield inside a try/catch, so we manually drive MoveNext() and + // do the catch around just the MoveNext call. + private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, IEnumerator inner, SemaphoreSlim semaphore) + { + try + { + while (true) + { + bool moved; + bool failed = false; + object current = null; + try + { + moved = inner.MoveNext(); + if (moved) + current = inner.Current; + } + catch (Exception e) + { + req.Exception = e; + req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; + req.Status = TextureLoadRequest.State.Failed; + moved = false; + failed = true; } + if (failed) + yield break; + if (!moved) + break; + yield return current; } - return new TextureInfo(file, texture2D, isNormalMap, false, true); + if (req.Status == TextureLoadRequest.State.Pending) + { + if (req.Result != null) + req.Status = TextureLoadRequest.State.Ready; + else + { + if (req.ErrorMessage == null) + req.ErrorMessage = "Loader produced no result"; + req.Status = TextureLoadRequest.State.Failed; + } + } + } + finally + { + semaphore.Release(); } + } - private TextureInfo LoadJPG() + private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) + { + string path = req.File.fullPath; + Task hdrTask = Task.Run(() => { - bool isNormal = file.name.EndsWith("NRM"); + using (s_pmParseDDSHeader.Auto()) + return ParseDDSHeader(path); + }); + while (!hdrTask.IsCompleted) + yield return null; + if (hdrTask.IsFaulted) + throw UnwrapFaultedTask(hdrTask, "DDS header parse failed"); + DDSPreparedHeader hdr = hdrTask.Result; + req.FileLength = hdr.FileLength; - if (isNormal) - { - Texture2D tex = new Texture2D(1, 1, TextureFormat.RGB24, false); - if (!ImageConversion.LoadImage(tex, buffer, false)) - return null; + if (!SupportedFormatCache.IsSupported(hdr.Format)) + { + req.ErrorMessage = $"DDS: format '{hdr.Format}' is not supported by your GPU"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } - Texture2D nrmTex = BitmapToCompressedNormalMapFast(tex); - return new TextureInfo(file, nrmTex, true, false, true); - } - else - { - Texture2D tex = new Texture2D(1, 1, TextureFormat.DXT1, false); - if (!ImageConversion.LoadImage(tex, buffer, false)) - return null; + // Unity rejects mipChain on NPOT textures for block-compressed formats. + bool mipChain = hdr.MipChain; + if (mipChain && IsBlockCompressed(hdr.Format) + && !(Numerics.IsPowerOfTwo(hdr.Width) && Numerics.IsPowerOfTwo(hdr.Height))) + { + mipChain = false; + } - return new TextureInfo(file, tex, false, true, true); - } + Texture2D tex = new Texture2D(hdr.Width, hdr.Height, hdr.Format, + mipChain ? TextureCreationFlags.MipChain : TextureCreationFlags.None); + if (tex.IsNullOrDestroyed()) + { + req.ErrorMessage = "DDS: Texture2D allocation failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private TextureInfo LoadMBM() + // Wait one frame so Unity's initial GPU resource creation completes + // before AsyncReadManager writes into the staging buffer. + yield return null; + + NativeArray dst = tex.GetRawTextureData(); + long expectedSize = dst.Length; + if (hdr.FileLength - hdr.DataOffset < expectedSize) { - memoryStream = new MemoryStream(buffer, 0, dataLength); - binaryReader = new BinaryReader(memoryStream); - Texture2D texture2D = MBMReader.ReadTexture2D(buffer, binaryReader, true, true, out bool isNormalMap); - return new TextureInfo(file, texture2D, isNormalMap, false, true); + UnityEngine.Object.Destroy(tex); + req.ErrorMessage = $"DDS: file is too small for declared format (need {expectedSize} bytes after offset {hdr.DataOffset}, have {hdr.FileLength - hdr.DataOffset})"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private static string mipMapsPNGTexturePath = Path.DirectorySeparatorChar + "Flags" + Path.DirectorySeparatorChar; + ReadHandle handle = BeginAsyncRead(path, dst, hdr.DataOffset, expectedSize); + + while (handle.Status == ReadStatus.InProgress) + yield return null; + + ReadStatus status = handle.Status; + handle.Dispose(); - private TextureInfo LoadPNG() + if (status != ReadStatus.Complete) { - if (!GetPNGSize(buffer, out uint width, out uint height)) - { - SetError("Invalid PNG file"); - return null; - } + UnityEngine.Object.Destroy(tex); + req.ErrorMessage = $"DDS: AsyncReadManager.Read failed (status={status})"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + tex.Apply(updateMipmaps: false, makeNoLongerReadable: true); + req.Result = new TextureInfo(req.File, tex, hdr.IsNormalMap, false, true); + req.Status = TextureLoadRequest.State.Ready; + } - bool isNormalMap = file.name.EndsWith("NRM"); - bool nonReadable = file.fullPath.Contains("@thumbs"); // KSPCF optimization : don't keep cargo icons in memory - bool hasMipMaps = file.fullPath.Contains(mipMapsPNGTexturePath); // only generate mipmaps for flags (stock behavior) - bool canCompress = hasMipMaps ? Numerics.IsPowerOfTwo(width) && Numerics.IsPowerOfTwo(height) : width % 4 == 0 && height % 4 == 0; + private static readonly string flagsPathSep = Path.DirectorySeparatorChar + "Flags" + Path.DirectorySeparatorChar; - // don't initially compress normal textures, as we need to swizzle the raw data first - TextureFormat textureFormat; - if (isNormalMap) - { - textureFormat = TextureFormat.ARGB32; - } - else if (!canCompress) + private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) + { + string filePath = req.File.fullPath; + req.FileLength = new FileInfo(filePath).Length; + string url = "file:///" + filePath.Replace('\\', '/'); + + UnityWebRequest uwr = UnityWebRequestTexture.GetTexture(url, nonReadable: false); + try + { + yield return uwr.SendWebRequest(); + + if (uwr.isNetworkError || uwr.isHttpError) { - textureFormat = TextureFormat.ARGB32; - SetWarning("Texture isn't eligible for DXT compression, width and height must be multiples of 4"); + req.ErrorMessage = $"UWR: {uwr.error}"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - else + + Texture2D src = DownloadHandlerTexture.GetContent(uwr); + if (src.IsNullOrDestroyed()) { - textureFormat = TextureFormat.DXT5; + req.ErrorMessage = "UWR: GetContent returned null"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - Texture2D texture = new Texture2D((int)width, (int)height, textureFormat, hasMipMaps); + bool isNormalMap = req.File.name.EndsWith("NRM"); + bool isFlag = filePath.Contains(flagsPathSep); + bool canCompress = src.width % 4 == 0 && src.height % 4 == 0; - if ((isNormalMap || canCompress) && textureCacheEnabled) + if (isNormalMap || isFlag) { - if (!ImageConversion.LoadImage(texture, buffer, false)) - return null; - + // Allocate a destination texture with mipchain (when POT) so that + // Apply(true) can generate mipmaps. UWR-loaded textures have no mipchain. + bool isPot = Numerics.IsPowerOfTwo(src.width) && Numerics.IsPowerOfTwo(src.height); + bool wantMipChain = isPot; + TextureFormat dstFormat = isNormalMap ? TextureFormat.RGBA32 : src.format; + Texture2D dst = new Texture2D(src.width, src.height, dstFormat, wantMipChain); if (isNormalMap) - texture = BitmapToCompressedNormalMapFast(texture, false); + dst.wrapMode = TextureWrapMode.Repeat; + + NativeArray srcData = src.GetRawTextureData(); + NativeArray dstData = dst.GetRawTextureData(); + TextureFormat srcFormat = src.format; - if (texture.graphicsFormat == GraphicsFormat.RGBA_DXT5_UNorm) + Task task; + if (isNormalMap) + { + task = Task.Run(() => + { + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(srcData, dstData, srcFormat); + }); + } + else { - SaveCachedTexture(file, texture, isNormalMap); + // Flag: copy level-0 raw data byte-for-byte (formats already match). + int level0Bytes = srcData.Length; + task = Task.Run(() => + { + using (s_pmCopyLevel0.Auto()) + NativeArray.Copy(srcData, 0, dstData, 0, level0Bytes); + }); + } - if (isNormalMap || nonReadable) - texture.Apply(true, true); + while (!task.IsCompleted) + yield return null; + if (task.IsFaulted) + { + UnityEngine.Object.Destroy(src); + UnityEngine.Object.Destroy(dst); + throw UnwrapFaultedTask(task, "texture pre-process task faulted"); } - } - else - { - if (!ImageConversion.LoadImage(texture, buffer, nonReadable)) - return null; - if (isNormalMap) - texture = BitmapToCompressedNormalMapFast(texture); + UnityEngine.Object.Destroy(src); + src = dst; + + if (wantMipChain) + src.Apply(updateMipmaps: true); } + if (canCompress) + src.Compress(highQuality: !isNormalMap); + else if (!isNormalMap) + Debug.LogWarning($"Texture '{req.File.url}' isn't eligible for DXT compression, width and height must be multiples of 4"); + + src.Apply(updateMipmaps: false, makeNoLongerReadable: true); - return new TextureInfo(file, texture, isNormalMap, !nonReadable, true); + bool isCompressed = + src.graphicsFormat == GraphicsFormat.RGBA_DXT5_UNorm + || src.graphicsFormat == GraphicsFormat.RGBA_DXT5_SRGB + || src.graphicsFormat == GraphicsFormat.RGBA_DXT1_UNorm + || src.graphicsFormat == GraphicsFormat.RGBA_DXT1_SRGB; + req.Result = new TextureInfo(req.File, src, isNormalMap, isReadable: false, isCompressed: isCompressed); + req.Status = TextureLoadRequest.State.Ready; + } + finally + { + uwr.Dispose(); } + } + + private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) + { + string path = req.File.fullPath; + Task sizeTask = Task.Run(() => + { + using (s_pmFileSize.Auto()) + return new FileInfo(path).Length; + }); + while (!sizeTask.IsCompleted) + yield return null; + if (sizeTask.IsFaulted) + throw UnwrapFaultedTask(sizeTask, "file size read failed"); - private TextureInfo LoadPNGCached() + long len = sizeTask.Result; + req.FileLength = len; + if (len <= 0 || len > int.MaxValue) { - if (cachedTextureInfo.TryCreateTexture(buffer, out Texture2D texture)) - return new TextureInfo(file, texture, cachedTextureInfo.normal, cachedTextureInfo.readable, true); + req.ErrorMessage = $"TRUECOLOR: invalid file length {len}"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } - buffer = System.IO.File.ReadAllBytes(file.fullPath); - return LoadPNG(); + NativeArray data = new NativeArray((int)len, Allocator.Persistent, NativeArrayOptions.UninitializedMemory); + ReadHandle handle = BeginAsyncRead(path, data, 0, len); + while (handle.Status == ReadStatus.InProgress) + yield return null; + ReadStatus rs = handle.Status; + handle.Dispose(); + + if (rs != ReadStatus.Complete) + { + data.Dispose(); + req.ErrorMessage = $"TRUECOLOR: AsyncReadManager.Read failed (status={rs})"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private TextureInfo LoadTGA() + byte[] managed = data.ToArray(); + data.Dispose(); + + Texture2D tex = new Texture2D(2, 2, TextureFormat.ARGB32, false); + if (!ImageConversion.LoadImage(tex, managed, markNonReadable: false)) { - if (dataLength < 18) + UnityEngine.Object.Destroy(tex); + req.ErrorMessage = "TRUECOLOR: ImageConversion.LoadImage failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + bool isNormalMap = req.File.name.EndsWith("NRM"); + if (isNormalMap) + { + bool isPot = Numerics.IsPowerOfTwo(tex.width) && Numerics.IsPowerOfTwo(tex.height); + Texture2D dst = new Texture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); + dst.wrapMode = TextureWrapMode.Repeat; + + NativeArray srcData = tex.GetRawTextureData(); + NativeArray dstData = dst.GetRawTextureData(); + TextureFormat srcFormat = tex.format; + Task swizzleTask = Task.Run(() => + { + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(srcData, dstData, srcFormat); + }); + while (!swizzleTask.IsCompleted) + yield return null; + if (swizzleTask.IsFaulted) { - SetError("TGA invalid length of only " + dataLength + "bytes"); - return null; + UnityEngine.Object.Destroy(tex); + UnityEngine.Object.Destroy(dst); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); } + UnityEngine.Object.Destroy(tex); + tex = dst; - TGAImage tgaImage = new TGAImage(); - TGAImage.header = new TGAHeader(buffer); - TGAImage.colorData = tgaImage.ReadImage(TGAImage.header, buffer); - if (TGAImage.colorData == null) - return null; + if (isPot) + { + tex.Apply(updateMipmaps: true); + tex.Compress(highQuality: false); + } + tex.Apply(updateMipmaps: false, makeNoLongerReadable: true); + + req.Result = new TextureInfo(req.File, tex, true, isReadable: false, isCompressed: isPot); + } + else + { + tex.Apply(updateMipmaps: false, makeNoLongerReadable: false); + req.Result = new TextureInfo(req.File, tex, false, isReadable: true, isCompressed: false); + } + req.Status = TextureLoadRequest.State.Ready; + } - Texture2D texture = tgaImage.CreateTexture(mipmap: true, linear: false, compress: true, compressHighQuality: false, allowRead: true); - if (texture.IsNullOrDestroyed()) - return null; + private static IEnumerator LoadMBMCoroutine(TextureLoadRequest req) + { + string path = req.File.fullPath; + Task readTask = Task.Run(() => + { + using (s_pmReadAllBytes.Auto()) + return File.ReadAllBytes(path); + }); + while (!readTask.IsCompleted) + yield return null; + if (readTask.IsFaulted) + throw UnwrapFaultedTask(readTask, "MBM file read failed"); - bool isNormalMap = file.name.EndsWith("NRM"); - if (isNormalMap) - texture = BitmapToCompressedNormalMapFast(texture); + byte[] buffer = readTask.Result; + req.FileLength = buffer.Length; - return new TextureInfo(file, texture, isNormalMap, !isNormalMap, true); + Texture2D texture; + bool isNormalMap; + using (MemoryStream ms = new MemoryStream(buffer, 0, buffer.Length)) + using (BinaryReader br = new BinaryReader(ms)) + { + texture = MBMReader.ReadTexture2D(buffer, br, true, true, out isNormalMap); } - - private TextureInfo LoadTRUECOLOR() + if (texture.IsNullOrDestroyed()) { - bool isNormalMap = file.name.EndsWith("NRM"); + req.ErrorMessage = "MBM: ReadTexture2D failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + req.Result = new TextureInfo(req.File, texture, isNormalMap, isReadable: false, isCompressed: true); + req.Status = TextureLoadRequest.State.Ready; + } - Texture2D texture = new Texture2D(1, 1, TextureFormat.ARGB32, false); - if (!ImageConversion.LoadImage(texture, buffer, false)) - return null; + private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) + { + string path = req.File.fullPath; + Task readTask = Task.Run(() => + { + using (s_pmReadAllBytes.Auto()) + return File.ReadAllBytes(path); + }); + while (!readTask.IsCompleted) + yield return null; + if (readTask.IsFaulted) + throw UnwrapFaultedTask(readTask, "TGA file read failed"); - if (isNormalMap) - texture = BitmapToCompressedNormalMapFast(texture); + byte[] buffer = readTask.Result; + req.FileLength = buffer.Length; + if (buffer.Length < 18) + { + req.ErrorMessage = $"TGA invalid length of only {buffer.Length} bytes"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } - return new TextureInfo(file, texture, isNormalMap, !isNormalMap, false); + TGAImage tgaImage = new TGAImage(); + TGAImage.header = new TGAHeader(buffer); + TGAImage.colorData = tgaImage.ReadImage(TGAImage.header, buffer); + if (TGAImage.colorData == null) + { + req.ErrorMessage = "TGA: ReadImage failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private GameObject LoadMU() + Texture2D texture = tgaImage.CreateTexture(mipmap: true, linear: false, compress: true, compressHighQuality: false, allowRead: true); + if (texture.IsNullOrDestroyed()) { - return MuParser.Parse(file.parent.url, buffer, dataLength); + req.ErrorMessage = "TGA: CreateTexture failed"; + req.Status = TextureLoadRequest.State.Failed; + yield break; } - private GameObject LoadDAE() + bool isNormalMap = req.File.name.EndsWith("NRM"); + if (isNormalMap) { - // given that this is a quite obsolete thing and that it's mess to reimplement, just call the stock - // stuff and re-load the file + bool isPot = Numerics.IsPowerOfTwo(texture.width) && Numerics.IsPowerOfTwo(texture.height); + Texture2D dst = new Texture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); + dst.wrapMode = TextureWrapMode.Repeat; - GameObject gameObject = new DatabaseLoaderModel_DAE.DAE().Load(file, new FileInfo(file.fullPath)); - if (gameObject.IsNotNullOrDestroyed()) + NativeArray srcData = texture.GetRawTextureData(); + NativeArray dstData = dst.GetRawTextureData(); + TextureFormat srcFormat = texture.format; + Task swizzleTask = Task.Run(() => { - MeshFilter[] componentsInChildren = gameObject.GetComponentsInChildren(); - foreach (MeshFilter meshFilter in componentsInChildren) - { - if (meshFilter.gameObject.name == "node_collider") - { - meshFilter.gameObject.AddComponent().sharedMesh = meshFilter.mesh; - MeshRenderer component = meshFilter.gameObject.GetComponent(); - UnityEngine.Object.Destroy(meshFilter); - UnityEngine.Object.Destroy(component); - } - } + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(srcData, dstData, srcFormat); + }); + while (!swizzleTask.IsCompleted) + yield return null; + if (swizzleTask.IsFaulted) + { + UnityEngine.Object.Destroy(texture); + UnityEngine.Object.Destroy(dst); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); } + UnityEngine.Object.Destroy(texture); + texture = dst; - return gameObject; + if (isPot) + { + texture.Apply(updateMipmaps: true); + texture.Compress(highQuality: false); + } + texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); + } + + req.Result = new TextureInfo(req.File, texture, isNormalMap, isReadable: !isNormalMap, isCompressed: true); + req.Status = TextureLoadRequest.State.Ready; + } + + private static IEnumerator LoadPNGCachedCoroutine(TextureLoadRequest req) + { + CachedTextureInfo info = req.CachedInfo; + string path = info.FilePath; + Task readTask = Task.Run(() => + { + using (s_pmReadAllBytes.Auto()) + return File.ReadAllBytes(path); + }); + while (!readTask.IsCompleted) + yield return null; + if (readTask.IsFaulted) + { + Debug.LogWarning($"[KSPCF] Cached PNG read failed for '{req.File.url}', falling back to fresh load"); + FallBackFromCache(req, info); + IEnumerator fallback = LoadUWRCoroutine(req); + while (fallback.MoveNext()) + yield return fallback.Current; + yield break; + } + + byte[] buffer = readTask.Result; + req.FileLength = buffer.Length; + + if (info.TryCreateTexture(buffer, out Texture2D texture)) + { + req.Result = new TextureInfo(req.File, texture, info.normal, isReadable: false, isCompressed: true); + req.Status = TextureLoadRequest.State.Ready; + yield break; } - private static Texture2D BitmapToCompressedNormalMapFast(Texture2D original, bool makeNoLongerReadable = true) + Debug.LogWarning($"[KSPCF] Cached PNG TryCreateTexture failed for '{req.File.url}', falling back to fresh load"); + FallBackFromCache(req, info); + IEnumerator fallback2 = LoadUWRCoroutine(req); + while (fallback2.MoveNext()) + yield return fallback2.Current; + } + + private static void FallBackFromCache(TextureLoadRequest req, CachedTextureInfo info) + { + if (loader.textureCacheData.Remove(info.name)) { - // ~6 times faster than the stock BitmapToUnityNormalMap() method - // Note that this would be a lot more efficient if we didn't have to create a new texture. - // Unfortunately, Unity doesn't provide any way to add mimaps to a texture that didn't - // have them initially (but I guess this is a deeper GPU related limitation)... + loader.textureDataIds.Remove(info.id); + try { File.Delete(info.FilePath); } catch { } + loader.cacheUpdated = true; + } + req.AssetType = RawAsset.AssetType.TexturePNG; + req.CameFromCache = false; + req.CachedInfo = null; + } - TextureFormat originalFormat = original.format; - Texture2D normalMap = new Texture2D(original.width, original.height, TextureFormat.RGBA32, true); - normalMap.wrapMode = TextureWrapMode.Repeat; + private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) + { + GameDatabase gdb = GameDatabase.Instance; + SemaphoreSlim semaphore = new SemaphoreSlim(MaxConcurrentTextures, MaxConcurrentTextures); + int spawnIdx = 0; + int completeIdx = 0; + int total = requests.Count; + double nextFrameTime = ElapsedTime + minFrameTimeD; - if (originalFormat == TextureFormat.RGBA32 - || originalFormat == TextureFormat.ARGB32 - || originalFormat == TextureFormat.RGB24) + while (completeIdx < total) + { + while (spawnIdx < total && semaphore.Wait(0)) { - NativeArray originalData = original.GetRawTextureData(); - NativeArray normalMapData = normalMap.GetRawTextureData(); - int size = originalData.Length; - byte r, g; - switch (originalFormat) + TextureLoadRequest req = requests[spawnIdx++]; + IEnumerator inner; + switch (req.AssetType) { - case TextureFormat.RGBA32: - // from (r, g, b, a) - // to (g, g, g, r); - for (int i = 0; i < size; i += 4) - { - r = originalData[i]; - g = originalData[i + 1]; - normalMapData[i] = g; - normalMapData[i + 1] = g; - normalMapData[i + 2] = g; - normalMapData[i + 3] = r; - } + case RawAsset.AssetType.TextureDDS: + inner = LoadDDSCoroutine(req); break; - case TextureFormat.ARGB32: - // from (a, r, g, b) - // to (g, g, g, r); - for (int i = 0; i < size; i += 4) - { - r = originalData[i + 1]; - g = originalData[i + 2]; - normalMapData[i] = g; - normalMapData[i + 1] = g; - normalMapData[i + 2] = g; - normalMapData[i + 3] = r; - } + case RawAsset.AssetType.TexturePNG: + case RawAsset.AssetType.TextureJPG: + inner = LoadUWRCoroutine(req); break; - case TextureFormat.RGB24: - // from (r, g, b) - // to (g, g, g, r); - int j = 0; - for (int i = 0; i < size; i += 3) - { - r = originalData[i]; - g = originalData[i + 1]; - normalMapData[j] = g; - normalMapData[j + 1] = g; - normalMapData[j + 2] = g; - normalMapData[j + 3] = r; - j += 4; - } + case RawAsset.AssetType.TextureTRUECOLOR: + inner = LoadTRUECOLORCoroutine(req); + break; + case RawAsset.AssetType.TextureMBM: + inner = LoadMBMCoroutine(req); + break; + case RawAsset.AssetType.TextureTGA: + inner = LoadTGACoroutine(req); + break; + case RawAsset.AssetType.TexturePNGCached: + inner = LoadPNGCachedCoroutine(req); + break; + default: + inner = null; break; } - } - else - { - Color32[] pixels = original.GetPixels32(); - for (int i = 0; i < pixels.Length; i++) + + if (inner == null) + { + req.ErrorMessage = $"Unknown asset type {req.AssetType}"; + req.Status = TextureLoadRequest.State.Failed; + semaphore.Release(); + } + else { - Color32 pixel = pixels[i]; - pixel.a = pixel.r; - pixel.r = pixel.g; - pixel.b = pixel.g; - pixels[i] = pixel; + Debug.Log($"Load Texture: {req.File.url}"); + gdb.StartCoroutine(LoadTextureWrapperCoroutine(req, inner, semaphore)); } - normalMap.SetPixels32(pixels); } - // Unity can't convert NPOT textures to DXT5 with mipmaps - if (Numerics.IsPowerOfTwo(normalMap.width) && Numerics.IsPowerOfTwo(normalMap.height)) - { - normalMap.Apply(true); // needed to generate mipmaps, must be done before compression - normalMap.Compress(false); - normalMap.Apply(true, makeNoLongerReadable); - } - else + TextureLoadRequest head = requests[completeIdx]; + if (head.Status == TextureLoadRequest.State.Pending) { - normalMap.Apply(true, makeNoLongerReadable); + if (ElapsedTime > nextFrameTime) + { + nextFrameTime = ElapsedTime + minFrameTimeD; + gdb.progressFraction = (float)(loadedAssetCount + completeIdx) / totalAssetCount; + gdb.progressTitle = $"Loading texture asset {completeIdx}/{total}"; + } + yield return null; + continue; } - Destroy(original); - return normalMap; + InsertReadyRequest(head, loadedUrls); + completeIdx++; + loadedAssetCount++; } } + private static void InsertReadyRequest(TextureLoadRequest req, HashSet loadedUrls) + { + if (req.Status == TextureLoadRequest.State.Failed) + { + Debug.LogWarning($"LOAD FAILED: {req.File.url}: {req.ErrorMessage}"); + if (req.Result != null && req.Result.texture.IsNotNullOrDestroyed()) + UnityEngine.Object.Destroy(req.Result.texture); + return; + } + + if (!loadedUrls.Add(req.File.url)) + { + Debug.LogWarning($"Duplicate texture asset '{req.File.url}' with extension '{req.File.fileExtension}' won't be loaded"); + if (req.Result != null && req.Result.texture.IsNotNullOrDestroyed()) + UnityEngine.Object.Destroy(req.Result.texture); + return; + } + + req.Result.name = req.File.url; + req.Result.texture.name = req.File.url; + GameDatabase.Instance.databaseTexture.Add(req.Result); + texturesByUrl[req.File.url] = req.Result; + KSPCFFastLoaderReport.texturesBytesLoaded += req.FileLength; + KSPCFFastLoaderReport.texturesLoaded++; + } + #endregion #region PartLoader reimplementation @@ -2315,12 +2706,12 @@ IEnumerable instructions #region PNG texture cache - private static void SetupTextureCacheThread(List textures) + private static void SetupTextureCacheThread(List requests) { loader.SetupTextureCache(); - foreach (RawAsset rawAsset in textures) - rawAsset.CheckTextureCache(); + foreach (TextureLoadRequest req in requests) + req.CheckTextureCache(); } private void SetupTextureCache() @@ -2444,7 +2835,9 @@ public CachedTextureInfo(UrlFile urlFile, Texture2D texture, bool isNormalMap, l height = texture.height; mipCount = texture.mipmapCount; normal = isNormalMap; - readable = !isNormalMap && !name.Contains("@thumbs"); + // Always non-readable: matches the unified DDS/PNG/JPG readability policy. + // The 'readable' field in older cache entries on disk is ignored on read. + readable = false; loaded = true; } @@ -2460,7 +2853,7 @@ public bool TryCreateTexture(byte[] buffer, out Texture2D texture) { texture = new Texture2D(width, height, GraphicsFormat.RGBA_DXT5_UNorm, mipCount, mipCount == 1 ? TextureCreationFlags.None : TextureCreationFlags.MipChain); texture.LoadRawTextureData(buffer); - texture.Apply(false, !readable); + texture.Apply(false, true); loaded = true; return true; } @@ -2508,7 +2901,7 @@ private static bool GetFileStats(string path, out long size, out long time) return true; } - private static void SaveCachedTexture(UrlDir.UrlFile urlFile, Texture2D texture, bool isNormalMap) + private static void SaveCachedTextureFromBytes(UrlDir.UrlFile urlFile, Texture2D texture, bool isNormalMap, byte[] rawData) { if (!GetFileStats(urlFile.fullPath, out long size, out long creationTime)) { @@ -2517,7 +2910,7 @@ private static void SaveCachedTexture(UrlDir.UrlFile urlFile, Texture2D texture, } CachedTextureInfo cachedTextureInfo = new CachedTextureInfo(urlFile, texture, isNormalMap, size, creationTime); - cachedTextureInfo.SaveRawTextureData(texture); + File.WriteAllBytes(Path.Combine(loader.textureCachePath, cachedTextureInfo.id.ToString()), rawData); loader.textureCacheData.Add(cachedTextureInfo.name, cachedTextureInfo); loader.textureDataIds.Add(cachedTextureInfo.id); loader.cacheUpdated = true; From 07bc11142037a26d1c0709180198af10d4c4ec02 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 01:23:34 -0700 Subject: [PATCH 02/21] Avoid initializing texture pixels when creating them --- KSPCommunityFixes/KSPCommunityFixes.csproj | 2 + KSPCommunityFixes/Performance/FastLoader.cs | 77 +++++++++++++++++++-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/KSPCommunityFixes/KSPCommunityFixes.csproj b/KSPCommunityFixes/KSPCommunityFixes.csproj index 508bfd9a..4a60b46b 100644 --- a/KSPCommunityFixes/KSPCommunityFixes.csproj +++ b/KSPCommunityFixes/KSPCommunityFixes.csproj @@ -102,6 +102,8 @@ + + diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 893257ac..3807d414 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -18,6 +18,7 @@ using System.Reflection; using System.Reflection.Emit; using System.Runtime.InteropServices; +using System.Runtime.Serialization; using System.Threading; using KSPCommunityFixes.Library; using TMPro; @@ -1602,6 +1603,66 @@ private static unsafe ReadHandle BeginAsyncRead(string path, NativeArray d return AsyncReadManager.Read(path, &cmd, 1); } + // Mirrors UnityEngine.Experimental.Rendering.TextureCreationFlags but with the + // additional flag values that exist in Unity's native code but aren't exposed in + // the public managed enum. DontInitializePixels skips the zero-fill that the + // normal Texture2D constructor performs — pointless work when we're about to + // overwrite the bytes via LoadRawTextureData / AsyncReadManager / LoadImage. + // Borrowed from KSPTextureLoader (../AsyncTextureLoad/src/KSPTextureLoader/TextureUtils.cs). + [Flags] + private enum InternalTextureCreationFlags + { + None = 0, + MipChain = 1 << 0, + DontInitializePixels = 1 << 2, + DontDestroyTexture = 1 << 3, + DontCreateSharedTextureData = 1 << 4, + APIShareable = 1 << 5, + Crunch = 1 << 6, + } + + // Allocates a Texture2D without zeroing its pixel buffer. Equivalent to the + // standard Texture2D constructor except for the DontInitializePixels flag, + // which the public managed API doesn't expose for the TextureFormat overload. + private static Texture2D CreateUninitializedTexture2D( + int width, + int height, + TextureFormat format = TextureFormat.RGBA32, + bool mipChain = false, + bool linear = false, + InternalTextureCreationFlags flags = InternalTextureCreationFlags.None) + { + if (GraphicsFormatUtility.IsCrunchFormat(format)) + flags |= InternalTextureCreationFlags.Crunch; + int mipCount = !mipChain ? 1 : -1; + return CreateUninitializedTexture2D( + width, height, mipCount, + GraphicsFormatUtility.GetGraphicsFormat(format, isSRGB: !linear), + flags); + } + + private static Texture2D CreateUninitializedTexture2D( + int width, + int height, + int mipCount, + GraphicsFormat format, + InternalTextureCreationFlags flags = InternalTextureCreationFlags.None) + { + Texture2D tex = (Texture2D)FormatterServices.GetUninitializedObject(typeof(Texture2D)); + if (!tex.ValidateFormat(GraphicsFormatUtility.GetTextureFormat(format))) + return tex; + + flags |= InternalTextureCreationFlags.DontInitializePixels; + if (mipCount != 1) + flags |= InternalTextureCreationFlags.MipChain; + + Texture2D.Internal_Create( + tex, width, height, mipCount, format, + (TextureCreationFlags)flags, IntPtr.Zero); + + return tex; + } + // Wraps an inner format-specific coroutine with exception capture and semaphore release. // C# does not allow yield inside a try/catch, so we manually drive MoveNext() and // do the catch around just the MoveNext call. @@ -1683,8 +1744,10 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) mipChain = false; } - Texture2D tex = new Texture2D(hdr.Width, hdr.Height, hdr.Format, - mipChain ? TextureCreationFlags.MipChain : TextureCreationFlags.None); + Texture2D tex = CreateUninitializedTexture2D( + hdr.Width, hdr.Height, + mipChain ? -1 : 1, + hdr.Format); if (tex.IsNullOrDestroyed()) { req.ErrorMessage = "DDS: Texture2D allocation failed"; @@ -1766,7 +1829,7 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) bool isPot = Numerics.IsPowerOfTwo(src.width) && Numerics.IsPowerOfTwo(src.height); bool wantMipChain = isPot; TextureFormat dstFormat = isNormalMap ? TextureFormat.RGBA32 : src.format; - Texture2D dst = new Texture2D(src.width, src.height, dstFormat, wantMipChain); + Texture2D dst = CreateUninitializedTexture2D(src.width, src.height, dstFormat, wantMipChain); if (isNormalMap) dst.wrapMode = TextureWrapMode.Repeat; @@ -1871,7 +1934,7 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) byte[] managed = data.ToArray(); data.Dispose(); - Texture2D tex = new Texture2D(2, 2, TextureFormat.ARGB32, false); + Texture2D tex = CreateUninitializedTexture2D(2, 2, TextureFormat.ARGB32, mipChain: false); if (!ImageConversion.LoadImage(tex, managed, markNonReadable: false)) { UnityEngine.Object.Destroy(tex); @@ -1884,7 +1947,7 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) if (isNormalMap) { bool isPot = Numerics.IsPowerOfTwo(tex.width) && Numerics.IsPowerOfTwo(tex.height); - Texture2D dst = new Texture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); + Texture2D dst = CreateUninitializedTexture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); dst.wrapMode = TextureWrapMode.Repeat; NativeArray srcData = tex.GetRawTextureData(); @@ -2001,7 +2064,7 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isNormalMap) { bool isPot = Numerics.IsPowerOfTwo(texture.width) && Numerics.IsPowerOfTwo(texture.height); - Texture2D dst = new Texture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); + Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); dst.wrapMode = TextureWrapMode.Repeat; NativeArray srcData = texture.GetRawTextureData(); @@ -2851,7 +2914,7 @@ public bool TryCreateTexture(byte[] buffer, out Texture2D texture) { try { - texture = new Texture2D(width, height, GraphicsFormat.RGBA_DXT5_UNorm, mipCount, mipCount == 1 ? TextureCreationFlags.None : TextureCreationFlags.MipChain); + texture = CreateUninitializedTexture2D(width, height, mipCount, GraphicsFormat.RGBA_DXT5_UNorm); texture.LoadRawTextureData(buffer); texture.Apply(false, true); loaded = true; From d0fa7ba5c07462654e0d5a1970b6c940c1fe125a Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 02:28:31 -0700 Subject: [PATCH 03/21] Add some yields to avoid UnshareTextureData overhead --- KSPCommunityFixes/Performance/FastLoader.cs | 222 +++++++++++--------- 1 file changed, 124 insertions(+), 98 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 3807d414..8bc5e9d3 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -35,6 +35,7 @@ using Debug = UnityEngine.Debug; using UnityEngine.Profiling; using System.Threading.Tasks; +using KSP.UI; namespace KSPCommunityFixes.Performance { @@ -157,9 +158,14 @@ internal class KSPCFFastLoader : MonoBehaviour // max concurrent per-texture coroutines spawned by TextureDriverCoroutine. // Each in-flight request holds at most one of {ReadHandle, UnityWebRequest, background Task}, - // so a single semaphore covers all resource bounds. + // so this caps total in-flight resource usage. private const int MaxConcurrentTextures = 512; + // max new texture-load coroutines spawned in any single frame, regardless of how + // many completions free up queue slots that frame. Bounds per-frame allocation / + // dispatch overhead independently of MaxConcurrentTextures. + private const int MaxTextureSpawnsPerFrame = 128; + private static Harmony persistentHarmony; private static string PersistentHarmonyID => typeof(KSPCFFastLoader).FullName; @@ -1270,6 +1276,7 @@ private GameObject LoadDAE() private static readonly ProfilerMarker s_pmCopyLevel0 = new ProfilerMarker("KSPCF.Tex.CopyLevel0"); private static readonly ProfilerMarker s_pmFileSize = new ProfilerMarker("KSPCF.Tex.FileSize"); private static readonly ProfilerMarker s_pmReadAllBytes = new ProfilerMarker("KSPCF.Tex.ReadAllBytes"); + private static readonly ProfilerMarker s_pmCompress = new ProfilerMarker("KSPCF.Tex.Compress"); // Result/error carrier for each texture file. Replaces RawAsset for textures. private sealed class TextureLoadRequest @@ -1663,54 +1670,48 @@ private static Texture2D CreateUninitializedTexture2D( return tex; } - // Wraps an inner format-specific coroutine with exception capture and semaphore release. + // Wraps an inner format-specific coroutine with exception capture. // C# does not allow yield inside a try/catch, so we manually drive MoveNext() and - // do the catch around just the MoveNext call. - private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, IEnumerator inner, SemaphoreSlim semaphore) + // do the catch around just the MoveNext call. The driver detects completion via + // req.Status, so no other signaling is required here. + private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, IEnumerator inner) { - try + while (true) { - while (true) + bool moved; + bool failed = false; + object current = null; + try { - bool moved; - bool failed = false; - object current = null; - try - { - moved = inner.MoveNext(); - if (moved) - current = inner.Current; - } - catch (Exception e) - { - req.Exception = e; - req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; - req.Status = TextureLoadRequest.State.Failed; - moved = false; - failed = true; - } - if (failed) - yield break; - if (!moved) - break; - yield return current; + moved = inner.MoveNext(); + if (moved) + current = inner.Current; } - - if (req.Status == TextureLoadRequest.State.Pending) + catch (Exception e) { - if (req.Result != null) - req.Status = TextureLoadRequest.State.Ready; - else - { - if (req.ErrorMessage == null) - req.ErrorMessage = "Loader produced no result"; - req.Status = TextureLoadRequest.State.Failed; - } + req.Exception = e; + req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; + req.Status = TextureLoadRequest.State.Failed; + moved = false; + failed = true; } + if (failed) + yield break; + if (!moved) + break; + yield return current; } - finally + + if (req.Status == TextureLoadRequest.State.Pending) { - semaphore.Release(); + if (req.Result != null) + req.Status = TextureLoadRequest.State.Ready; + else + { + if (req.ErrorMessage == null) + req.ErrorMessage = "Loader produced no result"; + req.Status = TextureLoadRequest.State.Failed; + } } } @@ -1833,6 +1834,9 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) if (isNormalMap) dst.wrapMode = TextureWrapMode.Repeat; + // Wait a frame to avoid UnshareTextureData overhead within Unity + yield return null; + NativeArray srcData = src.GetRawTextureData(); NativeArray dstData = dst.GetRawTextureData(); TextureFormat srcFormat = src.format; @@ -1874,7 +1878,10 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) } if (canCompress) - src.Compress(highQuality: !isNormalMap); + { + using (s_pmCompress.Auto()) + src.Compress(highQuality: !isNormalMap); + } else if (!isNormalMap) Debug.LogWarning($"Texture '{req.File.url}' isn't eligible for DXT compression, width and height must be multiples of 4"); @@ -1950,6 +1957,9 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) Texture2D dst = CreateUninitializedTexture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); dst.wrapMode = TextureWrapMode.Repeat; + // Wait a frame so we avoid a call to UnshareTextureData within unity + yield return null; + NativeArray srcData = tex.GetRawTextureData(); NativeArray dstData = dst.GetRawTextureData(); TextureFormat srcFormat = tex.format; @@ -1972,7 +1982,8 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) if (isPot) { tex.Apply(updateMipmaps: true); - tex.Compress(highQuality: false); + using (s_pmCompress.Auto()) + tex.Compress(highQuality: false); } tex.Apply(updateMipmaps: false, makeNoLongerReadable: true); @@ -2067,6 +2078,9 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); dst.wrapMode = TextureWrapMode.Repeat; + // Avoid UnshareTextureData overhead within Unity by waiting a frame. + yield return null; + NativeArray srcData = texture.GetRawTextureData(); NativeArray dstData = dst.GetRawTextureData(); TextureFormat srcFormat = texture.format; @@ -2089,7 +2103,8 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isPot) { texture.Apply(updateMipmaps: true); - texture.Compress(highQuality: false); + using (s_pmCompress.Auto()) + texture.Compress(highQuality: false); } texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); } @@ -2152,74 +2167,85 @@ private static void FallBackFromCache(TextureLoadRequest req, CachedTextureInfo private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) { GameDatabase gdb = GameDatabase.Instance; - SemaphoreSlim semaphore = new SemaphoreSlim(MaxConcurrentTextures, MaxConcurrentTextures); + Queue active = new Queue(MaxConcurrentTextures); int spawnIdx = 0; - int completeIdx = 0; int total = requests.Count; double nextFrameTime = ElapsedTime + minFrameTimeD; - while (completeIdx < total) + while (spawnIdx < total || active.Count > 0) { - while (spawnIdx < total && semaphore.Wait(0)) + // Drain completed requests at the front of the queue. + while (active.Count > 0 && active.Peek().Status != TextureLoadRequest.State.Pending) { - TextureLoadRequest req = requests[spawnIdx++]; - IEnumerator inner; - switch (req.AssetType) - { - case RawAsset.AssetType.TextureDDS: - inner = LoadDDSCoroutine(req); - break; - case RawAsset.AssetType.TexturePNG: - case RawAsset.AssetType.TextureJPG: - inner = LoadUWRCoroutine(req); - break; - case RawAsset.AssetType.TextureTRUECOLOR: - inner = LoadTRUECOLORCoroutine(req); - break; - case RawAsset.AssetType.TextureMBM: - inner = LoadMBMCoroutine(req); - break; - case RawAsset.AssetType.TextureTGA: - inner = LoadTGACoroutine(req); - break; - case RawAsset.AssetType.TexturePNGCached: - inner = LoadPNGCachedCoroutine(req); - break; - default: - inner = null; - break; - } + InsertReadyRequest(active.Peek(), loadedUrls); + active.Dequeue(); + loadedAssetCount++; + } - if (inner == null) - { - req.ErrorMessage = $"Unknown asset type {req.AssetType}"; - req.Status = TextureLoadRequest.State.Failed; - semaphore.Release(); - } - else - { - Debug.Log($"Load Texture: {req.File.url}"); - gdb.StartCoroutine(LoadTextureWrapperCoroutine(req, inner, semaphore)); - } + // Spawn new coroutines, bounded by the in-flight cap and the per-frame cap. + int spawnsThisFrame = 0; + while (spawnIdx < total + && active.Count < MaxConcurrentTextures + && spawnsThisFrame < MaxTextureSpawnsPerFrame) + { + SpawnTextureCoroutine(requests[spawnIdx++], active, gdb); + spawnsThisFrame++; } - TextureLoadRequest head = requests[completeIdx]; - if (head.Status == TextureLoadRequest.State.Pending) + if (active.Count == 0 && spawnIdx >= total) + break; + + if (ElapsedTime > nextFrameTime) { - if (ElapsedTime > nextFrameTime) - { - nextFrameTime = ElapsedTime + minFrameTimeD; - gdb.progressFraction = (float)(loadedAssetCount + completeIdx) / totalAssetCount; - gdb.progressTitle = $"Loading texture asset {completeIdx}/{total}"; - } - yield return null; - continue; + nextFrameTime = ElapsedTime + minFrameTimeD; + int completed = spawnIdx - active.Count; + gdb.progressFraction = (float)(loadedAssetCount + completed) / totalAssetCount; + gdb.progressTitle = $"Loading texture asset {completed}/{total}"; } + yield return null; + } + } - InsertReadyRequest(head, loadedUrls); - completeIdx++; - loadedAssetCount++; + private static void SpawnTextureCoroutine(TextureLoadRequest req, Queue active, GameDatabase gdb) + { + IEnumerator inner; + switch (req.AssetType) + { + case RawAsset.AssetType.TextureDDS: + inner = LoadDDSCoroutine(req); + break; + case RawAsset.AssetType.TexturePNG: + case RawAsset.AssetType.TextureJPG: + inner = LoadUWRCoroutine(req); + break; + case RawAsset.AssetType.TextureTRUECOLOR: + inner = LoadTRUECOLORCoroutine(req); + break; + case RawAsset.AssetType.TextureMBM: + inner = LoadMBMCoroutine(req); + break; + case RawAsset.AssetType.TextureTGA: + inner = LoadTGACoroutine(req); + break; + case RawAsset.AssetType.TexturePNGCached: + inner = LoadPNGCachedCoroutine(req); + break; + default: + inner = null; + break; + } + + if (inner == null) + { + req.ErrorMessage = $"Unknown asset type {req.AssetType}"; + req.Status = TextureLoadRequest.State.Failed; + } + else + { + Debug.Log($"Load Texture: {req.File.url}"); + gdb.StartCoroutine(LoadTextureWrapperCoroutine(req, inner)); } + active.Enqueue(req); } private static void InsertReadyRequest(TextureLoadRequest req, HashSet loadedUrls) From d6572c5b44a7119ac39c33e7c7228ef8db55a24b Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 19:34:28 -0700 Subject: [PATCH 04/21] Add some more tracing spans --- KSPCommunityFixes/Performance/FastLoader.cs | 45 ++++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 8bc5e9d3..e7b2d207 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -159,7 +159,7 @@ internal class KSPCFFastLoader : MonoBehaviour // max concurrent per-texture coroutines spawned by TextureDriverCoroutine. // Each in-flight request holds at most one of {ReadHandle, UnityWebRequest, background Task}, // so this caps total in-flight resource usage. - private const int MaxConcurrentTextures = 512; + private const int MaxConcurrentTextures = 16384; // max new texture-load coroutines spawned in any single frame, regardless of how // many completions free up queue slots that frame. Bounds per-frame allocation / @@ -1277,6 +1277,11 @@ private GameObject LoadDAE() private static readonly ProfilerMarker s_pmFileSize = new ProfilerMarker("KSPCF.Tex.FileSize"); private static readonly ProfilerMarker s_pmReadAllBytes = new ProfilerMarker("KSPCF.Tex.ReadAllBytes"); private static readonly ProfilerMarker s_pmCompress = new ProfilerMarker("KSPCF.Tex.Compress"); + private static readonly ProfilerMarker s_pmGetRawDataDDS = new ProfilerMarker("KSPCF.Tex.LoadDDS.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataUWR = new ProfilerMarker("KSPCF.Tex.LoadUWR.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataTGA = new ProfilerMarker("KSPCF.Tex.LoadTGA.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataPNGCached = new ProfilerMarker("KSPCF.Tex.LoadPNGCached.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataSaveCache = new ProfilerMarker("KSPCF.Tex.SaveCache.GetRawTextureData"); // Result/error carrier for each texture file. Replaces RawAsset for textures. private sealed class TextureLoadRequest @@ -1760,7 +1765,9 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) // before AsyncReadManager writes into the staging buffer. yield return null; - NativeArray dst = tex.GetRawTextureData(); + NativeArray dst; + using (s_pmGetRawDataDDS.Auto()) + dst = tex.GetRawTextureData(); long expectedSize = dst.Length; if (hdr.FileLength - hdr.DataOffset < expectedSize) { @@ -1837,8 +1844,13 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) // Wait a frame to avoid UnshareTextureData overhead within Unity yield return null; - NativeArray srcData = src.GetRawTextureData(); - NativeArray dstData = dst.GetRawTextureData(); + NativeArray srcData; + NativeArray dstData; + using (s_pmGetRawDataUWR.Auto()) + { + srcData = src.GetRawTextureData(); + dstData = dst.GetRawTextureData(); + } TextureFormat srcFormat = src.format; Task task; @@ -1942,7 +1954,7 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) data.Dispose(); Texture2D tex = CreateUninitializedTexture2D(2, 2, TextureFormat.ARGB32, mipChain: false); - if (!ImageConversion.LoadImage(tex, managed, markNonReadable: false)) + if (!tex.LoadImage(managed, markNonReadable: false)) { UnityEngine.Object.Destroy(tex); req.ErrorMessage = "TRUECOLOR: ImageConversion.LoadImage failed"; @@ -1960,8 +1972,13 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) // Wait a frame so we avoid a call to UnshareTextureData within unity yield return null; - NativeArray srcData = tex.GetRawTextureData(); - NativeArray dstData = dst.GetRawTextureData(); + NativeArray srcData; + NativeArray dstData; + using (s_pmGetRawDataTGA.Auto()) + { + srcData = tex.GetRawTextureData(); + dstData = dst.GetRawTextureData(); + } TextureFormat srcFormat = tex.format; Task swizzleTask = Task.Run(() => { @@ -2081,8 +2098,14 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) // Avoid UnshareTextureData overhead within Unity by waiting a frame. yield return null; - NativeArray srcData = texture.GetRawTextureData(); - NativeArray dstData = dst.GetRawTextureData(); + NativeArray srcData; + NativeArray dstData; + using (s_pmGetRawDataPNGCached.Auto()) + { + srcData = texture.GetRawTextureData(); + dstData = dst.GetRawTextureData(); + } + TextureFormat srcFormat = texture.format; Task swizzleTask = Task.Run(() => { @@ -2932,7 +2955,9 @@ public CachedTextureInfo(UrlFile urlFile, Texture2D texture, bool isNormalMap, l public void SaveRawTextureData(Texture2D texture) { - byte[] rawData = texture.GetRawTextureData(); + byte[] rawData; + using (s_pmGetRawDataSaveCache.Auto()) + rawData = texture.GetRawTextureData(); File.WriteAllBytes(Path.Combine(loader.textureCachePath, id.ToString()), rawData); } From 7fa2c5bc3c2ecfc3e115d14fbf6ae87674ef9480 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 20:09:18 -0700 Subject: [PATCH 05/21] Clean up some messy code --- KSPCommunityFixes/Performance/FastLoader.cs | 43 ++++++++++----------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index e7b2d207..7eb4a9bc 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -36,6 +36,7 @@ using UnityEngine.Profiling; using System.Threading.Tasks; using KSP.UI; +using System.Security.Cryptography; namespace KSPCommunityFixes.Performance { @@ -1683,40 +1684,36 @@ private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, I { while (true) { - bool moved; - bool failed = false; - object current = null; + object current; try { - moved = inner.MoveNext(); - if (moved) - current = inner.Current; + if (!inner.MoveNext()) + break; + + current = inner.Current; } catch (Exception e) { req.Exception = e; req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; req.Status = TextureLoadRequest.State.Failed; - moved = false; - failed = true; - } - if (failed) yield break; - if (!moved) - break; + } + yield return current; } - if (req.Status == TextureLoadRequest.State.Pending) + if (req.Status != TextureLoadRequest.State.Pending) + yield break; + + if (req.Result != null) { - if (req.Result != null) - req.Status = TextureLoadRequest.State.Ready; - else - { - if (req.ErrorMessage == null) - req.ErrorMessage = "Loader produced no result"; - req.Status = TextureLoadRequest.State.Failed; - } + req.Status = TextureLoadRequest.State.Ready; + } + else + { + req.ErrorMessage ??= "Loader produced no result"; + req.Status = TextureLoadRequest.State.Failed; } } @@ -1725,8 +1722,8 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) string path = req.File.fullPath; Task hdrTask = Task.Run(() => { - using (s_pmParseDDSHeader.Auto()) - return ParseDDSHeader(path); + using var scope = s_pmParseDDSHeader.Auto(); + return ParseDDSHeader(path); }); while (!hdrTask.IsCompleted) yield return null; From e5123194a6fc3b9a0be5640585555432c0e98125 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 20:36:02 -0700 Subject: [PATCH 06/21] Simplify a comment --- KSPCommunityFixes/Performance/FastLoader.cs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 7eb4a9bc..905bf792 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -157,14 +157,8 @@ internal class KSPCFFastLoader : MonoBehaviour // min amount of files to try to keep in memory, regardless of maxBufferSize private const int minFileRead = 10; - // max concurrent per-texture coroutines spawned by TextureDriverCoroutine. - // Each in-flight request holds at most one of {ReadHandle, UnityWebRequest, background Task}, - // so this caps total in-flight resource usage. - private const int MaxConcurrentTextures = 16384; - - // max new texture-load coroutines spawned in any single frame, regardless of how - // many completions free up queue slots that frame. Bounds per-frame allocation / - // dispatch overhead independently of MaxConcurrentTextures. + // Max number of new texture load coroutines that will be spawned each frame. + // This should roughly limit the max frame time spent on loading textures. private const int MaxTextureSpawnsPerFrame = 128; private static Harmony persistentHarmony; @@ -2187,7 +2181,7 @@ private static void FallBackFromCache(TextureLoadRequest req, CachedTextureInfo private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) { GameDatabase gdb = GameDatabase.Instance; - Queue active = new Queue(MaxConcurrentTextures); + Queue active = new Queue(); int spawnIdx = 0; int total = requests.Count; double nextFrameTime = ElapsedTime + minFrameTimeD; @@ -2204,9 +2198,7 @@ private static IEnumerator TextureDriverCoroutine(List reque // Spawn new coroutines, bounded by the in-flight cap and the per-frame cap. int spawnsThisFrame = 0; - while (spawnIdx < total - && active.Count < MaxConcurrentTextures - && spawnsThisFrame < MaxTextureSpawnsPerFrame) + while (spawnIdx < total && spawnsThisFrame < MaxTextureSpawnsPerFrame) { SpawnTextureCoroutine(requests[spawnIdx++], active, gdb); spawnsThisFrame++; From 08c32f5eb38ac676dcc98be44caca2eed58c2972 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Wed, 29 Apr 2026 23:48:04 -0700 Subject: [PATCH 07/21] Allow NPOT dds textures to have mipmaps --- KSPCommunityFixes/Performance/FastLoader.cs | 36 ++------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 905bf792..63c92eb1 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1366,31 +1366,7 @@ public static void Build() public static bool IsSupported(GraphicsFormat fmt) => supported != null && supported.Contains(fmt); } - // Block-compressed format check used to ignore mipChain on NPOT textures - // (Unity rejects DXT5 + mipmaps on non-power-of-two sources). - private static bool IsBlockCompressed(GraphicsFormat fmt) - { - switch (fmt) - { - case GraphicsFormat.RGBA_DXT1_UNorm: - case GraphicsFormat.RGBA_DXT1_SRGB: - case GraphicsFormat.RGBA_DXT5_UNorm: - case GraphicsFormat.RGBA_DXT5_SRGB: - case GraphicsFormat.R_BC4_UNorm: - case GraphicsFormat.R_BC4_SNorm: - case GraphicsFormat.RG_BC5_UNorm: - case GraphicsFormat.RG_BC5_SNorm: - case GraphicsFormat.RGBA_BC7_UNorm: - case GraphicsFormat.RGBA_BC7_SRGB: - case GraphicsFormat.RGB_BC6H_SFloat: - case GraphicsFormat.RGB_BC6H_UFloat: - return true; - default: - return false; - } - } - - // Background DDS header reader. Throws on bad magic or unsupported format. +// Background DDS header reader. Throws on bad magic or unsupported format. // Does not call any Unity API (so it is safe on a worker thread). private static DDSPreparedHeader ParseDDSHeader(string path) { @@ -1733,17 +1709,9 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) yield break; } - // Unity rejects mipChain on NPOT textures for block-compressed formats. - bool mipChain = hdr.MipChain; - if (mipChain && IsBlockCompressed(hdr.Format) - && !(Numerics.IsPowerOfTwo(hdr.Width) && Numerics.IsPowerOfTwo(hdr.Height))) - { - mipChain = false; - } - Texture2D tex = CreateUninitializedTexture2D( hdr.Width, hdr.Height, - mipChain ? -1 : 1, + hdr.MipChain ? -1 : 1, hdr.Format); if (tex.IsNullOrDestroyed()) { From 3e3b196f6eae96fc83cb5cdbe07414c7bac73db6 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 00:02:10 -0700 Subject: [PATCH 08/21] Cleanup cached PNG path --- KSPCommunityFixes/GlobalSuppressions.cs | 1 - KSPCommunityFixes/Internal/PatchSettings.cs | 18 -- KSPCommunityFixes/Performance/FastLoader.cs | 338 +------------------- 3 files changed, 9 insertions(+), 348 deletions(-) diff --git a/KSPCommunityFixes/GlobalSuppressions.cs b/KSPCommunityFixes/GlobalSuppressions.cs index 2c49b003..431d1572 100644 --- a/KSPCommunityFixes/GlobalSuppressions.cs +++ b/KSPCommunityFixes/GlobalSuppressions.cs @@ -9,6 +9,5 @@ [assembly: SuppressMessage("Style", "IDE0017:Simplify object initialization")] [assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types")] [assembly: SuppressMessage("Style", "IDE0016:Use 'throw' expression", Justification = "", Scope = "member", Target = "~M:KSPCommunityFixes.BasePatch.PatchInfo.#ctor(KSPCommunityFixes.BasePatch.PatchMethodType,System.Reflection.MethodBase,KSPCommunityFixes.BasePatch,System.String,System.Int32)")] -[assembly: SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "", Scope = "type", Target = "~T:KSPCommunityFixes.TextureLoaderOptimizations.CachedTextureInfo")] [assembly: SuppressMessage("Style", "IDE0028:Simplify collection initialization", Justification = "", Scope = "member", Target = "~M:KSPCommunityFixes.LocalizationUtils.GenerateLocTemplate(System.String)")] diff --git a/KSPCommunityFixes/Internal/PatchSettings.cs b/KSPCommunityFixes/Internal/PatchSettings.cs index 700076c4..554625f9 100644 --- a/KSPCommunityFixes/Internal/PatchSettings.cs +++ b/KSPCommunityFixes/Internal/PatchSettings.cs @@ -38,9 +38,6 @@ protected override void ApplyPatches() if (disableMHPatch != null) entryCount++; - if (KSPCFFastLoader.IsPatchEnabled) - entryCount++; - // NoIVA is always enabled entryCount++; } @@ -107,21 +104,6 @@ static void GameplaySettingsScreen_DrawMiniSettings_Postfix(ref DialogGUIBase[] count++; } - if (KSPCFFastLoader.IsPatchEnabled) - { - DialogGUIToggle toggle = new DialogGUIToggle(KSPCFFastLoader.TextureCacheEnabled, - () => (KSPCFFastLoader.TextureCacheEnabled) - ? Localizer.Format("#autoLOC_900889") //"Enabled" - : Localizer.Format("#autoLOC_900890"), //"Disabled" - KSPCFFastLoader.OnToggleCacheFromSettings, 150f); - toggle.tooltipText = KSPCFFastLoader.LOC_SettingsTooltip; - - modifiedResult[count] = new DialogGUIHorizontalLayout(TextAnchor.MiddleLeft, - new DialogGUILabel(() => KSPCFFastLoader.LOC_SettingsTitle, 150f), - toggle, new DialogGUIFlexibleSpace()); - count++; - } - DialogGUISlider noIVAslider = new DialogGUISlider(NoIVA.PatchStateToFloat, 0f, 2f, true, 100f, 20f, NoIVA.SwitchPatchState); noIVAslider.tooltipText = NoIVA.LOC_SettingsTooltip; DialogGUILabel valueLabel = new DialogGUILabel(NoIVA.PatchStateTitle); diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 63c92eb1..90d3a0fe 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -130,12 +130,6 @@ void OnDestroy() [KSPAddon(KSPAddon.Startup.Instantly, true)] internal class KSPCFFastLoader : MonoBehaviour { - public static string LOC_SettingsTitle = "Texture caching optimization"; - public static string LOC_SettingsTooltip = - "Cache PNG textures on disk instead of converting them on every KSP launch." + - "\nSpeedup loading time but increase disk space usage." + - "\nChanges will take effect after relaunching KSP"; - public static string LOC_PopupL1 = "KSPCommunityFixes can cache converted PNG textures on disk to speed up loading time."; public static string LOC_F_PopupL2 = @@ -173,22 +167,14 @@ internal class KSPCFFastLoader : MonoBehaviour public static KSPCFFastLoader loader; public static bool IsPatchEnabled { get; private set; } - public static bool TextureCacheEnabled => textureCacheEnabled; + // Vestigial: kept so the popup can persist its choice across launches once it is repurposed. private static bool textureCacheEnabled; private static string ModPath => Path.GetDirectoryName(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); private static string ConfigPath => Path.Combine(ModPath, "PluginData", "PNGTextureCache.cfg"); private bool userOptInChoiceDone; - private const string textureCacheVersion = "V2"; private string configPath; - private string textureCachePath; - private string textureCacheDataPath; - private string textureProgressMarkerPath; - - private Dictionary textureCacheData; - private HashSet textureDataIds; - private bool cacheUpdated = false; internal static Dictionary modelsByUrl; internal static Dictionary modelsByDirectoryUrl; @@ -257,7 +243,6 @@ private void Awake() GameEvents.OnGameDatabaseLoaded.Add(OnGameDatabaseLoaded); configPath = ConfigPath; - textureCachePath = Path.Combine(ModPath, "PluginData", "TextureCache"); if (File.Exists(configPath)) { @@ -269,17 +254,6 @@ private void Awake() if (!config.TryGetValue(nameof(textureCacheEnabled), ref textureCacheEnabled)) userOptInChoiceDone = false; } - -#if DEBUG && !DEBUG_TEXTURE_CACHE - userOptInChoiceDone = true; - textureCacheEnabled = false; -#endif - } - - void Start() - { - if (IsPatchEnabled && !userOptInChoiceDone) - StartCoroutine(WaitForUserOptIn()); } /// @@ -468,10 +442,6 @@ static IEnumerator FastAssetLoader(List configFileTypes) KSPCFFastLoaderReport.wConfigTranslate.Stop(); yield return null; - gdb.progressTitle = "Waiting for PNGTextureCache opt-in..."; - while (!loader.userOptInChoiceDone) - yield return null; - // Start load asset bundles in the background while we load other assets. PreloadAssetBundleObjects(gdb); @@ -577,9 +547,6 @@ static IEnumerator FastAssetLoader(List configFileTypes) } } - // PNG/DDS cache temporarily disabled — opt-in popup and settings are still - // wired up, but no cache I/O is performed during loading. - gdb.progressTitle = "Loading sound assets..."; KSPCFFastLoaderReport.wAudioLoading.Restart(); yield return null; @@ -1022,7 +989,6 @@ public enum AssetType TextureJPG, TextureMBM, TexturePNG, - TexturePNGCached, TextureTGA, TextureTRUECOLOR, ModelMU, @@ -1035,7 +1001,6 @@ public enum AssetType "JPG texture", "MBM texture", "PNG texture", - "Cached PNG Texture", "TGA texture", "TRUECOLOR texture", "MU model", @@ -1274,9 +1239,8 @@ private GameObject LoadDAE() private static readonly ProfilerMarker s_pmCompress = new ProfilerMarker("KSPCF.Tex.Compress"); private static readonly ProfilerMarker s_pmGetRawDataDDS = new ProfilerMarker("KSPCF.Tex.LoadDDS.GetRawTextureData"); private static readonly ProfilerMarker s_pmGetRawDataUWR = new ProfilerMarker("KSPCF.Tex.LoadUWR.GetRawTextureData"); + private static readonly ProfilerMarker s_pmGetRawDataTRUECOLOR = new ProfilerMarker("KSPCF.Tex.LoadTRUECOLOR.GetRawTextureData"); private static readonly ProfilerMarker s_pmGetRawDataTGA = new ProfilerMarker("KSPCF.Tex.LoadTGA.GetRawTextureData"); - private static readonly ProfilerMarker s_pmGetRawDataPNGCached = new ProfilerMarker("KSPCF.Tex.LoadPNGCached.GetRawTextureData"); - private static readonly ProfilerMarker s_pmGetRawDataSaveCache = new ProfilerMarker("KSPCF.Tex.SaveCache.GetRawTextureData"); // Result/error carrier for each texture file. Replaces RawAsset for textures. private sealed class TextureLoadRequest @@ -1286,8 +1250,6 @@ public enum State : byte { Pending, Ready, Failed } public UrlFile File; public RawAsset.AssetType AssetType; public long FileLength; - public CachedTextureInfo CachedInfo; - public bool CameFromCache; public volatile State Status; public TextureInfo Result; public string ErrorMessage; @@ -1299,17 +1261,6 @@ public TextureLoadRequest(UrlFile file, RawAsset.AssetType assetType) AssetType = assetType; Status = State.Pending; } - - // Called from the texture cache reader thread before the driver runs. - public void CheckTextureCache() - { - CachedTextureInfo info = GetCachedTextureInfo(File); - if (info == null) - return; - AssetType = RawAsset.AssetType.TexturePNGCached; - CachedInfo = info; - CameFromCache = true; - } } // Result of background DDS header parsing. @@ -1933,7 +1884,7 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) NativeArray srcData; NativeArray dstData; - using (s_pmGetRawDataTGA.Auto()) + using (s_pmGetRawDataTRUECOLOR.Auto()) { srcData = tex.GetRawTextureData(); dstData = dst.GetRawTextureData(); @@ -2059,7 +2010,7 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) NativeArray srcData; NativeArray dstData; - using (s_pmGetRawDataPNGCached.Auto()) + using (s_pmGetRawDataTGA.Auto()) { srcData = texture.GetRawTextureData(); dstData = dst.GetRawTextureData(); @@ -2095,57 +2046,6 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) req.Status = TextureLoadRequest.State.Ready; } - private static IEnumerator LoadPNGCachedCoroutine(TextureLoadRequest req) - { - CachedTextureInfo info = req.CachedInfo; - string path = info.FilePath; - Task readTask = Task.Run(() => - { - using (s_pmReadAllBytes.Auto()) - return File.ReadAllBytes(path); - }); - while (!readTask.IsCompleted) - yield return null; - if (readTask.IsFaulted) - { - Debug.LogWarning($"[KSPCF] Cached PNG read failed for '{req.File.url}', falling back to fresh load"); - FallBackFromCache(req, info); - IEnumerator fallback = LoadUWRCoroutine(req); - while (fallback.MoveNext()) - yield return fallback.Current; - yield break; - } - - byte[] buffer = readTask.Result; - req.FileLength = buffer.Length; - - if (info.TryCreateTexture(buffer, out Texture2D texture)) - { - req.Result = new TextureInfo(req.File, texture, info.normal, isReadable: false, isCompressed: true); - req.Status = TextureLoadRequest.State.Ready; - yield break; - } - - Debug.LogWarning($"[KSPCF] Cached PNG TryCreateTexture failed for '{req.File.url}', falling back to fresh load"); - FallBackFromCache(req, info); - IEnumerator fallback2 = LoadUWRCoroutine(req); - while (fallback2.MoveNext()) - yield return fallback2.Current; - } - - private static void FallBackFromCache(TextureLoadRequest req, CachedTextureInfo info) - { - if (loader.textureCacheData.Remove(info.name)) - { - loader.textureDataIds.Remove(info.id); - try { File.Delete(info.FilePath); } catch { } - loader.cacheUpdated = true; - } - req.AssetType = RawAsset.AssetType.TexturePNG; - req.CameFromCache = false; - req.CachedInfo = null; - } - private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) { GameDatabase gdb = GameDatabase.Instance; @@ -2207,9 +2107,6 @@ private static void SpawnTextureCoroutine(TextureLoadRequest req, Queue instructions } #endregion - #region PNG texture cache - - private static void SetupTextureCacheThread(List requests) - { - loader.SetupTextureCache(); - - foreach (TextureLoadRequest req in requests) - req.CheckTextureCache(); - } - - private void SetupTextureCache() - { - textureCacheDataPath = Path.Combine(textureCachePath, "textureData.json"); - textureProgressMarkerPath = Path.Combine(textureCachePath, "progressMarker"); - - textureCacheData = new Dictionary(2000); - textureDataIds = new HashSet(2000); - - if (Directory.Exists(textureCachePath)) - { - if (File.Exists(textureProgressMarkerPath)) - { - // If progress marker is still here, the game somehow crashed during loading on - // the previous run, so we delete the whole cache to avoid orphan cached texture - // files from lying around - Directory.Delete(textureCachePath, true); - Directory.CreateDirectory(textureCachePath); - } - else if (File.Exists(textureCacheDataPath)) - { - string[] textureCacheDataContent = File.ReadAllLines(textureCacheDataPath); - - if (textureCacheDataContent.Length > 0 && textureCacheDataContent[0].StartsWith(textureCacheVersion)) - { - for (int i = 1; i < textureCacheDataContent.Length; i++) - { - string json = textureCacheDataContent[i]; - CachedTextureInfo cachedTextureInfo = JsonUtility.FromJson(json); - textureCacheData.Add(cachedTextureInfo.name, cachedTextureInfo); - textureDataIds.Add(cachedTextureInfo.id); - } - } - else - { - Directory.Delete(textureCachePath, true); - Directory.CreateDirectory(textureCachePath); - } - } - } - else - { - Directory.CreateDirectory(textureCachePath); - } - - File.WriteAllText(textureProgressMarkerPath, string.Empty); - } - - private void WriteTextureCache() - { - if (!userOptInChoiceDone || !textureCacheEnabled) - { - if (Directory.Exists(textureCachePath)) - Directory.Delete(textureCachePath, true); - } - else - { - foreach (CachedTextureInfo cachedTextureInfo in textureCacheData.Values) - { - if (!cachedTextureInfo.loaded) - { - cacheUpdated = true; - File.Delete(cachedTextureInfo.FilePath); - } - } + #region User opt-in popup (vestigial) - if (cacheUpdated) - { - File.Delete(textureCacheDataPath); - - List textureCacheDataContent = new List(textureCacheData.Count + 1); - textureCacheDataContent.Add(textureCacheVersion); - - foreach (CachedTextureInfo cachedTextureInfo in textureCacheData.Values) - if (cachedTextureInfo.loaded) - textureCacheDataContent.Add(JsonUtility.ToJson(cachedTextureInfo)); - - File.WriteAllLines(textureCacheDataPath, textureCacheDataContent); - } - - File.Delete(textureProgressMarkerPath); - } - } - - [Serializable] - private class CachedTextureInfo - { - private static readonly System.Random random = new System.Random(); - - public string name; - public uint id; - public long time; - public long size; - public int width; - public int height; - public int mipCount; - public bool readable; - public bool normal; - [NonSerialized] public bool loaded = false; - - public string FilePath => Path.Combine(loader.textureCachePath, id.ToString()); - - public CachedTextureInfo() { } - - public CachedTextureInfo(UrlFile urlFile, Texture2D texture, bool isNormalMap, long size, long time) - { - name = urlFile.url; - do - { - unchecked - { - id = (uint)random.Next(); - } - } - while (loader.textureDataIds.Contains(id)); - - this.size = size; - this.time = time; - width = texture.width; - height = texture.height; - mipCount = texture.mipmapCount; - normal = isNormalMap; - // Always non-readable: matches the unified DDS/PNG/JPG readability policy. - // The 'readable' field in older cache entries on disk is ignored on read. - readable = false; - loaded = true; - } - - public void SaveRawTextureData(Texture2D texture) - { - byte[] rawData; - using (s_pmGetRawDataSaveCache.Auto()) - rawData = texture.GetRawTextureData(); - File.WriteAllBytes(Path.Combine(loader.textureCachePath, id.ToString()), rawData); - } - - public bool TryCreateTexture(byte[] buffer, out Texture2D texture) - { - try - { - texture = CreateUninitializedTexture2D(width, height, mipCount, GraphicsFormat.RGBA_DXT5_UNorm); - texture.LoadRawTextureData(buffer); - texture.Apply(false, true); - loaded = true; - return true; - } - catch (Exception e) - { - Debug.LogWarning($"[KSPCF] Failed to load cached PNG texture '{name}'\n{e}"); - texture = null; - return false; - } - } - } - - private static CachedTextureInfo GetCachedTextureInfo(UrlDir.UrlFile file) - { - if (!loader.textureCacheData.TryGetValue(file.url, out CachedTextureInfo cachedTextureInfo)) - return null; - - if (!GetFileStats(file.fullPath, out long size, out long time) || size != cachedTextureInfo.size || time != cachedTextureInfo.time) - { - loader.textureCacheData.Remove(file.url); - loader.textureDataIds.Remove(cachedTextureInfo.id); - File.Delete(cachedTextureInfo.FilePath); - loader.cacheUpdated = true; - return null; - } - - return cachedTextureInfo; - } - - private static bool GetFileStats(string path, out long size, out long time) - { - MonoIO.GetFileStat(path, out MonoIOStat stat, out MonoIOError error); - if (error == MonoIOError.ERROR_FILE_NOT_FOUND || error == MonoIOError.ERROR_PATH_NOT_FOUND || error == MonoIOError.ERROR_NOT_READY) - { - size = 0; - time = 0; - return false; - } - - size = stat.Length; - time = Math.Max(stat.CreationTime, stat.LastWriteTime); - if (size <= 0 || time <= 0) - return false; - - return true; - } - - private static void SaveCachedTextureFromBytes(UrlDir.UrlFile urlFile, Texture2D texture, bool isNormalMap, byte[] rawData) - { - if (!GetFileStats(urlFile.fullPath, out long size, out long creationTime)) - { - Debug.LogWarning($"[KSPCF] PNG texture '{urlFile.url}' couldn't be cached : IO error"); - return; - } - - CachedTextureInfo cachedTextureInfo = new CachedTextureInfo(urlFile, texture, isNormalMap, size, creationTime); - File.WriteAllBytes(Path.Combine(loader.textureCachePath, cachedTextureInfo.id.ToString()), rawData); - loader.textureCacheData.Add(cachedTextureInfo.name, cachedTextureInfo); - loader.textureDataIds.Add(cachedTextureInfo.id); - loader.cacheUpdated = true; - Debug.Log($"[KSPCF] PNG texture '{urlFile.url}' was converted to DXT5 and has been cached for future reloads"); - } + // The popup, opt-in flow, and PNG-cache-size estimator below are intentionally + // kept around — the cache they were originally tied to has been removed, but the + // popup is going to be repurposed for an upcoming feature. Nothing currently + // triggers WaitForUserOptIn; it must be invoked explicitly by the new feature. private static IEnumerator WaitForUserOptIn() { @@ -3068,15 +2757,6 @@ private static void SetOptIn(bool optIn, ref bool? choosed) config.Save(ConfigPath); } - internal static void OnToggleCacheFromSettings(bool cacheEnabled) - { - textureCacheEnabled = cacheEnabled; - ConfigNode config = new ConfigNode(); - config.AddValue(nameof(userOptInChoiceDone), true); - config.AddValue(nameof(textureCacheEnabled), cacheEnabled); - config.Save(ConfigPath); - } - private static readonly string flagsPath = Path.DirectorySeparatorChar + "Flags" + Path.DirectorySeparatorChar; private static bool GetPngCacheSize(string path, out int cacheSize, out bool isNormal) From 8825c33bb4ae6c52e45cdba1f13d771ee4872231 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 01:00:43 -0700 Subject: [PATCH 09/21] Avoid allocating secondary textures for swizzles --- KSPCommunityFixes/Performance/FastLoader.cs | 220 +++++++++++--------- 1 file changed, 119 insertions(+), 101 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 90d3a0fe..3e7f3674 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1233,7 +1233,6 @@ private GameObject LoadDAE() // appears under that thread in the Unity profiler. private static readonly ProfilerMarker s_pmParseDDSHeader = new ProfilerMarker("KSPCF.Tex.ParseDDSHeader"); private static readonly ProfilerMarker s_pmSwizzleNormalMap = new ProfilerMarker("KSPCF.Tex.SwizzleNormalMap"); - private static readonly ProfilerMarker s_pmCopyLevel0 = new ProfilerMarker("KSPCF.Tex.CopyLevel0"); private static readonly ProfilerMarker s_pmFileSize = new ProfilerMarker("KSPCF.Tex.FileSize"); private static readonly ProfilerMarker s_pmReadAllBytes = new ProfilerMarker("KSPCF.Tex.ReadAllBytes"); private static readonly ProfilerMarker s_pmCompress = new ProfilerMarker("KSPCF.Tex.Compress"); @@ -1317,7 +1316,7 @@ public static void Build() public static bool IsSupported(GraphicsFormat fmt) => supported != null && supported.Contains(fmt); } -// Background DDS header reader. Throws on bad magic or unsupported format. + // Background DDS header reader. Throws on bad magic or unsupported format. // Does not call any Unity API (so it is safe on a worker thread). private static DDSPreparedHeader ParseDDSHeader(string path) { @@ -1463,10 +1462,33 @@ private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeade } } - // Channel-swizzle for normal maps (extracted from BitmapToCompressedNormalMapFast). - // src must hold pixel data in srcFormat; dst is written as RGBA32 (level 0 only — - // if dst has a mip chain, higher mip levels are left untouched and are populated - // by the caller's Apply(updateMipmaps: true)). + // In-place channel-swizzle for RGBA32 normal maps. Operates on the texture's + // entire raw byte buffer, which means every populated mip level when the texture + // was created with a mip chain. + // + // Channel swizzle is per-pixel and the box-filter mip generator is linear, so + // swizzling pre-built mips matches what stock KSP produced by swizzling level-0 + // and regenerating from there (BitmapToCompressedNormalMapFast). + private static unsafe void SwizzleNormalMap(NativeArray data) + { + byte* p = (byte*)NativeArrayUnsafeUtility.GetUnsafePtr(data); + int len = data.Length; + // (r, g, b, a) -> (g, g, g, r) + for (int i = 0; i < len; i += 4) + { + byte r = p[i]; + byte g = p[i + 1]; + p[i] = g; + p[i + 1] = g; + p[i + 2] = g; + p[i + 3] = r; + } + } + + // Legacy src->dst swizzle, kept for the rare TGA RGB24 path (where source and + // destination have different pixel sizes so an in-place transform is impossible). + // Writes level 0 only; the caller is expected to call Apply(updateMipmaps: true) + // afterwards if a mip chain is wanted. private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) { byte* s = (byte*)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(src); @@ -1708,8 +1730,6 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) req.Status = TextureLoadRequest.State.Ready; } - private static readonly string flagsPathSep = Path.DirectorySeparatorChar + "Flags" + Path.DirectorySeparatorChar; - private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) { string filePath = req.File.fullPath; @@ -1737,66 +1757,33 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) } bool isNormalMap = req.File.name.EndsWith("NRM"); - bool isFlag = filePath.Contains(flagsPathSep); bool canCompress = src.width % 4 == 0 && src.height % 4 == 0; - if (isNormalMap || isFlag) + // UWR returns a Texture2D with a mipchain already populated, so for normal + // maps we swizzle every level of its CPU buffer in place — no dst alloc, + // no copy, no Apply(true). + if (isNormalMap) { - // Allocate a destination texture with mipchain (when POT) so that - // Apply(true) can generate mipmaps. UWR-loaded textures have no mipchain. - bool isPot = Numerics.IsPowerOfTwo(src.width) && Numerics.IsPowerOfTwo(src.height); - bool wantMipChain = isPot; - TextureFormat dstFormat = isNormalMap ? TextureFormat.RGBA32 : src.format; - Texture2D dst = CreateUninitializedTexture2D(src.width, src.height, dstFormat, wantMipChain); - if (isNormalMap) - dst.wrapMode = TextureWrapMode.Repeat; + src.wrapMode = TextureWrapMode.Repeat; // Wait a frame to avoid UnshareTextureData overhead within Unity yield return null; - NativeArray srcData; - NativeArray dstData; + NativeArray allLevels; using (s_pmGetRawDataUWR.Auto()) + allLevels = src.GetRawTextureData(); + Task swizzleTask = Task.Run(() => { - srcData = src.GetRawTextureData(); - dstData = dst.GetRawTextureData(); - } - TextureFormat srcFormat = src.format; - - Task task; - if (isNormalMap) - { - task = Task.Run(() => - { - using (s_pmSwizzleNormalMap.Auto()) - SwizzleNormalMap(srcData, dstData, srcFormat); - }); - } - else - { - // Flag: copy level-0 raw data byte-for-byte (formats already match). - int level0Bytes = srcData.Length; - task = Task.Run(() => - { - using (s_pmCopyLevel0.Auto()) - NativeArray.Copy(srcData, 0, dstData, 0, level0Bytes); - }); - } - - while (!task.IsCompleted) + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(allLevels); + }); + while (!swizzleTask.IsCompleted) yield return null; - if (task.IsFaulted) + if (swizzleTask.IsFaulted) { UnityEngine.Object.Destroy(src); - UnityEngine.Object.Destroy(dst); - throw UnwrapFaultedTask(task, "texture pre-process task faulted"); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); } - - UnityEngine.Object.Destroy(src); - src = dst; - - if (wantMipChain) - src.Apply(updateMipmaps: true); } if (canCompress) @@ -1863,7 +1850,11 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) byte[] managed = data.ToArray(); data.Dispose(); - Texture2D tex = CreateUninitializedTexture2D(2, 2, TextureFormat.ARGB32, mipChain: false); + // Create as RGBA32 with mipchain when this is a normal map: LoadImage will + // populate every mip level for us, so we can swizzle the whole thing in place. + // Non-normals keep the existing single-mip readable behavior. + bool isNormalMap = req.File.name.EndsWith("NRM"); + Texture2D tex = CreateUninitializedTexture2D(2, 2, TextureFormat.RGBA32, mipChain: isNormalMap); if (!tex.LoadImage(managed, markNonReadable: false)) { UnityEngine.Object.Destroy(tex); @@ -1872,43 +1863,32 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) yield break; } - bool isNormalMap = req.File.name.EndsWith("NRM"); if (isNormalMap) { bool isPot = Numerics.IsPowerOfTwo(tex.width) && Numerics.IsPowerOfTwo(tex.height); - Texture2D dst = CreateUninitializedTexture2D(tex.width, tex.height, TextureFormat.RGBA32, mipChain: isPot); - dst.wrapMode = TextureWrapMode.Repeat; + tex.wrapMode = TextureWrapMode.Repeat; // Wait a frame so we avoid a call to UnshareTextureData within unity yield return null; - NativeArray srcData; - NativeArray dstData; + NativeArray allLevels; using (s_pmGetRawDataTRUECOLOR.Auto()) - { - srcData = tex.GetRawTextureData(); - dstData = dst.GetRawTextureData(); - } - TextureFormat srcFormat = tex.format; + allLevels = tex.GetRawTextureData(); Task swizzleTask = Task.Run(() => { using (s_pmSwizzleNormalMap.Auto()) - SwizzleNormalMap(srcData, dstData, srcFormat); + SwizzleNormalMap(allLevels); }); while (!swizzleTask.IsCompleted) yield return null; if (swizzleTask.IsFaulted) { UnityEngine.Object.Destroy(tex); - UnityEngine.Object.Destroy(dst); throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); } - UnityEngine.Object.Destroy(tex); - tex = dst; if (isPot) { - tex.Apply(updateMipmaps: true); using (s_pmCompress.Auto()) tex.Compress(highQuality: false); } @@ -2002,44 +1982,82 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isNormalMap) { bool isPot = Numerics.IsPowerOfTwo(texture.width) && Numerics.IsPowerOfTwo(texture.height); - Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); - dst.wrapMode = TextureWrapMode.Repeat; - - // Avoid UnshareTextureData overhead within Unity by waiting a frame. - yield return null; - NativeArray srcData; - NativeArray dstData; - using (s_pmGetRawDataTGA.Auto()) + if (texture.format == TextureFormat.RGBA32) { - srcData = texture.GetRawTextureData(); - dstData = dst.GetRawTextureData(); - } + // tgaImage.CreateTexture(mipmap: true, ...) already calls Apply(true) + // and the texture is readable, so the CPU buffer holds every populated + // mip level. Swizzle the whole thing in place. + texture.wrapMode = TextureWrapMode.Repeat; - TextureFormat srcFormat = texture.format; - Task swizzleTask = Task.Run(() => - { - using (s_pmSwizzleNormalMap.Auto()) - SwizzleNormalMap(srcData, dstData, srcFormat); - }); - while (!swizzleTask.IsCompleted) + // Avoid UnshareTextureData overhead within Unity by waiting a frame. yield return null; - if (swizzleTask.IsFaulted) + + NativeArray allLevels; + using (s_pmGetRawDataTGA.Auto()) + allLevels = texture.GetRawTextureData(); + Task swizzleTask = Task.Run(() => + { + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(allLevels); + }); + while (!swizzleTask.IsCompleted) + yield return null; + if (swizzleTask.IsFaulted) + { + UnityEngine.Object.Destroy(texture); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); + } + + if (isPot) + { + using (s_pmCompress.Auto()) + texture.Compress(highQuality: false); + } + texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); + } + else { + // RGB24 (24bpp TGA): pixel size differs from RGBA32, so we can't + // swizzle in place. Fall back to the legacy src->dst expansion path. + Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); + dst.wrapMode = TextureWrapMode.Repeat; + + yield return null; + + NativeArray srcData; + NativeArray dstData; + using (s_pmGetRawDataTGA.Auto()) + { + srcData = texture.GetRawTextureData(); + dstData = dst.GetRawTextureData(); + } + + TextureFormat srcFormat = texture.format; + Task swizzleTask = Task.Run(() => + { + using (s_pmSwizzleNormalMap.Auto()) + SwizzleNormalMap(srcData, dstData, srcFormat); + }); + while (!swizzleTask.IsCompleted) + yield return null; + if (swizzleTask.IsFaulted) + { + UnityEngine.Object.Destroy(texture); + UnityEngine.Object.Destroy(dst); + throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); + } UnityEngine.Object.Destroy(texture); - UnityEngine.Object.Destroy(dst); - throw UnwrapFaultedTask(swizzleTask, "swizzle task faulted"); - } - UnityEngine.Object.Destroy(texture); - texture = dst; + texture = dst; - if (isPot) - { - texture.Apply(updateMipmaps: true); - using (s_pmCompress.Auto()) - texture.Compress(highQuality: false); + if (isPot) + { + texture.Apply(updateMipmaps: true); + using (s_pmCompress.Auto()) + texture.Compress(highQuality: false); + } + texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); } - texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); } req.Result = new TextureInfo(req.File, texture, isNormalMap, isReadable: !isNormalMap, isCompressed: true); From 36269c8bcb70f55f1dc45729cf318cf08c55f2f7 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 01:02:40 -0700 Subject: [PATCH 10/21] Actually compress TGA images --- KSPCommunityFixes/Performance/FastLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 3e7f3674..a9bdd610 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1970,7 +1970,7 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) yield break; } - Texture2D texture = tgaImage.CreateTexture(mipmap: true, linear: false, compress: true, compressHighQuality: false, allowRead: true); + Texture2D texture = tgaImage.CreateTexture(mipmap: true, linear: false, compress: true, compressHighQuality: true, allowRead: true); if (texture.IsNullOrDestroyed()) { req.ErrorMessage = "TGA: CreateTexture failed"; From 9a5717323e14dd90ddcb37e209803e844c113530 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 01:30:35 -0700 Subject: [PATCH 11/21] Fix a double-count in loaded textures --- KSPCommunityFixes/Performance/FastLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index a9bdd610..5e48bae0 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -2097,7 +2097,7 @@ private static IEnumerator TextureDriverCoroutine(List reque { nextFrameTime = ElapsedTime + minFrameTimeD; int completed = spawnIdx - active.Count; - gdb.progressFraction = (float)(loadedAssetCount + completed) / totalAssetCount; + gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; gdb.progressTitle = $"Loading texture asset {completed}/{total}"; } yield return null; From 4e6901351a8b09449fba19323b7d03c7fc682933 Mon Sep 17 00:00:00 2001 From: Sean Lynch Date: Thu, 30 Apr 2026 01:48:18 -0700 Subject: [PATCH 12/21] Fix an OOB write in the TGA loader --- KSPCommunityFixes/Performance/FastLoader.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 5e48bae0..2b7e5078 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1487,8 +1487,8 @@ private static unsafe void SwizzleNormalMap(NativeArray data) // Legacy src->dst swizzle, kept for the rare TGA RGB24 path (where source and // destination have different pixel sizes so an in-place transform is impossible). - // Writes level 0 only; the caller is expected to call Apply(updateMipmaps: true) - // afterwards if a mip chain is wanted. + // Walks src.Length end-to-end; the caller must size dst with a mip chain that + // matches src's so the constant 3:4 (or 4:4) byte-count ratio fills dst exactly. private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) { byte* s = (byte*)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(src); @@ -2019,8 +2019,11 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) else { // RGB24 (24bpp TGA): pixel size differs from RGBA32, so we can't - // swizzle in place. Fall back to the legacy src->dst expansion path. - Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: isPot); + // swizzle in place. Fall back to the legacy src->dst expansion + // path. dst is allocated with a full mip chain so its byte layout + // matches the mipmapped src (CreateTexture(mipmap: true) populates + // every level), letting the swizzle fill dst end-to-end. + Texture2D dst = CreateUninitializedTexture2D(texture.width, texture.height, TextureFormat.RGBA32, mipChain: true); dst.wrapMode = TextureWrapMode.Repeat; yield return null; @@ -2052,7 +2055,6 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isPot) { - texture.Apply(updateMipmaps: true); using (s_pmCompress.Auto()) texture.Compress(highQuality: false); } From aad8abe8ba5edd5188e6b757ab72ca29cf45ed3d Mon Sep 17 00:00:00 2001 From: Phantomical Date: Sat, 2 May 2026 14:45:58 -0700 Subject: [PATCH 13/21] Delay Load Texture log until the texture load has actually completed --- KSPCommunityFixes/Performance/FastLoader.cs | 177 ++++++++++++++------ 1 file changed, 126 insertions(+), 51 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 2b7e5078..667b9c5d 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1316,8 +1316,6 @@ public static void Build() public static bool IsSupported(GraphicsFormat fmt) => supported != null && supported.Contains(fmt); } - // Background DDS header reader. Throws on bad magic or unsupported format. - // Does not call any Unity API (so it is safe on a worker thread). private static DDSPreparedHeader ParseDDSHeader(string path) { FileInfo fi = new FileInfo(path); @@ -1336,7 +1334,7 @@ private static DDSPreparedHeader ParseDDSHeader(string path) bool isNormalMap = (hdr.ddspf.dwFlags & 0x80000u) != 0 || (hdr.ddspf.dwFlags & 0x80000000u) != 0; DDSHeaderDX10 dx10Header = default; - bool hasDx10 = (DDSFourCCBg)hdr.ddspf.dwFourCC == DDSFourCCBg.DX10; + bool hasDx10 = (DDSFourCC)hdr.ddspf.dwFourCC == DDSFourCC.DX10; if (hasDx10) { if (fileLength < 148) @@ -1361,8 +1359,7 @@ private static DDSPreparedHeader ParseDDSHeader(string path) }; } - // Background-thread-safe FourCC enum (mirrors RawAsset.DDSFourCC, which is private to RawAsset). - private enum DDSFourCCBg : uint + private enum DDSFourCC : uint { DXT1 = 0x31545844, DXT2 = 0x32545844, @@ -1395,26 +1392,26 @@ private enum DDSFourCCBg : uint private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeaderDX10 dx10, out string error) { error = null; - DDSFourCCBg fourCC = (DDSFourCCBg)hdr.ddspf.dwFourCC; + DDSFourCC fourCC = (DDSFourCC)hdr.ddspf.dwFourCC; switch (fourCC) { - case DDSFourCCBg.DXT1: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT1, true); - case DDSFourCCBg.DXT5: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT5, true); - case DDSFourCCBg.BC4U_ATI: - case DDSFourCCBg.BC4U: return GraphicsFormat.R_BC4_UNorm; - case DDSFourCCBg.BC4S: return GraphicsFormat.R_BC4_SNorm; - case DDSFourCCBg.BC5U_ATI: - case DDSFourCCBg.BC5U: return GraphicsFormat.RG_BC5_UNorm; - case DDSFourCCBg.BC5S: return GraphicsFormat.RG_BC5_SNorm; - case DDSFourCCBg.R16G16B16A16_UNORM: return GraphicsFormat.R16G16B16A16_UNorm; - case DDSFourCCBg.R16G16B16A16_SNORM: return GraphicsFormat.R16G16B16A16_SNorm; - case DDSFourCCBg.R16_FLOAT: return GraphicsFormat.R16_SFloat; - case DDSFourCCBg.R16G16_FLOAT: return GraphicsFormat.R16G16_SFloat; - case DDSFourCCBg.R16G16B16A16_FLOAT: return GraphicsFormat.R16G16B16A16_SFloat; - case DDSFourCCBg.R32_FLOAT: return GraphicsFormat.R32_SFloat; - case DDSFourCCBg.R32G32_FLOAT: return GraphicsFormat.R32G32_SFloat; - case DDSFourCCBg.R32G32B32A32_FLOAT: return GraphicsFormat.R32G32B32A32_SFloat; - case DDSFourCCBg.DX10: + case DDSFourCC.DXT1: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT1, true); + case DDSFourCC.DXT5: return GraphicsFormatUtility.GetGraphicsFormat(TextureFormat.DXT5, true); + case DDSFourCC.BC4U_ATI: + case DDSFourCC.BC4U: return GraphicsFormat.R_BC4_UNorm; + case DDSFourCC.BC4S: return GraphicsFormat.R_BC4_SNorm; + case DDSFourCC.BC5U_ATI: + case DDSFourCC.BC5U: return GraphicsFormat.RG_BC5_UNorm; + case DDSFourCC.BC5S: return GraphicsFormat.RG_BC5_SNorm; + case DDSFourCC.R16G16B16A16_UNORM: return GraphicsFormat.R16G16B16A16_UNorm; + case DDSFourCC.R16G16B16A16_SNORM: return GraphicsFormat.R16G16B16A16_SNorm; + case DDSFourCC.R16_FLOAT: return GraphicsFormat.R16_SFloat; + case DDSFourCC.R16G16_FLOAT: return GraphicsFormat.R16G16_SFloat; + case DDSFourCC.R16G16B16A16_FLOAT: return GraphicsFormat.R16G16B16A16_SFloat; + case DDSFourCC.R32_FLOAT: return GraphicsFormat.R32_SFloat; + case DDSFourCC.R32G32_FLOAT: return GraphicsFormat.R32G32_SFloat; + case DDSFourCC.R32G32B32A32_FLOAT: return GraphicsFormat.R32G32B32A32_SFloat; + case DDSFourCC.DX10: if (!hasDx10) { error = "DX10 marker without DX10 header"; @@ -1446,14 +1443,14 @@ private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeade error = $"DXT10 format '{dx10.dxgiFormat}' is not supported"; return GraphicsFormat.None; } - case DDSFourCCBg.DXT2: - case DDSFourCCBg.DXT3: - case DDSFourCCBg.DXT4: - case DDSFourCCBg.RGBG: - case DDSFourCCBg.GRGB: - case DDSFourCCBg.UYVY: - case DDSFourCCBg.YUY2: - case DDSFourCCBg.CxV8U8: + case DDSFourCC.DXT2: + case DDSFourCC.DXT3: + case DDSFourCC.DXT4: + case DDSFourCC.RGBG: + case DDSFourCC.GRGB: + case DDSFourCC.UYVY: + case DDSFourCC.YUY2: + case DDSFourCC.CxV8U8: error = $"format '{fourCC}' is not supported, use DXT1 for RGB textures or DXT5 for RGBA textures"; return GraphicsFormat.None; default: @@ -1471,6 +1468,8 @@ private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeade // and regenerating from there (BitmapToCompressedNormalMapFast). private static unsafe void SwizzleNormalMap(NativeArray data) { + using var scope = s_pmSwizzleNormalMap.Auto(); + byte* p = (byte*)NativeArrayUnsafeUtility.GetUnsafePtr(data); int len = data.Length; // (r, g, b, a) -> (g, g, g, r) @@ -1491,6 +1490,8 @@ private static unsafe void SwizzleNormalMap(NativeArray data) // matches src's so the constant 3:4 (or 4:4) byte-count ratio fills dst exactly. private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) { + using var scope = s_pmSwizzleNormalMap.Auto(); + byte* s = (byte*)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(src); byte* d = (byte*)NativeArrayUnsafeUtility.GetUnsafePtr(dst); int srcLen = src.Length; @@ -1533,9 +1534,7 @@ private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray reque { GameDatabase gdb = GameDatabase.Instance; Queue active = new Queue(); - int spawnIdx = 0; int total = requests.Count; - double nextFrameTime = ElapsedTime + minFrameTimeD; + var iter = requests.GetEnumerator(); + int completed = 0; - while (spawnIdx < total || active.Count > 0) + while (true) { - // Drain completed requests at the front of the queue. - while (active.Count > 0 && active.Peek().Status != TextureLoadRequest.State.Pending) + while (active.TryPeek(out var pending)) { - InsertReadyRequest(active.Peek(), loadedUrls); + if (pending.Status == TextureLoadRequest.State.Pending) + break; + active.Dequeue(); + InsertReadyRequest(pending, loadedUrls); loadedAssetCount++; + completed++; } - // Spawn new coroutines, bounded by the in-flight cap and the per-frame cap. - int spawnsThisFrame = 0; - while (spawnIdx < total && spawnsThisFrame < MaxTextureSpawnsPerFrame) + for (int i = 0; i < MaxTextureSpawnsPerFrame; ++i) { - SpawnTextureCoroutine(requests[spawnIdx++], active, gdb); - spawnsThisFrame++; + if (!iter.MoveNext()) + goto WINDDOWN; + var request = iter.Current; + + gdb.StartCoroutine(LoadTextureCoroutine(request)); + active.Enqueue(request); } - if (active.Count == 0 && spawnIdx >= total) - break; + gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; + gdb.progressTitle = $"Loading texture asset {completed}/{total}"; + yield return null; + } - if (ElapsedTime > nextFrameTime) + WINDDOWN: + while (active.TryDequeue(out var pending)) + { + while (pending.Status == TextureLoadRequest.State.Pending) { - nextFrameTime = ElapsedTime + minFrameTimeD; - int completed = spawnIdx - active.Count; gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; gdb.progressTitle = $"Loading texture asset {completed}/{total}"; + yield return null; } - yield return null; + + InsertReadyRequest(pending, loadedUrls); + loadedAssetCount++; + completed++; + } + } + + private static IEnumerator LoadTextureCoroutine(TextureLoadRequest req) + { + IEnumerator inner; + switch (req.AssetType) + { + case RawAsset.AssetType.TextureDDS: + inner = LoadDDSCoroutine(req); + break; + case RawAsset.AssetType.TexturePNG: + case RawAsset.AssetType.TextureJPG: + inner = LoadUWRCoroutine(req); + break; + case RawAsset.AssetType.TextureTRUECOLOR: + inner = LoadTRUECOLORCoroutine(req); + break; + case RawAsset.AssetType.TextureMBM: + inner = LoadMBMCoroutine(req); + break; + case RawAsset.AssetType.TextureTGA: + inner = LoadTGACoroutine(req); + break; + default: + req.ErrorMessage = $"Unknown asset type {req.AssetType}"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + while (true) + { + object current; + try + { + if (!inner.MoveNext()) + break; + + current = inner.Current; + } + catch (Exception e) + { + req.Exception = e; + req.ErrorMessage = $"{e.GetType().Name}: {e.Message}"; + req.Status = TextureLoadRequest.State.Failed; + yield break; + } + + yield return current; + } + + if (req.Status != TextureLoadRequest.State.Pending) + yield break; + + if (req.Result != null) + { + req.Status = TextureLoadRequest.State.Ready; + } + else + { + req.ErrorMessage ??= "Loader produced no result"; + req.Status = TextureLoadRequest.State.Failed; } } @@ -2147,6 +2220,8 @@ private static void SpawnTextureCoroutine(TextureLoadRequest req, Queue loadedUrls) { + Debug.Log($"Load Texture: {req.File.url}"); + if (req.Status == TextureLoadRequest.State.Failed) { Debug.LogWarning($"LOAD FAILED: {req.File.url}: {req.ErrorMessage}"); From ba3d33a7343f7da0bc2d37ed2238bb03b5306c7b Mon Sep 17 00:00:00 2001 From: Phantomical Date: Sat, 2 May 2026 15:36:04 -0700 Subject: [PATCH 14/21] Properly delay until render thread is finished --- KSPCommunityFixes/Performance/FastLoader.cs | 78 +++++++++++++++++---- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 667b9c5d..aa0eb093 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -37,6 +37,7 @@ using System.Threading.Tasks; using KSP.UI; using System.Security.Cryptography; +using UnityEngine.Rendering; namespace KSPCommunityFixes.Performance { @@ -153,7 +154,7 @@ internal class KSPCFFastLoader : MonoBehaviour // Max number of new texture load coroutines that will be spawned each frame. // This should roughly limit the max frame time spent on loading textures. - private const int MaxTextureSpawnsPerFrame = 128; + private const int MaxTextureSpawnsPerFrame = 512; private static Harmony persistentHarmony; private static string PersistentHarmonyID => typeof(KSPCFFastLoader).FullName; @@ -1692,9 +1693,9 @@ private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) yield break; } - // Wait one frame so Unity's initial GPU resource creation completes - // before AsyncReadManager writes into the staging buffer. - yield return null; + // Wait until the texture is finished uploading so unity doesn't + // copy its internal buffer when we call GetRawTextureData + yield return WaitForGraphicsThread(); NativeArray dst; using (s_pmGetRawDataDDS.Auto()) @@ -1755,6 +1756,10 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) yield break; } + // Wait until the texture is finished uploading so unity doesn't + // copy its internal buffer when we operate on it. + yield return WaitForGraphicsThread(); + bool isNormalMap = req.File.name.EndsWith("NRM"); bool canCompress = src.width % 4 == 0 && src.height % 4 == 0; @@ -1765,9 +1770,6 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) { src.wrapMode = TextureWrapMode.Repeat; - // Wait a frame to avoid UnshareTextureData overhead within Unity - yield return null; - NativeArray allLevels; using (s_pmGetRawDataUWR.Auto()) allLevels = src.GetRawTextureData(); @@ -1867,8 +1869,9 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) bool isPot = Numerics.IsPowerOfTwo(tex.width) && Numerics.IsPowerOfTwo(tex.height); tex.wrapMode = TextureWrapMode.Repeat; - // Wait a frame so we avoid a call to UnshareTextureData within unity - yield return null; + // Wait until the texture is finished uploading so unity doesn't + // copy its internal buffer when we call GetRawTextureData + yield return WaitForGraphicsThread(); NativeArray allLevels; using (s_pmGetRawDataTRUECOLOR.Auto()) @@ -1989,8 +1992,9 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) // mip level. Swizzle the whole thing in place. texture.wrapMode = TextureWrapMode.Repeat; - // Avoid UnshareTextureData overhead within Unity by waiting a frame. - yield return null; + // Wait until the texture is finished uploading so unity doesn't + // copy its internal buffer when we call GetRawTextureData + yield return WaitForGraphicsThread(); NativeArray allLevels; using (s_pmGetRawDataTGA.Auto()) @@ -2101,7 +2105,7 @@ private static IEnumerator TextureDriverCoroutine(List reque yield return null; } - WINDDOWN: + WINDDOWN: while (active.TryDequeue(out var pending)) { while (pending.Status == TextureLoadRequest.State.Pending) @@ -3036,6 +3040,56 @@ public void Show() } } + // A helper that yields until it has been processed on the render thread. + // Use this to delay until the render thread is no longer using a texture + // (or any other resource). + private unsafe class WaitForGraphicsThreadInst : CustomYieldInstruction + { + static CommandBuffer DispatchCB; + static readonly IntPtr NotifyPtr = (IntPtr)Marshal.GetFunctionPointerForDelegate((Action)Notify); + static readonly int GchandleOffset = UnsafeUtility.GetFieldOffset( + typeof(WaitForGraphicsThreadInst).GetField(nameof(gchandle), BindingFlags.Instance | BindingFlags.NonPublic)); + static readonly int ReadyOffset = UnsafeUtility.GetFieldOffset( + typeof(WaitForGraphicsThreadInst).GetField(nameof(ready), BindingFlags.Instance | BindingFlags.NonPublic)); + + ulong gchandle = 0; + bool ready = false; + + public override bool keepWaiting => !ready; + + public WaitForGraphicsThreadInst() + { + DispatchCB ??= new CommandBuffer() + { + name = "KSPCF.WaitForGraphicsThreadCB" + }; + + void* addr = UnsafeUtility.PinGCObjectAndGetAddress(this, out gchandle); + try + { + DispatchCB.Clear(); + DispatchCB.IssuePluginEventAndData(NotifyPtr, 0, (IntPtr)addr); + Graphics.ExecuteCommandBuffer(DispatchCB); + } + catch + { + UnsafeUtility.ReleaseGCObject(gchandle); + throw; + } + } + + static void Notify(int _, IntPtr data) + { + ulong gchandle = *(ulong*)((byte*)data + GchandleOffset); + bool* ready = (bool*)((byte*)data + ReadyOffset); + + *ready = true; + UnsafeUtility.ReleaseGCObject(gchandle); + } + } + + private static WaitForGraphicsThreadInst WaitForGraphicsThread() => + new WaitForGraphicsThreadInst(); #endregion From 7076a6d34143f5e5c612bfe83c6dc313aa8f38a6 Mon Sep 17 00:00:00 2001 From: Phantomical Date: Sat, 2 May 2026 22:18:27 -0700 Subject: [PATCH 15/21] Move completed texture loading after dispatches --- KSPCommunityFixes/Performance/FastLoader.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index aa0eb093..db07aabf 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -2079,6 +2079,16 @@ private static IEnumerator TextureDriverCoroutine(List reque while (true) { + for (int i = 0; i < MaxTextureSpawnsPerFrame; ++i) + { + if (!iter.MoveNext()) + goto WINDDOWN; + var request = iter.Current; + + gdb.StartCoroutine(LoadTextureCoroutine(request)); + active.Enqueue(request); + } + while (active.TryPeek(out var pending)) { if (pending.Status == TextureLoadRequest.State.Pending) @@ -2090,16 +2100,6 @@ private static IEnumerator TextureDriverCoroutine(List reque completed++; } - for (int i = 0; i < MaxTextureSpawnsPerFrame; ++i) - { - if (!iter.MoveNext()) - goto WINDDOWN; - var request = iter.Current; - - gdb.StartCoroutine(LoadTextureCoroutine(request)); - active.Enqueue(request); - } - gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; gdb.progressTitle = $"Loading texture asset {completed}/{total}"; yield return null; From ae7b6587ff36ac0f05af7577df3552d8f3ec0393 Mon Sep 17 00:00:00 2001 From: Phantomical Date: Mon, 13 Jul 2026 17:10:35 -0700 Subject: [PATCH 16/21] Load dds textures by building an asset bundle in unity --- KSPCommunityFixes/KSPCommunityFixes.csproj | 6 + .../TextureBundle/BundleBufferWriter.cs | 239 +++++++++ .../Library/TextureBundle/BundleWriter.cs | 160 ++++++ .../TextureBundle/ReferenceTypeTrees.cs | 98 ++++ .../TextureBundle/SerializedFileWriter.cs | 197 ++++++++ .../TextureBundle/SerializedTypeTrees.cs | 14 + .../TextureBundle/TextureBundleBuilder.cs | 470 ++++++++++++++++++ .../TextureBundle/TextureTypeTrees.bin | Bin 0 -> 3840 bytes KSPCommunityFixes/Performance/FastLoader.cs | 466 +++++++++++++---- Tools/GenerateTextureTypeTrees.py | 194 ++++++++ 10 files changed, 1743 insertions(+), 101 deletions(-) create mode 100644 KSPCommunityFixes/Library/TextureBundle/BundleBufferWriter.cs create mode 100644 KSPCommunityFixes/Library/TextureBundle/BundleWriter.cs create mode 100644 KSPCommunityFixes/Library/TextureBundle/ReferenceTypeTrees.cs create mode 100644 KSPCommunityFixes/Library/TextureBundle/SerializedFileWriter.cs create mode 100644 KSPCommunityFixes/Library/TextureBundle/SerializedTypeTrees.cs create mode 100644 KSPCommunityFixes/Library/TextureBundle/TextureBundleBuilder.cs create mode 100644 KSPCommunityFixes/Library/TextureBundle/TextureTypeTrees.bin create mode 100644 Tools/GenerateTextureTypeTrees.py diff --git a/KSPCommunityFixes/KSPCommunityFixes.csproj b/KSPCommunityFixes/KSPCommunityFixes.csproj index 4a60b46b..13d449a0 100644 --- a/KSPCommunityFixes/KSPCommunityFixes.csproj +++ b/KSPCommunityFixes/KSPCommunityFixes.csproj @@ -44,6 +44,12 @@ + + + + + $(KSPBT_ModRoot)\$(AVCFilename) diff --git a/KSPCommunityFixes/Library/TextureBundle/BundleBufferWriter.cs b/KSPCommunityFixes/Library/TextureBundle/BundleBufferWriter.cs new file mode 100644 index 00000000..b783fa24 --- /dev/null +++ b/KSPCommunityFixes/Library/TextureBundle/BundleBufferWriter.cs @@ -0,0 +1,239 @@ +using System; +using System.Text; + +namespace KSPCommunityFixes.Library.TextureBundle +{ + /// + /// A single growable byte buffer that a bundle prefix is written into, with primitive + /// writers, deferred-length helpers and back-patching. It is the allocation-light write + /// path: one buffer grown in place, values encoded straight into it (strings included, + /// with no intermediate arrays), and a single right-sized copy at the end via + /// . + /// + /// + /// Both little-endian and big-endian values are supported through : + /// the UnityFS container header and the serialized-file header are big-endian, while + /// serialized metadata and object data are little-endian. + /// + /// makes pad relative to a chosen origin + /// rather than the buffer start. The serialized file is written into this same buffer at a + /// non-16-aligned offset, but Unity aligns object data relative to the serialized file's own + /// start; pointing at that start keeps the padding identical to what + /// a file written from offset zero would have. + /// + /// Borrowed from KSPTextureLoader + /// (../KSPTextureLoader/src/KSPTextureLoader/Format/Bundle/BundleBufferWriter.cs). + /// + internal sealed class BundleBufferWriter + { + byte[] buffer; + int length; + + /// When true multi-byte integers are written most-significant-byte first. + public bool BigEndian; + + /// The origin pads relative to (see the type remarks). + public int AlignBase; + + public BundleBufferWriter(int initialCapacity = 256) + { + if (initialCapacity < 0) + throw new ArgumentOutOfRangeException(nameof(initialCapacity)); + buffer = new byte[Math.Max(initialCapacity, 64)]; + } + + /// The number of bytes written so far (the current write position). + public int Length => length; + + void EnsureCapacity(int additional) + { + long required = (long)length + additional; + if (required <= buffer.Length) + return; + + long capacity = buffer.Length; + while (capacity < required) + capacity *= 2; + if (capacity > int.MaxValue) + capacity = int.MaxValue; + + Array.Resize(ref buffer, (int)capacity); + } + + void WriteUInt(ulong value, int size) + { + EnsureCapacity(size); + if (BigEndian) + { + for (int i = 0; i < size; ++i) + buffer[length + i] = (byte)(value >> (8 * (size - 1 - i))); + } + else + { + for (int i = 0; i < size; ++i) + buffer[length + i] = (byte)(value >> (8 * i)); + } + length += size; + } + + public void WriteByte(byte value) + { + EnsureCapacity(1); + buffer[length++] = value; + } + + public void WriteBool(bool value) => WriteByte((byte)(value ? 1 : 0)); + + public void WriteUInt16(ushort value) => WriteUInt(value, 2); + + public void WriteInt32(int value) => WriteUInt((uint)value, 4); + + public void WriteUInt32(uint value) => WriteUInt(value, 4); + + public void WriteInt64(long value) => WriteUInt((ulong)value, 8); + + public unsafe void WriteSingle(float value) => WriteUInt32(*(uint*)&value); + + public void WriteBytes(byte[] value) + { + if (value is null || value.Length == 0) + return; + EnsureCapacity(value.Length); + Buffer.BlockCopy(value, 0, buffer, length, value.Length); + length += value.Length; + } + + /// Write zero bytes. + public void WriteZeros(int count) + { + if (count < 0) + throw new ArgumentOutOfRangeException(nameof(count)); + EnsureCapacity(count); + Array.Clear(buffer, length, count); + length += count; + } + + /// + /// Pad with zero bytes until the write position is a multiple of + /// relative to . + /// + public void Align(int alignment = 4) + { + int rem = (length - AlignBase) % alignment; + if (rem != 0) + WriteZeros(alignment - rem); + } + + /// + /// Write an Int32 length-prefixed UTF-8 string and align to 4 bytes afterwards, the way + /// strings are stored inside serialized object data. The bytes are encoded straight into + /// the buffer with no intermediate array. + /// + public void WriteAlignedString(string value) + { + int count = Utf8ByteCount(value); + WriteInt32(count); + WriteUtf8(value, count); + Align(4); + } + + /// Write a null-terminated UTF-8 string (no intermediate array). + public void WriteCString(string value) + { + WriteUtf8(value, Utf8ByteCount(value)); + WriteByte(0); + } + + static int Utf8ByteCount(string value) => + string.IsNullOrEmpty(value) ? 0 : Encoding.UTF8.GetByteCount(value); + + void WriteUtf8(string value, int count) + { + if (count > 0) + { + EnsureCapacity(count); + Encoding.UTF8.GetBytes(value, 0, value.Length, buffer, length); + length += count; + } + } + + /// Reserve four bytes to be back-patched later, returning their position. + public int ReserveUInt32() + { + int pos = length; + WriteZeros(4); + return pos; + } + + /// Reserve eight bytes to be back-patched later, returning their position. + public int ReserveInt64() + { + int pos = length; + WriteZeros(8); + return pos; + } + + /// Back-patch four bytes at an earlier position, with the given endianness. + public void PatchUInt32(int position, uint value, bool bigEndian) + { + if (bigEndian) + for (int i = 0; i < 4; ++i) + buffer[position + 3 - i] = (byte)(value >> (8 * i)); + else + for (int i = 0; i < 4; ++i) + buffer[position + i] = (byte)(value >> (8 * i)); + } + + /// Back-patch eight bytes at an earlier position, with the given endianness. + public void PatchInt64(int position, long value, bool bigEndian) + { + ulong v = (ulong)value; + if (bigEndian) + for (int i = 0; i < 8; ++i) + buffer[position + 7 - i] = (byte)(v >> (8 * i)); + else + for (int i = 0; i < 8; ++i) + buffer[position + i] = (byte)(v >> (8 * i)); + } + + /// Begin a length-prefixed array whose element count is filled in on . + public ArrayScope BeginArray() => new ArrayScope(this); + + /// + /// A length-prefixed array wrapper: reserves the Int32 count up front, tallies elements + /// as they are written and back-patches the count when the scope ends. + /// + public struct ArrayScope + { + readonly BundleBufferWriter writer; + readonly int countPosition; + int count; + + internal ArrayScope(BundleBufferWriter writer) + { + this.writer = writer; + countPosition = writer.ReserveUInt32(); + count = 0; + } + + /// Record that one element is about to be written. + public void Add() => count++; + + /// Patch the element count, optionally aligning the writer to 4 bytes after. + public void End(bool align = false) + { + writer.PatchUInt32(countPosition, (uint)count, bigEndian: false); + if (align) + writer.Align(4); + } + } + + /// Copy the written bytes into a new, exactly-sized array. + public byte[] ToArray() + { + var result = new byte[length]; + Buffer.BlockCopy(buffer, 0, result, 0, length); + return result; + } + } +} diff --git a/KSPCommunityFixes/Library/TextureBundle/BundleWriter.cs b/KSPCommunityFixes/Library/TextureBundle/BundleWriter.cs new file mode 100644 index 00000000..c12570b8 --- /dev/null +++ b/KSPCommunityFixes/Library/TextureBundle/BundleWriter.cs @@ -0,0 +1,160 @@ +using System; + +namespace KSPCommunityFixes.Library.TextureBundle +{ + /// + /// Writes the framing of a UnityFS bundle wrapping a single serialized file (the + /// CAB-<hash> entry) and its streamed resource (CAB-<hash>.resS) + /// directly into a shared : the container header and the + /// blocks-info directory. The serialized file is written straight after (by + /// ) into the same buffer. For textures streamed from an + /// existing DDS file on disk the resS is empty (its pixel bytes live in the DDS file, which + /// Unity opens itself). Sizes that depend on the serialized file length are back-patched by + /// . + /// + /// + /// The layout matches what Unity 2019.4.18f1 itself produces, except that the bundle is left + /// uncompressed: it is loaded once and discarded, so compressing it would only add work. + /// + /// Borrowed from KSPTextureLoader + /// (../KSPTextureLoader/src/KSPTextureLoader/Format/Bundle/BundleWriter.cs). + /// + internal static class BundleWriter + { + const string Signature = "UnityFS"; + const uint FormatVersion = 7; + const string PlayerMinVersion = "5.x.x"; + const string EngineRevision = "2019.4.18f1"; + + // Low 6 bits are the blocks-info compression type (0 == none); 0x40 is + // kArchiveBlocksAndDirectoryInfoCombined, set by real 2019.4 bundles. + const uint BundleFlags = 0x40; + + // Marks a directory node as a serialized file. + const uint SerializedFileNodeFlag = 0x4; + + /// + /// The reserved header/blocks-info slots that depend on the serialized file length, + /// back-patched by . + /// + public readonly struct PrefixScope + { + internal readonly int TotalSizePosition; + internal readonly int BlockUncompressedPosition; + internal readonly int BlockCompressedPosition; + internal readonly int Node0SizePosition; + internal readonly int Node1OffsetPosition; + internal readonly long ResSLength; + + internal PrefixScope( + int totalSizePosition, + int blockUncompressedPosition, + int blockCompressedPosition, + int node0SizePosition, + int node1OffsetPosition, + long resSLength + ) + { + TotalSizePosition = totalSizePosition; + BlockUncompressedPosition = blockUncompressedPosition; + BlockCompressedPosition = blockCompressedPosition; + Node0SizePosition = node0SizePosition; + Node1OffsetPosition = node1OffsetPosition; + ResSLength = resSLength; + } + } + + /// + /// Write the container header and blocks-info directory into (which + /// must be empty). is the internal archive name; the resource node + /// is named ".resS" to match the m_StreamData.path written + /// into the texture object. is the length of the streamed + /// pixel data appended after the prefix (0 when streaming from an external file). + /// + public static PrefixScope WriteHeaderAndBlocksInfo( + BundleBufferWriter w, + string cab, + long resSLength + ) + { + if (string.IsNullOrEmpty(cab)) + throw new ArgumentException("cab name is required", nameof(cab)); + if (resSLength < 0) + throw new ArgumentOutOfRangeException(nameof(resSLength)); + + w.AlignBase = 0; + w.BigEndian = true; + + w.WriteCString(Signature); + w.WriteUInt32(FormatVersion); + w.WriteCString(PlayerMinVersion); + w.WriteCString(EngineRevision); + + int totalSizePosition = w.ReserveInt64(); // total bundle size, patched once known + int blocksInfoCompPosition = w.ReserveUInt32(); // compressed blocks-info size + int blocksInfoUncompPosition = w.ReserveUInt32(); // uncompressed blocks-info size + w.WriteUInt32(BundleFlags); + + // Bundle version 7 pads the header to a 16-byte boundary. + w.Align(16); + + int blocksInfoStart = w.Length; + + w.WriteZeros(16); // uncompressed data hash + + // A single uncompressed block spanning the whole virtual file system; the block sizes + // span the serialized file plus the resS and are patched in Finish. + w.WriteInt32(1); + int blockUncompressedPosition = w.ReserveUInt32(); + int blockCompressedPosition = w.ReserveUInt32(); + w.WriteUInt16(0); // compression type 0 == none + + // Two directory nodes: the serialized file, then its resS right after. + w.WriteInt32(2); + + w.WriteInt64(0); // node 0 offset within the block data + int node0SizePosition = w.ReserveInt64(); // serialized file size + w.WriteUInt32(SerializedFileNodeFlag); + w.WriteCString(cab); + + int node1OffsetPosition = w.ReserveInt64(); // resS offset == serialized file size + w.WriteInt64(resSLength); + w.WriteUInt32(0); + w.WriteCString(cab + ".resS"); + + int blocksInfoLength = w.Length - blocksInfoStart; + w.PatchUInt32(blocksInfoCompPosition, (uint)blocksInfoLength, bigEndian: true); + w.PatchUInt32(blocksInfoUncompPosition, (uint)blocksInfoLength, bigEndian: true); + + return new PrefixScope( + totalSizePosition, + blockUncompressedPosition, + blockCompressedPosition, + node0SizePosition, + node1OffsetPosition, + resSLength + ); + } + + /// + /// Back-patch every size that depends on the serialized file length, plus the total bundle + /// size. Call once the serialized file has been written into the same buffer. + /// + public static void Finish(BundleBufferWriter w, in PrefixScope scope, long serializedFileLength) + { + // The blocks-info block sizes are unsigned 32-bit, and everything lives in a single + // uncompressed block. + long blockDataSize = serializedFileLength + scope.ResSLength; + if (blockDataSize > uint.MaxValue) + throw new InvalidOperationException( + "texture data is too large to fit in a streamed bundle" + ); + + w.PatchUInt32(scope.BlockUncompressedPosition, (uint)blockDataSize, bigEndian: true); + w.PatchUInt32(scope.BlockCompressedPosition, (uint)blockDataSize, bigEndian: true); + w.PatchInt64(scope.Node0SizePosition, serializedFileLength, bigEndian: true); + w.PatchInt64(scope.Node1OffsetPosition, serializedFileLength, bigEndian: true); + w.PatchInt64(scope.TotalSizePosition, w.Length + scope.ResSLength, bigEndian: true); + } + } +} diff --git a/KSPCommunityFixes/Library/TextureBundle/ReferenceTypeTrees.cs b/KSPCommunityFixes/Library/TextureBundle/ReferenceTypeTrees.cs new file mode 100644 index 00000000..acf222af --- /dev/null +++ b/KSPCommunityFixes/Library/TextureBundle/ReferenceTypeTrees.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Text; +using System.Threading; + +namespace KSPCommunityFixes.Library.TextureBundle +{ + /// + /// The verbatim serialized type-tree entries for the classes the texture-bundle writer emits + /// (Texture2D and AssetBundle), loaded once from the embedded + /// TextureTypeTrees.bin artifact. copies each entry + /// straight into the bundles it generates and enables the type tree, so Unity deserializes the + /// objects from the embedded tree rather than its compiled-in layout. + /// + /// + /// The artifact is produced offline (see Tools/GenerateTextureTypeTrees.py) by + /// extracting the two type entries from a Unity-generated reference bundle, so it needs no + /// runtime bundle parsing. Its format (all little-endian) is: magic "KCTT", u32 version, an + /// int32-length-prefixed UTF-8 unity version string, an int32 entry count, then per entry an + /// int32 class id, an int32 byte length and that many raw type-entry bytes. + /// + internal static class ReferenceTypeTrees + { + const string ResourceSuffix = "TextureTypeTrees.bin"; + static readonly byte[] Magic = { (byte)'K', (byte)'C', (byte)'T', (byte)'T' }; + const uint SupportedVersion = 1; + + sealed class Data + { + public readonly Dictionary ByClassId = new Dictionary(); + public string UnityVersion; + } + + static readonly Lazy data = new Lazy(Load, LazyThreadSafetyMode.ExecutionAndPublication); + + /// The engine version string of the reference artifact (e.g. "2019.4.18f1"). + public static string UnityVersion => data.Value.UnityVersion; + + /// The verbatim type-entry bytes for a class, to copy into a generated file's type list. + public static byte[] TypeEntry(int classId) + { + if (data.Value.ByClassId.TryGetValue(classId, out var entry)) + return entry; + throw new InvalidOperationException( + $"the embedded texture type-tree artifact has no type tree for class {classId}" + ); + } + + static Data Load() + { + var asm = Assembly.GetExecutingAssembly(); + string name = null; + foreach (var candidate in asm.GetManifestResourceNames()) + if (candidate.EndsWith(ResourceSuffix, StringComparison.OrdinalIgnoreCase)) + { + name = candidate; + break; + } + if (name is null) + throw new InvalidOperationException( + $"embedded texture type-tree artifact ('{ResourceSuffix}') not found" + ); + + using (var stream = asm.GetManifestResourceStream(name)) + using (var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: false)) + { + var magic = reader.ReadBytes(Magic.Length); + if (magic.Length != Magic.Length + || magic[0] != Magic[0] || magic[1] != Magic[1] + || magic[2] != Magic[2] || magic[3] != Magic[3]) + throw new InvalidDataException("texture type-tree artifact has a bad magic"); + + uint version = reader.ReadUInt32(); + if (version != SupportedVersion) + throw new InvalidDataException( + $"texture type-tree artifact version {version} is unsupported" + ); + + var result = new Data(); + + int unityVersionLength = reader.ReadInt32(); + result.UnityVersion = Encoding.UTF8.GetString(reader.ReadBytes(unityVersionLength)); + + int count = reader.ReadInt32(); + for (int i = 0; i < count; ++i) + { + int classId = reader.ReadInt32(); + int length = reader.ReadInt32(); + result.ByClassId[classId] = reader.ReadBytes(length); + } + + return result; + } + } + } +} diff --git a/KSPCommunityFixes/Library/TextureBundle/SerializedFileWriter.cs b/KSPCommunityFixes/Library/TextureBundle/SerializedFileWriter.cs new file mode 100644 index 00000000..0681f76e --- /dev/null +++ b/KSPCommunityFixes/Library/TextureBundle/SerializedFileWriter.cs @@ -0,0 +1,197 @@ +using System; + +namespace KSPCommunityFixes.Library.TextureBundle +{ + /// + /// Writes the framing of a serialized Unity asset-bundle file (the CAB-<hash> + /// entry) directly into a shared : the header, the + /// little-endian metadata (unity version, type list, object table) and the padding up to the + /// object-data section. Callers write each object body between + /// and , and the byte + /// offsets/sizes are back-patched into the reserved object-table slots. + /// + /// + /// The emitted file enables the type tree and copies each class's type entry verbatim from the + /// embedded reference artifact (see ), so Unity deserializes + /// objects from the embedded tree. It targets serialized file format 21 (Unity 2019.4), the + /// version the reference type entries were captured for. + /// + /// Borrowed from KSPTextureLoader + /// (../KSPTextureLoader/src/KSPTextureLoader/Format/Bundle/SerializedFileWriter.cs). + /// + internal static class SerializedFileWriter + { + // 2019.4 writes serialized-file format 21. The reference type entries copied into the + // metadata were captured from a format-21 file, so this must match. + const uint FormatVersion = 21; + + const int ObjectAlignment = 16; + + // metadataSize + fileSize + version + dataOffset + endianness byte + 3 reserved. + const int HeaderLength = 20; + + /// Identifies one object to place in the serialized file. + public readonly struct ObjectMeta + { + public readonly long PathId; + public readonly int ClassId; + + public ObjectMeta(long pathId, int classId) + { + PathId = pathId; + ClassId = classId; + } + } + + /// The reserved object-table slots for one object, patched as its body is written. + public struct ObjectSlot + { + internal int OffsetPosition; + internal int SizePosition; + internal int Start; + } + + /// + /// Bookkeeping returned by : the reserved header slots and the + /// object-data section origin, used to place object bodies and finalize the file. + /// + public readonly struct FileScope + { + internal readonly int FileStart; + internal readonly int FileSizePosition; + internal readonly int DataOffset; + + internal FileScope(int fileStart, int fileSizePosition, int dataOffset) + { + FileStart = fileStart; + FileSizePosition = fileSizePosition; + DataOffset = dataOffset; + } + + /// Align to the next object slot and record where this object's body begins. + public void BeginObject(BundleBufferWriter w, ref ObjectSlot slot) + { + w.Align(ObjectAlignment); + slot.Start = w.Length; + // The object-table offset is relative to the data section (dataOffset). + w.PatchUInt32( + slot.OffsetPosition, + (uint)(w.Length - FileStart - DataOffset), + bigEndian: false + ); + } + + /// Back-patch this object's byte size once its body has been written. + public void EndObject(BundleBufferWriter w, in ObjectSlot slot) => + w.PatchUInt32(slot.SizePosition, (uint)(w.Length - slot.Start), bigEndian: false); + + /// Patch the total file size and return it. + public long End(BundleBufferWriter w) + { + int fileLength = w.Length - FileStart; + w.PatchUInt32(FileSizePosition, (uint)fileLength, bigEndian: true); + return fileLength; + } + } + + /// + /// Write the header, metadata and object-data padding into at its + /// current position, reserving one per object in + /// . is the Unity + /// BuildTarget the bundle is tagged for; it must match the running player or Unity + /// rejects the bundle at load. + /// + public static FileScope BeginFile( + BundleBufferWriter w, + int targetPlatform, + ReadOnlySpan objects, + Span slots + ) + { + if (objects.Length == 0) + throw new ArgumentException("at least one object is required", nameof(objects)); + if (slots.Length < objects.Length) + throw new ArgumentException("slots must have one entry per object", nameof(slots)); + + int fileStart = w.Length; + // Object-data alignment is relative to the serialized file's own start. + w.AlignBase = fileStart; + + // Header (big-endian), sizes back-patched once known. + w.BigEndian = true; + int metaSizePosition = w.ReserveUInt32(); + int fileSizePosition = w.ReserveUInt32(); + w.WriteUInt32(FormatVersion); + int dataOffsetPosition = w.ReserveUInt32(); + w.WriteByte(0); // m_Endianess: 0 == little-endian data + w.WriteZeros(3); // reserved + + int metaStart = w.Length; // == fileStart + HeaderLength + + // Metadata (little-endian). + w.BigEndian = false; + w.WriteCString(ReferenceTypeTrees.UnityVersion); + w.WriteInt32(targetPlatform); + w.WriteBool(true); // m_EnableTypeTree + + // The distinct class ids, in first-seen order, form the type list. Each entry (class + // id, strip/script fields, hashes, type-tree blob and dependencies) is copied verbatim + // from the reference artifact so objects carry their full tree. This is sized to + // objects.Length (which can be tens of thousands for a combined texture bundle), so it + // lives on the heap rather than the stack. + Span typeOrder = new int[objects.Length]; + int typeCount = 0; + for (int i = 0; i < objects.Length; ++i) + if (IndexOf(typeOrder, typeCount, objects[i].ClassId) < 0) + typeOrder[typeCount++] = objects[i].ClassId; + + w.WriteInt32(typeCount); + for (int i = 0; i < typeCount; ++i) + w.WriteBytes(ReferenceTypeTrees.TypeEntry(typeOrder[i])); + + // Object table: reserve the byte offset/size of each object, patched as bodies are + // written. + w.WriteInt32(objects.Length); + for (int i = 0; i < objects.Length; ++i) + { + w.Align(4); + w.WriteInt64(objects[i].PathId); + slots[i].OffsetPosition = w.ReserveUInt32(); + slots[i].SizePosition = w.ReserveUInt32(); + w.WriteInt32(IndexOf(typeOrder, typeCount, objects[i].ClassId)); + } + + w.WriteInt32(0); // m_ScriptTypes count + w.WriteInt32(0); // m_Externals count + w.WriteInt32(0); // m_RefTypes count + w.WriteCString(""); // userInformation + + int metaLength = w.Length - metaStart; + int dataOffset = Align(HeaderLength + metaLength, ObjectAlignment); + + // Pad up to the object-data section. + int padding = dataOffset - (w.Length - fileStart); + if (padding > 0) + w.WriteZeros(padding); + + w.PatchUInt32(metaSizePosition, (uint)metaLength, bigEndian: true); + w.PatchUInt32(dataOffsetPosition, (uint)dataOffset, bigEndian: true); + + return new FileScope(fileStart, fileSizePosition, dataOffset); + } + + static int IndexOf(Span values, int count, int value) + { + for (int i = 0; i < count; ++i) + if (values[i] == value) + return i; + return -1; + } + + static int Align(int value, int alignment) + { + int rem = value % alignment; + return rem == 0 ? value : value + (alignment - rem); + } + } +} diff --git a/KSPCommunityFixes/Library/TextureBundle/SerializedTypeTrees.cs b/KSPCommunityFixes/Library/TextureBundle/SerializedTypeTrees.cs new file mode 100644 index 00000000..d0107566 --- /dev/null +++ b/KSPCommunityFixes/Library/TextureBundle/SerializedTypeTrees.cs @@ -0,0 +1,14 @@ +namespace KSPCommunityFixes.Library.TextureBundle +{ + /// + /// The Unity runtime class ids the texture-bundle writer can emit into a generated bundle. The + /// serialized field layouts these ids map to are not transcribed here: the writer copies each + /// class's type tree verbatim from the embedded reference artifact (see + /// ). + /// + internal static class SerializedTypeTrees + { + public const int Texture2DClassId = 28; + public const int AssetBundleClassId = 142; + } +} diff --git a/KSPCommunityFixes/Library/TextureBundle/TextureBundleBuilder.cs b/KSPCommunityFixes/Library/TextureBundle/TextureBundleBuilder.cs new file mode 100644 index 00000000..7afc327a --- /dev/null +++ b/KSPCommunityFixes/Library/TextureBundle/TextureBundleBuilder.cs @@ -0,0 +1,470 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace KSPCommunityFixes.Library.TextureBundle +{ + /// + /// Builds a minimal UnityFS bundle wrapping a single streamed Texture2D and the + /// AssetBundle that references it, where the texture's pixel data lives in an existing + /// DDS file on disk. The generated bundle carries only ~1 KB of metadata: the texture's + /// m_StreamData.path is the absolute path of the DDS file and m_StreamData.offset + /// is its data offset, so Unity opens the file and streams the compressed pixels itself. Pure + /// CPU work; safe to call from a background thread. + /// + /// + /// The whole prefix is written into a single : the UnityFS + /// framing (), the serialized-file framing + /// () and the two object bodies written here by hand. The + /// bodies reproduce the exact field order, sizes and alignment padding of Unity 2019.4's own + /// layout — the same layout the embedded type tree encodes. + /// + /// Borrowed from KSPTextureLoader + /// (../KSPTextureLoader/src/KSPTextureLoader/Format/Bundle/TextureBundleBuilder.cs), stripped + /// to the classic Texture2D + external-file path. + /// + internal static class TextureBundleBuilder + { + // Unity BuildTarget.StandaloneWindows64. + public const int StandaloneWindows64 = 19; + + // UnityEngine.Rendering.TextureDimension.Tex2D. + const int TextureDimensionTex2D = 2; + + // Unity's usual default fallback, TextureFormat.ARGB32. + const int ForcedFallbackFormat = 4; + + const long AssetBundlePathId = 1; + const long TexturePathId = 2; + + // Texture objects in a multi-texture bundle get path ids 2, 3, 4, ... (the AssetBundle + // object is path id 1). + const long FirstTexturePathId = 2; + + // Unity canonicalizes the name passed to LoadAsset (lowercase, forward slashes) but + // compares it against the stored container key verbatim, so a key with uppercase + // characters or backslashes can never be looked up. A fixed already-canonical key + // sidesteps that entirely; the caller renames the loaded texture afterwards anyway. + const string ContainerKey = "texture"; + + /// The texture to wrap in a bundle. + public sealed class TextureRequest + { + /// Serialized m_Name. Cosmetic only: the asset is looked up by the fixed + /// container key in , and callers overwrite the texture's + /// name after loading it. + public string Name = "texture"; + + public int Width; + public int Height; + public int MipCount = 1; + + /// The legacy TextureFormat written into m_TextureFormat. + public int Format; + + /// 0 == linear, 1 == sRGB. + public int ColorSpace = 1; + + /// Whether Unity should keep a CPU-side copy of the pixels. + public bool Readable; + } + + /// The built bundle prefix plus the name to request from it. + public readonly struct Built + { + public readonly byte[] Prefix; + public readonly string AssetName; + public readonly long PixelsLength; + + public Built(byte[] prefix, string assetName, long pixelsLength) + { + Prefix = prefix; + AssetName = assetName; + PixelsLength = pixelsLength; + } + } + + /// + /// One streamed texture to place in a combined bundle built by . + /// is both the serialized m_Name and the m_Container key, + /// so the caller can look the loaded texture back up by its name. + /// + public readonly struct TextureEntry + { + public readonly string Name; + public readonly int Width; + public readonly int Height; + public readonly int MipCount; + + /// The legacy TextureFormat written into m_TextureFormat. + public readonly int Format; + + /// 0 == linear, 1 == sRGB. + public readonly int ColorSpace; + + public readonly bool Readable; + + /// Absolute path of the DDS file the pixels are streamed from. + public readonly string ExternalPath; + + /// Byte offset of the pixel data within . + public readonly long ExternalOffset; + + /// Total mip-chain byte size streamed from the file. + public readonly long PixelsLength; + + public TextureEntry( + string name, + int width, + int height, + int mipCount, + int format, + int colorSpace, + bool readable, + string externalPath, + long externalOffset, + long pixelsLength) + { + Name = name; + Width = width; + Height = height; + MipCount = mipCount; + Format = format; + ColorSpace = colorSpace; + Readable = readable; + ExternalPath = externalPath; + ExternalOffset = externalOffset; + PixelsLength = pixelsLength; + } + } + + /// + /// Build a bundle whose streamed pixel data lives in the file at + /// , starting at and + /// spanning bytes. + /// + public static Built Build( + TextureRequest req, + long pixelsLength, + string externalPath, + long externalOffset, + int targetPlatform = StandaloneWindows64 + ) + { + if (req is null) + throw new ArgumentNullException(nameof(req)); + if (externalPath is null) + throw new ArgumentNullException(nameof(externalPath)); + if (pixelsLength < 0) + throw new ArgumentOutOfRangeException(nameof(pixelsLength)); + if (externalOffset < 0) + throw new ArgumentOutOfRangeException(nameof(externalOffset)); + + // Use a unique guid so that concurrent loads don't collide. + string cab = "CAB-" + Guid.NewGuid().ToString("N"); + + // UnityFS uses forward slashes and runs into issues with backslashes, so normalize. + string streamPath = Path.GetFullPath(externalPath).Replace('\\', '/'); + long streamOffset = externalOffset; + long streamSize = pixelsLength; + if (streamOffset + streamSize > uint.MaxValue) + throw new InvalidOperationException( + "texture data exceeds 4 GB; stream offsets are 32-bit in Unity bundles" + ); + + var w = new BundleBufferWriter(EstimateSize()); + + // No pixel bytes are appended: they live in the external DDS file. + var prefix = BundleWriter.WriteHeaderAndBlocksInfo(w, cab, resSLength: 0); + + Span objects = + stackalloc SerializedFileWriter.ObjectMeta[2]; + objects[0] = new SerializedFileWriter.ObjectMeta( + AssetBundlePathId, SerializedTypeTrees.AssetBundleClassId); + objects[1] = new SerializedFileWriter.ObjectMeta( + TexturePathId, SerializedTypeTrees.Texture2DClassId); + Span slots = + stackalloc SerializedFileWriter.ObjectSlot[2]; + + var file = SerializedFileWriter.BeginFile(w, targetPlatform, objects, slots); + + file.BeginObject(w, ref slots[0]); + WriteAssetBundleBody(w, cab, TexturePathId); + file.EndObject(w, slots[0]); + + file.BeginObject(w, ref slots[1]); + WriteClassicTextureBody(w, req, streamPath, streamOffset, streamSize); + file.EndObject(w, slots[1]); + + long serializedFileLength = file.End(w); + + w.AlignBase = 0; + BundleWriter.Finish(w, prefix, serializedFileLength); + + return new Built(w.ToArray(), ContainerKey, pixelsLength); + } + + /// + /// Build a single UnityFS bundle wrapping every texture in , each + /// streaming its pixels from its own DDS file on disk, plus the one AssetBundle object + /// that references them all. Loading this bundle once (a single + /// AssetBundle.LoadFromMemoryAsync + LoadAllAssetsAsync) realizes them all, + /// which avoids the native-allocator exhaustion of loading tens of thousands of one-texture + /// bundles. Returns null when is empty. Pure CPU work; safe + /// to call from a background thread. + /// + public static byte[] BuildMany( + IReadOnlyList entries, + int targetPlatform = StandaloneWindows64 + ) + { + if (entries is null) + throw new ArgumentNullException(nameof(entries)); + + int n = entries.Count; + if (n == 0) + return null; + + // Use a unique guid so that concurrent loads don't collide. + string cab = "CAB-" + Guid.NewGuid().ToString("N"); + + var w = new BundleBufferWriter(EstimateSizeMany(n)); + + // No pixel bytes are appended: they live in the external DDS files. + var prefix = BundleWriter.WriteHeaderAndBlocksInfo(w, cab, resSLength: 0); + + // One AssetBundle object plus one Texture2D object per entry. N is in the tens of + // thousands for a heavy install, so these live on the heap, not the stack. + var objects = new SerializedFileWriter.ObjectMeta[n + 1]; + var slots = new SerializedFileWriter.ObjectSlot[n + 1]; + objects[0] = new SerializedFileWriter.ObjectMeta( + AssetBundlePathId, SerializedTypeTrees.AssetBundleClassId); + for (int i = 0; i < n; ++i) + objects[i + 1] = new SerializedFileWriter.ObjectMeta( + FirstTexturePathId + i, SerializedTypeTrees.Texture2DClassId); + + var file = SerializedFileWriter.BeginFile(w, targetPlatform, objects, slots); + + file.BeginObject(w, ref slots[0]); + WriteAssetBundleBodyMany(w, cab, entries); + file.EndObject(w, slots[0]); + + for (int i = 0; i < n; ++i) + { + TextureEntry e = entries[i]; + + // UnityFS uses forward slashes and runs into issues with backslashes, so normalize. + string streamPath = Path.GetFullPath(e.ExternalPath).Replace('\\', '/'); + long streamOffset = e.ExternalOffset; + long streamSize = e.PixelsLength; + if (streamOffset + streamSize > uint.MaxValue) + throw new InvalidOperationException( + "texture data exceeds 4 GB; stream offsets are 32-bit in Unity bundles" + ); + + var req = new TextureRequest + { + Name = e.Name, + Width = e.Width, + Height = e.Height, + MipCount = e.MipCount, + Format = e.Format, + ColorSpace = e.ColorSpace, + Readable = e.Readable, + }; + + file.BeginObject(w, ref slots[i + 1]); + WriteClassicTextureBody(w, req, streamPath, streamOffset, streamSize); + file.EndObject(w, slots[i + 1]); + } + + long serializedFileLength = file.End(w); + + w.AlignBase = 0; + BundleWriter.Finish(w, prefix, serializedFileLength); + + return w.ToArray(); + } + + // Framing plus the two verbatim type entries (which dominate) plus room for N object bodies. + // Each body is m_Name + the fixed Texture2D fields + the m_StreamData path (an absolute file + // path); ~512 bytes/texture is a generous per-entry estimate so the buffer rarely regrows. + static int EstimateSizeMany(int n) => + 1024 + + ReferenceTypeTrees.TypeEntry(SerializedTypeTrees.AssetBundleClassId).Length + + ReferenceTypeTrees.TypeEntry(SerializedTypeTrees.Texture2DClassId).Length + + n * 512; + + // A generous starting capacity so the buffer rarely regrows: the framing plus the two + // verbatim type entries (the type trees dominate) plus room for the bodies. + static int EstimateSize() => + 1024 + + ReferenceTypeTrees.TypeEntry(SerializedTypeTrees.AssetBundleClassId).Length + + ReferenceTypeTrees.TypeEntry(SerializedTypeTrees.Texture2DClassId).Length; + + static void WriteClassicTextureBody( + BundleBufferWriter w, + TextureRequest req, + string streamPath, + long streamOffset, + long streamSize + ) + { + // Unlike m_StreamData.size, m_CompleteImageSize is a signed int in the 2019.4 type + // tree, so the classic types cap at 2 GB per image. + if (streamSize > int.MaxValue) + throw new InvalidOperationException( + "texture data exceeds 2 GB; m_CompleteImageSize is a signed 32-bit int" + ); + + w.WriteAlignedString(req.Name); // m_Name + w.WriteInt32(ForcedFallbackFormat); // m_ForcedFallbackFormat + w.WriteBool(false); // m_DownscaleFallback + w.Align(4); + w.WriteInt32(req.Width); // m_Width + w.WriteInt32(req.Height); // m_Height + // The size of one image (a single face's full mip chain). Unity reads + // m_ImageCount * m_CompleteImageSize from the stream; for a 2D texture m_ImageCount is 1. + w.WriteInt32((int)streamSize); // m_CompleteImageSize + w.WriteInt32(req.Format); // m_TextureFormat + w.WriteInt32(req.MipCount); // m_MipCount + w.WriteBool(req.Readable); // m_IsReadable + w.WriteBool(false); // m_IgnoreMasterTextureLimit + w.WriteBool(false); // m_IsPreProcessed + w.WriteBool(false); // m_StreamingMipmaps + w.Align(4); + w.WriteInt32(0); // m_StreamingMipmapsPriority + w.Align(4); + w.WriteInt32(1); // m_ImageCount + w.WriteInt32(TextureDimensionTex2D); // m_TextureDimension + WriteTextureSettings(w); // m_TextureSettings + w.WriteInt32(0); // m_LightmapFormat + w.WriteInt32(req.ColorSpace); // m_ColorSpace + WriteEmptyImageData(w); // image data + WriteStreamData(w, streamOffset, streamSize, streamPath); // m_StreamData + } + + static void WriteTextureSettings(BundleBufferWriter w) + { + w.WriteInt32(1); // m_FilterMode (bilinear) + w.WriteInt32(1); // m_Aniso + w.WriteSingle(0f); // m_MipBias + w.WriteInt32(0); // m_WrapU (repeat) + w.WriteInt32(0); // m_WrapV + w.WriteInt32(0); // m_WrapW + } + + // The inline pixel array is always empty (pixels are streamed), but the node still carries + // a count prefix and the align flag. + static void WriteEmptyImageData(BundleBufferWriter w) => w.BeginArray().End(align: true); + + // offset and size are unsigned ints; the 4 GB bound is checked in Build. + static void WriteStreamData( + BundleBufferWriter w, + long streamOffset, + long streamSize, + string streamPath + ) + { + w.WriteUInt32((uint)streamOffset); // offset + w.WriteUInt32((uint)streamSize); // size + w.WriteAlignedString(streamPath); // path + } + + static void WriteAssetBundleBody(BundleBufferWriter w, string identity, long texturePathId) + { + w.WriteAlignedString(identity); // m_Name + + // m_PreloadTable is what LoadAssetAsync actually loads during its asynchronous phase: + // the preload thread reads the objects listed for the requested asset, including their + // streamed data. Without an entry the request completes having loaded nothing, and the + // first access to its `asset` property then performs the entire load (pixel read + + // upload) synchronously on the main thread. + var preload = w.BeginArray(); + preload.Add(); + WritePPtr(w, fileId: 0, pathId: texturePathId); + preload.End(align: true); + + // m_Container: map. Its Array is not aligned. + var container = w.BeginArray(); + container.Add(); + w.WriteAlignedString(ContainerKey); // pair.first + WriteAssetInfo(w, preloadIndex: 0, preloadSize: 1, fileId: 0, pathId: texturePathId); + container.End(); + + WriteAssetInfo(w, preloadIndex: 0, preloadSize: 0, fileId: 0, pathId: 0); // m_MainAsset + w.WriteUInt32(1); // m_RuntimeCompatibility + w.WriteAlignedString(identity); // m_AssetBundleName + w.BeginArray().End(align: true); // m_Dependencies (empty) + w.WriteBool(false); // m_IsStreamedSceneAssetBundle + w.Align(4); + w.WriteInt32(0); // m_ExplicitDataLayout + w.WriteInt32(0); // m_PathFlags + w.BeginArray().End(); // m_SceneHashes (empty map, Array not aligned) + } + + // The multi-texture variant of WriteAssetBundleBody: the preload table and container each + // carry one entry per texture, and each container entry references its own single-entry + // preload slice so LoadAllAssetsAsync streams every texture. + static void WriteAssetBundleBodyMany( + BundleBufferWriter w, + string identity, + IReadOnlyList entries + ) + { + int n = entries.Count; + + w.WriteAlignedString(identity); // m_Name + + // m_PreloadTable: one PPtr per texture. + var preload = w.BeginArray(); + for (int i = 0; i < n; ++i) + { + preload.Add(); + WritePPtr(w, fileId: 0, pathId: FirstTexturePathId + i); + } + preload.End(align: true); + + // m_Container: map. Its Array is not aligned. Each texture is keyed by + // its entry name and points at its own single-entry preload slice. + var container = w.BeginArray(); + for (int i = 0; i < n; ++i) + { + container.Add(); + w.WriteAlignedString(entries[i].Name); // pair.first + WriteAssetInfo( + w, preloadIndex: i, preloadSize: 1, fileId: 0, pathId: FirstTexturePathId + i); + } + container.End(); + + WriteAssetInfo(w, preloadIndex: 0, preloadSize: 0, fileId: 0, pathId: 0); // m_MainAsset + w.WriteUInt32(1); // m_RuntimeCompatibility + w.WriteAlignedString(identity); // m_AssetBundleName + w.BeginArray().End(align: true); // m_Dependencies (empty) + w.WriteBool(false); // m_IsStreamedSceneAssetBundle + w.Align(4); + w.WriteInt32(0); // m_ExplicitDataLayout + w.WriteInt32(0); // m_PathFlags + w.BeginArray().End(); // m_SceneHashes (empty map, Array not aligned) + } + + static void WritePPtr(BundleBufferWriter w, int fileId, long pathId) + { + w.WriteInt32(fileId); // m_FileID + w.WriteInt64(pathId); // m_PathID + } + + static void WriteAssetInfo( + BundleBufferWriter w, + int preloadIndex, + int preloadSize, + int fileId, + long pathId + ) + { + w.WriteInt32(preloadIndex); + w.WriteInt32(preloadSize); + WritePPtr(w, fileId, pathId); + } + } +} diff --git a/KSPCommunityFixes/Library/TextureBundle/TextureTypeTrees.bin b/KSPCommunityFixes/Library/TextureBundle/TextureTypeTrees.bin new file mode 100644 index 0000000000000000000000000000000000000000..805235bb5b2cb55704f5cf18c07573dc47d450fa GIT binary patch literal 3840 zcmd5;OK%)S5bp8X0mA!{@JJvAhi9;fLjnl`Y2$T_SJ<$L9V{V)8Sl33iDze4vtzJ1 zpgDlV34u5m2_Zm2xp0ld6$uG(MnXa$!6knHB+lmh`mx)++sPS9NtycYXCr zZE-OWLfnk!#PP{96DKDoPp?i6;qx$_VxR8C?(X^VpAN+1D~G?TT=}E*$-mKKfWL$( zhhD0P;(3hTh2Rr!s#Q%O0x=8xyFlZdEhuj64G6?gFbNAjw5PwI>f#mgC$K~CF$}Ci zT=@8T;Fs{wH^=yRpzNVm*a%-7VSID2y%###W5q&L*=Qlo`H$g~zHh*DhlW&Z@n`Wx zoRRH?zi9BAu7NiUz8^i~uwwnc4F&!4PYwEK{2S2I?iM_**!68eN&Hq9R64!>&kTN> z>FqFU|E2TUBFhE&dDMiQkE5Fn;duUk1Mmy}SM)eEmZV%KOmY zjmH(7AN79@0;m6bToCUTA2axYJ@9WZ|1pCf+yiHPrwq<>;)?bElEE3HD;7Uz@cUg5 z?{M) z|3_$Bu+c*Qmtp@YaPI%8(c8}AcK(MgV0z-r51$+Vk5FG25M3ZDUVa#(uIl2$UVb>L zY<%3!504Ju?d6#?Ys%J47{@DNa|2)PFcGa*C+YDoA5$KkpR=$pYRz_><98^o& zoml3w-VWE~LiC=5Zc%RKo0;r6nvXiQbTi@bdiR11Tj5H~arLz%&E$O8&1Gh6=Aw3_ z$2Bt9$kL|lc4Z4=7xGMo?I>A;vv%0&?xAXAQJO{hHa#;MJEw6y9kpfBjnYKCJZDr3 zGSA_X&dQpZC`MfKX^WwrO`>j!9BVb=~#bH6G~TCyD}G@5Iarz7|%syRrWaHJHIc_e024VrJt{izcy9R zc3yy%`GLGZ`9Jf}uQ=z-&%CFKhYCil1^2)^=Vc!9d5}Eh3hs&69@6)ibMzTfj|y-* zna7wvcu&bgpk%aKh)*Cs-XGd(9x615mU-%L*pK6#@so!fbD)0wiiZjg(!%)E{FsCD zmwBinj+prApFHIIA69wd3uuWe9x6CUOCUzP{UgWtd19Y8KY0i#rzZiJ-^~+)JjDGu zgVFmDKRKq%LnF$64H(x~<{^zY3|{7;5%vGx1kSxH^N@+(*z0jFs>}@P6+=&-ss{zux@hA@Ktn zExr9Q_k;DdE5x}z@-+QFPO#{Y!1?sf{}X`x_386xdH6}ee8T^0GP+e$@?cn|$??dX1xSL-l zsVx+w8s2a2ugxzfs4Wzvn!)_Sdf(q)KfjzL-Z#g)pI=Uua6Lcs3)lY~Z3{M92ImLQ zU3dP|#EmVx!TFiXT>LC5jeKe|X{p)- typeof(KSPCFFastLoader).FullName; @@ -454,13 +456,15 @@ static IEnumerator FastAssetLoader(List configFileTypes) // Files loaded by our custom loaders List audioFiles = new List(1000); - List textureRequests = new List(10000); - List modelAssets = new List(5000); + // Textures that need to be loaded on the main thread go through here. + BlockingCollection textureQueue = []; + List bundleRequests = new(10000); + List modelAssets = new(5000); // Files loaded by mod-defined loaders (ex : Shabby *.shab files) - List unsupportedAudioFiles = new List(100); - List unsupportedTextureFiles = new List(100); - List unsupportedModelFiles = new List(100); + List unsupportedAudioFiles = new(100); + List unsupportedTextureFiles = new(100); + List unsupportedModelFiles = new(100); // Keeping track of already loaded files to avoid loading duplicates. // Note that to replicate stock behavior, we can't populate those @@ -500,23 +504,23 @@ static IEnumerator FastAssetLoader(List configFileTypes) switch (file.fileExtension) { case "dds": - textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureDDS)); + bundleRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureDDS)); break; case "jpg": case "jpeg": - textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureJPG)); + textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureJPG)); break; case "mbm": - textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureMBM)); + textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureMBM)); break; case "png": - textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TexturePNG)); + textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TexturePNG)); break; case "tga": - textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTGA)); + textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTGA)); break; case "truecolor": - textureRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTRUECOLOR)); + textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTRUECOLOR)); break; default: unsupportedTextureFiles.Add(file); @@ -548,6 +552,18 @@ static IEnumerator FastAssetLoader(List configFileTypes) } } + SupportedFormatCache.Build(); + + // Tune the AUP for much better throughput + QualitySettings.asyncUploadTimeSlice = 10; // ms per frame spent on async uploads (default 2) + QualitySettings.asyncUploadBufferSize = 128; // MB ring buffer for async uploads (default 16) + + int textureCount = bundleRequests.Count + textureQueue.Count; + + // Kick off the background bundle build + BundleState bundleState = new(); + gdb.StartCoroutine(LoadBundledAssets(bundleState, bundleRequests, textureQueue)); + gdb.progressTitle = "Loading sound assets..."; KSPCFFastLoaderReport.wAudioLoading.Restart(); yield return null; @@ -638,9 +654,9 @@ static IEnumerator FastAssetLoader(List configFileTypes) // start texture loading gdb.progressFraction = 0.25f; KSPCFFastLoaderReport.wAudioLoading.Stop(); - SupportedFormatCache.Build(); KSPCFFastLoaderReport.wTextureLoading.Restart(); gdb.progressTitle = "Loading texture assets..."; + yield return null; // call non-stock texture loaders @@ -688,8 +704,11 @@ static IEnumerator FastAssetLoader(List configFileTypes) } // call our custom loader + yield return gdb.StartCoroutine(TextureDriverCoroutine(textureQueue, allTextureFiles, bundleState, textureCount)); - yield return gdb.StartCoroutine(TextureDriverCoroutine(textureRequests, allTextureFiles)); + while (!bundleState.Done) + yield return null; + InsertBundledTextures(bundleState, allTextureFiles); // start model loading gdb.progressFraction = 0.75f; @@ -1227,6 +1246,222 @@ private GameObject LoadDAE() #endregion + #region Texture bundle loader + private struct BundleItem + { + public TextureLoadRequest Request; + public bool IsNormalMap; + } + + // Result of the background DDS bucketing + bundle-building task. + private sealed class BundleBuildResult + { + // The combined bundle bytes, or null when no DDS texture was bundle-eligible. + public byte[] Bytes; + // The eligible textures, to be looked up in the loaded bundle by File.url. + public List Items; + } + + private sealed class BundleState + { + // How many textures are getting loaded from the bundle? + public int Count => Items.Count; + // What's the current progress of the bundle + public float Progress; + // Did the bundle load fail? + public bool Failed; + public bool Done; + public Dictionary Map = []; + public List Items = []; + } + + // Background task: parse every DDS header in parallel, route the bundle-eligible textures into + // one combined bundle (assembled here on this thread) and push the rest onto textureQueue for + // the per-request driver. Returns the combined bundle bytes (null when none were eligible) plus + // the eligible items, which the main thread resolves against the loaded bundle afterwards. + private static BundleBuildResult BuildDDSBundle( + List ddsRequests, + BlockingCollection textureQueue) + { + // One (TextureEntry for the writer, BundleItem for the resolve) pair per eligible texture. + ConcurrentQueue<(TextureBundleBuilder.TextureEntry entry, BundleItem item)> eligible = + new(); + + // The reads are tiny (a 128-148 byte header each) but there are tens of thousands of them, + // so parse them across the thread pool. ParseDDSHeader opens its own stream per call and + // only touches thread-safe GraphicsFormatUtility APIs, so this is safe to run concurrently; + // each request is written by exactly one iteration, and textureQueue/eligible are both + // thread-safe producers. + try + { + Parallel.ForEach(ddsRequests, req => + { + DDSPreparedHeader hdr; + try + { + using (s_pmParseDDSHeader.Auto()) + hdr = ParseDDSHeader(req.File.fullPath); + } + catch + { + // Couldn't parse: let the per-request DDS loader re-parse and surface the error. + textureQueue.Add(req); + return; + } + + req.FileLength = hdr.FileLength; + + if (hdr.BundleEligible && SupportedFormatCache.IsSupported(hdr.Format)) + { + req.Bundled = true; + TextureBundleBuilder.TextureEntry entry = new( + req.File.url, + hdr.Width, hdr.Height, hdr.MipCount, + hdr.ClassicTextureFormat, hdr.ColorSpace, readable: false, + req.File.fullPath, hdr.DataOffset, hdr.StreamedSize); + eligible.Enqueue((entry, new BundleItem { Request = req, IsNormalMap = hdr.IsNormalMap })); + } + else + { + textureQueue.Add(req); + } + }); + } + finally + { + // The main thread finished adding non-DDS before dispatch and this task is the only + // other producer, so once bucketing is done nothing else will be added. This runs even + // if bucketing throws unexpectedly, so the driver's IsCompleted wait can never hang. + textureQueue.CompleteAdding(); + } + + List entries = new List(eligible.Count); + List items = new List(eligible.Count); + foreach ((TextureBundleBuilder.TextureEntry entry, BundleItem item) in eligible) + { + entries.Add(entry); + items.Add(item); + } + + return new BundleBuildResult + { + Bytes = entries.Count == 0 ? null : TextureBundleBuilder.BuildMany(entries), + Items = items, + }; + } + + private static IEnumerator LoadBundledAssets( + BundleState state, + List requests, + BlockingCollection textureQueue + ) + { + var inner = LoadBundledAssetsImpl(state, requests, textureQueue); + + while (true) + { + object current; + try + { + if (!inner.MoveNext()) + break; + + current = inner.Current; + } + catch (Exception e) + { + if (e is AggregateException agg) + e = agg.InnerException ?? e; + + Debug.LogError("Failed to load bundled textres"); + Debug.LogException(e); + + state.Progress = 1f; + state.Failed = true; + break; + } + + yield return current; + } + + state.Done = true; + } + + private static IEnumerator LoadBundledAssetsImpl( + BundleState state, + List requests, + BlockingCollection textureQueue) + { + var task = Task.Run(() => BuildDDSBundle(requests, textureQueue)); + while (!task.IsCompleted) + yield return null; + + var built = task.Result; + state.Items = built.Items; + if (state.Count == 0) + { + state.Progress = 1f; + yield break; + } + + var bundleRequest = AssetBundle.LoadFromMemoryAsync(built.Bytes); + yield return bundleRequest; + + var bundle = bundleRequest.assetBundle; + if (bundle == null) + throw new Exception("failed to load texture asset bundle"); + + var request = bundle.LoadAllAssetsAsync(); + while (!request.isDone) + { + state.Progress = request.progress; + yield return null; + } + state.Progress = 1f; + + UnityEngine.Object[] assets = request.allAssets; + Dictionary map = new(assets.Length); + for (int i = 0; i < assets.Length; ++i) + { + if (assets[i] is Texture2D tex) + map[tex.name] = tex; + } + + state.Map = map; + } + + private static void InsertBundledTextures( + BundleState state, + HashSet loadedUrls) + { + List items = state.Items; + if (items == null || items.Count == 0) + return; + + Dictionary map = state.Map; + + foreach (var item in items) + { + TextureLoadRequest req = item.Request; + + if (!state.Failed && map != null + && map.TryGetValue(req.File.url, out Texture2D tex) && tex.IsNotNullOrDestroyed()) + { + req.Result = new TextureInfo(req.File, tex, item.IsNormalMap, isReadable: false, isCompressed: true); + req.Status = TextureLoadRequest.State.Ready; + } + else + { + req.ErrorMessage ??= "DDS: streamed texture missing from combined bundle"; + req.Status = TextureLoadRequest.State.Failed; + } + + InsertReadyRequest(req, loadedUrls); + loadedAssetCount++; + } + } + #endregion + #region Per-texture coroutine loader // Profiling markers for the work scheduled on background threads via Task.Run. @@ -1254,6 +1489,7 @@ public enum State : byte { Pending, Ready, Failed } public TextureInfo Result; public string ErrorMessage; public Exception Exception; + public bool Bundled; public TextureLoadRequest(UrlFile file, RawAsset.AssetType assetType) { @@ -1273,6 +1509,14 @@ private struct DDSPreparedHeader public GraphicsFormat Format; public long DataOffset; public long FileLength; + + // Whether this texture can be loaded through the streamed asset-bundle path (see + // LoadDDSCoroutine). When true, the fields below are populated for the bundle body. + public bool BundleEligible; + public int ClassicTextureFormat; // legacy TextureFormat as int + public int ColorSpace; // 0 == linear, 1 == sRGB + public int MipCount; // full mip count Unity will allocate + public long StreamedSize; // total mip-chain byte size read from the file } // Probes which GraphicsFormats are actually usable on the running GPU. @@ -1348,15 +1592,35 @@ private static DDSPreparedHeader ParseDDSHeader(string path) throw new IOException($"DDS: {error ?? "unknown format"}"); long dataOffset = hasDx10 ? 148 : 128; + int width = (int)hdr.dwWidth; + int height = (int)hdr.dwHeight; + int mipCount = mipChain ? ComputeMipCount(width, height) : 1; + + // Can we load this texture directly through an asset bundle? + bool bundleEligible = TryGetClassicFormat(fmt, out int classicFormat, out int colorSpace) + && IsBlockAligned(fmt, width, height); + long streamedSize = 0; + if (bundleEligible) + { + streamedSize = ComputeMipChainSize(fmt, width, height, mipCount); + if (streamedSize > int.MaxValue || fileLength - dataOffset < streamedSize) + bundleEligible = false; + } + return new DDSPreparedHeader { - Width = (int)hdr.dwWidth, - Height = (int)hdr.dwHeight, + Width = width, + Height = height, MipChain = mipChain, IsNormalMap = isNormalMap, Format = fmt, DataOffset = dataOffset, FileLength = fileLength, + BundleEligible = bundleEligible, + ClassicTextureFormat = classicFormat, + ColorSpace = colorSpace, + MipCount = mipCount, + StreamedSize = streamedSize, }; } @@ -1460,13 +1724,7 @@ private static GraphicsFormat MapDDSFormat(DDSHeader hdr, bool hasDx10, DDSHeade } } - // In-place channel-swizzle for RGBA32 normal maps. Operates on the texture's - // entire raw byte buffer, which means every populated mip level when the texture - // was created with a mip chain. - // - // Channel swizzle is per-pixel and the box-filter mip generator is linear, so - // swizzling pre-built mips matches what stock KSP produced by swizzling level-0 - // and regenerating from there (BitmapToCompressedNormalMapFast). + // In-place swizzle for RGBA32 normal maps. Goes from rgba -> gggr. private static unsafe void SwizzleNormalMap(NativeArray data) { using var scope = s_pmSwizzleNormalMap.Auto(); @@ -1485,10 +1743,7 @@ private static unsafe void SwizzleNormalMap(NativeArray data) } } - // Legacy src->dst swizzle, kept for the rare TGA RGB24 path (where source and - // destination have different pixel sizes so an in-place transform is impossible). - // Walks src.Length end-to-end; the caller must size dst with a mip chain that - // matches src's so the constant 3:4 (or 4:4) byte-count ratio fills dst exactly. + // Channel swizzle for RGB24, allocates and goes from rgb -> gggr. private static unsafe void SwizzleNormalMap(NativeArray src, NativeArray dst, TextureFormat srcFormat) { using var scope = s_pmSwizzleNormalMap.Auto(); @@ -1559,12 +1814,8 @@ private static unsafe ReadHandle BeginAsyncRead(string path, NativeArray d return AsyncReadManager.Read(path, &cmd, 1); } - // Mirrors UnityEngine.Experimental.Rendering.TextureCreationFlags but with the - // additional flag values that exist in Unity's native code but aren't exposed in - // the public managed enum. DontInitializePixels skips the zero-fill that the - // normal Texture2D constructor performs — pointless work when we're about to - // overwrite the bytes via LoadRawTextureData / AsyncReadManager / LoadImage. - // Borrowed from KSPTextureLoader (../AsyncTextureLoad/src/KSPTextureLoader/TextureUtils.cs). + // An extended version of TextureCreationFlags that contains additional values + // that are not exposed publically by unity. [Flags] private enum InternalTextureCreationFlags { @@ -1660,19 +1911,74 @@ private static IEnumerator LoadTextureWrapperCoroutine(TextureLoadRequest req, I } } + // The classic Texture2D object serializes a legacy TextureFormat plus a colour space; only + // graphics formats that survive the round-trip can go through the bundle path. + private static bool TryGetClassicFormat(GraphicsFormat format, out int textureFormat, out int colorSpace) + { + TextureFormat tf = GraphicsFormatUtility.GetTextureFormat(format); + bool srgb = GraphicsFormatUtility.IsSRGBFormat(format); + textureFormat = (int)tf; + colorSpace = srgb ? 1 : 0; + return GraphicsFormatUtility.GetGraphicsFormat(tf, srgb) == format; + } + + // A texture is "block aligned" when it is uncompressed (block size 1x1, always aligned) or + // its dimensions are a multiple of the compression block size. Only misaligned compressed + // textures must avoid the background-upload bundle path. + private static bool IsBlockAligned(GraphicsFormat format, int width, int height) + { + if (!GraphicsFormatUtility.IsCompressedFormat(format)) + return true; + int blockWidth = (int)GraphicsFormatUtility.GetBlockWidth(format); + int blockHeight = (int)GraphicsFormatUtility.GetBlockHeight(format); + return width % blockWidth == 0 && height % blockHeight == 0; + } + + // The number of mip levels Unity allocates for a full mip chain. + private static int ComputeMipCount(int width, int height) + { + int size = Math.Max(width, height); + int count = 1; + while (size > 1) + { + size >>= 1; + count++; + } + return count; + } + + // Total byte size of the mip chain as laid out in a Texture2D's raw data: for each level, + // ceil(w/blockW) * ceil(h/blockH) * blockSize. Matches Unity's GetRawTextureData layout. + private static long ComputeMipChainSize(GraphicsFormat format, int width, int height, int mipCount) + { + int blockWidth = (int)GraphicsFormatUtility.GetBlockWidth(format); + int blockHeight = (int)GraphicsFormatUtility.GetBlockHeight(format); + int blockSize = (int)GraphicsFormatUtility.GetBlockSize(format); + long total = 0; + for (int mip = 0; mip < mipCount; ++mip) + { + int mipWidth = Math.Max(1, width >> mip); + int mipHeight = Math.Max(1, height >> mip); + int blocksX = Math.Max(1, (mipWidth + blockWidth - 1) / blockWidth); + int blocksY = Math.Max(1, (mipHeight + blockHeight - 1) / blockHeight); + total += (long)blocksX * blocksY * blockSize; + } + return total; + } + private static IEnumerator LoadDDSCoroutine(TextureLoadRequest req) { string path = req.File.fullPath; - Task hdrTask = Task.Run(() => + Task prepTask = Task.Run(() => { - using var scope = s_pmParseDDSHeader.Auto(); - return ParseDDSHeader(path); + using (s_pmParseDDSHeader.Auto()) + return ParseDDSHeader(path); }); - while (!hdrTask.IsCompleted) + while (!prepTask.IsCompleted) yield return null; - if (hdrTask.IsFaulted) - throw UnwrapFaultedTask(hdrTask, "DDS header parse failed"); - DDSPreparedHeader hdr = hdrTask.Result; + if (prepTask.IsFaulted) + throw UnwrapFaultedTask(prepTask, "DDS header parse failed"); + DDSPreparedHeader hdr = prepTask.Result; req.FileLength = hdr.FileLength; if (!SupportedFormatCache.IsSupported(hdr.Format)) @@ -2069,21 +2375,26 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) req.Status = TextureLoadRequest.State.Ready; } - private static IEnumerator TextureDriverCoroutine(List requests, HashSet loadedUrls) + // Drains the shared texture queue, spawning up to MaxTextureSpawnsPerFrame concurrent + // per-request loaders per frame and inserting finished ones in FIFO order. The queue is fed + // by the enumeration loop (non-DDS) and the background bucketer (non-bundled DDS); the + // bucketer calls CompleteAdding when done, so IsCompleted && empty means no more will arrive. + private static IEnumerator TextureDriverCoroutine( + BlockingCollection requests, + HashSet loadedUrls, + BundleState state, + int totalTextureCount) { GameDatabase gdb = GameDatabase.Instance; - Queue active = new Queue(); - int total = requests.Count; - var iter = requests.GetEnumerator(); + Queue active = new(); int completed = 0; while (true) { for (int i = 0; i < MaxTextureSpawnsPerFrame; ++i) { - if (!iter.MoveNext()) - goto WINDDOWN; - var request = iter.Current; + if (!requests.TryTake(out var request)) + break; gdb.StartCoroutine(LoadTextureCoroutine(request)); active.Enqueue(request); @@ -2100,24 +2411,16 @@ private static IEnumerator TextureDriverCoroutine(List reque completed++; } + int progress = completed + (int)(state.Progress * state.Count); + gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; - gdb.progressTitle = $"Loading texture asset {completed}/{total}"; - yield return null; - } + gdb.progressTitle = $"Loading texture asset {progress}/{totalTextureCount}"; - WINDDOWN: - while (active.TryDequeue(out var pending)) - { - while (pending.Status == TextureLoadRequest.State.Pending) - { - gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; - gdb.progressTitle = $"Loading texture asset {completed}/{total}"; - yield return null; - } + // Done when the producers have finished and everything spawned has been drained. + if (requests.IsCompleted && active.Count == 0) + break; - InsertReadyRequest(pending, loadedUrls); - loadedAssetCount++; - completed++; + yield return null; } } @@ -2183,45 +2486,6 @@ private static IEnumerator LoadTextureCoroutine(TextureLoadRequest req) } } - private static void SpawnTextureCoroutine(TextureLoadRequest req, Queue active, GameDatabase gdb) - { - IEnumerator inner; - switch (req.AssetType) - { - case RawAsset.AssetType.TextureDDS: - inner = LoadDDSCoroutine(req); - break; - case RawAsset.AssetType.TexturePNG: - case RawAsset.AssetType.TextureJPG: - inner = LoadUWRCoroutine(req); - break; - case RawAsset.AssetType.TextureTRUECOLOR: - inner = LoadTRUECOLORCoroutine(req); - break; - case RawAsset.AssetType.TextureMBM: - inner = LoadMBMCoroutine(req); - break; - case RawAsset.AssetType.TextureTGA: - inner = LoadTGACoroutine(req); - break; - default: - inner = null; - break; - } - - if (inner == null) - { - req.ErrorMessage = $"Unknown asset type {req.AssetType}"; - req.Status = TextureLoadRequest.State.Failed; - } - else - { - Debug.Log($"Load Texture: {req.File.url}"); - gdb.StartCoroutine(LoadTextureWrapperCoroutine(req, inner)); - } - active.Enqueue(req); - } - private static void InsertReadyRequest(TextureLoadRequest req, HashSet loadedUrls) { Debug.Log($"Load Texture: {req.File.url}"); diff --git a/Tools/GenerateTextureTypeTrees.py b/Tools/GenerateTextureTypeTrees.py new file mode 100644 index 00000000..cf4dce82 --- /dev/null +++ b/Tools/GenerateTextureTypeTrees.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# One-time developer tool. NOT part of the build. +# +# Regenerates KSPCommunityFixes/Performance/TextureBundle/TextureTypeTrees.bin, the +# embedded artifact that supplies the invariant Unity type-tree bytes spliced into the +# in-memory asset bundles KSPCFFastLoader builds to stream DDS textures from disk. +# +# It extracts the type entries for the classes we emit (Texture2D = 28, AssetBundle = 142) +# from a proven, Unity-generated reference bundle -- the sibling KSPTextureLoader project's +# typetrees.bundle (itself produced offline from Unity's class database). Only these two +# top-level entries are needed: each carries its complete nested field tree (StreamingInfo, +# GLTextureSettings, m_Container/AssetInfo/PPtr, etc.) inline. +# +# Source bundle is an uncompressed UnityFS archive (serialized file format 21, Unity +# 2019.4.18f1). If KSP ever moves to a different Unity version, regenerate from a matching +# reference bundle. +# +# Usage: python Tools/GenerateTextureTypeTrees.py [path-to-typetrees.bundle] +# +# Output format of TextureTypeTrees.bin (all little-endian): +# magic : 4 bytes "KCTT" +# version : u32 = 1 +# unityVersionLen : i32 +# unityVersion : UTF-8 bytes (no terminator) +# count : i32 +# per entry: classId i32, length i32, raw type-entry bytes + +import os +import struct +import sys + +# The classes whose type entries we emit into generated bundles. +WANTED = [28, 142] # Texture2D, AssetBundle + +DEFAULT_SOURCE = os.path.join( + os.path.dirname(__file__), "..", "..", + "KSPTextureLoader", "src", "KSPTextureLoader", "Format", "Bundle", "typetrees.bundle", +) +OUTPUT = os.path.join( + os.path.dirname(__file__), "..", + "KSPCommunityFixes", "Library", "TextureBundle", "TextureTypeTrees.bin", +) + + +class Cursor: + def __init__(self, buf, pos=0, big_endian=True): + self.b = buf + self.pos = pos + self.be = big_endian + + def u(self, n): + v = int.from_bytes(self.b[self.pos:self.pos + n], "big" if self.be else "little") + self.pos += n + return v + + def i(self, n): + v = int.from_bytes(self.b[self.pos:self.pos + n], "big" if self.be else "little", signed=True) + self.pos += n + return v + + def cstr(self): + start = self.pos + while self.b[self.pos] != 0: + self.pos += 1 + v = self.b[start:self.pos].decode("utf-8") + self.pos += 1 + return v + + def skip(self, n): + self.pos += n + + +def extract(bundle): + c = Cursor(bundle, big_endian=True) + + # --- UnityFS container header --- + sig = c.cstr() + if sig != "UnityFS": + raise ValueError(f"not a UnityFS bundle (signature {sig!r})") + c.u(4) # format version (7) + c.cstr() # player min version + c.cstr() # engine revision + c.i(8) # total size + c.u(4) # blocks-info compressed size + c.u(4) # blocks-info uncompressed size + flags = c.u(4) + if (flags & 0x3F) != 0: + raise ValueError("reference bundle must have uncompressed blocks-info") + if c.pos % 16: # bundle format 7 aligns the header to 16 bytes + c.skip(16 - (c.pos % 16)) + + # --- blocks-info directory --- + c.skip(16) # data hash + block_count = c.i(4) + for _ in range(block_count): + c.u(4) # uncompressed size + c.u(4) # compressed size + block_flags = c.u(2) + if block_flags & 0x3F: + raise ValueError("reference bundle block must be uncompressed") + node_count = c.i(4) + nodes = [] + for _ in range(node_count): + off = c.i(8) + size = c.i(8) + nflags = c.u(4) + c.cstr() # node path + nodes.append((off, size, nflags)) + block_data_start = c.pos + + sf_nodes = [n for n in nodes if n[2] & 0x4] + if len(sf_nodes) != 1: + raise ValueError(f"expected exactly one serialized-file node, found {len(sf_nodes)}") + off, size, _ = sf_nodes[0] + sf = bundle[block_data_start + off: block_data_start + off + size] + + # --- serialized file header (big-endian) --- + s = Cursor(sf, big_endian=True) + s.u(4) # metadata size + s.u(4) # file size + version = s.u(4) + if version != 21: + raise ValueError(f"expected serialized file format 21, got {version}") + s.u(4) # data offset + endian = s.u(1) + s.skip(3) + + # --- metadata --- + s.be = (endian == 1) + unity_version = s.cstr() + s.i(4) # target platform + enable_type_tree = s.u(1) + if not enable_type_tree: + raise ValueError("reference bundle must have the type tree enabled") + + type_count = s.i(4) + entries = {} + for _ in range(type_count): + start = s.pos + class_id = s.i(4) + s.u(1) # is stripped type + s.i(2) # script type index + if class_id == 114: + s.skip(16) # script id (MonoBehaviour only) + s.skip(16) # old type hash + node_n = s.i(4) + str_buf = s.i(4) + s.skip(node_n * 32) # type-tree nodes (32 bytes each for format 21) + s.skip(str_buf) # string buffer + dep_n = s.i(4) + s.skip(dep_n * 4) # type dependencies + entries[class_id] = sf[start:s.pos] + + return unity_version, entries + + +def main(): + source = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_SOURCE + source = os.path.abspath(source) + if not os.path.exists(source): + sys.exit(f"source bundle not found: {source}") + + with open(source, "rb") as f: + bundle = f.read() + + unity_version, entries = extract(bundle) + + missing = [cid for cid in WANTED if cid not in entries] + if missing: + sys.exit(f"reference bundle is missing type entries for classes {missing}") + + out = bytearray() + out += b"KCTT" + out += struct.pack(" Date: Mon, 13 Jul 2026 17:14:36 -0700 Subject: [PATCH 17/21] Lower the priority of the texture bundle request --- KSPCommunityFixes/Performance/FastLoader.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 7c2e46b8..82ec3b1a 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -1412,6 +1412,10 @@ private static IEnumerator LoadBundledAssetsImpl( throw new Exception("failed to load texture asset bundle"); var request = bundle.LoadAllAssetsAsync(); + + // This should (maybe?) allow other concurrent asset bundle requests + // to not be blocked by this one. + request.priority = -10; while (!request.isDone) { state.Progress = request.progress; From bd769c2b9470f1cd8c338937f3d641566e0d2176 Mon Sep 17 00:00:00 2001 From: Phantomical Date: Mon, 13 Jul 2026 18:53:56 -0700 Subject: [PATCH 18/21] Add the PNG texture cache back in --- KSPCommunityFixes/Performance/FastLoader.cs | 373 ++++++++++++++++---- 1 file changed, 313 insertions(+), 60 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 82ec3b1a..30569d02 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -448,6 +448,13 @@ static IEnumerator FastAssetLoader(List configFileTypes) // Start load asset bundles in the background while we load other assets. PreloadAssetBundleObjects(gdb); + // If the user hasn't chosen yet then wati for the opt-in + if (!loader.userOptInChoiceDone) + { + gdb.progressTitle = "Waiting for texture cache opt-in..."; + yield return gdb.StartCoroutine(WaitForUserOptIn()); + } + gdb.progressTitle = "Searching assets to load..."; yield return null; @@ -455,7 +462,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) double nextFrameTime = ElapsedTime + minFrameTimeD; // Files loaded by our custom loaders - List audioFiles = new List(1000); + List audioFiles = new(1000); // Textures that need to be loaded on the main thread go through here. BlockingCollection textureQueue = []; List bundleRequests = new(10000); @@ -472,9 +479,9 @@ static IEnumerator FastAssetLoader(List configFileTypes) // before flaging a same-url file as duplicate. Not doing this can break // mods relying on that implementation detail, looking at you, Shabby // and ConformalDecals - HashSet allAudioFiles = new HashSet(1000); - HashSet allTextureFiles = new HashSet(10000); - HashSet allModelFiles = new HashSet(5000); + HashSet allAudioFiles = new(1000); + HashSet allTextureFiles = new(10000); + HashSet allModelFiles = new(5000); foreach (UrlDir dir in gdb.root.AllDirectories) { @@ -514,7 +521,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureMBM)); break; case "png": - textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TexturePNG)); + bundleRequests.Add(new TextureLoadRequest(file, RawAsset.AssetType.TexturePNG)); break; case "tga": textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTGA)); @@ -1275,74 +1282,34 @@ private sealed class BundleState public List Items = []; } - // Background task: parse every DDS header in parallel, route the bundle-eligible textures into - // one combined bundle (assembled here on this thread) and push the rest onto textureQueue for - // the per-request driver. Returns the combined bundle bytes (null when none were eligible) plus - // the eligible items, which the main thread resolves against the loaded bundle afterwards. private static BundleBuildResult BuildDDSBundle( - List ddsRequests, + List bundleRequests, BlockingCollection textureQueue) { - // One (TextureEntry for the writer, BundleItem for the resolve) pair per eligible texture. - ConcurrentQueue<(TextureBundleBuilder.TextureEntry entry, BundleItem item)> eligible = - new(); + List entries = new(bundleRequests.Count); + List items = new(bundleRequests.Count); - // The reads are tiny (a 128-148 byte header each) but there are tens of thousands of them, - // so parse them across the thread pool. ParseDDSHeader opens its own stream per call and - // only touches thread-safe GraphicsFormatUtility APIs, so this is safe to run concurrently; - // each request is written by exactly one iteration, and textureQueue/eligible are both - // thread-safe producers. try { - Parallel.ForEach(ddsRequests, req => + foreach (BundleClassification result in bundleRequests.AsParallel().AsOrdered().Select(ClassifyBundleRequest)) { - DDSPreparedHeader hdr; - try - { - using (s_pmParseDDSHeader.Auto()) - hdr = ParseDDSHeader(req.File.fullPath); - } - catch - { - // Couldn't parse: let the per-request DDS loader re-parse and surface the error. - textureQueue.Add(req); - return; - } - - req.FileLength = hdr.FileLength; - - if (hdr.BundleEligible && SupportedFormatCache.IsSupported(hdr.Format)) + if (result.Eligible) { - req.Bundled = true; - TextureBundleBuilder.TextureEntry entry = new( - req.File.url, - hdr.Width, hdr.Height, hdr.MipCount, - hdr.ClassicTextureFormat, hdr.ColorSpace, readable: false, - req.File.fullPath, hdr.DataOffset, hdr.StreamedSize); - eligible.Enqueue((entry, new BundleItem { Request = req, IsNormalMap = hdr.IsNormalMap })); + entries.Add(result.Entry); + items.Add(result.Item); } else { - textureQueue.Add(req); + textureQueue.Add(result.Request); } - }); + } } finally { - // The main thread finished adding non-DDS before dispatch and this task is the only - // other producer, so once bucketing is done nothing else will be added. This runs even - // if bucketing throws unexpectedly, so the driver's IsCompleted wait can never hang. + // Signal to the main thread that no new requests are coming textureQueue.CompleteAdding(); } - List entries = new List(eligible.Count); - List items = new List(eligible.Count); - foreach ((TextureBundleBuilder.TextureEntry entry, BundleItem item) in eligible) - { - entries.Add(entry); - items.Add(item); - } - return new BundleBuildResult { Bytes = entries.Count == 0 ? null : TextureBundleBuilder.BuildMany(entries), @@ -1350,6 +1317,86 @@ private static BundleBuildResult BuildDDSBundle( }; } + // Outcome of classifying one bundle candidate: either it is bundle-eligible (Entry + Item are set and + // it goes into the combined bundle) or it isn't (only Request is set and it goes to the driver queue). + private readonly struct BundleClassification + { + public readonly TextureLoadRequest Request; + public readonly bool Eligible; + public readonly TextureBundleBuilder.TextureEntry Entry; + public readonly BundleItem Item; + + public BundleClassification(TextureLoadRequest request) + { + Request = request; + Eligible = false; + Entry = default; + Item = default; + } + + public BundleClassification(TextureBundleBuilder.TextureEntry entry, BundleItem item) + { + Request = item.Request; + Eligible = true; + Entry = entry; + Item = item; + } + } + + /// + /// Can we include this request in the asset bundle? + /// + private static BundleClassification ClassifyBundleRequest(TextureLoadRequest req) + { + // Resolve the file whose pixels this request streams from. DDS streams from its own file; a PNG + // streams from its DXT cache, but only when caching is on and a valid, up-to-date cache exists. A + // PNG without one goes to the regular decode path (which rebuilds the cache), so its normal-map + // status comes from the file name, not a header. + string sourcePath; + bool isNormalMap; + if (req.AssetType == RawAsset.AssetType.TexturePNG) + { + if (!textureCacheEnabled || !TryGetValidPngCache(req.File, out sourcePath)) + return new BundleClassification(req); + isNormalMap = req.File.name.EndsWith("NRM"); + } + else + { + sourcePath = req.File.fullPath; + isNormalMap = false; + } + + DDSPreparedHeader hdr; + try + { + using (s_pmParseDDSHeader.Auto()) + hdr = ParseDDSHeader(sourcePath); + } + catch + { + // Couldn't parse: let the per-request loader re-parse (rebuilding the cache for a PNG) and + // surface the error. + return new BundleClassification(req); + } + + req.FileLength = hdr.FileLength; + if (req.AssetType != RawAsset.AssetType.TexturePNG) + isNormalMap = hdr.IsNormalMap; + + if (hdr.BundleEligible && SupportedFormatCache.IsSupported(hdr.Format)) + { + req.Bundled = true; + TextureBundleBuilder.TextureEntry entry = new( + req.File.url, + hdr.Width, hdr.Height, hdr.MipCount, + hdr.ClassicTextureFormat, hdr.ColorSpace, readable: false, + sourcePath, hdr.DataOffset, hdr.StreamedSize); + return new BundleClassification(entry, new BundleItem { Request = req, IsNormalMap = isNormalMap }); + } + + return new BundleClassification(req); + } + private static IEnumerator LoadBundledAssets( BundleState state, List requests, @@ -1466,6 +1513,207 @@ private static void InsertBundledTextures( } #endregion + #region PNG texture cache + + // If the user opts into it, we cache compressed versions of PNG files as DDS files under + // GameData/KSPCommunityFixes/PluginData/TextureCache. Later, when textures are loaded from + // the cache, they can go through the asset bundle path. + + // Marker written into dwReserved1[4] so a cache file is recognisably ours and not some unrelated DDS + // that happens to hash to the same name. + private const uint PngCacheMarker = 0x4643_534Bu; // "KSCF" + + private static string PngCacheDir => Path.Combine(ModPath, "PluginData", "TextureCache"); + + // Deterministic, collision-free (SHA1) and length-safe cache path for a texture URL. + private static string GetPngCachePath(string url) + { + byte[] hash; + using (SHA1 sha = SHA1.Create()) + hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(url)); + return Path.Combine(PngCacheDir, BitConverter.ToString(hash).Replace("-", "") + ".dds"); + } + + // Source-file identity stamp: byte size + last-write-time. A cache is valid only while both still match. + private static bool GetPngStamp(string path, out long size, out long time) + { + size = 0; + time = 0; + try + { + FileInfo fi = new FileInfo(path); + if (!fi.Exists) + return false; + size = fi.Length; + time = fi.LastWriteTimeUtc.ToFileTimeUtc(); + return size > 0; + } + catch + { + return false; + } + } + + // Reads the (size, time) stamp embedded in a cache file's DDS reserved header. Returns false if the + // file is missing, too small, isn't a DDS, or wasn't written by us. + private static bool TryReadPngCacheStamp(string path, out long size, out long time) + { + size = 0; + time = 0; + try + { + using FileStream fs = File.OpenRead(path); + if (fs.Length < 148) + return false; + using BinaryReader br = new BinaryReader(fs); + if (br.ReadUInt32() != DDSValues.uintMagic) + return false; + // dwReserved1 starts 28 bytes into the 124-byte header, i.e. at file offset 32. + fs.Position = 32; + long s = br.ReadInt64(); // dwReserved1[0..1] + long t = br.ReadInt64(); // dwReserved1[2..3] + if (br.ReadUInt32() != PngCacheMarker) // dwReserved1[4] + return false; + size = s; + time = t; + return true; + } + catch + { + return false; + } + } + + // True when a valid, up-to-date cache DDS exists for the given PNG. cachePath is set to the resolved + // cache location on success so the caller can stream from it. + private static bool TryGetValidPngCache(UrlFile file, out string cachePath) + { + cachePath = GetPngCachePath(file.url); + if (!GetPngStamp(file.fullPath, out long size, out long time)) + return false; + return TryReadPngCacheStamp(cachePath, out long cachedSize, out long cachedTime) + && cachedSize == size && cachedTime == time; + } + + // Maps the four DXT graphics formats the PNG loader can produce to their DXGI equivalents. Returns + // false for anything else (e.g. uncompressed), which is never cached. + private static bool TryGetCacheDxgiFormat(GraphicsFormat format, out uint dxgiFormat) + { + switch (format) + { + case GraphicsFormat.RGBA_DXT1_UNorm: dxgiFormat = (uint)DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM; return true; + case GraphicsFormat.RGBA_DXT1_SRGB: dxgiFormat = (uint)DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM_SRGB; return true; + case GraphicsFormat.RGBA_DXT5_UNorm: dxgiFormat = (uint)DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM; return true; + case GraphicsFormat.RGBA_DXT5_SRGB: dxgiFormat = (uint)DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM_SRGB; return true; + default: dxgiFormat = 0; return false; + } + } + + // Captures the compressed pixels of a just-loaded PNG (main thread) and writes them to the on-disk + // cache as a DDS on a background thread. No-op unless the texture is in a cacheable DXT format with a + // round-trippable mip layout. Must be called before Apply(makeNoLongerReadable) frees the pixels. + private static void TryWritePngCache(UrlFile file, Texture2D src) + { + try + { + if (!TryGetCacheDxgiFormat(src.graphicsFormat, out uint dxgiFormat)) + return; + + int width = src.width; + int height = src.height; + int mipCount = src.mipmapCount; + // The bundle path reconstructs either a full mip chain or a single level; anything else + // (a partial chain) can't be round-tripped, so don't cache it. + if (mipCount != 1 && mipCount != ComputeMipCount(width, height)) + return; + + if (!GetPngStamp(file.fullPath, out long size, out long time)) + return; + + byte[] data = src.GetRawTextureData().ToArray(); + string path = GetPngCachePath(file.url); + Task.Run(() => WritePngCacheFile(path, width, height, mipCount, dxgiFormat, size, time, data)); + } + catch (Exception e) + { + Debug.LogWarning($"[KSPCFFastLoader] Couldn't cache PNG '{file.url}': {e.Message}"); + } + } + + private static void WritePngCacheFile( + string path, int width, int height, int mipCount, uint dxgiFormat, long srcSize, long srcTime, byte[] data) + { + try + { + Directory.CreateDirectory(Path.GetDirectoryName(path)); + using FileStream fs = File.Create(path); + using BinaryWriter bw = new BinaryWriter(fs); + WriteDdsCacheHeader(bw, width, height, mipCount, dxgiFormat, srcSize, srcTime); + bw.Write(data, 0, data.Length); + } + catch (Exception e) + { + Debug.LogWarning($"[KSPCFFastLoader] Couldn't write PNG cache '{path}': {e.Message}"); + } + } + + // Writes a DX10 DDS header (magic + 124-byte DDS_HEADER + 20-byte DDS_HEADER_DXT10 = 148 bytes) that + // ParseDDSHeader reads back to the exact GraphicsFormat / dimensions / mip count. The source PNG's + // (size, time) stamp plus our marker live in the otherwise-unused dwReserved1 words; Unity streams + // only the pixel bytes past offset 148 and never parses this header, so those words are free to use. + private static void WriteDdsCacheHeader( + BinaryWriter bw, int width, int height, int mipCount, uint dxgiFormat, long srcSize, long srcTime) + { + bool hasMips = mipCount > 1; + + const uint DDSD_CAPS = 0x1, DDSD_HEIGHT = 0x2, DDSD_WIDTH = 0x4, DDSD_PIXELFORMAT = 0x1000; + const uint DDSD_MIPMAPCOUNT = 0x20000, DDSD_LINEARSIZE = 0x80000; + const uint DDPF_FOURCC = 0x4; + const uint DDSCAPS_TEXTURE = 0x1000, DDSCAPS_COMPLEX = 0x8, DDSCAPS_MIPMAP = 0x400000; + + bool isBc1 = dxgiFormat == (uint)DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM + || dxgiFormat == (uint)DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM_SRGB; + int blockBytes = isBc1 ? 8 : 16; + uint topLinearSize = (uint)(Math.Max(1, (width + 3) / 4) * Math.Max(1, (height + 3) / 4) * blockBytes); + + bw.Write(DDSValues.uintMagic); // "DDS " + bw.Write(124u); // dwSize + uint flags = DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | DDSD_PIXELFORMAT | DDSD_LINEARSIZE; + if (hasMips) flags |= DDSD_MIPMAPCOUNT; + bw.Write(flags); // dwFlags + bw.Write((uint)height); // dwHeight + bw.Write((uint)width); // dwWidth + bw.Write(topLinearSize); // dwPitchOrLinearSize + bw.Write(0u); // dwDepth + bw.Write((uint)mipCount); // dwMipMapCount + // dwReserved1[11]: source stamp + marker, rest zero. + bw.Write(srcSize); // [0..1] + bw.Write(srcTime); // [2..3] + bw.Write(PngCacheMarker); // [4] + for (int i = 5; i < 11; i++) + bw.Write(0u); // [5..10] + // DDS_PIXELFORMAT (32 bytes): FourCC "DX10". + bw.Write(32u); // dwSize + bw.Write(DDPF_FOURCC); // dwFlags + bw.Write(DDSValues.uintDX10); // dwFourCC + bw.Write(0u); bw.Write(0u); bw.Write(0u); bw.Write(0u); bw.Write(0u); // bit count + channel masks + uint caps = DDSCAPS_TEXTURE; + if (hasMips) caps |= DDSCAPS_COMPLEX | DDSCAPS_MIPMAP; + bw.Write(caps); // dwCaps + bw.Write(0u); // dwCaps2 + bw.Write(0u); // dwCaps3 + bw.Write(0u); // dwCaps4 + bw.Write(0u); // dwReserved2 + // DDS_HEADER_DXT10 (20 bytes). + bw.Write(dxgiFormat); // dxgiFormat + bw.Write(3u); // resourceDimension = D3D10_RESOURCE_DIMENSION_TEXTURE2D + bw.Write(0u); // miscFlag + bw.Write(1u); // arraySize + bw.Write(0u); // miscFlags2 + } + + #endregion + #region Per-texture coroutine loader // Profiling markers for the work scheduled on background threads via Task.Run. @@ -2105,6 +2353,11 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) else if (!isNormalMap) Debug.LogWarning($"Texture '{req.File.url}' isn't eligible for DXT compression, width and height must be multiples of 4"); + // Persist the compressed PNG to the on-disk cache (before the pixels are freed below) so + // future loads can stream it straight from the combined bundle instead of decoding again. + if (textureCacheEnabled && req.AssetType == RawAsset.AssetType.TexturePNG) + TryWritePngCache(req.File, src); + src.Apply(updateMipmaps: false, makeNoLongerReadable: true); bool isCompressed = @@ -3037,12 +3290,12 @@ IEnumerable instructions } #endregion - #region User opt-in popup (vestigial) + #region User opt-in popup - // The popup, opt-in flow, and PNG-cache-size estimator below are intentionally - // kept around — the cache they were originally tied to has been removed, but the - // popup is going to be repurposed for an upcoming feature. Nothing currently - // triggers WaitForUserOptIn; it must be invoked explicitly by the new feature. + // Shown once on first launch (from FastAssetLoader, gated on userOptInChoiceDone) to let the user + // enable the on-disk PNG texture cache. The choice is persisted to PNGTextureCache.cfg and drives + // textureCacheEnabled; the popup also estimates the loading-time saving and disk cost from the + // install's PNG textures. private static IEnumerator WaitForUserOptIn() { From 9b4248f4b56abd82e94c84927a3c4f3fd7300e7a Mon Sep 17 00:00:00 2001 From: Phantomical Date: Mon, 13 Jul 2026 21:51:57 -0700 Subject: [PATCH 19/21] Tweak QualitySettings --- KSPCommunityFixes/Performance/FastLoader.cs | 197 +++++++++++++++----- 1 file changed, 149 insertions(+), 48 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 30569d02..764c113a 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -470,7 +470,6 @@ static IEnumerator FastAssetLoader(List configFileTypes) // Files loaded by mod-defined loaders (ex : Shabby *.shab files) List unsupportedAudioFiles = new(100); - List unsupportedTextureFiles = new(100); List unsupportedModelFiles = new(100); // Keeping track of already loaded files to avoid loading duplicates. @@ -530,7 +529,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureTRUECOLOR)); break; default: - unsupportedTextureFiles.Add(file); + textureQueue.Add(new TextureLoadRequest(file, RawAsset.AssetType.TextureCUSTOM)); break; } break; @@ -562,8 +561,8 @@ static IEnumerator FastAssetLoader(List configFileTypes) SupportedFormatCache.Build(); // Tune the AUP for much better throughput - QualitySettings.asyncUploadTimeSlice = 10; // ms per frame spent on async uploads (default 2) - QualitySettings.asyncUploadBufferSize = 128; // MB ring buffer for async uploads (default 16) + QualitySettings.asyncUploadTimeSlice = 25; + QualitySettings.asyncUploadBufferSize = 128; int textureCount = bundleRequests.Count + textureQueue.Count; @@ -666,49 +665,11 @@ static IEnumerator FastAssetLoader(List configFileTypes) yield return null; - // call non-stock texture loaders - // note : we could use the StringComparer.OrdinalIgnoreCase comparer as the dictionary key comparer, // as this is the comparison that stock is doing. However, profiling show that casing mismatches rarely happen // (never in stock, 0.22% of calls in a very heavily modded install with a bunch of part mods of varying quality) // and the overhead of the OrdinalIgnoreCase comparer is offsetting the gains (but a small margin, but still). texturesByUrl = new Dictionary(allTextureFiles.Count); - unsupportedFilesCount = unsupportedTextureFiles.Count; - loadersCount = gdb.loadersTexture.Count; - - if (loadersCount > 0 && unsupportedFilesCount > 0) - { - for (int i = 0; i < unsupportedFilesCount; i++) - { - UrlFile file = unsupportedTextureFiles[i]; - - if (allTextureFiles.Contains(file.url)) - { - Debug.LogWarning($"Duplicate texture asset '{file.url}' with extension '{file.fileExtension}' won't be loaded"); - continue; - } - - Debug.Log($"Load Texture: {file.url}"); - for (int k = 0; k < loadersCount; k++) - { - DatabaseLoader loader = gdb.loadersTexture[k]; - if (!loader.extensions.Contains(file.fileExtension)) - continue; - - yield return gdb.StartCoroutine(loader.Load(file, new FileInfo(file.fullPath))); - if (loader.successful) - { - loader.obj.name = file.url; - loader.obj.texture.name = file.url; - gdb.databaseTexture.Add(loader.obj); - allTextureFiles.Add(file.url); - loadedAssetCount++; - gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; - } - break; - } - } - } // call our custom loader yield return gdb.StartCoroutine(TextureDriverCoroutine(textureQueue, allTextureFiles, bundleState, textureCount)); @@ -717,6 +678,8 @@ static IEnumerator FastAssetLoader(List configFileTypes) yield return null; InsertBundledTextures(bundleState, allTextureFiles); + QualitySettings.asyncUploadTimeSlice = 2; + // start model loading gdb.progressFraction = 0.75f; KSPCFFastLoaderReport.wTextureLoading.Stop(); @@ -1018,6 +981,7 @@ public enum AssetType TexturePNG, TextureTGA, TextureTRUECOLOR, + TextureCUSTOM, ModelMU, ModelDAE } @@ -1732,7 +1696,15 @@ private static void WriteDdsCacheHeader( // Result/error carrier for each texture file. Replaces RawAsset for textures. private sealed class TextureLoadRequest { - public enum State : byte { Pending, Ready, Failed } + public enum State : byte + { + Pending, + Ready, + Failed, + // Used for custom loaders, they are responsible for printing + // their own error messages on failure (unless they throw). + Skip + } public UrlFile File; public RawAsset.AssetType AssetType; @@ -1815,7 +1787,7 @@ public static void Build() private static DDSPreparedHeader ParseDDSHeader(string path) { - FileInfo fi = new FileInfo(path); + FileInfo fi = new(path); long fileLength = fi.Length; if (fileLength < 128) throw new IOException($"DDS file '{path}' is too small ({fileLength} bytes)"); @@ -2057,7 +2029,7 @@ private static Exception UnwrapFaultedTask(Task task, string fallbackMessage) // pointer setup goes through this static helper. private static unsafe ReadHandle BeginAsyncRead(string path, NativeArray dst, long offset, long size) { - ReadCommand cmd = new ReadCommand + ReadCommand cmd = new() { Buffer = NativeArrayUnsafeUtility.GetUnsafePtr(dst), Offset = offset, @@ -2347,6 +2319,16 @@ private static IEnumerator LoadUWRCoroutine(TextureLoadRequest req) if (canCompress) { + // Avoid making the compress call if the frame time is already > 25ms + while (true) + { + float frameTime = Time.realtimeSinceStartup - Time.unscaledTime; + if (frameTime < 0.025) + break; + + yield return null; + } + using (s_pmCompress.Auto()) src.Compress(highQuality: !isNormalMap); } @@ -2454,6 +2436,16 @@ private static IEnumerator LoadTRUECOLORCoroutine(TextureLoadRequest req) if (isPot) { + // Avoid making the compress call if the frame time is already > 25ms + while (true) + { + float frameTime = Time.realtimeSinceStartup - Time.unscaledTime; + if (frameTime < 0.025) + break; + + yield return null; + } + using (s_pmCompress.Auto()) tex.Compress(highQuality: false); } @@ -2577,6 +2569,16 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isPot) { + // Avoid making the compress call if the frame time is already > 25ms + while (true) + { + float frameTime = Time.realtimeSinceStartup - Time.unscaledTime; + if (frameTime < 0.025) + break; + + yield return null; + } + using (s_pmCompress.Auto()) texture.Compress(highQuality: false); } @@ -2621,9 +2623,20 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) if (isPot) { + // Avoid making the compress call if the frame time is already > 25ms + while (true) + { + float frameTime = Time.realtimeSinceStartup - Time.unscaledTime; + if (frameTime < 0.025) + break; + + yield return null; + } + using (s_pmCompress.Auto()) texture.Compress(highQuality: false); } + texture.Apply(updateMipmaps: false, makeNoLongerReadable: true); } } @@ -2632,6 +2645,88 @@ private static IEnumerator LoadTGACoroutine(TextureLoadRequest req) req.Status = TextureLoadRequest.State.Ready; } + // Custom database loaders are singletons, so we cannot load more than + // one texture at a time with them. This class serves to ensure that + // doesn't happen. + class CustomLoaderGuard : CustomYieldInstruction, IDisposable + { + static readonly Dictionary> queues = []; + + object loader; + + public CustomLoaderGuard(object loader) + { + if (!queues.TryGetValue(loader, out var queue)) + { + queue = []; + queues.Add(loader, queue); + } + + queue.Enqueue(this); + this.loader = loader; + } + + public override bool keepWaiting + { + get + { + if (loader is null) + return false; + + var queue = queues[loader]; + var head = queue.Peek(); + return !ReferenceEquals(head, this); + } + } + + public void Dispose() + { + if (loader is null) + return; + + queues[loader].Dequeue(); + loader = null; + } + + public static void Clear() => queues.Clear(); + } + + private static IEnumerator LoadCUSTOMCoroutine(TextureLoadRequest req) + { + UrlFile file = req.File; + var gdb = GameDatabase.Instance; + + foreach (var loader in gdb.loadersTexture) + { + if (!loader.extensions.Contains(file.fileExtension)) + continue; + + using var guard = new CustomLoaderGuard(loader); + yield return guard; + + var inner = loader.Load(file, new FileInfo(file.fullPath)); + using var _guard = inner as IDisposable; + while (inner.MoveNext()) + yield return inner.Current; + + if (!loader.successful) + break; + + loader.obj.name = file.url; + loader.obj.texture.name = file.url; + req.Result = loader.obj; + req.Status = TextureLoadRequest.State.Ready; + yield break; + } + + // Some modded loaders (e.g. shabby) use the texture loader to load + // non-texture things. In this case they'll load the file but not + // mark the load as successful. KSP does nothing in this case, so + // we reproduce these by explicitly skipping them, which prints the + // "Loaded texture: ..." message but doesn't print any error messages. + req.Status = TextureLoadRequest.State.Skip; + } + // Drains the shared texture queue, spawning up to MaxTextureSpawnsPerFrame concurrent // per-request loaders per frame and inserting finished ones in FIFO order. The queue is fed // by the enumeration loop (non-DDS) and the background bucketer (non-bundled DDS); the @@ -2679,6 +2774,8 @@ private static IEnumerator TextureDriverCoroutine( yield return null; } + + CustomLoaderGuard.Clear(); } private static IEnumerator LoadTextureCoroutine(TextureLoadRequest req) @@ -2703,11 +2800,12 @@ private static IEnumerator LoadTextureCoroutine(TextureLoadRequest req) inner = LoadTGACoroutine(req); break; default: - req.ErrorMessage = $"Unknown asset type {req.AssetType}"; - req.Status = TextureLoadRequest.State.Failed; - yield break; + inner = LoadCUSTOMCoroutine(req); + break; } + using var _guard = inner as IDisposable; + while (true) { object current; @@ -2747,6 +2845,9 @@ private static void InsertReadyRequest(TextureLoadRequest req, HashSet l { Debug.Log($"Load Texture: {req.File.url}"); + if (req.Status == TextureLoadRequest.State.Skip) + return; + if (req.Status == TextureLoadRequest.State.Failed) { Debug.LogWarning($"LOAD FAILED: {req.File.url}: {req.ErrorMessage}"); From 75532e0d7827ee2c6e5da44b3cb25584c2a352ba Mon Sep 17 00:00:00 2001 From: Phantomical Date: Wed, 15 Jul 2026 23:31:20 -0700 Subject: [PATCH 20/21] Bump up the async upload buffer to 256MB --- KSPCommunityFixes/Performance/FastLoader.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 764c113a..bdedd9d2 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -562,7 +562,7 @@ static IEnumerator FastAssetLoader(List configFileTypes) // Tune the AUP for much better throughput QualitySettings.asyncUploadTimeSlice = 25; - QualitySettings.asyncUploadBufferSize = 128; + QualitySettings.asyncUploadBufferSize = 256; int textureCount = bundleRequests.Count + textureQueue.Count; @@ -739,6 +739,8 @@ static IEnumerator FastAssetLoader(List configFileTypes) arrayPool = null; MuParser.ReleaseBuffers(); + QualitySettings.asyncUploadBufferSize = 32; + // stock stuff gdb.lastLoadTime = KSPUtil.SystemDateTime.DateTimeNow(); gdb.progressFraction = 1f; @@ -1274,6 +1276,9 @@ private static BundleBuildResult BuildDDSBundle( textureQueue.CompleteAdding(); } + // Put the largest textures first so unity loads them during the audio phase. + entries.Sort(static (a, b) => -a.PixelsLength.CompareTo(b.PixelsLength)); + return new BundleBuildResult { Bytes = entries.Count == 0 ? null : TextureBundleBuilder.BuildMany(entries), From 9b84c46f3eecef033489dba3305285c1159ed4c5 Mon Sep 17 00:00:00 2001 From: Phantomical Date: Thu, 16 Jul 2026 00:08:09 -0700 Subject: [PATCH 21/21] Get the loaded count for textures working right --- KSPCommunityFixes/Performance/FastLoader.cs | 45 +++++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index bdedd9d2..fe50afbc 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -674,9 +674,8 @@ static IEnumerator FastAssetLoader(List configFileTypes) // call our custom loader yield return gdb.StartCoroutine(TextureDriverCoroutine(textureQueue, allTextureFiles, bundleState, textureCount)); - while (!bundleState.Done) - yield return null; - InsertBundledTextures(bundleState, allTextureFiles); + // Now wait for all asset bundle textures to finish + yield return gdb.StartCoroutine(InsertBundledTextures(bundleState, allTextureFiles, textureCount)); QualitySettings.asyncUploadTimeSlice = 2; @@ -1450,13 +1449,23 @@ private static IEnumerator LoadBundledAssetsImpl( state.Map = map; } - private static void InsertBundledTextures( + private static IEnumerator InsertBundledTextures( BundleState state, - HashSet loadedUrls) + HashSet loadedUrls, + int totalTextureCount) { + var gdb = GameDatabase.Instance; + while (!state.Done) + { + int progress = (int)(state.Progress * state.Count); + gdb.progressFraction = (float)(loadedAssetCount + progress) / totalAssetCount; + gdb.progressTitle = $"Loading texture asset {progress}/{totalTextureCount}"; + yield return null; + } + List items = state.Items; if (items == null || items.Count == 0) - return; + yield break; Dictionary map = state.Map; @@ -1478,6 +1487,10 @@ private static void InsertBundledTextures( InsertReadyRequest(req, loadedUrls); loadedAssetCount++; + + float frameTime = Time.realtimeSinceStartup - Time.unscaledTime; + if (frameTime > 0.1) + yield return null; } } #endregion @@ -2732,10 +2745,6 @@ private static IEnumerator LoadCUSTOMCoroutine(TextureLoadRequest req) req.Status = TextureLoadRequest.State.Skip; } - // Drains the shared texture queue, spawning up to MaxTextureSpawnsPerFrame concurrent - // per-request loaders per frame and inserting finished ones in FIFO order. The queue is fed - // by the enumeration loop (non-DDS) and the background bucketer (non-bundled DDS); the - // bucketer calls CompleteAdding when done, so IsCompleted && empty means no more will arrive. private static IEnumerator TextureDriverCoroutine( BlockingCollection requests, HashSet loadedUrls, @@ -2744,7 +2753,7 @@ private static IEnumerator TextureDriverCoroutine( { GameDatabase gdb = GameDatabase.Instance; Queue active = new(); - int completed = 0; + int start = loadedAssetCount; while (true) { @@ -2764,10 +2773,13 @@ private static IEnumerator TextureDriverCoroutine( active.Dequeue(); InsertReadyRequest(pending, loadedUrls); - loadedAssetCount++; - completed++; + + float frameTime = Time.realtimeSinceStartup - Time.unscaledTime; + if (frameTime > 0.1) + break; } + int completed = loadedAssetCount - start; int progress = completed + (int)(state.Progress * state.Count); gdb.progressFraction = (float)loadedAssetCount / totalAssetCount; @@ -2783,8 +2795,15 @@ private static IEnumerator TextureDriverCoroutine( CustomLoaderGuard.Clear(); } + struct AssetCountGuard() : IDisposable + { + public void Dispose() => loadedAssetCount++; + } + private static IEnumerator LoadTextureCoroutine(TextureLoadRequest req) { + using var guard = new AssetCountGuard(); + IEnumerator inner; switch (req.AssetType) {