From bdbffaef231bd3fdd3e52f1a186c0fe91ce79e3d Mon Sep 17 00:00:00 2001 From: Chris Hausky Date: Wed, 6 May 2026 23:40:21 +0200 Subject: [PATCH 01/27] feat: allow simplified readback of xnb asset type --- Core/XNB/XnbCompressor.cs | 71 +------ Core/XNB/XnbDecompressStream.cs | 174 ++++++++++++++++++ Core/XNB/XnbSerializer.cs | 85 ++++++--- Interface/Actions/ListPackageContentAction.cs | 19 ++ 4 files changed, 250 insertions(+), 99 deletions(-) create mode 100644 Core/XNB/XnbDecompressStream.cs diff --git a/Core/XNB/XnbCompressor.cs b/Core/XNB/XnbCompressor.cs index 908777b..34db180 100644 --- a/Core/XNB/XnbCompressor.cs +++ b/Core/XNB/XnbCompressor.cs @@ -1,12 +1,7 @@ -using System.Text; - -using Microsoft.Xna.Framework.Content; - -namespace FEZRepacker.Core.XNB +namespace FEZRepacker.Core.XNB { public class XnbCompressor { - /// /// Attempts to decompress given stream containing XNB file. /// @@ -18,69 +13,7 @@ public class XnbCompressor /// Thrown when compressed data is invalid. public static Stream Decompress(Stream xnbStream) { - var decompressedStream = new MemoryStream(); - - if (!XnbHeader.TryRead(xnbStream, out var header) || (header.Flags & XnbHeader.XnbFlags.Compressed) == 0) - { - xnbStream.Position = 0; - xnbStream.CopyTo(decompressedStream); - } - else - { - header.Flags -= XnbHeader.XnbFlags.Compressed; - header.Write(decompressedStream); - - using var decompressedDataStream = new MemoryStream(); - - using var xnbReader = new BinaryReader(xnbStream, Encoding.UTF8, true); - LzxDecoder decoder = new LzxDecoder(16); - - int compressedSize = xnbReader.ReadInt32(); - int decompressedSize = xnbReader.ReadInt32(); - - long startPos = xnbStream.Position; - long pos = startPos; - - while (pos - startPos < compressedSize) - { - // all of these shorts are big endian - int flag = xnbStream.ReadByte(); - int frameSize, blockSize; - if (flag == 0xFF) - { - frameSize = (xnbStream.ReadByte() << 8) | xnbStream.ReadByte(); - blockSize = (xnbStream.ReadByte() << 8) | xnbStream.ReadByte(); - pos += 5; - } - else - { - frameSize = 0x8000; - blockSize = (flag << 8) | xnbStream.ReadByte(); - pos += 2; - } - - - if (blockSize == 0 || frameSize == 0) break; - - decoder.Decompress(xnbStream, blockSize, decompressedDataStream, frameSize); - pos += blockSize; - - xnbStream.Position = pos; - } - - if (decompressedDataStream.Position != decompressedSize) - { - throw new InvalidDataException("XNBDecompressor failed!"); - } - - new BinaryWriter(decompressedStream).Write(decompressedSize); - - decompressedDataStream.Position = 0; - decompressedDataStream.CopyTo(decompressedStream); - } - - decompressedStream.Position = 0; - return decompressedStream; + return new XnbDecompressStream(xnbStream); } public static Stream Compress(Stream xnbStream) diff --git a/Core/XNB/XnbDecompressStream.cs b/Core/XNB/XnbDecompressStream.cs new file mode 100644 index 0000000..2a3a244 --- /dev/null +++ b/Core/XNB/XnbDecompressStream.cs @@ -0,0 +1,174 @@ +using System.Text; + +using Microsoft.Xna.Framework.Content; + +namespace FEZRepacker.Core.XNB +{ + internal class XnbDecompressStream : Stream + { + private readonly Stream _source; + private readonly bool _copyOriginalSource; + private readonly LzxDecoder? _decoder; + private readonly long _sourceEndPosition; + private readonly int _decompressedSize; + private readonly byte[] _decompressedHeaderBytes; + private readonly MemoryStream? _decompressionBuffer; + private long _readPosition; + private long _sourcePosition; + private bool _finished; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _copyOriginalSource ? _source.Length : _decompressedHeaderBytes.Length + _decompressedSize; + public override long Position + { + get => _copyOriginalSource ? _source.Position : _readPosition; + set => throw new NotSupportedException(); + } + + public XnbDecompressStream(Stream source) + { + _source = source; + + if (!TryProcessHeader(out _decompressedHeaderBytes, out var compressedSize, out var decompressedSize)) + { + _copyOriginalSource = true; + return; + } + + _decoder = new LzxDecoder(16); + _sourcePosition = source.Position; + _sourceEndPosition = _sourcePosition + compressedSize; + _decompressedSize = decompressedSize; + _decompressionBuffer = new MemoryStream(); + } + + private bool TryProcessHeader(out byte[] decompressedHeaderBytes, out int compressedSize, out int decompressedSize) + { + var sourcePositionPreHeaderRead = _source.Position; + if (!XnbHeader.TryRead(_source, out var header) || (header.Flags & XnbHeader.XnbFlags.Compressed) == 0) + { + decompressedHeaderBytes = Array.Empty(); + compressedSize = 0; + decompressedSize = 0; + _source.Position = sourcePositionPreHeaderRead; + return false; + } + + using var reader = new BinaryReader(_source, Encoding.UTF8, true); + compressedSize = reader.ReadInt32(); + decompressedSize = reader.ReadInt32(); + + header.Flags &= ~XnbHeader.XnbFlags.Compressed; + using var headerBufferStream = new MemoryStream(); + header.Write(headerBufferStream); + new BinaryWriter(headerBufferStream).Write(decompressedSize); + decompressedHeaderBytes = headerBufferStream.ToArray(); + + return true; + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_copyOriginalSource) + { + return _source.Read(buffer, offset, count); + } + + int bytesRead = 0; + + if (_readPosition < _decompressedHeaderBytes.Length) + { + var headerBytesToReadCount = (int)Math.Min(_decompressedHeaderBytes.Length - _readPosition, count); + Array.Copy(_decompressedHeaderBytes, (int)_readPosition, buffer, offset, headerBytesToReadCount); + _readPosition += headerBytesToReadCount; + offset += headerBytesToReadCount; + count -= headerBytesToReadCount; + bytesRead += headerBytesToReadCount; + } + + long decompressionReadPosition = _readPosition - _decompressedHeaderBytes.Length; + + while (_decompressionBuffer!.Length - decompressionReadPosition < count && !_finished) + { + DecompressNextBlock(); + } + + int decompressedBytesToReadCount = (int)Math.Min(count, _decompressionBuffer.Length - decompressionReadPosition); + if (decompressedBytesToReadCount > 0) + { + _decompressionBuffer.Position = decompressionReadPosition; + int read = _decompressionBuffer.Read(buffer, offset, decompressedBytesToReadCount); + _readPosition += read; + bytesRead += read; + } + + return bytesRead; + } + + private void DecompressNextBlock() + { + if (_sourcePosition >= _sourceEndPosition) + { + MarkFinished(); + return; + } + + _source.Position = _sourcePosition; + + // all of these shorts are big-endian + int flag = _source.ReadByte(); + int frameSize, blockSize; + if (flag == 0xFF) + { + frameSize = (_source.ReadByte() << 8) | _source.ReadByte(); + blockSize = (_source.ReadByte() << 8) | _source.ReadByte(); + _sourcePosition += 5; + } + else + { + frameSize = 0x8000; + blockSize = (flag << 8) | _source.ReadByte(); + _sourcePosition += 2; + } + + if (blockSize == 0 || frameSize == 0) + { + MarkFinished(); + return; + } + + _decompressionBuffer!.Position = _decompressionBuffer.Length; + _decoder!.Decompress(_source, blockSize, _decompressionBuffer, frameSize); + _sourcePosition += blockSize; + } + + private void MarkFinished() + { + _finished = true; + var finalSize = _decompressionBuffer!.Length; + if (finalSize != _decompressedSize) + { + throw new XnbSerializationException( + $"XNB decompression data size mismatch - expected {_decompressedSize}, got {finalSize}" + ); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _decompressionBuffer?.Dispose(); + } + + base.Dispose(disposing); + } + + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} diff --git a/Core/XNB/XnbSerializer.cs b/Core/XNB/XnbSerializer.cs index 4bbe21e..4909e00 100644 --- a/Core/XNB/XnbSerializer.cs +++ b/Core/XNB/XnbSerializer.cs @@ -30,40 +30,27 @@ public static class XnbSerializer public static object? Deserialize(Stream xnbStream) { using Stream decompressedInput = XnbCompressor.Decompress(xnbStream); - - if (!XnbHeader.TryRead(decompressedInput, out var header)) - { - throw new XnbSerializationException("Unable to parse XNB header."); - } - - using var binaryReader = new BinaryReader(decompressedInput, Encoding.UTF8, true); - - int fileSize = binaryReader.ReadInt32(); - - var usedTypesQualifiers = ReadAssemblyQualifiersList(binaryReader); - - int additionalResourcesCount = binaryReader.Read7BitEncodedInt(); - - // FEZ XNB files shouldn't have additional resources - if (additionalResourcesCount > 0) - { - throw new XnbSerializationException("Additional XNB resources in a single file detected."); - } - - int mainResourceTypeID = binaryReader.Read7BitEncodedInt(); - var mainResourceQualifier = usedTypesQualifiers[mainResourceTypeID - 1]; - - var primaryContentType = XnbPrimaryContents.FindByQualifier(mainResourceQualifier); - - if (primaryContentType == null) - { - throw new XnbSerializationException($"Cannot find XNB primary format identity for type {mainResourceQualifier.Name}"); - } - + var primaryContentType = ExtractPrimaryContentIdentity(decompressedInput); using var xnbReader = new XnbContentReader(decompressedInput, primaryContentType, true); return xnbReader.ReadContent(primaryContentType.PrimaryContentType.ContentType, true); } + /// + /// Decompresses and deserializes XNB asset only enough to determine primary content type. + /// Useful for efficiently determining a type of asset from XNB stream. + /// + /// A stream containing serialized XNB asset + /// A type assigned as a primary content type of given XNB asset. + /// + /// Thrown when the stream does not contain a valid XNB file. + /// + public static Type DeserializePrimaryContentTypeOnly(Stream xnbStream) + { + using Stream decompressedInput = XnbCompressor.Decompress(xnbStream); + var primaryContentType = ExtractPrimaryContentIdentity(decompressedInput); + return primaryContentType.PrimaryContentType.ContentType; + } + /// /// Produces a stream containing a given object, serialized into XNB format. /// @@ -105,6 +92,44 @@ public static Stream Serialize(object obj) return outputStream; } + private static XnbPrimaryContentIdentity ExtractPrimaryContentIdentity(Stream xnbStream) + { + if (!XnbHeader.TryRead(xnbStream, out var header)) + { + throw new XnbSerializationException("Unable to parse XNB header."); + } + + if ((header.Flags & XnbHeader.XnbFlags.Compressed) != 0) + { + throw new XnbSerializationException("Cannot extract primary content identity from compressed XNB stream."); + } + + using var binaryReader = new BinaryReader(xnbStream, Encoding.UTF8, true); + + int fileSize = binaryReader.ReadInt32(); + + var usedTypesQualifiers = ReadAssemblyQualifiersList(binaryReader); + + int additionalResourcesCount = binaryReader.Read7BitEncodedInt(); + + // FEZ XNB files shouldn't have additional resources + if (additionalResourcesCount > 0) + { + throw new XnbSerializationException("Additional XNB resources in a single file detected."); + } + + int mainResourceTypeID = binaryReader.Read7BitEncodedInt(); + var mainResourceQualifier = usedTypesQualifiers[mainResourceTypeID - 1]; + + var primaryContentType = XnbPrimaryContents.FindByQualifier(mainResourceQualifier); + + if (primaryContentType == null) + { + throw new XnbSerializationException($"Cannot find XNB primary format identity for type {mainResourceQualifier.Name}"); + } + + return primaryContentType; + } private static List ReadAssemblyQualifiersList(BinaryReader xnbReader) { diff --git a/Interface/Actions/ListPackageContentAction.cs b/Interface/Actions/ListPackageContentAction.cs index fbf3a5c..df27103 100644 --- a/Interface/Actions/ListPackageContentAction.cs +++ b/Interface/Actions/ListPackageContentAction.cs @@ -1,4 +1,5 @@ using FEZRepacker.Core.FileSystem; +using FEZRepacker.Core.XNB; namespace FEZRepacker.Interface.Actions { @@ -24,6 +25,24 @@ public void Execute(Dictionary args) foreach (var entry in pakPackage.Entries) { var extension = entry.FindExtension(); + if (extension == ".xnb") + { + using var pakFile = entry.Open(); + string assetTypeName; + try + { + var assetType = XnbSerializer.DeserializePrimaryContentTypeOnly(pakFile); + assetTypeName = assetType.Name; + } + catch + { + assetTypeName = "unknown"; + } + + Console.WriteLine($"{entry.Path} ({assetTypeName} XNB asset, size: {entry.Length} bytes)"); + continue; + } + var typeText = extension.Length == 0 ? "unknown" : extension; Console.WriteLine($"{entry.Path} ({typeText} file, size: {entry.Length} bytes)"); } From bd33622f652a31dba76b0ba2b10de53cdb1091d2 Mon Sep 17 00:00:00 2001 From: Chris Hausky Date: Thu, 7 May 2026 11:04:03 +0200 Subject: [PATCH 02/27] feat: improve performance of lzx decoder --- Core/Helpers/LzxDecoder.cs | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Core/Helpers/LzxDecoder.cs b/Core/Helpers/LzxDecoder.cs index a4c55c1..22336bf 100644 --- a/Core/Helpers/LzxDecoder.cs +++ b/Core/Helpers/LzxDecoder.cs @@ -3,6 +3,8 @@ * (C) 2003-2004 Stuart Caie. * (C) 2011 Ali Scissons. * + * Slightly modified with performance in mind for FEZRepacker + * * The LZX method was created by Jonathan Forbes and Tomi Poutanen, adapted * by Microsoft Corporation. * @@ -52,6 +54,8 @@ namespace Microsoft.Xna.Framework.Content class LzxDecoder { + private const int UINT_BITS = sizeof(uint) * 8; + public static uint[] position_base; public static byte[] extra_bits; @@ -87,7 +91,7 @@ public LzxDecoder(int window) m_state = new LzxState(); m_state.actual_size = 0; m_state.window = new byte[wndsize]; - for (int i = 0; i < wndsize; i++) m_state.window[i] = 0xDC; + m_state.window.AsSpan().Fill(0xDC); m_state.actual_size = wndsize; m_state.window_size = wndsize; m_state.window_posn = 0; @@ -115,9 +119,9 @@ public LzxDecoder(int window) m_state.LENGTH_len = new byte[LzxConstants.LENGTH_MAXSYMBOLS + LzxConstants.LENTABLE_SAFETY]; m_state.ALIGNED_table = new ushort[(1 << LzxConstants.ALIGNED_TABLEBITS) + (LzxConstants.ALIGNED_MAXSYMBOLS << 1)]; m_state.ALIGNED_len = new byte[LzxConstants.ALIGNED_MAXSYMBOLS + LzxConstants.LENTABLE_SAFETY]; - /* initialise tables to 0 (because deltas will be applied to them) */ - for (int i = 0; i < LzxConstants.MAINTREE_MAXSYMBOLS; i++) m_state.MAINTREE_len[i] = 0; - for (int i = 0; i < LzxConstants.LENGTH_MAXSYMBOLS; i++) m_state.LENGTH_len[i] = 0; + /* initialize tables to 0 (because deltas will be applied to them) */ + m_state.MAINTREE_len.AsSpan().Clear(); + m_state.LENGTH_len.AsSpan().Clear(); } public int Decompress(Stream inData, int inLen, Stream outData, int outLen) @@ -445,9 +449,7 @@ public int Decompress(Stream inData, int inLen, Stream outData, int outLen) case LzxConstants.BLOCKTYPE.UNCOMPRESSED: if ((inData.Position + this_run) > endpos) return -1; //TODO throw proper exception - byte[] temp_buffer = new byte[this_run]; - inData.Read(temp_buffer, 0, this_run); - temp_buffer.CopyTo(window, (int)window_posn); + inData.Read(window, (int)window_posn, this_run); window_posn += (uint)this_run; break; @@ -522,8 +524,7 @@ private int MakeDecodeTable(uint nsyms, uint nbits, byte[] length, ushort[] tabl if ((pos += bit_mask) > table_mask) return 1; /* table overrun */ /* fill all possible lookups of this symbol with the symbol itself */ - fill = bit_mask; - while (fill-- > 0) table[leaf++] = sym; + table.AsSpan((int)leaf, (int)bit_mask).Fill(sym); } } bit_mask >>= 1; @@ -534,7 +535,7 @@ private int MakeDecodeTable(uint nsyms, uint nbits, byte[] length, ushort[] tabl if (pos != table_mask) { /* clear the remainder of the table */ - for (sym = (ushort)pos; sym < table_mask; sym++) table[sym] = 0; + table.AsSpan((int)pos, (int)(table_mask - pos)).Clear(); /* give ourselves room for codes to grow by up to 16 more bits */ pos <<= 16; @@ -601,13 +602,15 @@ private void ReadLengths(byte[] lens, uint first, uint last, BitBuffer bitbuf) LzxConstants.PRETREE_MAXSYMBOLS, LzxConstants.PRETREE_TABLEBITS, bitbuf); if (z == 17) { - y = bitbuf.ReadBits(4); y += 4; - while (y-- != 0) lens[x++] = 0; + y = bitbuf.ReadBits(4) + 4; + lens.AsSpan((int)x, (int)y).Clear(); + x += y; } else if (z == 18) { - y = bitbuf.ReadBits(5); y += 20; - while (y-- != 0) lens[x++] = 0; + y = bitbuf.ReadBits(5) + 20; + lens.AsSpan((int)x, (int)y).Clear(); + x += y; } else if (z == 19) { @@ -631,7 +634,7 @@ private uint ReadHuffSym(ushort[] table, byte[] lengths, uint nsyms, uint nbits, bitbuf.EnsureBits(16); if ((i = table[bitbuf.PeekBits((byte)nbits)]) >= nsyms) { - j = (uint)(1 << (int)((sizeof(uint) * 8) - nbits)); + j = (uint)(1 << (int)(UINT_BITS - nbits)); do { j >>= 1; i <<= 1; i |= (bitbuf.GetBuffer() & j) != 0 ? (uint)1 : 0; @@ -670,14 +673,14 @@ public void EnsureBits(byte bits) int lo = (byte)byteStream.ReadByte(); int hi = (byte)byteStream.ReadByte(); //int amount2shift = sizeof(uint)*8 - 16 - bitsleft; - buffer |= (uint)(((hi << 8) | lo) << (sizeof(uint) * 8 - 16 - bitsleft)); + buffer |= (uint)(((hi << 8) | lo) << (UINT_BITS - 16 - bitsleft)); bitsleft += 16; } } public uint PeekBits(byte bits) { - return (buffer >> ((sizeof(uint) * 8) - bits)); + return (buffer >> (UINT_BITS - bits)); } public void RemoveBits(byte bits) From 1f8b2b4297022e42618b8e80f01ccb0ad4f9249b Mon Sep 17 00:00:00 2001 From: Chris Hausky Date: Thu, 7 May 2026 18:49:28 +0200 Subject: [PATCH 03/27] feat: change reflection serialization to sourcegen --- Core.SourceGen/CodeStringBuilder.cs | 53 +++ Core.SourceGen/ContentSerializerGenerator.cs | 306 ++++++++++++++++++ .../FEZRepacker.Core.SourceGen.csproj | 16 + Core/FEZRepacker.Core.csproj | 4 + .../GenericContentSerializer.cs | 146 --------- .../AnimatedTextureContentIdentity.cs | 7 +- .../ContentTypes/ArtObjectContentIdentity.cs | 10 +- .../XNB/ContentTypes/EffectContentIdentity.cs | 3 +- Core/XNB/ContentTypes/LevelContentIdentity.cs | 48 +-- Core/XNB/ContentTypes/MapContentIdentity.cs | 8 +- .../NpcMetadataContentIdentity.cs | 2 +- Core/XNB/ContentTypes/SkyContentIdentity.cs | 4 +- .../SoundEffectContentIdentity.cs | 4 +- .../ContentTypes/SpriteFontContentIdentity.cs | 8 +- .../ContentTypes/TextureContentIdentity.cs | 2 +- .../TrackedSongContentIdentity.cs | 4 +- .../ContentTypes/TrileSetContentIdentity.cs | 12 +- FEZRepacker.sln | 6 + 18 files changed, 439 insertions(+), 204 deletions(-) create mode 100644 Core.SourceGen/CodeStringBuilder.cs create mode 100644 Core.SourceGen/ContentSerializerGenerator.cs create mode 100644 Core.SourceGen/FEZRepacker.Core.SourceGen.csproj delete mode 100644 Core/XNB/ContentSerialization/GenericContentSerializer.cs diff --git a/Core.SourceGen/CodeStringBuilder.cs b/Core.SourceGen/CodeStringBuilder.cs new file mode 100644 index 0000000..c896c5b --- /dev/null +++ b/Core.SourceGen/CodeStringBuilder.cs @@ -0,0 +1,53 @@ +using System.Text; + +namespace FEZRepacker.Core.SourceGen +{ + public class CodeStringBuilder + { + private readonly StringBuilder _stringBuilder; + private int _currentIndent; + private bool _shouldIndent = false; + + public CodeStringBuilder() + { + _stringBuilder = new StringBuilder(); + _currentIndent = 0; + } + + public void Append(string s) + { + if (_shouldIndent) + { + _stringBuilder.Append(new string(' ', _currentIndent * 4)); + _shouldIndent = false; + } + _stringBuilder.Append(s); + } + + public void AppendLine() + { + _stringBuilder.AppendLine(); + _shouldIndent = true; + } + + public void AppendLine(string s) + { + Append(s); + AppendLine(); + } + + public void BeginCodeBlock() + { + AppendLine("{"); + _currentIndent++; + } + + public void EndCodeBlock() + { + if(_currentIndent > 0) _currentIndent--; + AppendLine("}"); + } + + public override string ToString() => _stringBuilder.ToString(); + } +} diff --git a/Core.SourceGen/ContentSerializerGenerator.cs b/Core.SourceGen/ContentSerializerGenerator.cs new file mode 100644 index 0000000..6f43979 --- /dev/null +++ b/Core.SourceGen/ContentSerializerGenerator.cs @@ -0,0 +1,306 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace FEZRepacker.Core.SourceGen; + +[Generator] +public sealed class ContentSerializerGenerator : IIncrementalGenerator +{ + private const string XnbReaderTypeAttributeName = "FEZRepacker.Core.Definitions.Game.XnbReaderTypeAttribute"; + private const string XnbPropertyAttributeName = "FEZRepacker.Core.Definitions.Game.XnbPropertyAttribute"; + + private struct XnbTypeInfo + { + public string TypeName; + public string TypeFullName; + public string QualifierString; + public List GenericParameters; + public List Properties; + } + + private struct XnbPropertyInfo + { + public string Name; + public string TypeFullName; + public bool IsNullable; + public bool IsReferenceType; + public int Order; + public bool UseConverter; + public bool Optional; + public bool SkipIdentifier; + } + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var xnbTypeInfos = context.SyntaxProvider + .CreateSyntaxProvider((node, _) => node is TypeDeclarationSyntax { AttributeLists.Count: > 0 }, GetXnbType) + .Where(m => m != null); + + context.RegisterSourceOutput(xnbTypeInfos, (ctx, xnbTypeInfo) => + CreateSerializerSourceFile(ctx, xnbTypeInfo!.Value)); + } + + private static XnbTypeInfo? GetXnbType(GeneratorSyntaxContext ctx, CancellationToken ct) + { + if (ctx.SemanticModel.GetDeclaredSymbol(ctx.Node, ct) is not INamedTypeSymbol typeSymbol) + { + return null; + } + + var xnbReaderTypeAttribute = typeSymbol.GetAttributes() + .FirstOrDefault(a => a.AttributeClass?.ToDisplayString() == XnbReaderTypeAttributeName); + + if (xnbReaderTypeAttribute is not {ConstructorArguments.Length: > 0}) + { + return null; + } + + var qualifierString = xnbReaderTypeAttribute.ConstructorArguments[0].Value as string ?? string.Empty; + var genericParameters = typeSymbol.TypeParameters.Select(tp => tp.Name).ToList(); + var properties = new List(); + + foreach (var member in typeSymbol.GetMembers().OfType()) + { + ct.ThrowIfCancellationRequested(); + + var xnbPropertyAttribute = member.GetAttributes() + .FirstOrDefault(a => a.AttributeClass?.ToDisplayString() == XnbPropertyAttributeName); + + if (xnbPropertyAttribute == null) + { + continue; + } + + int order = (int?)(xnbPropertyAttribute.ConstructorArguments.FirstOrDefault()).Value ?? 0; + + bool useConverter = xnbPropertyAttribute.NamedArguments + .FirstOrDefault(x => x.Key == "UseConverter").Value.Value is true; + bool optional = xnbPropertyAttribute.NamedArguments + .FirstOrDefault(x => x.Key == "Optional").Value.Value is true; + bool skipIdentifier = xnbPropertyAttribute.NamedArguments + .FirstOrDefault(x => x.Key == "SkipIdentifier").Value.Value is true; + + ITypeSymbol underlyingType = member.Type; + bool isNullableValueType = false; + + if (member.Type is INamedTypeSymbol {ConstructedFrom.SpecialType: SpecialType.System_Nullable_T} namedType) + { + underlyingType = namedType.TypeArguments[0]; + isNullableValueType = true; + } + + properties.Add(new XnbPropertyInfo + { + Name = member.Name, + TypeFullName = underlyingType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), + IsNullable = isNullableValueType, + IsReferenceType = !isNullableValueType && member.Type.IsReferenceType, + Order = order, + UseConverter = useConverter, + Optional = optional, + SkipIdentifier = skipIdentifier + }); + } + + properties.Sort((a, b) => a.Order.CompareTo(b.Order)); + + return new XnbTypeInfo + { + TypeName = typeSymbol.Name, + TypeFullName = typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), + QualifierString = qualifierString, + Properties = properties, + GenericParameters = genericParameters + }; + } + + private static void CreateSerializerSourceFile(SourceProductionContext ctx, XnbTypeInfo model) + { + var cb = new CodeStringBuilder(); + EmitSerializer(cb, model); + ctx.AddSource($"{model.TypeName}ContentSerializer.g.cs", cb.ToString()); + } + + private static void EmitSerializer(CodeStringBuilder cb, XnbTypeInfo xnbTypeInfo) + { + cb.AppendLine("// "); + cb.AppendLine("// FEZRepacker.Core.SourceGen output"); + cb.AppendLine("#nullable enable"); + cb.AppendLine(); + cb.AppendLine("using FEZRepacker.Core;"); + cb.AppendLine("using FEZRepacker.Core.Helpers;"); + cb.AppendLine("using FEZRepacker.Core.XNB;"); + cb.AppendLine(); + cb.AppendLine("namespace FEZRepacker.Core.XNB.ContentSerialization;"); + cb.AppendLine(); + cb.Append($"internal sealed class {ConstructSerializerName(xnbTypeInfo)}"); + cb.AppendLine($" : XnbContentSerializer<{xnbTypeInfo.TypeFullName}>"); + cb.BeginCodeBlock(); + { + cb.AppendLine($"public override XnbAssemblyQualifier Name => \"{xnbTypeInfo.QualifierString}\";"); + cb.AppendLine(); + EmitDeserialize(cb, xnbTypeInfo); + cb.AppendLine(); + EmitSerialize(cb, xnbTypeInfo); + } + cb.EndCodeBlock(); + } + + private static string ConstructSerializerName(XnbTypeInfo xnbTypeInfo) + { + var name = $"{xnbTypeInfo.TypeName}ContentSerializer"; + if (xnbTypeInfo.GenericParameters.Count > 0) + { + var genericParametersList = string.Join(", " , xnbTypeInfo.GenericParameters); + name += $"<{genericParametersList}>"; + } + return name; + } + + private static void EmitDeserialize(CodeStringBuilder cb, XnbTypeInfo model) + { + cb.AppendLine("public override object Deserialize(XnbContentReader reader)"); + cb.BeginCodeBlock(); + { + cb.AppendLine($"var content = new {model.TypeFullName}();"); + + foreach (var prop in model.Properties) + { + EmitPropertyDeserialize(cb, prop); + } + + cb.AppendLine("return content;"); + } + cb.EndCodeBlock(); + } + + private static void EmitPropertyDeserialize(CodeStringBuilder cb, XnbPropertyInfo prop) + { + if (prop.Optional) + { + cb.AppendLine($"if (reader.ReadBoolean())"); + cb.BeginCodeBlock(); + } + + cb.Append($"content.{prop.Name} = "); + + if (prop.UseConverter) + { + cb.Append($"({prop.TypeFullName})reader.ReadContent(typeof({prop.TypeFullName}), "); + cb.Append($"{(prop.SkipIdentifier ? "true" : "false")})!"); + } + else + { + cb.Append(prop.TypeFullName switch + { + "bool" => "reader.ReadBoolean()", + "int" => "reader.ReadInt32()", + "byte" => "reader.ReadByte()", + "short" => "reader.ReadInt16()", + "float" => "reader.ReadSingle()", + "char" => "reader.ReadChar()", + "string" => "reader.ReadString()", + "global::FEZRepacker.Core.Definitions.Game.XNA.Vector2" => "reader.ReadVector2()", + "global::FEZRepacker.Core.Definitions.Game.XNA.Vector3" => "reader.ReadVector3()", + "global::FEZRepacker.Core.Definitions.Game.XNA.Quaternion" => "reader.ReadQuaternion()", + "global::FEZRepacker.Core.Definitions.Game.XNA.Color" => "reader.ReadColor()", + "global::System.TimeSpan" => "new global::System.TimeSpan(reader.ReadInt64())", + _ => $"default! /* unsupported type: {prop.TypeFullName} */" + }); + } + + cb.AppendLine(";"); + + if (prop.Optional) + { + cb.EndCodeBlock(); + } + } + + private static void EmitSerialize(CodeStringBuilder cb, XnbTypeInfo model) + { + cb.AppendLine("public override void Serialize(object data, XnbContentWriter writer)"); + cb.BeginCodeBlock(); + { + cb.AppendLine($"var content = ({model.TypeFullName})data;"); + + foreach (var prop in model.Properties) + { + EmitPropertySerialize(cb, prop); + } + } + cb.EndCodeBlock(); + } + + private static void EmitPropertySerialize(CodeStringBuilder cb, XnbPropertyInfo prop) + { + var valueExpression = $"content.{prop.Name}"; + var propertyType = $"typeof({prop.TypeFullName})"; + + if (prop.Optional) + { + if (prop.IsNullable) + { + cb.AppendLine($"if ({valueExpression}.HasValue)"); + valueExpression += ".Value"; + } + else if (prop.IsReferenceType) + { + cb.AppendLine($"if ({valueExpression} != null)"); + } + else + { + cb.AppendLine($"// {prop.Name}"); + } + + cb.BeginCodeBlock(); + cb.AppendLine($"int typeId = writer.Identity.ContentTypes.FindIndex(t => t.ContentType == {propertyType}) + 1;"); + cb.AppendLine("if (typeId > 0) writer.Write7BitEncodedInt(typeId);"); + cb.AppendLine("else writer.Write(true);"); + } + + if (prop.UseConverter) + { + cb.Append($"writer.WriteContent({propertyType}, {valueExpression}, "); + cb.Append($"{(prop.SkipIdentifier ? "true" : "false")})"); + } + else + { + cb.Append(prop.TypeFullName switch + { + "bool" => $"writer.Write({valueExpression})", + "int" => $"writer.Write({valueExpression})", + "byte" => $"writer.Write({valueExpression})", + "short" => $"writer.Write({valueExpression})", + "float" => $"writer.Write({valueExpression})", + "char" => $"writer.Write({valueExpression})", + "string" => $"writer.Write({valueExpression})", + "global::FEZRepacker.Core.Definitions.Game.XNA.Vector2" => $"writer.Write({valueExpression})", + "global::FEZRepacker.Core.Definitions.Game.XNA.Vector3" => $"writer.Write({valueExpression})", + "global::FEZRepacker.Core.Definitions.Game.XNA.Quaternion" => $"writer.Write({valueExpression})", + "global::FEZRepacker.Core.Definitions.Game.XNA.Color" => $"writer.Write({valueExpression})", + "global::System.TimeSpan" => $"writer.Write({valueExpression}.Ticks)", + _ => $"_ = {valueExpression} /* unsupported type: {prop.TypeFullName} */" + }); + } + cb.AppendLine(";"); + + if (prop.Optional) + { + cb.EndCodeBlock(); + + if (prop.IsNullable || prop.IsReferenceType) + { + cb.AppendLine("else"); + cb.BeginCodeBlock(); + cb.AppendLine("writer.Write(false);"); + cb.EndCodeBlock(); + } + } + } +} + diff --git a/Core.SourceGen/FEZRepacker.Core.SourceGen.csproj b/Core.SourceGen/FEZRepacker.Core.SourceGen.csproj new file mode 100644 index 0000000..27719ec --- /dev/null +++ b/Core.SourceGen/FEZRepacker.Core.SourceGen.csproj @@ -0,0 +1,16 @@ + + + netstandard2.0 + preview + enable + FEZRepacker.Core.SourceGen + true + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + \ No newline at end of file diff --git a/Core/FEZRepacker.Core.csproj b/Core/FEZRepacker.Core.csproj index b3eba64..9801b3c 100644 --- a/Core/FEZRepacker.Core.csproj +++ b/Core/FEZRepacker.Core.csproj @@ -25,6 +25,10 @@ + + + + diff --git a/Core/XNB/ContentSerialization/GenericContentSerializer.cs b/Core/XNB/ContentSerialization/GenericContentSerializer.cs deleted file mode 100644 index a69481b..0000000 --- a/Core/XNB/ContentSerialization/GenericContentSerializer.cs +++ /dev/null @@ -1,146 +0,0 @@ -using System.Linq.Expressions; -using System.Reflection; - -using FEZRepacker.Core.Definitions.Game; -using FEZRepacker.Core.Definitions.Game.XNA; -using FEZRepacker.Core.Helpers; - -namespace FEZRepacker.Core.XNB.ContentSerialization -{ - /// - /// Generalizes content type creation by using reflection on given type containing XnbTypeAttribute - /// by parsing each public property with XnbPropertyAttribute based on parameters contained in it. - /// This replaces the need of creating a content type for each custom asset type. - /// - /// Class type with XnbTypeAttribute to form content type from. - internal class GenericContentSerializer : XnbContentSerializer - { - private XnbAssemblyQualifier _name; - - private readonly Func _typeBuilder; - private Dictionary _propertyMap = new(); - private Dictionary _underlyingTypeMap = new(); - - public override XnbAssemblyQualifier Name => _name; - - public GenericContentSerializer() : base() - { - var qualifier = XnbAssemblyQualifier.GetFromXnbReaderType(typeof(T)); - if (qualifier.HasValue) _name = qualifier.Value; - PopulateReflectionMaps(); - _typeBuilder = CreateContainedTypeConstructor(); - } - - private void PopulateReflectionMaps() - { - _propertyMap = typeof(T).GetProperties() - .Where(property => Attribute.IsDefined(property, typeof(XnbPropertyAttribute))) - .ToDictionary( - property => property, - property => (property.GetCustomAttributes(typeof(XnbPropertyAttribute), false).Single() as XnbPropertyAttribute)! - ).OrderBy(pair => pair.Value.Order) - .ToDictionary(pair => pair.Key, pair => pair.Value); - - _underlyingTypeMap = _propertyMap.ToDictionary( - pair => pair.Key, - pair => Nullable.GetUnderlyingType(pair.Key.PropertyType) ?? pair.Key.PropertyType - ); - } - - private Func CreateContainedTypeConstructor() - { - var t = typeof(T); - var ex = new Expression[] { Expression.New(typeof(T)) }; - var block = Expression.Block(t, ex); - return Expression.Lambda>(block).Compile(); - } - - public override object Deserialize(XnbContentReader reader) - { - object content = _typeBuilder()!; - - foreach (var propertyMapRecord in _propertyMap) - { - var property = propertyMapRecord.Key; - var attribute = propertyMapRecord.Value; - - Type propertyType = _underlyingTypeMap[property]; - - if (attribute.Optional) - { - if (!reader.ReadBoolean()) continue; - } - - object? readValue = null; - - if (attribute.UseConverter) - { - readValue = reader.ReadContent(propertyType, attribute.SkipIdentifier); - } - else if (propertyType == typeof(bool)) readValue = reader.ReadBoolean(); - else if (propertyType == typeof(int)) readValue = reader.ReadInt32(); - else if (propertyType == typeof(byte)) readValue = reader.ReadByte(); - else if (propertyType == typeof(short)) readValue = reader.ReadInt16(); - else if (propertyType == typeof(float)) readValue = reader.ReadSingle(); - else if (propertyType == typeof(char)) readValue = reader.ReadChar(); - else if (propertyType == typeof(string)) readValue = reader.ReadString(); - else if (propertyType == typeof(Vector2)) readValue = reader.ReadVector2(); - else if (propertyType == typeof(Vector3)) readValue = reader.ReadVector3(); - else if (propertyType == typeof(Quaternion)) readValue = reader.ReadQuaternion(); - else if (propertyType == typeof(Color)) readValue = reader.ReadColor(); - else if (propertyType == typeof(TimeSpan)) readValue = new TimeSpan(reader.ReadInt64()); - else throw new NotSupportedException($"Type {propertyType.FullName} is not supported"); - - if (readValue != null) property.SetValue(content, readValue); - } - - return content; - } - - public override void Serialize(object data, XnbContentWriter writer) - { - foreach (var propertyMapRecord in _propertyMap) - { - var property = propertyMapRecord.Key; - var attribute = propertyMapRecord.Value; - - Type propertyType = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; - - object? writeValue = property.GetValue(data); - - if (attribute.Optional) - { - int typeID = writer.Identity.ContentTypes.FindIndex(t => t.ContentType == propertyType) + 1; - if (writeValue != null) - { - if (typeID > 0) writer.Write7BitEncodedInt(typeID); - else writer.Write(true); - } - else writer.Write(false); - - } - - if (writeValue != null) - { - if (attribute.UseConverter) - { - writer.WriteContent(propertyType, writeValue, attribute.SkipIdentifier); - } - else if (propertyType == typeof(bool)) writer.Write((bool)writeValue!); - else if (propertyType == typeof(int)) writer.Write((int)writeValue!); - else if (propertyType == typeof(byte)) writer.Write((byte)writeValue!); - else if (propertyType == typeof(short)) writer.Write((short)writeValue!); - else if (propertyType == typeof(float)) writer.Write((float)writeValue!); - else if (propertyType == typeof(char)) writer.Write(((char)writeValue!)); - else if (propertyType == typeof(string)) writer.Write((string)writeValue!); - else if (propertyType == typeof(Vector2)) writer.Write((Vector2)writeValue!); - else if (propertyType == typeof(Vector3)) writer.Write((Vector3)writeValue!); - else if (propertyType == typeof(Quaternion)) writer.Write((Quaternion)writeValue!); - else if (propertyType == typeof(Color)) writer.Write((Color)writeValue!); - else if (propertyType == typeof(TimeSpan)) writer.Write(((TimeSpan)writeValue!).Ticks); - else throw new NotSupportedException($"Type {propertyType.FullName} is not supported"); - } - } - } - } -} diff --git a/Core/XNB/ContentTypes/AnimatedTextureContentIdentity.cs b/Core/XNB/ContentTypes/AnimatedTextureContentIdentity.cs index 4eb1250..f20162f 100644 --- a/Core/XNB/ContentTypes/AnimatedTextureContentIdentity.cs +++ b/Core/XNB/ContentTypes/AnimatedTextureContentIdentity.cs @@ -2,7 +2,6 @@ using FEZRepacker.Core.XNB.ContentSerialization.System; using FEZRepacker.Core.Definitions.Game.Graphics; -using FEZRepacker.Core.Definitions.Game.XNA; namespace FEZRepacker.Core.XNB.ContentTypes { @@ -10,12 +9,12 @@ internal class AnimatedTextureContentIdentity : XnbPrimaryContentIdentity { protected override List ContentTypesFactory => new() { - new GenericContentSerializer(), + new AnimatedTextureContentSerializer(), new ByteArrayContentSerializer(), new ListContentSerializer(), - new GenericContentSerializer(), + new FrameContentContentSerializer(), new TimeSpanContentSerializer(), - new GenericContentSerializer() + new RectangleContentSerializer() }; } } diff --git a/Core/XNB/ContentTypes/ArtObjectContentIdentity.cs b/Core/XNB/ContentTypes/ArtObjectContentIdentity.cs index b1bc389..b930105 100644 --- a/Core/XNB/ContentTypes/ArtObjectContentIdentity.cs +++ b/Core/XNB/ContentTypes/ArtObjectContentIdentity.cs @@ -12,13 +12,13 @@ internal class ArtObjectContentIdentity : XnbPrimaryContentIdentity { protected override List ContentTypesFactory => new() { - new GenericContentSerializer(), - new GenericContentSerializer(), + new ArtObjectContentSerializer(), + new Texture2DContentSerializer(), new EnumContentSerializer(), new ByteArrayContentSerializer(), - new GenericContentSerializer>(), - new GenericContentSerializer(), - new GenericContentSerializer(), + new IndexedPrimitivesContentSerializer(), + new VertexInstanceContentSerializer(), + new MatrixContentSerializer(), new EnumContentSerializer(), new Int32ContentSerializer(), new ArrayContentSerializer(), diff --git a/Core/XNB/ContentTypes/EffectContentIdentity.cs b/Core/XNB/ContentTypes/EffectContentIdentity.cs index 6e3a746..d1932ab 100644 --- a/Core/XNB/ContentTypes/EffectContentIdentity.cs +++ b/Core/XNB/ContentTypes/EffectContentIdentity.cs @@ -1,5 +1,4 @@  -using FEZRepacker.Core.Definitions.Game.XNA; using FEZRepacker.Core.XNB.ContentSerialization; using FEZRepacker.Core.XNB.ContentSerialization.System; @@ -9,7 +8,7 @@ internal class EffectContentIdentity : XnbPrimaryContentIdentity { protected override List ContentTypesFactory => new() { - new GenericContentSerializer(), + new EffectContentSerializer(), new ByteArrayContentSerializer() }; diff --git a/Core/XNB/ContentTypes/LevelContentIdentity.cs b/Core/XNB/ContentTypes/LevelContentIdentity.cs index a5e80f8..53a088f 100644 --- a/Core/XNB/ContentTypes/LevelContentIdentity.cs +++ b/Core/XNB/ContentTypes/LevelContentIdentity.cs @@ -11,66 +11,66 @@ internal class LevelContentIdentity : XnbPrimaryContentIdentity { protected override List ContentTypesFactory => new() { - new GenericContentSerializer(), + new LevelContentSerializer(), new StringContentSerializer(), - new GenericContentSerializer(), - new GenericContentSerializer(), + new TrileFaceContentSerializer(), + new TrileEmplacementContentSerializer(), new EnumContentSerializer(), new Int32ContentSerializer(), new BooleanContentSerializer(), new EnumContentSerializer(), new DictionaryContentSerializer(), - new GenericContentSerializer(), + new VolumeContentSerializer(), new ArrayContentSerializer(), - new GenericContentSerializer(), + new VolumeActorSettingsContentSerializer(), new ListContentSerializer(), - new GenericContentSerializer(), + new DotDialogueLineContentSerializer(), new ArrayContentSerializer(), new EnumContentSerializer(), new DictionaryContentSerializer(), - new GenericContentSerializer