From 2216c80e3e29c25eeee7bc875bc240b8c780a9a1 Mon Sep 17 00:00:00 2001 From: Michael Staib Date: Tue, 28 Jul 2026 08:42:11 +0000 Subject: [PATCH] Add round 2 Fusion hot path micro benchmarks --- .../AbstractSelectionSetCacheBenchmark.cs | 440 ++++++ .../CompositeContextFusedDecodeBenchmark.cs | 900 +++++++++++++ .../ForwardedVariableBlitBenchmark.cs | 1180 +++++++++++++++++ .../PropertyNameLengthPrefilterBenchmark.cs | 753 +++++++++++ .../SelectionChildMapBenchmark.cs | 798 +++++++++++ .../SelectionTypeShapeBenchmark.cs | 894 +++++++++++++ .../SourceEnumeratorRowCarryBenchmark.cs | 526 ++++++++ 7 files changed, 5491 insertions(+) create mode 100644 src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/AbstractSelectionSetCacheBenchmark.cs create mode 100644 src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/CompositeContextFusedDecodeBenchmark.cs create mode 100644 src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/ForwardedVariableBlitBenchmark.cs create mode 100644 src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/PropertyNameLengthPrefilterBenchmark.cs create mode 100644 src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SelectionChildMapBenchmark.cs create mode 100644 src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SelectionTypeShapeBenchmark.cs create mode 100644 src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SourceEnumeratorRowCarryBenchmark.cs diff --git a/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/AbstractSelectionSetCacheBenchmark.cs b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/AbstractSelectionSetCacheBenchmark.cs new file mode 100644 index 00000000000..ee472ac1426 --- /dev/null +++ b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/AbstractSelectionSetCacheBenchmark.cs @@ -0,0 +1,440 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using HotChocolate.Fusion.Execution.Nodes; +using HotChocolate.Fusion.Logging; +using HotChocolate.Fusion.Options; +using HotChocolate.Fusion.Types; +using HotChocolate.Language; +using HotChocolate.Types; +using Microsoft.Extensions.ObjectPool; + +#nullable enable + +namespace HotChocolate.Fusion.Execution.Benchmarks; + +/// +/// Measures child selection-set resolution for abstract-typed selections on the value +/// completion hot path. Selection.GetSelectionSet(IComplexTypeDefinition) +/// (src/HotChocolate/Fusion/src/Fusion.Execution/Execution/Nodes/Selection.cs lines 128-153) +/// caches the child selection set in _childSelectionSet only when the field's named +/// type is a concrete object type (lines 138-150). Interface and union typed fields fall +/// through at line 152 to Operation.GetSelectionSet(selection, typeContext) +/// (Execution/Nodes/Operation.cs lines 160-189), which builds a +/// (selection.Id, typeContext.Name) tuple key (line 166) and probes a +/// ConcurrentDictionary (line 168), hashing the type-name string on every probe. +/// The hot callers pay this once per newly materialized abstract object element +/// (ValueCompletion.cs line 1118) and per merge target initialization (lines 239 and 272), +/// so a 1000-element interface list pays 1000 string-hash dictionary probes to fetch the +/// same handful of plan-stable SelectionSet instances. +/// +/// The candidate adds a plan-lifetime copy-on-write array of (type, selection set) entries +/// on Selection, scanned by reference identity and capped at 8 entries. Misses fall +/// back to the operation dictionary unchanged, so behavior is identical by construction: +/// Operation.GetSelectionSet only ever publishes one sealed instance per key. +/// The baseline calls the real product method; the optimized variant is a benchmark-local +/// replica of Selection.GetSelectionSet with the candidate scan inserted at the +/// abstract fall-through. DistinctTypes cycles 1 (homogeneous list, the common +/// case), 2 (alternating types), 3, and 10 (two types beyond the 8-entry cap, the +/// worst-case regression shape) concrete implementers across 1000 probes per invocation. +/// +[MemoryDiagnoser] +[Config(typeof(BenchmarkConfig))] +public class AbstractSelectionSetCacheBenchmark +{ + /// + /// BenchmarkDotNet 0.15.8 has no RuntimeMoniker for the net11.0 preview host and + /// this project pins TargetFramework to net11.0, so out-of-process toolchains can + /// neither validate nor build a child process here. The job therefore runs in + /// process with the intended 3 warmup and 10 measurement iterations. + /// + private sealed class BenchmarkConfig : ManualConfig + { + public BenchmarkConfig() + => AddJob( + Job.Default + .WithWarmupCount(3) + .WithIterationCount(10) + .WithToolchain(InProcessEmitToolchain.Instance)); + } + + private const int ProbeCount = 1000; + private const int MaxCachedTypeContexts = 8; + private const string OperationId = "123456789101112"; + + private static readonly string[] s_implementerNames = + [ + "Product", + "Article", + "Brand", + "Category", + "Promotion", + "Vendor", + "Review", + "Store", + "Banner", + "Coupon" + ]; + + private static readonly int[] s_allDistinctTypeShapes = [1, 2, 3, 10]; + + private Selection _selection = null!; + private CachedSelectionResolver _cachedResolver = null!; + private IComplexTypeDefinition[] _allTypeContexts = null!; + private IComplexTypeDefinition[] _probeTypeContexts = null!; + + public long Consumed; + + /// + /// The number of distinct concrete types cycling through the 1000 probes: + /// 1 is a homogeneous list, 2 the alternating shape that thrashes single-slot + /// designs, 3 a small heterogeneous list, and 10 exceeds the 8-entry cap so two + /// of the ten types always miss the candidate cache and fall back to the + /// operation dictionary after a full 8-entry scan. + /// + [Params(1, 2, 3, 10)] + public int DistinctTypes; + + [GlobalSetup] + public void Setup() + { + var schema = ComposeSchema(); + + var document = Utf8GraphQLParser.Parse( + """ + { + searchContent(query: "bench") { + id + title + } + } + """); + var operationDefinition = (OperationDefinitionNode)document.Definitions[0]; + + var compiler = new OperationCompiler( + schema, + new NoOpObjectPool>>()); + var operation = compiler.Compile(OperationId, OperationId, operationDefinition); + + if (!operation.RootSelectionSet.TryGetSelection("searchContent", out var selection)) + { + throw new InvalidOperationException( + "The compiled operation has no searchContent selection."); + } + + if (selection.NamedType is not FusionInterfaceTypeDefinition) + { + throw new InvalidOperationException( + "searchContent must return an interface so the abstract fall-through " + + "(Selection.cs line 152) is what both variants measure."); + } + + _selection = selection; + _cachedResolver = new CachedSelectionResolver(selection); + + _allTypeContexts = new IComplexTypeDefinition[s_implementerNames.Length]; + + for (var i = 0; i < s_implementerNames.Length; i++) + { + _allTypeContexts[i] = + schema.Types.GetType(s_implementerNames[i]); + } + + // Warm both paths once per type context in a fixed order. This forces the lazy + // CompileSelectionSet under the operation lock (Operation.cs lines 170-185) + // exactly once per type, so the measured loops pay pure probe cost. It also + // fills the candidate cache deterministically: the first 8 types occupy the + // capped entries and the last 2 always miss, which is the beyond-cap shape + // of the 10-type parameter. + foreach (var typeContext in _allTypeContexts) + { + var expected = _selection.GetSelectionSet(typeContext); + var actual = _cachedResolver.GetSelectionSet(typeContext); + + if (expected is null || !ReferenceEquals(expected, actual)) + { + throw new InvalidOperationException( + $"Warmup mismatch for type '{typeContext.Name}': the cached variant " + + "did not return the product selection set instance."); + } + } + + if (_cachedResolver.CachedEntryCount != MaxCachedTypeContexts) + { + throw new InvalidOperationException( + $"The candidate cache holds {_cachedResolver.CachedEntryCount} entries " + + $"after warmup but the cap is {MaxCachedTypeContexts}."); + } + + // Verify equivalence over every parameter shape, not just the active one. + foreach (var distinctTypes in s_allDistinctTypeShapes) + { + VerifyEquivalence(BuildProbeSequence(distinctTypes)); + } + + _probeTypeContexts = BuildProbeSequence(DistinctTypes); + } + + /// + /// Current product behavior: every probe calls the real + /// Selection.GetSelectionSet(IComplexTypeDefinition), whose abstract + /// fall-through (Selection.cs line 152) probes the operation's + /// string-keyed ConcurrentDictionary per call (Operation.cs lines 166-168). + /// + [Benchmark(Baseline = true)] + public long Baseline_OperationDictionaryProbe() + { + var sum = RunBaseline(_probeTypeContexts); + Consumed = sum; + return sum; + } + + /// + /// Candidate: the same resolution fronted by a plan-lifetime copy-on-write array + /// of (type, selection set) entries scanned by reference identity; only misses + /// beyond the 8-entry cap still reach the operation dictionary. + /// + [Benchmark] + public long Cached_CopyOnWriteArrayScan() + { + var sum = RunCached(_probeTypeContexts); + Consumed = sum; + return sum; + } + + private long RunBaseline(IComplexTypeDefinition[] probes) + { + var selection = _selection; + var sum = 0L; + + for (var i = 0; i < probes.Length; i++) + { + sum += selection.GetSelectionSet(probes[i])!.Id; + } + + return sum; + } + + private long RunCached(IComplexTypeDefinition[] probes) + { + var resolver = _cachedResolver; + var sum = 0L; + + for (var i = 0; i < probes.Length; i++) + { + sum += resolver.GetSelectionSet(probes[i])!.Id; + } + + return sum; + } + + private IComplexTypeDefinition[] BuildProbeSequence(int distinctTypes) + { + var probes = new IComplexTypeDefinition[ProbeCount]; + + for (var i = 0; i < probes.Length; i++) + { + probes[i] = _allTypeContexts[i % distinctTypes]; + } + + return probes; + } + + private void VerifyEquivalence(IComplexTypeDefinition[] probes) + { + for (var i = 0; i < probes.Length; i++) + { + var expected = _selection.GetSelectionSet(probes[i]); + var actual = _cachedResolver.GetSelectionSet(probes[i]); + + if (expected is null || !ReferenceEquals(expected, actual)) + { + throw new InvalidOperationException( + $"Probe {i} ('{probes[i].Name}'): baseline and cached variants " + + "resolved different selection set instances."); + } + } + + var baselineSum = RunBaseline(probes); + var cachedSum = RunCached(probes); + + if (baselineSum != cachedSum) + { + throw new InvalidOperationException( + $"Checksum mismatch: baseline {baselineSum}, cached {cachedSum}."); + } + } + + /// + /// Benchmark-local stand-in for a Selection carrying the candidate cache field. + /// The product change would add the copy-on-write array as a field on Selection itself; + /// this wrapper reproduces Selection.GetSelectionSet(IComplexTypeDefinition) + /// (Selection.cs lines 128-153) statement for statement, with the candidate scan + /// inserted at the abstract fall-through (line 152). + /// + private sealed class CachedSelectionResolver + { + private readonly Selection _selection; + + // Stand-in for the private Selection._childSelectionSet field. It stays null for + // abstract named types exactly as in the product, so the measured abstract path + // executes the same null check and type test as the product method. + private SelectionSet? _childSelectionSet; + + // The candidate field: a copy-on-write array of (type, set) entries for abstract + // named types, initially null, published via Interlocked.CompareExchange. A lost + // race drops an entry but never changes results because the operation dictionary + // returns the identical instance on the retry. + private SelectionSetCacheEntry[]? _abstractSelectionSets; + + public CachedSelectionResolver(Selection selection) + { + _selection = selection; + } + + public int CachedEntryCount + => Volatile.Read(ref _abstractSelectionSets)?.Length ?? 0; + + public SelectionSet? GetSelectionSet(IComplexTypeDefinition typeContext) + { + var selection = _selection; + + // Mirrors the leaf gate at Selection.cs lines 130-133. IsLeaf reads the same + // flag bit the product method tests. + if (selection.IsLeaf) + { + return null; + } + + // Mirrors the concrete-type fast path at Selection.cs lines 138-150. + var childSelectionSet = _childSelectionSet; + + if (childSelectionSet is not null) + { + return childSelectionSet; + } + + if (selection.NamedType is IObjectTypeDefinition) + { + childSelectionSet = + selection.DeclaringSelectionSet.DeclaringOperation + .GetSelectionSet(selection, typeContext); + _childSelectionSet = childSelectionSet; + return childSelectionSet; + } + + // Candidate change: scan the copy-on-write array by reference identity + // before falling through to Operation.GetSelectionSet (Selection.cs + // line 152). Type definition instances are schema singletons, so + // reference equality is sound; a miss degrades to today's behavior. + var entries = Volatile.Read(ref _abstractSelectionSets); + + if (entries is not null) + { + for (var i = 0; i < entries.Length; i++) + { + var entry = entries[i]; + + if (ReferenceEquals(entry.Type, typeContext)) + { + return entry.Set; + } + } + } + + var selectionSet = + selection.DeclaringSelectionSet.DeclaringOperation + .GetSelectionSet(selection, typeContext); + + if (entries is null) + { + var initial = new SelectionSetCacheEntry[1]; + initial[0] = new SelectionSetCacheEntry(typeContext, selectionSet); + Interlocked.CompareExchange(ref _abstractSelectionSets, initial, null); + } + else if (entries.Length < MaxCachedTypeContexts) + { + var updated = new SelectionSetCacheEntry[entries.Length + 1]; + Array.Copy(entries, updated, entries.Length); + updated[entries.Length] = new SelectionSetCacheEntry(typeContext, selectionSet); + Interlocked.CompareExchange(ref _abstractSelectionSets, updated, entries); + } + + return selectionSet; + } + } + + private readonly struct SelectionSetCacheEntry + { + public SelectionSetCacheEntry(ITypeDefinition type, SelectionSet set) + { + Type = type; + Set = set; + } + + public readonly ITypeDefinition Type; + + public readonly SelectionSet Set; + } + + /// + /// Composes a fusion schema whose SearchResult interface has ten implementers so the + /// benchmark can cycle more runtime types than the candidate's 8-entry cap. + /// Composition recipe follows FusionBenchmarkBase.CreateFusionSchema. + /// + private static FusionSchemaDefinition ComposeSchema() + { + var sdl = new StringBuilder(); + sdl.Append( + """ + type Query { + searchContent(query: String!): [SearchResult!]! + } + + interface SearchResult { + id: ID! + title: String! + } + """); + + foreach (var name in s_implementerNames) + { + sdl.Append("\n\n"); + sdl.Append("type ").Append(name).Append(" implements SearchResult {\n"); + sdl.Append(" id: ID!\n"); + sdl.Append(" title: String!\n"); + sdl.Append('}'); + } + + List sourceSchemas = [new SourceSchemaText("search", sdl.ToString())]; + + var compositionLog = new CompositionLog(); + var composerOptions = new SchemaComposerOptions(); + var composer = new SchemaComposer(sourceSchemas, composerOptions, compositionLog); + var result = composer.Compose(); + + if (!result.IsSuccess) + { + throw new InvalidOperationException(result.Errors[0].Message); + } + + return FusionSchemaDefinition.Create(result.Value.ToSyntaxNode()); + } + + private sealed class NoOpObjectPool : ObjectPool where T : class, new() + { + public override T Get() + { + return new T(); + } + + public override void Return(T obj) + { + } + } +} diff --git a/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/CompositeContextFusedDecodeBenchmark.cs b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/CompositeContextFusedDecodeBenchmark.cs new file mode 100644 index 00000000000..6e475a31e7c --- /dev/null +++ b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/CompositeContextFusedDecodeBenchmark.cs @@ -0,0 +1,900 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using HotChocolate.Buffers; +using HotChocolate.Fusion.Execution.Nodes; +using HotChocolate.Fusion.Execution.Rewriters; +using HotChocolate.Fusion.Text.Json; +using HotChocolate.Language; +using HotChocolate.Types; +using Microsoft.Extensions.ObjectPool; +using CompositeCursor = HotChocolate.Fusion.Text.Json.CompositeResultDocument.Cursor; +using OperationReferenceType = HotChocolate.Fusion.Text.Json.CompositeResultDocument.OperationReferenceType; + +#nullable enable + +namespace HotChocolate.Fusion.Execution.Benchmarks; + +/// +/// Measures the redundant MetaDb row decodes ValueCompletion pays to classify an +/// already-initialized merge target and then acquire its object or array context. +/// +/// Object path: BuildResult's already-initialized branch (ValueCompletion.cs lines 82-90) +/// reads target.ValueKind (CompositeResultDocument.MetaDb.cs GetElementTokenType, lines +/// 903-920, which resolves the Reference row and recomputes the chunk lookup for a row it +/// already holds a ref to), then TryUpgradeOpaqueTarget reads target.SelectionSet +/// (ValueCompletion.cs line 252 into CompositeResultDocument.cs lines 52-67, two more full +/// row reads of the same rows), then target.GetObjectContext() +/// (CompositeResultDocument.TryGetProperty.cs lines 370-390) resolves the same Reference a +/// third time via MetaDb.GetValue (MetaDb.cs lines 752-764) and fetches the selection set +/// again. TryCompleteObjectValue (ValueCompletion.cs lines 1116-1126) pays ValueKind plus +/// GetObjectContext per re-merged nested object. +/// +/// List path: TryCompleteList (ValueCompletion.cs lines 935-957) decodes the same target +/// row five times on the merge branch: ValueKind at line 935, ValueKind again at line 939, +/// GetArrayLength at line 940, the EnumerateArray token re-check +/// (CompositeResultElement.cs lines 969-987) and the ArrayEnumerator constructor's GetValue +/// (CompositeResultElement.ArrayEnumerator.cs lines 19-27). +/// +/// The fused candidate resolves the target row exactly once per node with the real +/// MetaDb.GetValue and derives everything from that row: a tri-state object context +/// (TokenType.None reports the initialize path, StartObject yields the context plus its +/// selection set, anything else throws exactly like CheckExpectedType) and an array context +/// that surfaces the resolved token type plus length and row span so the enumerator starts +/// with zero further decodes while the non-array error branch stays non-throwing. Fresh +/// (TokenType.None) slots are benchmarked as their own pair to show the false path is cost +/// neutral, which is the main regression scenario of the candidate. +/// +/// The baselines call only real product internals on a real CompositeResultDocument whose +/// targets were materialized through SetObjectValue/SetArrayValue, so every measured target +/// is a Reference row exactly like a steady-state merge target. +/// +[MemoryDiagnoser] +[Config(typeof(BenchmarkConfig))] +public class CompositeContextFusedDecodeBenchmark : FusionBenchmarkBase +{ + /// + /// BenchmarkDotNet 0.15.8 has no RuntimeMoniker for the net11.0 preview host and + /// this project pins TargetFramework to net11.0, so out-of-process toolchains can + /// neither validate nor build a child process here. The job therefore runs in + /// process with the intended 3 warmup and 10 measurement iterations. Benchmarks are + /// grouped by category so every baseline/fused pair gets its own Ratio column. + /// + private sealed class BenchmarkConfig : ManualConfig + { + public BenchmarkConfig() + { + AddJob( + Job.Default + .WithWarmupCount(3) + .WithIterationCount(10) + .WithToolchain(InProcessEmitToolchain.Instance)); + AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByCategory); + AddColumn(CategoriesColumn.Default); + } + } + + private const string OperationId = "123456789101112"; + private const int ObjectCount = 1000; + private const int ListCount = 100; + private const int ListLength = 50; + private const int TinyListCount = 100; + private const int TinyListLength = 1; + private const int FreshCount = 1000; + + private MemoryArena _arena = null!; + private CompositeResultDocument _document = null!; + private CompositeResultElement[] _objectTargets = null!; + private CompositeResultElement[] _listTargets = null!; + private CompositeResultElement[] _tinyListTargets = null!; + private CompositeResultElement[] _freshTargets = null!; + + // Stand-in for CompositeResultDocument._disposed, which is private. The fused product + // method would keep exactly one ObjectDisposedException.ThrowIf field-load-plus-branch + // guard (CompositeResultDocument.TryGetProperty.cs line 372); the fused accessors below + // pay the equivalent cost through this field so the candidate is not flattered. + private int _disposedGuard; + + [GlobalSetup] + public void Setup() + { + _disposedGuard = 0; + + var schema = CreateFusionSchema(); + var documentRewriter = new DocumentRewriter(schema); + var operationDefinition = documentRewriter + .RewriteDocument( + Utf8GraphQLParser.Parse( + """ + { + productById(id: "1") { + id + name + description + price + dimension { height width } + } + } + """)) + .Definitions + .OfType() + .Single(); + var fieldMapPool = new DefaultObjectPool< + OrderedDictionary>>( + new DefaultPooledObjectPolicy< + OrderedDictionary>>()); + var operationCompiler = new OperationCompiler(schema, fieldMapPool); + var operation = operationCompiler.Compile( + OperationId, + OperationId, + operationDefinition); + + var productSelection = operation.RootSelectionSet.Selections[0]; + var productSelectionSet = operation.GetSelectionSet(productSelection); + + if (!productSelectionSet.TryGetSelection("dimension"u8, out var dimensionSelection)) + { + throw new InvalidOperationException("The dimension selection is missing."); + } + + var dimensionSelectionSet = operation.GetSelectionSet(dimensionSelection); + + _arena = new MemoryArena(); + _document = new CompositeResultDocument(_arena, operation, includeFlags: 0); + + var rootContext = _document.Data.GetObjectContext(); + + if (!rootContext.TryGetProperty("productById"u8, out var productSlot, out _)) + { + throw new InvalidOperationException("The productById slot is missing."); + } + + productSlot.SetObjectValue(productSelectionSet, out var productContext); + + // Every target below is an array element value row that SetObjectValue/SetArrayValue + // turned into a Reference row (AssignCompositeValue, CompositeResultDocument.cs lines + // 519-525), matching the steady-state merge targets BuildResult and TryCompleteList + // see after the first subgraph result initialized them. The fresh targets stay + // TokenType.None, matching first-materialization slots. + _objectTargets = CreateArrayElementTargets( + productContext, + "dimension"u8, + ObjectCount, + element => element.SetObjectValue(dimensionSelectionSet)); + _listTargets = CreateArrayElementTargets( + productContext, + "name"u8, + ListCount, + element => element.SetArrayValue(ListLength)); + _tinyListTargets = CreateArrayElementTargets( + productContext, + "price"u8, + TinyListCount, + element => element.SetArrayValue(TinyListLength)); + _freshTargets = CreateArrayElementTargets( + productContext, + "description"u8, + FreshCount, + initialize: null); + + VerifyEquivalence(); + } + + [GlobalCleanup] + public void Cleanup() + { + _disposedGuard = 1; + _document.Dispose(); + _arena.Dispose(); + } + + /// + /// Current product sequence of BuildResult's already-initialized branch + /// (ValueCompletion.cs lines 82-90): ValueKind, then the TryUpgradeOpaqueTarget + /// SelectionSet read (line 252), then GetObjectContext, three resolutions of the same + /// Reference row and two selection-set fetches per target. + /// + [Benchmark(Baseline = true)] + [BenchmarkCategory("ObjectMerge")] + public long ObjectMerge_Baseline() + { + var targets = _objectTargets; + var sum = 0L; + + for (var i = 0; i < targets.Length; i++) + { + var target = targets[i]; + + if (target.ValueKind is JsonValueKind.Undefined) + { + // InitializeTargetObject (ValueCompletion.cs line 84); never taken here. + sum--; + } + else + { + // TryUpgradeOpaqueTarget entry read (ValueCompletion.cs line 252). + var selectionSet = target.SelectionSet; + + if (selectionSet is { Type.Kind: TypeKind.Interface }) + { + // The upgrade flow; never taken for these object-typed selection sets. + sum -= 2; + } + + var objectContext = target.GetObjectContext(); + sum += Consume(in objectContext) + (selectionSet?.Id ?? -3); + } + } + + return sum; + } + + /// + /// Fused candidate for the same branch: one tri-state context acquisition that also + /// hands the selection set to the upgrade check, exactly one row resolution per target. + /// + [Benchmark] + [BenchmarkCategory("ObjectMerge")] + public long ObjectMerge_Fused() + { + var document = _document; + var targets = _objectTargets; + var sum = 0L; + + for (var i = 0; i < targets.Length; i++) + { + var target = targets[i]; + + if (!TryGetObjectContextFused( + document, + target.Cursor, + out var objectContext, + out var selectionSet)) + { + sum--; + } + else + { + if (selectionSet is { Type.Kind: TypeKind.Interface }) + { + sum -= 2; + } + + sum += Consume(in objectContext) + (selectionSet?.Id ?? -3); + } + } + + return sum; + } + + /// + /// Current product sequence of TryCompleteObjectValue's re-merge branch + /// (ValueCompletion.cs lines 1116-1126): ValueKind, then GetObjectContext. + /// + [Benchmark(Baseline = true)] + [BenchmarkCategory("NestedObjectMerge")] + public long NestedObjectMerge_Baseline() + { + var targets = _objectTargets; + var sum = 0L; + + for (var i = 0; i < targets.Length; i++) + { + var target = targets[i]; + + if (target.ValueKind is JsonValueKind.Undefined) + { + // The SetObjectValue initialize path (lines 1117-1121); never taken here. + sum--; + } + else + { + var objectContext = target.GetObjectContext(); + sum += Consume(in objectContext); + } + } + + return sum; + } + + /// + /// Fused candidate for the same branch: the tri-state acquisition replaces both the + /// ValueKind probe and the GetObjectContext re-resolution. + /// + [Benchmark] + [BenchmarkCategory("NestedObjectMerge")] + public long NestedObjectMerge_Fused() + { + var document = _document; + var targets = _objectTargets; + var sum = 0L; + + for (var i = 0; i < targets.Length; i++) + { + var target = targets[i]; + + if (!TryGetObjectContextFused(document, target.Cursor, out var objectContext, out _)) + { + sum--; + } + else + { + sum += Consume(in objectContext); + } + } + + return sum; + } + + /// + /// Current product sequence of TryCompleteList's merge branch (ValueCompletion.cs lines + /// 935-957): ValueKind twice, GetArrayLength, then EnumerateArray, which re-checks the + /// token type and whose enumerator constructor resolves the row once more. The constant + /// list length stands in for source.GetArrayLength(), whose cost is source-side and + /// excluded from both variants; each list is consumed through one MoveNext plus Current, + /// the per-element loop itself being identical in both variants. + /// + [Benchmark(Baseline = true)] + [BenchmarkCategory("ListMerge")] + public long ListMerge_Baseline() + => ListBaselineCore(_listTargets, ListLength); + + /// + /// Fused candidate for the same branch: one row resolution yields the token type, the + /// length and the row span, and the enumerator is built from the resolved cursor with + /// zero further decodes. + /// + [Benchmark] + [BenchmarkCategory("ListMerge")] + public long ListMerge_Fused() + => ListFusedCore(_listTargets, ListLength); + + /// + /// The list merge pair on single-element lists, the tiny shape where fixed per-node + /// costs dominate and a fused-path regression would show. + /// + [Benchmark(Baseline = true)] + [BenchmarkCategory("TinyListMerge")] + public long TinyListMerge_Baseline() + => ListBaselineCore(_tinyListTargets, TinyListLength); + + /// + /// Fused candidate on single-element lists. + /// + [Benchmark] + [BenchmarkCategory("TinyListMerge")] + public long TinyListMerge_Fused() + => ListFusedCore(_tinyListTargets, TinyListLength); + + /// + /// Current product classification of fresh (TokenType.None) slots: the ValueKind probe + /// is a targeted 4-byte read with no reference to resolve, after which the real code + /// would initialize the slot. + /// + [Benchmark(Baseline = true)] + [BenchmarkCategory("FreshSlot")] + public long FreshSlot_Baseline() + { + var targets = _freshTargets; + var sum = 0L; + + for (var i = 0; i < targets.Length; i++) + { + if (targets[i].ValueKind is JsonValueKind.Undefined) + { + // The initialize path would run here; classification stops at the probe. + sum++; + } + else + { + sum -= 5; + } + } + + return sum; + } + + /// + /// Fused candidate on fresh slots: the tri-state acquisition reads the full 20-byte row + /// where the baseline reads 4 bytes of the same row. This pair bounds the regression + /// risk on fresh-object-dominant payloads and must stay close to a 1.0 ratio. + /// + [Benchmark] + [BenchmarkCategory("FreshSlot")] + public long FreshSlot_Fused() + { + var document = _document; + var targets = _freshTargets; + var sum = 0L; + + for (var i = 0; i < targets.Length; i++) + { + if (!TryGetObjectContextFused(document, targets[i].Cursor, out _, out _)) + { + sum++; + } + else + { + sum -= 5; + } + } + + return sum; + } + + private long ListBaselineCore(CompositeResultElement[] targets, int expectedLength) + { + var sum = 0L; + + for (var i = 0; i < targets.Length; i++) + { + var target = targets[i]; + + // Decode 1 (ValueCompletion.cs line 935). + if (target.ValueKind is JsonValueKind.Undefined) + { + // The SetArrayValue initialize path (line 937); never taken here. + sum--; + } + + // Decodes 2 and 3 (lines 939-940). + else if (target.ValueKind is not JsonValueKind.Array + || target.GetArrayLength() != expectedLength) + { + // The non-throwing length-mismatch error branch (lines 941-953); + // never taken here. + sum -= 2; + } + else + { + // Decodes 4 and 5 (line 957): EnumerateArray re-checks the token type and + // the ArrayEnumerator constructor resolves the row again. + var enumerator = target.EnumerateArray().GetEnumerator(); + + if (enumerator.MoveNext()) + { + sum += enumerator.Current.Cursor.Value; + } + } + } + + return sum; + } + + private long ListFusedCore(CompositeResultElement[] targets, int expectedLength) + { + var document = _document; + var sum = 0L; + + for (var i = 0; i < targets.Length; i++) + { + var target = targets[i]; + + var tokenType = GetArrayContextFused( + document, + target.Cursor, + out var startCursor, + out var length, + out var numberOfRows); + + if (tokenType is ElementTokenType.None) + { + sum--; + } + else if (tokenType is not ElementTokenType.StartArray || length != expectedLength) + { + sum -= 2; + } + else + { + var enumerator = + new FusedArrayEnumerator(document, startCursor, numberOfRows).GetEnumerator(); + + if (enumerator.MoveNext()) + { + sum += enumerator.Current.Cursor.Value; + } + } + } + + return sum; + } + + /// + /// Candidate fused object classification. One real MetaDb.GetValue + /// (CompositeResultDocument.MetaDb.cs lines 752-764) replaces the ValueKind probe, the + /// SelectionSet read and the GetObjectContext re-resolution: TokenType.None reports the + /// initialize path, StartObject yields the context built from the row already in hand + /// (mirroring CompositeResultDocument.GetObjectContext, + /// CompositeResultDocument.TryGetProperty.cs lines 370-390) plus its selection set for + /// the upgrade check, and any other token throws exactly like CheckExpectedType. + /// + private bool TryGetObjectContextFused( + CompositeResultDocument document, + CompositeCursor cursor, + out CompositeObjectContext objectContext, + out SelectionSet? selectionSet) + { + // Mirrors CompositeResultElement.CheckValidInstance + // (CompositeResultElement.cs lines 1099-1105). + if (document is null) + { + throw new InvalidOperationException(); + } + + ObjectDisposedException.ThrowIf(_disposedGuard != 0, document); + + var row = document._metaDb.GetValue(ref cursor); + + if (row.TokenType is ElementTokenType.None) + { + objectContext = default; + selectionSet = null; + return false; + } + + CheckExpectedTypeCopy(ElementTokenType.StartObject, row.TokenType); + + // Mirrors the selection-set derivation of CompositeResultDocument.GetObjectContext + // (CompositeResultDocument.TryGetProperty.cs lines 382-387). + selectionSet = row.OperationReferenceType is OperationReferenceType.SelectionSet + ? document.GetOperation().GetSelectionSetById(row.OperationReferenceId) + : null; + + objectContext = new CompositeObjectContext( + document, + cursor, + selectionSet, + row.NumberOfRows); + return true; + } + + /// + /// Candidate fused list classification. One real MetaDb.GetValue resolves the target + /// and surfaces the token type so the caller keeps TryCompleteList's non-throwing + /// length-mismatch error branch (ValueCompletion.cs lines 939-954) exactly as today; + /// length and row span are only derived for StartArray rows. + /// + private ElementTokenType GetArrayContextFused( + CompositeResultDocument document, + CompositeCursor cursor, + out CompositeCursor startCursor, + out int length, + out int numberOfRows) + { + // Mirrors CompositeResultElement.CheckValidInstance + // (CompositeResultElement.cs lines 1099-1105). + if (document is null) + { + throw new InvalidOperationException(); + } + + ObjectDisposedException.ThrowIf(_disposedGuard != 0, document); + + var row = document._metaDb.GetValue(ref cursor); + startCursor = cursor; + + if (row.TokenType is ElementTokenType.StartArray) + { + length = row.SizeOrLength; + numberOfRows = row.NumberOfRows; + return ElementTokenType.StartArray; + } + + length = 0; + numberOfRows = 0; + return row.TokenType; + } + + // Byte-faithful copy of the private CompositeResultDocument.CheckExpectedType + // (CompositeResultDocument.cs lines 590-596), which the fused product method would call. + private static void CheckExpectedTypeCopy(ElementTokenType expected, ElementTokenType actual) + { + if (expected != actual) + { + throw new ArgumentOutOfRangeException($"Expected {expected} but found {actual}."); + } + } + + /// + /// Opaque sink that forces both variants to fully materialize the acquired context so + /// neither side can be dead-code eliminated. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private static long Consume(in CompositeObjectContext objectContext) => 1; + + /// + /// Candidate enumerator shape: mirrors CompositeResultElement.ArrayEnumerator + /// (CompositeResultElement.ArrayEnumerator.cs) except that the constructor takes the + /// pre-resolved start cursor and row count instead of re-running MetaDb.GetValue + /// (lines 19-27), which is exactly the internal constructor the recommended design adds. + /// + private struct FusedArrayEnumerator + { + private readonly CompositeResultDocument _document; + private readonly CompositeCursor _start; + private readonly CompositeCursor _end; + private CompositeCursor _cursor; + + internal FusedArrayEnumerator( + CompositeResultDocument document, + CompositeCursor startCursor, + int numberOfRows) + { + _document = document; + _start = startCursor; + _end = startCursor + numberOfRows; + _cursor = startCursor; + } + + // Mirrors ArrayEnumerator.Current (lines 30-42). + public CompositeResultElement Current + { + get + { + var cursor = _cursor; + if (cursor == _start || cursor >= _end) + { + return default; + } + + return new CompositeResultElement(_document, cursor); + } + } + + // Mirrors ArrayEnumerator.GetEnumerator (lines 50-55). + public FusedArrayEnumerator GetEnumerator() + { + var enumerator = this; + enumerator._cursor = enumerator._start; + return enumerator; + } + + // Mirrors ArrayEnumerator.MoveNext (lines 66-93). + public bool MoveNext() + { + var start = _start; + var end = _end; + + if (_cursor == start) + { + var first = start + 1; + if (first < end) + { + _cursor = first; + return true; + } + + _cursor = end; + return false; + } + + var next = _cursor + 1; + if (next < end) + { + _cursor = next; + return true; + } + + _cursor = end; + return false; + } + } + + private static CompositeResultElement[] CreateArrayElementTargets( + in CompositeObjectContext parentContext, + ReadOnlySpan slotName, + int count, + Action? initialize) + { + if (!parentContext.TryGetProperty(slotName, out var slot, out _)) + { + throw new InvalidOperationException("A benchmark slot is missing."); + } + + slot.SetArrayValue(count); + + var targets = new CompositeResultElement[count]; + var i = 0; + + foreach (var element in slot.EnumerateArray()) + { + initialize?.Invoke(element); + targets[i++] = element; + } + + if (i != count) + { + throw new InvalidOperationException("Target materialization is incomplete."); + } + + return targets; + } + + private void VerifyEquivalence() + { + for (var i = 0; i < _objectTargets.Length; i++) + { + VerifyObjectTarget(_objectTargets[i], i); + } + + for (var i = 0; i < _listTargets.Length; i++) + { + VerifyListTarget(_listTargets[i], ListLength, i); + } + + for (var i = 0; i < _tinyListTargets.Length; i++) + { + VerifyListTarget(_tinyListTargets[i], TinyListLength, i); + } + + for (var i = 0; i < _freshTargets.Length; i++) + { + VerifyFreshTarget(_freshTargets[i], i); + } + + VerifyChecksum("ObjectMerge", ObjectMerge_Baseline(), ObjectMerge_Fused()); + VerifyChecksum("NestedObjectMerge", NestedObjectMerge_Baseline(), NestedObjectMerge_Fused()); + VerifyChecksum("ListMerge", ListMerge_Baseline(), ListMerge_Fused()); + VerifyChecksum("TinyListMerge", TinyListMerge_Baseline(), TinyListMerge_Fused()); + VerifyChecksum("FreshSlot", FreshSlot_Baseline(), FreshSlot_Fused()); + } + + private void VerifyObjectTarget(CompositeResultElement target, int index) + { + if (target.ValueKind is not JsonValueKind.Object) + { + throw new InvalidOperationException( + $"Object target {index} is not an object but {target.ValueKind}."); + } + + var baselineSelectionSet = target.SelectionSet; + var baselineContext = target.GetObjectContext(); + + if (!TryGetObjectContextFused( + _document, + target.Cursor, + out var fusedContext, + out var fusedSelectionSet)) + { + throw new InvalidOperationException( + $"The fused acquisition reported object target {index} as uninitialized."); + } + + if (!ReferenceEquals(baselineSelectionSet, fusedSelectionSet)) + { + throw new InvalidOperationException( + $"Object target {index} resolves different selection sets between the " + + "baseline and the fused variant."); + } + + CompareContextProperty(in baselineContext, in fusedContext, "height"u8, index); + CompareContextProperty(in baselineContext, in fusedContext, "width"u8, index); + CompareContextProperty(in baselineContext, in fusedContext, "missing"u8, index); + } + + private static void CompareContextProperty( + in CompositeObjectContext baselineContext, + in CompositeObjectContext fusedContext, + ReadOnlySpan propertyName, + int index) + { + var baselineFound = baselineContext.TryGetProperty( + propertyName, + out var baselineValue, + out var baselineSelection); + var fusedFound = fusedContext.TryGetProperty( + propertyName, + out var fusedValue, + out var fusedSelection); + + if (baselineFound != fusedFound + || baselineValue.Cursor != fusedValue.Cursor + || !ReferenceEquals(baselineSelection, fusedSelection)) + { + throw new InvalidOperationException( + $"Object target {index} resolves a property differently between the " + + "baseline context and the fused context."); + } + } + + private void VerifyListTarget(CompositeResultElement target, int expectedLength, int index) + { + if (target.ValueKind is not JsonValueKind.Array + || target.GetArrayLength() != expectedLength) + { + throw new InvalidOperationException( + $"List target {index} is not an array of length {expectedLength}."); + } + + var tokenType = GetArrayContextFused( + _document, + target.Cursor, + out var startCursor, + out var length, + out var numberOfRows); + + if (tokenType is not ElementTokenType.StartArray || length != expectedLength) + { + throw new InvalidOperationException( + $"The fused acquisition misclassifies list target {index}: " + + $"{tokenType} with length {length}."); + } + + var baselineEnumerator = target.EnumerateArray().GetEnumerator(); + var fusedEnumerator = + new FusedArrayEnumerator(_document, startCursor, numberOfRows).GetEnumerator(); + var elementCount = 0; + + while (true) + { + var baselineMoved = baselineEnumerator.MoveNext(); + var fusedMoved = fusedEnumerator.MoveNext(); + + if (baselineMoved != fusedMoved) + { + throw new InvalidOperationException( + $"List target {index} enumerators disagree after {elementCount} elements."); + } + + if (!baselineMoved) + { + break; + } + + if (baselineEnumerator.Current.Cursor != fusedEnumerator.Current.Cursor) + { + throw new InvalidOperationException( + $"List target {index} yields a different element cursor at " + + $"position {elementCount}."); + } + + elementCount++; + } + + if (elementCount != expectedLength) + { + throw new InvalidOperationException( + $"List target {index} enumerated {elementCount} elements " + + $"instead of {expectedLength}."); + } + } + + private void VerifyFreshTarget(CompositeResultElement target, int index) + { + if (target.ValueKind is not JsonValueKind.Undefined) + { + throw new InvalidOperationException($"Fresh target {index} is not undefined."); + } + + if (TryGetObjectContextFused(_document, target.Cursor, out _, out _)) + { + throw new InvalidOperationException( + $"The fused acquisition reported fresh target {index} as initialized."); + } + + var tokenType = GetArrayContextFused(_document, target.Cursor, out _, out _, out _); + + if (tokenType is not ElementTokenType.None) + { + throw new InvalidOperationException( + $"The fused list acquisition misclassifies fresh target {index} as {tokenType}."); + } + } + + private static void VerifyChecksum(string pair, long baseline, long fused) + { + if (baseline != fused) + { + throw new InvalidOperationException( + $"{pair} checksum mismatch: baseline {baseline} vs fused {fused}."); + } + } +} diff --git a/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/ForwardedVariableBlitBenchmark.cs b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/ForwardedVariableBlitBenchmark.cs new file mode 100644 index 00000000000..a5499519c31 --- /dev/null +++ b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/ForwardedVariableBlitBenchmark.cs @@ -0,0 +1,1180 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.Json; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using HotChocolate.Buffers; +using HotChocolate.Execution; +using HotChocolate.Fusion; +using HotChocolate.Fusion.Execution; +using HotChocolate.Fusion.Execution.Nodes; +using HotChocolate.Fusion.Execution.Results; +using HotChocolate.Fusion.Language; +using HotChocolate.Fusion.Text.Json; +using HotChocolate.Language; +using HotChocolate.Text.Json; +using FusionNameNode = HotChocolate.Fusion.Language.NameNode; +using IntValueNode = HotChocolate.Language.IntValueNode; +using FloatValueNode = HotChocolate.Language.FloatValueNode; +using StringValueNode = HotChocolate.Language.StringValueNode; +using BooleanValueNode = HotChocolate.Language.BooleanValueNode; +using NullValueNode = HotChocolate.Language.NullValueNode; +using EnumValueNode = HotChocolate.Language.EnumValueNode; +using ListValueNode = HotChocolate.Language.ListValueNode; +using ObjectValueNode = HotChocolate.Language.ObjectValueNode; +using ObjectFieldNode = HotChocolate.Language.ObjectFieldNode; +using IValueNode = HotChocolate.Language.IValueNode; + +#nullable enable + +namespace HotChocolate.Fusion.Execution.Benchmarks; + +/// +/// Measures the forwarded-variable serialization waste in the per-entry loops of +/// FetchResultStore.BuildVariableValueSets (FetchResultStore.cs lines +/// 1142-1147) and FetchResultStore.BuildVariableValueSetsFromSnapshot +/// (lines 1049-1054). When a fetch node forwards request variables, the requirement +/// fast paths are bypassed (guard at line 1094) and every entry re-serializes the +/// identical forwarded-variables AST through WritePropertyName plus +/// WriteValueNode (lines 1782-1833), a recursive type switch with string +/// escaping and number transcoding whose output bytes are constant across the loop. +/// +/// The candidate serializes the forwarded variables once for the first non-empty +/// entry, copies the interior bytes between the object braces into a pooled buffer, +/// and splices those bytes into every further entry with one raw write. The public +/// JsonWriter.WriteRawValue (JsonWriter.cs lines 459-479) writes the same +/// bytes and leaves the same writer state as the internal WriteRawValueStart +/// plus WriteRawValueEnd pair (lines 487-523) that the product change would +/// use. As the accepted rider, requirement keys are transcoded to UTF-8 once per +/// call and written through the public byte-span WritePropertyName overload; +/// the product change would use the internal WritePropertyNameUnescaped +/// (JsonWriter.WriteValues.PropertyName.cs lines 230-244), which is not visible to +/// this project, so the measured rider keeps the escape scan and is slightly +/// conservative. +/// +/// The workload drives the snapshot-path loop because its baseline is fully +/// reachable through the real internal CreateVariableValueSetsFromSnapshot +/// (FetchResultStore.cs lines 859-894) without seeding a composite result document, +/// and its per-entry serialization is identical in shape to the general element +/// loop. The product method is the reference baseline and includes the store's +/// pooling lifecycle (Clean, FetchResultStore.Pooling.cs lines 104-139); the +/// two replica methods share benchmark-owned writer machinery and an identical +/// writer-only clean, so the candidate's effect reads from the replica pair. +/// +[MemoryDiagnoser] +[Config(typeof(BenchmarkConfig))] +public class ForwardedVariableBlitBenchmark +{ + /// + /// BenchmarkDotNet 0.15.8 has no RuntimeMoniker for the net11.0 preview host and + /// this project pins TargetFramework to net11.0, so out-of-process toolchains can + /// neither validate nor build a child process here. The job therefore runs in + /// process with the intended 3 warmup and 10 measurement iterations. + /// + private sealed class BenchmarkConfig : ManualConfig + { + public BenchmarkConfig() + => AddJob( + Job.Default + .WithWarmupCount(3) + .WithIterationCount(10) + .WithToolchain(InProcessEmitToolchain.Instance)); + } + + private const int MaxRetainedLength = 256; + private const int RequirementKeyBufferSize = 64; + private const string RequirementKey = "__fusion_1_id"; + + // Mirrors FetchResultStore.cs line 37. + private static readonly ArrayPool s_variableValuePool = + ArrayPool.Shared; + + private static readonly OperationRequirement[] s_requirements = + [Requirement(RequirementKey)]; + + private static readonly HashSet s_importedKeys = + new([RequirementKey], StringComparer.Ordinal); + + private FetchResultStore _productStore = null!; + private FetchResultStore _sourceStore = null!; + private ChunkedArrayWriter _replicaWriter = null!; + private JsonWriter _replicaJsonWriter = null!; + private ReplicaVariableDedupTable _replicaDedup = null!; + private ChunkedArrayWriter _blitWriter = null!; + private JsonWriter _blitJsonWriter = null!; + private ReplicaVariableDedupTable _blitDedup = null!; + private ImmutableArray _importedEntries; + private IReadOnlyList _requestVariables = null!; + + public long Consumed; + + /// + /// The number of imported entries the merge iterates. 1 is the regression + /// shape where the blit variant pays its one-time capture without ever + /// splicing; 256 is the entity-batch shape where the win amortizes. + /// + [Params(1, 16, 256)] + public int EntryCount { get; set; } + + /// + /// Small is one forwarded int variable. Large is three forwarded variables + /// including a nested object filter of roughly 25 AST nodes with strings that + /// need escaping, the shape where per-entry re-serialization is most costly. + /// + [Params("Small", "Large")] + public string VariableShape { get; set; } = "Small"; + + [GlobalSetup] + public void Setup() + { + _productStore = new FetchResultStore(); + _sourceStore = new FetchResultStore(); + + // Mirrors the writer wiring of the FetchResultStore constructor + // (FetchResultStore.cs lines 48-50 and 70-74). + _replicaWriter = new ChunkedArrayWriter(); + _replicaJsonWriter = new JsonWriter(_replicaWriter, new JsonWriterOptions { Indented = false }); + _replicaDedup = new ReplicaVariableDedupTable(_replicaWriter); + _blitWriter = new ChunkedArrayWriter(); + _blitJsonWriter = new JsonWriter(_blitWriter, new JsonWriterOptions { Indented = false }); + _blitDedup = new ReplicaVariableDedupTable(_blitWriter); + + _requestVariables = VariableShape == "Small" + ? BuildSmallVariables() + : BuildLargeVariables(); + + var keyBytes = 0; + + foreach (var requirement in s_requirements) + { + keyBytes += Encoding.UTF8.GetByteCount(requirement.Key); + } + + if (keyBytes > RequirementKeyBufferSize) + { + throw new InvalidOperationException( + "The requirement keys do not fit the pre-encode buffer; " + + "increase RequirementKeyBufferSize."); + } + + _importedEntries = BuildImportedEntries(EntryCount); + + VerifyEquivalence(); + } + + [GlobalCleanup] + public void Cleanup() + { + _productStore.Dispose(); + _sourceStore.Dispose(); + _replicaDedup.Dispose(); + _replicaWriter.Dispose(); + _blitDedup.Dispose(); + _blitWriter.Dispose(); + } + + /// + /// Current product behavior through the real internals: one snapshot merge via + /// CreateVariableValueSetsFromSnapshot followed by the pooling + /// Clean that production performs between requests. The store's Clean + /// also resets collection buffers the replicas do not carry, so this method is + /// the fidelity reference rather than the isolation partner of the blit. + /// + [Benchmark(Baseline = true)] + public long Merge_Product() + { + var result = _productStore.CreateVariableValueSetsFromSnapshot( + _importedEntries, + s_importedKeys, + _requestVariables, + s_requirements); + + var acc = Accumulate(result); + _productStore.Clean(MaxRetainedLength, MaxRetainedLength); + + Consumed = acc; + return acc; + } + + /// + /// Byte-faithful replica of the current per-entry loop over benchmark-owned + /// writer machinery: every entry re-serializes the forwarded-variables AST. + /// Its delta against bounds the replica's fidelity; + /// its delta against isolates the + /// candidate on identical machinery and lifecycle. + /// + [Benchmark] + public long Merge_ReplicaPerEntrySerialize() + { + var result = BuildFromSnapshotCurrent( + _replicaWriter, + _replicaJsonWriter, + _replicaDedup, + _importedEntries, + _requestVariables, + s_requirements); + + var acc = Accumulate(result); + _replicaWriter.Clean(); + + Consumed = acc; + return acc; + } + + /// + /// Candidate optimization: the forwarded variables serialize once for the first + /// non-empty entry, the interior bytes are captured into a pooled buffer, and + /// every further entry splices them with one raw write; requirement keys are + /// UTF-8 encoded once per call. + /// + [Benchmark] + public long Merge_ReplicaPrefixBlit() + { + var result = BuildFromSnapshotPrefixBlit( + _blitWriter, + _blitJsonWriter, + _blitDedup, + _importedEntries, + _requestVariables, + s_requirements); + + var acc = Accumulate(result); + _blitWriter.Clean(); + + Consumed = acc; + return acc; + } + + private static long Accumulate(ImmutableArray values) + { + var acc = 0L; + + foreach (var entry in values) + { + acc += entry.Path.Length + entry.Values.Length + entry.AdditionalPaths.Length; + } + + return acc; + } + + /// + /// Byte-faithful copy of FetchResultStore.BuildVariableValueSetsFromSnapshot + /// (FetchResultStore.cs lines 1027-1084) over benchmark-owned writer machinery. + /// The entry-point lock and argument guards of CreateVariableValueSetsFromSnapshot + /// (lines 859-894) are constant per-call costs outside the measured loop and are + /// omitted from both replicas. + /// + private static ImmutableArray BuildFromSnapshotCurrent( + ChunkedArrayWriter variableWriter, + JsonWriter jsonWriter, + ReplicaVariableDedupTable dedupTable, + ImmutableArray importedEntries, + IReadOnlyList requestVariables, + ReadOnlySpan requiredData) + { + dedupTable.Initialize(importedEntries.Length); + + VariableValues[]? variableValueSets = null; + var additionalPaths = new AdditionalPathAccumulator(); + var nextIndex = 0; + + foreach (var importedEntry in importedEntries) + { + if (importedEntry.IsEmpty) + { + continue; + } + + jsonWriter.Reset(variableWriter); + var startPosition = variableWriter.Position; + jsonWriter.WriteStartObject(); + + // The per-entry forwarded-variables re-serialization this candidate + // removes (FetchResultStore.cs lines 1049-1054). + for (var i = 0; i < requestVariables.Count; i++) + { + var field = requestVariables[i]; + jsonWriter.WritePropertyName(field.Name.Value); + WriteValueNode(jsonWriter, field.Value); + } + + if (!TryWriteRequestedRequirementValues(jsonWriter, importedEntry.Values, requiredData)) + { + variableWriter.ResetTo(startPosition); + continue; + } + + jsonWriter.WriteEndObject(); + + var entry = TryCreateVariableValues( + variableWriter, + dedupTable, + importedEntry.Path, + startPosition, + ref additionalPaths, + nextIndex, + out var dedupIndex); + + if (entry is null) + { + additionalPaths.AddRange(dedupIndex, importedEntry.AdditionalPaths.AsSpan()); + continue; + } + + variableValueSets ??= s_variableValuePool.Rent(importedEntries.Length); + variableValueSets[nextIndex] = entry.Value; + additionalPaths.AddRange(nextIndex, importedEntry.AdditionalPaths.AsSpan()); + nextIndex++; + } + + return FinalizeVariableValueSets(variableValueSets, ref additionalPaths, nextIndex); + } + + /// + /// The recommended design: identical to + /// except that the forwarded variables + /// serialize once, per-entry splicing is a raw byte copy, and requirement keys + /// are pre-encoded once per call. + /// + private static ImmutableArray BuildFromSnapshotPrefixBlit( + ChunkedArrayWriter variableWriter, + JsonWriter jsonWriter, + ReplicaVariableDedupTable dedupTable, + ImmutableArray importedEntries, + IReadOnlyList requestVariables, + ReadOnlySpan requiredData) + { + dedupTable.Initialize(importedEntries.Length); + + VariableValues[]? variableValueSets = null; + var additionalPaths = new AdditionalPathAccumulator(); + var nextIndex = 0; + + // Rider: requirement keys are GraphQL names (always ASCII), so they are + // transcoded to UTF-8 once per call instead of per entry inside + // TryWriteRequirementValue (FetchResultStore.cs line 1762). + Span keyBytes = stackalloc byte[RequirementKeyBufferSize]; + Span keyEnds = stackalloc int[requiredData.Length]; + var keyOffset = 0; + + for (var i = 0; i < requiredData.Length; i++) + { + keyOffset += Encoding.UTF8.GetBytes(requiredData[i].Key, keyBytes[keyOffset..]); + keyEnds[i] = keyOffset; + } + + byte[]? prefix = null; + var prefixLength = -1; + + try + { + foreach (var importedEntry in importedEntries) + { + if (importedEntry.IsEmpty) + { + continue; + } + + jsonWriter.Reset(variableWriter); + var startPosition = variableWriter.Position; + jsonWriter.WriteStartObject(); + + if (requestVariables.Count > 0) + { + if (prefixLength < 0) + { + // First non-empty entry: serialize the forwarded variables + // exactly like the baseline and capture the interior bytes + // between the object braces before any requirement write can + // fail and rewind the writer. Writer positions are gap-free + // (ChunkedArrayWriter.cs lines 18-20), so the length is plain + // position arithmetic and CopyTo handles chunk crossings. + for (var i = 0; i < requestVariables.Count; i++) + { + var field = requestVariables[i]; + jsonWriter.WritePropertyName(field.Name.Value); + WriteValueNode(jsonWriter, field.Value); + } + + prefixLength = variableWriter.Position - startPosition - 1; + prefix = ArrayPool.Shared.Rent(prefixLength); + variableWriter.CopyTo(prefix.AsSpan(0, prefixLength), startPosition + 1, prefixLength); + } + else + { + // The public WriteRawValue (JsonWriter.cs lines 459-479) + // writes the same bytes and leaves the same writer state as + // the internal WriteRawValueStart plus WriteRawValueEnd pair + // (lines 487-523) for a contiguous span: no leading comma + // right after '{', and the list separator flag set so the + // next property name emits its comma. + jsonWriter.WriteRawValue(prefix!.AsSpan(0, prefixLength)); + } + } + + if (!TryWriteRequestedRequirementValuesPreEncoded( + jsonWriter, + importedEntry.Values, + requiredData, + keyBytes, + keyEnds)) + { + variableWriter.ResetTo(startPosition); + continue; + } + + jsonWriter.WriteEndObject(); + + var entry = TryCreateVariableValues( + variableWriter, + dedupTable, + importedEntry.Path, + startPosition, + ref additionalPaths, + nextIndex, + out var dedupIndex); + + if (entry is null) + { + additionalPaths.AddRange(dedupIndex, importedEntry.AdditionalPaths.AsSpan()); + continue; + } + + variableValueSets ??= s_variableValuePool.Rent(importedEntries.Length); + variableValueSets[nextIndex] = entry.Value; + additionalPaths.AddRange(nextIndex, importedEntry.AdditionalPaths.AsSpan()); + nextIndex++; + } + + return FinalizeVariableValueSets(variableValueSets, ref additionalPaths, nextIndex); + } + finally + { + if (prefix is not null) + { + ArrayPool.Shared.Return(prefix); + } + } + } + + // Byte-faithful copy of FetchResultStore.TryCreateVariableValues + // (FetchResultStore.cs lines 1679-1704). + private static VariableValues? TryCreateVariableValues( + ChunkedArrayWriter variableWriter, + ReplicaVariableDedupTable dedupTable, + CompactPath path, + int startPosition, + ref AdditionalPathAccumulator additionalPaths, + int nextIndex, + out int dedupIndex) + { + var length = variableWriter.Position - startPosition; + var hash = variableWriter.GetHashCode(startPosition, length); + + if (dedupTable.TryGet(hash, startPosition, length, out var existingIndex)) + { + dedupIndex = existingIndex; + additionalPaths.Add(existingIndex, path); + variableWriter.ResetTo(startPosition); + return null; + } + + dedupIndex = nextIndex; + dedupTable.Add(hash, nextIndex, startPosition, length); + return new VariableValues(path, JsonSegment.Create(variableWriter, startPosition, length)); + } + + // Byte-faithful copy of FetchResultStore.TryWriteRequestedRequirementValues + // (FetchResultStore.cs lines 1706-1726). + private static bool TryWriteRequestedRequirementValues( + JsonWriter jsonWriter, + JsonSegment values, + ReadOnlySpan requiredData) + { + if (values.IsEmpty) + { + return false; + } + + var sequence = values.AsSequence(); + + foreach (var requirement in requiredData) + { + if (!TryWriteRequirementValue(jsonWriter, sequence, requirement.Key)) + { + return false; + } + } + + return true; + } + + private static bool TryWriteRequestedRequirementValuesPreEncoded( + JsonWriter jsonWriter, + JsonSegment values, + ReadOnlySpan requiredData, + ReadOnlySpan keyBytes, + ReadOnlySpan keyEnds) + { + if (values.IsEmpty) + { + return false; + } + + var sequence = values.AsSequence(); + var keyStart = 0; + + for (var i = 0; i < requiredData.Length; i++) + { + if (!TryWriteRequirementValuePreEncoded( + jsonWriter, + sequence, + requiredData[i].Key, + keyBytes[keyStart..keyEnds[i]])) + { + return false; + } + + keyStart = keyEnds[i]; + } + + return true; + } + + // Byte-faithful copy of FetchResultStore.TryWriteRequirementValue + // (FetchResultStore.cs lines 1728-1769). + private static bool TryWriteRequirementValue( + JsonWriter jsonWriter, + ReadOnlySequence values, + string key) + { + var reader = new Utf8JsonReader(values); + + if (!reader.Read() || reader.TokenType is not JsonTokenType.StartObject) + { + return false; + } + + while (reader.Read()) + { + if (reader.TokenType is JsonTokenType.EndObject) + { + return false; + } + + if (reader.TokenType is not JsonTokenType.PropertyName) + { + return false; + } + + var matches = reader.ValueTextEquals(key); + + if (!reader.Read()) + { + return false; + } + + var start = reader.TokenStartIndex; + reader.Skip(); + var length = reader.BytesConsumed - start; + + if (matches) + { + jsonWriter.WritePropertyName(key); + WriteRawJsonValue(jsonWriter, values.Slice(start, length)); + return true; + } + } + + return false; + } + + // Same scan as TryWriteRequirementValue; the only change (the accepted rider) + // is that the property name write uses the pre-encoded UTF-8 key. + private static bool TryWriteRequirementValuePreEncoded( + JsonWriter jsonWriter, + ReadOnlySequence values, + string key, + ReadOnlySpan utf8Key) + { + var reader = new Utf8JsonReader(values); + + if (!reader.Read() || reader.TokenType is not JsonTokenType.StartObject) + { + return false; + } + + while (reader.Read()) + { + if (reader.TokenType is JsonTokenType.EndObject) + { + return false; + } + + if (reader.TokenType is not JsonTokenType.PropertyName) + { + return false; + } + + var matches = reader.ValueTextEquals(key); + + if (!reader.Read()) + { + return false; + } + + var start = reader.TokenStartIndex; + reader.Skip(); + var length = reader.BytesConsumed - start; + + if (matches) + { + jsonWriter.WritePropertyName(utf8Key); + WriteRawJsonValue(jsonWriter, values.Slice(start, length)); + return true; + } + } + + return false; + } + + // Byte-faithful copy of FetchResultStore.WriteRawJsonValue + // (FetchResultStore.cs lines 1771-1780). + private static void WriteRawJsonValue(JsonWriter jsonWriter, ReadOnlySequence value) + { + if (value.IsSingleSegment) + { + jsonWriter.WriteRawValue(value.FirstSpan); + return; + } + + jsonWriter.WriteRawValue(value.ToArray()); + } + + // Byte-faithful copy of FetchResultStore.WriteValueNode + // (FetchResultStore.cs lines 1782-1833). + private static void WriteValueNode(JsonWriter jsonWriter, IValueNode value) + { + switch (value) + { + case NullValueNode: + jsonWriter.WriteNullValue(); + break; + + case StringValueNode sv: + jsonWriter.WriteStringValue(sv.Value); + break; + + case IntValueNode iv: + WriteRawAscii(jsonWriter, iv.Value); + break; + + case FloatValueNode fv: + WriteRawAscii(jsonWriter, fv.Value); + break; + + case BooleanValueNode bv: + jsonWriter.WriteBooleanValue(bv.Value); + break; + + case EnumValueNode ev: + jsonWriter.WriteStringValue(ev.Value); + break; + + case ObjectValueNode ov: + jsonWriter.WriteStartObject(); + foreach (var field in ov.Fields) + { + jsonWriter.WritePropertyName(field.Name.Value); + WriteValueNode(jsonWriter, field.Value); + } + jsonWriter.WriteEndObject(); + break; + + case ListValueNode lv: + jsonWriter.WriteStartArray(); + foreach (var item in lv.Items) + { + WriteValueNode(jsonWriter, item); + } + jsonWriter.WriteEndArray(); + break; + + default: + jsonWriter.WriteNullValue(); + break; + } + } + + // Byte-faithful copy of FetchResultStore.WriteRawAscii + // (FetchResultStore.cs lines 1835-1840). + private static void WriteRawAscii(JsonWriter jsonWriter, string value) + { + Span buffer = stackalloc byte[value.Length]; + Encoding.UTF8.GetBytes(value.AsSpan(), buffer); + jsonWriter.WriteRawValue(buffer); + } + + // Byte-faithful copy of FetchResultStore.FinalizeVariableValueSets + // (FetchResultStore.cs lines 2545-2570). + private static ImmutableArray FinalizeVariableValueSets( + VariableValues[]? variableValueSets, + ref AdditionalPathAccumulator additionalPaths, + int nextIndex) + { + if (variableValueSets is null || nextIndex == 0) + { + if (variableValueSets is not null) + { + s_variableValuePool.Return(variableValueSets, clearArray: true); + } + + additionalPaths.Dispose(); + return []; + } + + additionalPaths.ApplyTo(variableValueSets, nextIndex); + additionalPaths.Dispose(); + + var span = variableValueSets.AsSpan(0, nextIndex); + var result = span.ToArray(); + span.Clear(); + s_variableValuePool.Return(variableValueSets); + + return ImmutableCollectionsMarshal.AsImmutableArray(result); + } + + private static IReadOnlyList BuildSmallVariables() + => [new ObjectFieldNode("limit", new IntValueNode(10))]; + + private static IReadOnlyList BuildLargeVariables() + => + [ + new ObjectFieldNode("first", new IntValueNode(25)), + new ObjectFieldNode("locale", new StringValueNode("en-US")), + new ObjectFieldNode( + "filter", + new ObjectValueNode( + new ObjectFieldNode("category", new EnumValueNode("ELECTRONICS")), + new ObjectFieldNode("minPrice", new IntValueNode(10)), + new ObjectFieldNode("maxPrice", new FloatValueNode(999.99)), + new ObjectFieldNode( + "tags", + new ListValueNode( + new StringValueNode("sale"), + new StringValueNode("new"), + new StringValueNode("featured"))), + new ObjectFieldNode("inStock", new BooleanValueNode(true)), + new ObjectFieldNode("cursor", NullValueNode.Default), + new ObjectFieldNode( + "nested", + new ObjectValueNode( + new ObjectFieldNode("brand", new StringValueNode("Acme \"Pro\" line")), + new ObjectFieldNode("rating", new IntValueNode(4)))))) + ]; + + private ImmutableArray BuildImportedEntries(int count) + { + var builder = ImmutableArray.CreateBuilder(count); + + for (var i = 0; i < count; i++) + { + builder.Add( + _sourceStore.CreateVariableValueSets( + Path(i), + [new ObjectFieldNode(RequirementKey, new StringValueNode($"id-{i}"))])); + } + + return builder.MoveToImmutable(); + } + + /// + /// A verification-only snapshot exercising every loop edge: an empty entry + /// (skipped), a first non-empty entry whose values lack the requirement key + /// (capture happens, then the requirement write fails and the writer rewinds), + /// a duplicate pair (dedup hit accumulating an additional path), and a unique + /// tail entry. + /// + private ImmutableArray BuildEdgeCaseSnapshot() + => + [ + default, + _sourceStore.CreateVariableValueSets( + Path(1000), + [new ObjectFieldNode("unrelated", new StringValueNode("x"))]), + _sourceStore.CreateVariableValueSets( + Path(0), + [new ObjectFieldNode(RequirementKey, new StringValueNode("dup"))]), + _sourceStore.CreateVariableValueSets( + Path(1), + [new ObjectFieldNode(RequirementKey, new StringValueNode("dup"))]), + _sourceStore.CreateVariableValueSets( + Path(2), + [new ObjectFieldNode(RequirementKey, new StringValueNode("unique"))]) + ]; + + private void VerifyEquivalence() + { + VerifySnapshotEquivalence(_importedEntries, "measured", expectedEntryCount: EntryCount); + VerifySnapshotEquivalence(BuildEdgeCaseSnapshot(), "edge", expectedEntryCount: 2); + } + + private void VerifySnapshotEquivalence( + ImmutableArray snapshot, + string workload, + int expectedEntryCount) + { + var product = _productStore.CreateVariableValueSetsFromSnapshot( + snapshot, s_importedKeys, _requestVariables, s_requirements); + var productEntries = ExtractEntries(product); + _productStore.Clean(MaxRetainedLength, MaxRetainedLength); + + var replica = BuildFromSnapshotCurrent( + _replicaWriter, _replicaJsonWriter, _replicaDedup, + snapshot, _requestVariables, s_requirements); + var replicaEntries = ExtractEntries(replica); + _replicaWriter.Clean(); + + var blit = BuildFromSnapshotPrefixBlit( + _blitWriter, _blitJsonWriter, _blitDedup, + snapshot, _requestVariables, s_requirements); + var blitEntries = ExtractEntries(blit); + _blitWriter.Clean(); + + if (productEntries.Count != expectedEntryCount) + { + throw new InvalidOperationException( + $"The {workload} workload produced {productEntries.Count} entries instead of " + + $"{expectedEntryCount}; the benchmark would measure an unintended path."); + } + + CompareEntries(productEntries, replicaEntries, workload, "per-entry replica"); + CompareEntries(productEntries, blitEntries, workload, "prefix-blit replica"); + } + + private static List ExtractEntries(ImmutableArray entries) + { + var extracted = new List(entries.Length); + + foreach (var entry in entries) + { + var additionalPaths = new int[entry.AdditionalPaths.Length][]; + var index = 0; + + foreach (var path in entry.AdditionalPaths) + { + additionalPaths[index++] = path.Segments.ToArray(); + } + + extracted.Add( + new ExtractedEntry( + entry.Path.Segments.ToArray(), + entry.Values.AsSequence().ToArray(), + additionalPaths)); + } + + return extracted; + } + + private static void CompareEntries( + List expected, + List actual, + string workload, + string variant) + { + if (expected.Count != actual.Count) + { + throw new InvalidOperationException( + $"{workload}/{variant}: expected {expected.Count} entries but got {actual.Count}."); + } + + for (var i = 0; i < expected.Count; i++) + { + var e = expected[i]; + var a = actual[i]; + + if (!e.Path.AsSpan().SequenceEqual(a.Path)) + { + throw new InvalidOperationException( + $"{workload}/{variant}: entry {i} resolved a different path."); + } + + if (!e.Values.AsSpan().SequenceEqual(a.Values)) + { + throw new InvalidOperationException( + $"{workload}/{variant}: entry {i} produced different variable bytes. Expected " + + $"'{Encoding.UTF8.GetString(e.Values)}' but got '{Encoding.UTF8.GetString(a.Values)}'."); + } + + if (e.AdditionalPaths.Length != a.AdditionalPaths.Length) + { + throw new InvalidOperationException( + $"{workload}/{variant}: entry {i} accumulated {a.AdditionalPaths.Length} " + + $"additional paths instead of {e.AdditionalPaths.Length}."); + } + + for (var j = 0; j < e.AdditionalPaths.Length; j++) + { + if (!e.AdditionalPaths[j].AsSpan().SequenceEqual(a.AdditionalPaths[j])) + { + throw new InvalidOperationException( + $"{workload}/{variant}: entry {i} additional path {j} differs."); + } + } + } + } + + private static OperationRequirement Requirement(string key) + => new( + key, + new NamedTypeNode("String"), + SelectionPath.Root, + new PathNode(new PathSegmentNode(new FusionNameNode(key))), + null); + + private static CompactPath Path(params int[] segments) + { + var buffer = new int[segments.Length + 1]; + buffer[0] = segments.Length; + segments.CopyTo(buffer.AsSpan(1)); + return new CompactPath(buffer); + } + + private sealed record ExtractedEntry(int[] Path, byte[] Values, int[][] AdditionalPaths); + + /// + /// Byte-faithful copy of the private nested FetchResultStore.VariableDedupTable + /// (FetchResultStore.cs lines 2572-2793), which is not reachable through + /// InternalsVisibleTo. Both replica variants dedup through this copy so the + /// hashing and probing costs stay identical between them. + /// + private sealed class ReplicaVariableDedupTable(ChunkedArrayWriter writer) : IDisposable + { + private const int DefaultBucketSize = 4; + private const int DefaultBucketCount = 16; + private const int TrackedSlotCapacity = 16; + + private readonly ChunkedArrayWriter _writer = writer; + private Entry[] _table = RentClearedTable(DefaultBucketCount * DefaultBucketSize); + private int[] _writtenSlots = []; + private int _bucketCount = DefaultBucketCount; + private readonly int _bucketSize = DefaultBucketSize; + private int _writtenCount; + private bool _clearFullTable; + + public void Initialize(int capacity) + { + var bucketCount = NextPowerOfTwo(Math.Max(capacity, DefaultBucketCount)); + var totalSize = bucketCount * _bucketSize; + + ClearPreviousEntries(); + + if (_table.Length < totalSize) + { + Resize(totalSize); + } + + _bucketCount = bucketCount; + } + + public bool TryGet( + int hash, + int location, + int length, + out int existingIndex) + { + var bucket = hash & 0x7FFFFFFF & (_bucketCount - 1); + var start = bucket * _bucketSize; + var end = start + _bucketSize; + + for (var s = start; s < end; s++) + { + ref var entry = ref _table[s]; + + if (entry.Index == 0) + { + existingIndex = -1; + return false; + } + + if (entry.Hash == hash + && entry.Length == length + && _writer.SequenceEqual(entry.Location, location, length)) + { + existingIndex = entry.Index - 1; + return true; + } + } + + existingIndex = -1; + return false; + } + + public void Add(int hash, int index, int location, int length) + { + var bucket = hash & 0x7FFFFFFF & (_bucketCount - 1); + var start = bucket * _bucketSize; + var end = start + _bucketSize; + + for (var s = start; s < end; s++) + { + ref var entry = ref _table[s]; + + if (entry.Index == 0) + { + RecordWrittenSlot(s); + + entry.Hash = hash; + entry.Index = index + 1; + entry.Location = location; + entry.Length = length; + return; + } + } + + Grow(); + Add(hash, index, location, length); + } + + public void Dispose() + { + if (_table.Length > 0) + { + ArrayPool.Shared.Return(_table); + _table = []; + } + + if (_writtenSlots.Length > 0) + { + ArrayPool.Shared.Return(_writtenSlots); + _writtenSlots = []; + } + + ResetClearState(); + } + + private void Grow() + { + var oldTable = _table; + var oldTotal = _bucketCount * _bucketSize; + var newBucketCount = _bucketCount * 2; + var newTotal = newBucketCount * _bucketSize; + var newTable = RentClearedTable(newTotal); + + _bucketCount = newBucketCount; + _table = newTable; + _clearFullTable = true; + _writtenCount = 0; + + try + { + for (var i = 0; i < oldTotal; i++) + { + var entry = oldTable[i]; + + if (entry.Index != 0) + { + Add(entry.Hash, entry.Index - 1, entry.Location, entry.Length); + } + } + } + finally + { + ArrayPool.Shared.Return(oldTable); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void Resize(int totalSize) + { + var newTable = RentClearedTable(totalSize); + var oldTable = _table; + _table = newTable; + ArrayPool.Shared.Return(oldTable); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void RecordWrittenSlot(int slot) + { + if (_clearFullTable) + { + return; + } + + if (_writtenCount == TrackedSlotCapacity) + { + _clearFullTable = true; + _writtenCount = 0; + return; + } + + if (_writtenSlots.Length == 0) + { + RentWrittenSlots(); + } + + _writtenSlots[_writtenCount++] = slot; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void RentWrittenSlots() + => _writtenSlots = ArrayPool.Shared.Rent(TrackedSlotCapacity); + + private void ClearPreviousEntries() + { + if (_clearFullTable) + { + _table.AsSpan(0, _bucketCount * _bucketSize).Clear(); + } + else + { + for (var i = 0; i < _writtenCount; i++) + { + _table[_writtenSlots[i]] = default; + } + } + + ResetClearState(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ResetClearState() + { + _writtenCount = 0; + _clearFullTable = false; + } + + private static Entry[] RentClearedTable(int minimumLength) + { + var table = ArrayPool.Shared.Rent(minimumLength); + table.AsSpan().Clear(); + return table; + } + + private static int NextPowerOfTwo(int n) + { + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + return n + 1; + } + + private struct Entry + { + public int Hash; + public int Index; // 1-based (0 = empty) + public int Location; + public int Length; + } + } +} diff --git a/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/PropertyNameLengthPrefilterBenchmark.cs b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/PropertyNameLengthPrefilterBenchmark.cs new file mode 100644 index 00000000000..c3c593717f3 --- /dev/null +++ b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/PropertyNameLengthPrefilterBenchmark.cs @@ -0,0 +1,753 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using HotChocolate.Buffers; +using HotChocolate.Fusion.Text.Json; + +#nullable enable + +namespace HotChocolate.Fusion.Execution.Benchmarks; + +/// +/// The payload shape a benchmark invocation scans. Each shape bounds a different +/// scenario of the length prefilter: the win case, the worst-case overhead case, +/// and the tiny-object case. +/// +public enum PropertyNameShape +{ + /// + /// 200 objects with 10 properties of varied name lengths, __typename first, + /// one escaped name and one container value. The win case: most candidates + /// are rejected by length before their name bytes are read. + /// + VariedLengths, + + /// + /// 200 objects whose 10 property names are all exactly 10 bytes, the length + /// of __typename. Every candidate passes the prefilter, so this bounds the + /// overhead at one integer compare per candidate on top of today's work. + /// + UniformLength, + + /// + /// 200 objects with a single short property. Bounds the tiny-object case: + /// one added compare on a hit, one saved name read on a miss. + /// + SingleProperty +} + +/// +/// Measures the backward property-name scan of +/// SourceResultDocument.TryGetNamedPropertyValueCore +/// (SourceResultDocument.TryGetProperty.cs lines 127-210). Today the scan decodes +/// each candidate's name row and then unconditionally calls ReadRawValue +/// (SourceResultDocument.cs lines 384-396 and 402-439), a packed-location decode, +/// segment-table load, bounds branch, span creation and [1..^1] quote slice, +/// before the SequenceEqual at line 197 can reject the candidate on length. +/// +/// Names are stored quote-inclusive (AppendStringToken, +/// SourceResultDocument.Parse.cs lines 518-534), so for unescaped names +/// (!row.HasComplexChildren) the row's SizeOrLength equals the +/// unquoted name length plus 2. The optimized variant compares +/// row.SizeOrLength against propertyName.Length + 2 on the already +/// loaded row and reads the name bytes only when the lengths agree, which rejects +/// exactly the candidates SequenceEqual would reject on length. Escaped +/// names keep today's unescape path byte for byte. +/// +/// Hot callers of this scan: __typename resolution once per abstract-typed +/// element (ValueCompletion.cs line 1291), where the planner injects __typename +/// as the first selection (OperationPlanner.cs lines 3878-3883) so the backward +/// scan visits it last, source path navigation per merged result +/// (FetchResultStore.cs lines 2060, 2116 and 2140-2146), and @interfaceObject +/// upgrade probes (ValueCompletion.cs line 258). +/// +/// The product baseline calls the real internal +/// TryGetNamedPropertyValue(Cursor, ReadOnlySpan<byte>, out ...) +/// (SourceResultDocument.TryGetProperty.cs lines 99-125). The core copy is a +/// byte-faithful benchmark-local replica of the same entry and scan; its delta +/// against the product baseline shows replica fidelity, and its delta against +/// the prefilter variant isolates the candidate. Each invocation runs the +/// shape's hit and miss lookups against every object element. +/// +[MemoryDiagnoser] +[Config(typeof(BenchmarkConfig))] +public class PropertyNameLengthPrefilterBenchmark +{ + /// + /// BenchmarkDotNet 0.15.8 has no RuntimeMoniker for the net11.0 preview host and + /// this project pins TargetFramework to net11.0, so out-of-process toolchains can + /// neither validate nor build a child process here. The job therefore runs in + /// process with the intended 3 warmup and 10 measurement iterations. + /// + private sealed class BenchmarkConfig : ManualConfig + { + public BenchmarkConfig() + => AddJob( + Job.Default + .WithWarmupCount(3) + .WithIterationCount(10) + .WithToolchain(InProcessEmitToolchain.Instance)); + } + + private const int ItemCount = 200; + + // Local copies of HotChocolate.Text.Json JsonConstants.StackallocByteThreshold + // (JsonConstants.cs line 5) and JsonConstants.BackSlash (line 22). The type is + // internal to HotChocolate.Text.Json, which grants InternalsVisibleTo to + // HotChocolate.Fusion.Execution but not to this benchmarks assembly. + private const int StackallocByteThreshold = 256; + private const byte BackSlash = (byte)'\\'; + + private sealed class Workload + { + public SourceResultDocument Document = null!; + public SourceResultDocument.Cursor[] ObjectCursors = null!; + public byte[][] LookupNames = null!; + public bool[] ExpectedFound = null!; + } + + private MemoryArena _arena = null!; + private Dictionary _workloads = null!; + private SourceResultDocument _document = null!; + private SourceResultDocument.Cursor[] _objectCursors = null!; + private byte[][] _lookupNames = null!; + + public long Consumed; + + [Params( + PropertyNameShape.VariedLengths, + PropertyNameShape.UniformLength, + PropertyNameShape.SingleProperty)] + public PropertyNameShape Shape { get; set; } + + [GlobalSetup] + public void Setup() + { + _arena = new MemoryArena(); + _workloads = new Dictionary(); + + foreach (var shape in new[] + { + PropertyNameShape.VariedLengths, + PropertyNameShape.UniformLength, + PropertyNameShape.SingleProperty + }) + { + _workloads[shape] = CreateWorkload(shape); + } + + // The equivalence check runs over every shape, not only the measured one. + VerifyEquivalence(); + + var current = _workloads[Shape]; + _document = current.Document; + _objectCursors = current.ObjectCursors; + _lookupNames = current.LookupNames; + + VerifyChecksums(); + } + + [GlobalCleanup] + public void Cleanup() + { + foreach (var workload in _workloads.Values) + { + workload.Document.Dispose(); + } + + _arena.Dispose(); + } + + /// + /// Current product behavior: the real internal + /// SourceResultDocument.TryGetNamedPropertyValue byte overload, which + /// runs the unmodified private core scan. This entry also pays the + /// ObjectDisposedException guard the replicas cannot reach (it reads the + /// private _disposed field); the guard is one branch per lookup. + /// + [Benchmark(Baseline = true)] + public long TryGetProperty_Product() + { + var document = _document; + var cursors = _objectCursors; + var lookups = _lookupNames; + var sum = 0L; + + for (var i = 0; i < cursors.Length; i++) + { + for (var n = 0; n < lookups.Length; n++) + { + if (document.TryGetNamedPropertyValue(cursors[i], lookups[n], out var value)) + { + sum += value._cursor.Value + 1; + } + } + } + + Consumed = sum; + return sum; + } + + /// + /// Byte-faithful benchmark-local copy of today's entry and core scan. Its delta + /// against the product baseline shows the replica is equivalent; its delta + /// against the prefilter variant isolates the candidate on identical code shape. + /// + [Benchmark] + public long TryGetProperty_CoreCopy() + { + var document = _document; + var cursors = _objectCursors; + var lookups = _lookupNames; + var sum = 0L; + + for (var i = 0; i < cursors.Length; i++) + { + for (var n = 0; n < lookups.Length; n++) + { + if (TryGetPropertyCoreCopy(document, cursors[i], lookups[n], out var value)) + { + sum += value._cursor.Value + 1; + } + } + } + + Consumed = sum; + return sum; + } + + /// + /// Candidate optimization: the same scan with the quoted-length prefilter. For + /// unescaped candidates the name bytes are read only when + /// row.SizeOrLength == propertyName.Length + 2; escaped candidates keep + /// today's path unchanged. + /// + [Benchmark] + public long TryGetProperty_LengthPrefilter() + { + var document = _document; + var cursors = _objectCursors; + var lookups = _lookupNames; + var sum = 0L; + + for (var i = 0; i < cursors.Length; i++) + { + for (var n = 0; n < lookups.Length; n++) + { + if (TryGetPropertyLengthPrefilter(document, cursors[i], lookups[n], out var value)) + { + sum += value._cursor.Value + 1; + } + } + } + + Consumed = sum; + return sum; + } + + // Byte-faithful copy of the internal byte overload of + // SourceResultDocument.TryGetNamedPropertyValue + // (SourceResultDocument.TryGetProperty.cs lines 99-125). The + // ObjectDisposedException guard reads the private _disposed field and is + // omitted; the documents here live for the whole run. CheckExpectedType is + // private and never fires in this workload (every scanned cursor is a + // StartObject row), so it is mirrored as a plain guard. + private static bool TryGetPropertyCoreCopy( + SourceResultDocument document, + SourceResultDocument.Cursor objectCursor, + ReadOnlySpan propertyName, + out SourceResultElement value) + { + var row = document.GetDbRow(objectCursor); + + if (row.TokenType != JsonTokenType.StartObject) + { + throw new InvalidOperationException("The element is not an object."); + } + + // Only one row means it was EndObject. + if (row.NumberOfRows == 1) + { + value = default; + return false; + } + + return TryGetNamedPropertyValueCoreCopy( + document, + objectCursor + 1, + objectCursor + (row.NumberOfRows - 1), + propertyName, + out value); + } + + // Byte-faithful copy of SourceResultDocument.TryGetNamedPropertyValueCore + // (SourceResultDocument.TryGetProperty.cs lines 127-210). GetDbRow is the + // internal accessor for the same _parsedData.Get the product calls + // (SourceResultDocument.Text.cs line 331). The Debug.Asserts of the product + // are conditional on DEBUG and compile away in the release builds benchmarks + // run under, so they are omitted here. + private static bool TryGetNamedPropertyValueCoreCopy( + SourceResultDocument document, + SourceResultDocument.Cursor startCursor, + SourceResultDocument.Cursor endCursor, + ReadOnlySpan propertyName, + out SourceResultElement value) + { + Span utf8UnescapedStack = stackalloc byte[StackallocByteThreshold]; + + // Move to the row before the EndObject + var cursor = endCursor - 1; + + while (cursor > startCursor) + { + var row = document.GetDbRow(cursor); + + // Move before the value + cursor -= row.IsSimpleValue ? 1 : row.NumberOfRows; + + row = document.GetDbRow(cursor); + + var currentPropertyName = ReadUnquotedRawValue(document, row); + + if (row.HasComplexChildren) + { + // An escaped property name will be longer than an unescaped candidate, + // so only unescape when the lengths are compatible. + if (currentPropertyName.Length > propertyName.Length) + { + var idx = currentPropertyName.IndexOf(BackSlash); + + // If everything up to where the property name has a backslash matches, keep going. + if (propertyName.Length > idx + && currentPropertyName[..idx].SequenceEqual(propertyName[..idx])) + { + var remaining = currentPropertyName.Length - idx; + var written = 0; + byte[]? rented = null; + + try + { + var utf8Unescaped = + remaining <= utf8UnescapedStack.Length + ? utf8UnescapedStack + : (rented = ArrayPool.Shared.Rent(remaining)); + + // Only unescape the part we haven't processed. + JsonReaderHelper.Unescape( + currentPropertyName[idx..], utf8Unescaped, 0, out written); + + // If the unescaped remainder matches the input remainder, it's a match. + if (utf8Unescaped[..written].SequenceEqual(propertyName[idx..])) + { + value = new SourceResultElement(document, cursor + 1); + return true; + } + } + finally + { + if (rented is not null) + { + rented.AsSpan(0, written).Clear(); + ArrayPool.Shared.Return(rented); + } + } + } + } + } + else if (currentPropertyName.SequenceEqual(propertyName)) + { + // If the property name is a match, the answer is the next element. + value = new SourceResultElement(document, cursor + 1); + return true; + } + + // Move to the previous value (name row is at 'id', previous value ends at id - 1) + cursor -= 1; + } + + value = default; + return false; + } + + // Optimized entry: identical to TryGetPropertyCoreCopy but dispatching into the + // prefilter core. + private static bool TryGetPropertyLengthPrefilter( + SourceResultDocument document, + SourceResultDocument.Cursor objectCursor, + ReadOnlySpan propertyName, + out SourceResultElement value) + { + var row = document.GetDbRow(objectCursor); + + if (row.TokenType != JsonTokenType.StartObject) + { + throw new InvalidOperationException("The element is not an object."); + } + + if (row.NumberOfRows == 1) + { + value = default; + return false; + } + + return TryGetNamedPropertyValueCorePrefilter( + document, + objectCursor + 1, + objectCursor + (row.NumberOfRows - 1), + propertyName, + out value); + } + + // The recommended design: the loop body branches on HasComplexChildren first, + // the escaped path is unchanged, and the unescaped path reads the name bytes + // only when row.SizeOrLength equals the sought name's quoted length. Names are + // stored quote-inclusive, so for unescaped names SizeOrLength - 2 is the exact + // unquoted length and the compare rejects exactly what SequenceEqual would + // reject on length. Everything else is byte-identical to the core copy above. + private static bool TryGetNamedPropertyValueCorePrefilter( + SourceResultDocument document, + SourceResultDocument.Cursor startCursor, + SourceResultDocument.Cursor endCursor, + ReadOnlySpan propertyName, + out SourceResultElement value) + { + Span utf8UnescapedStack = stackalloc byte[StackallocByteThreshold]; + + var targetQuotedLength = propertyName.Length + 2; + + // Move to the row before the EndObject + var cursor = endCursor - 1; + + while (cursor > startCursor) + { + var row = document.GetDbRow(cursor); + + // Move before the value + cursor -= row.IsSimpleValue ? 1 : row.NumberOfRows; + + row = document.GetDbRow(cursor); + + if (row.HasComplexChildren) + { + var currentPropertyName = ReadUnquotedRawValue(document, row); + + // An escaped property name will be longer than an unescaped candidate, + // so only unescape when the lengths are compatible. + if (currentPropertyName.Length > propertyName.Length) + { + var idx = currentPropertyName.IndexOf(BackSlash); + + // If everything up to where the property name has a backslash matches, keep going. + if (propertyName.Length > idx + && currentPropertyName[..idx].SequenceEqual(propertyName[..idx])) + { + var remaining = currentPropertyName.Length - idx; + var written = 0; + byte[]? rented = null; + + try + { + var utf8Unescaped = + remaining <= utf8UnescapedStack.Length + ? utf8UnescapedStack + : (rented = ArrayPool.Shared.Rent(remaining)); + + // Only unescape the part we haven't processed. + JsonReaderHelper.Unescape( + currentPropertyName[idx..], utf8Unescaped, 0, out written); + + // If the unescaped remainder matches the input remainder, it's a match. + if (utf8Unescaped[..written].SequenceEqual(propertyName[idx..])) + { + value = new SourceResultElement(document, cursor + 1); + return true; + } + } + finally + { + if (rented is not null) + { + rented.AsSpan(0, written).Clear(); + ArrayPool.Shared.Return(rented); + } + } + } + } + } + else if (row.SizeOrLength == targetQuotedLength) + { + var currentPropertyName = ReadUnquotedRawValue(document, row); + + if (currentPropertyName.SequenceEqual(propertyName)) + { + // If the property name is a match, the answer is the next element. + value = new SourceResultElement(document, cursor + 1); + return true; + } + } + + // Move to the previous value (name row is at 'id', previous value ends at id - 1) + cursor -= 1; + } + + value = default; + return false; + } + + // Benchmark-local copy of the private SourceResultDocument.ReadRawValue(DbRow, bool) + // with includeQuotes: false (SourceResultDocument.cs lines 383-396, including its + // AggressiveInlining). Property names are stored quote-inclusive, so the unquoted + // name slices one byte in on each side; the chunk read is the internal + // ReadRawValue(int, int) the private overload delegates to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlySpan ReadUnquotedRawValue( + SourceResultDocument document, + in SourceResultDocument.DbRow row) + { + if (row.TokenType is JsonTokenType.String or JsonTokenType.PropertyName) + { + return document.ReadRawValue(row.Location, row.SizeOrLength)[1..^1]; + } + + return document.ReadRawValue(row.Location, row.SizeOrLength); + } + + private Workload CreateWorkload(PropertyNameShape shape) + { + var payload = BuildPayload(shape); + var document = SourceResultDocument.Parse(_arena, payload, payload.Length); + + var products = document.Root + .GetProperty("data"u8) + .GetProperty("products"u8); + + var cursors = new SourceResultDocument.Cursor[products.GetArrayLength()]; + var i = 0; + + foreach (var element in products.EnumerateArray()) + { + cursors[i++] = element._cursor; + } + + if (i != ItemCount) + { + throw new InvalidOperationException( + $"Shape {shape}: expected {ItemCount} objects but materialized {i}."); + } + + byte[][] lookupNames; + bool[] expectedFound; + + switch (shape) + { + case PropertyNameShape.VariedLengths: + // __typename is physically first, so the backward scan visits it last + // (full scan hit); sku is physically last (first visited hit); escaped + // resolves through the unescape branch; doesNotExist misses every + // candidate. + lookupNames = + [ + "__typename"u8.ToArray(), + "sku"u8.ToArray(), + "escaped"u8.ToArray(), + "doesNotExist"u8.ToArray() + ]; + expectedFound = [true, true, true, false]; + break; + + case PropertyNameShape.UniformLength: + // Every candidate name is exactly 10 bytes, so both probes pass the + // prefilter on every candidate: the hit still scans all candidates and + // the 10-byte miss compares full names on every candidate. + lookupNames = + [ + "__typename"u8.ToArray(), + "notpresent"u8.ToArray() + ]; + expectedFound = [true, false]; + break; + + case PropertyNameShape.SingleProperty: + lookupNames = + [ + "id"u8.ToArray(), + "nope"u8.ToArray() + ]; + expectedFound = [true, false]; + break; + + default: + throw new InvalidOperationException($"Unknown shape {shape}."); + } + + return new Workload + { + Document = document, + ObjectCursors = cursors, + LookupNames = lookupNames, + ExpectedFound = expectedFound + }; + } + + private void VerifyEquivalence() + { + foreach (var (shape, workload) in _workloads) + { + var document = workload.Document; + + for (var i = 0; i < workload.ObjectCursors.Length; i++) + { + var cursor = workload.ObjectCursors[i]; + var element = new SourceResultElement(document, cursor); + + // Every present property must be found identically by all three + // variants; property.Name is the unescaped name, so this also + // exercises the escaped-name branch of both replicas. + foreach (var property in element.EnumerateObject()) + { + AssertAllVariantsAgree( + document, + cursor, + Encoding.UTF8.GetBytes(property.Name), + expectedFound: true, + shape, + i); + } + + for (var n = 0; n < workload.LookupNames.Length; n++) + { + AssertAllVariantsAgree( + document, + cursor, + workload.LookupNames[n], + workload.ExpectedFound[n], + shape, + i); + } + } + } + } + + private static void AssertAllVariantsAgree( + SourceResultDocument document, + SourceResultDocument.Cursor objectCursor, + ReadOnlySpan propertyName, + bool expectedFound, + PropertyNameShape shape, + int elementIndex) + { + var name = Encoding.UTF8.GetString(propertyName); + + var productFound = document.TryGetNamedPropertyValue( + objectCursor, propertyName, out var productValue); + var copyFound = TryGetPropertyCoreCopy( + document, objectCursor, propertyName, out var copyValue); + var prefilterFound = TryGetPropertyLengthPrefilter( + document, objectCursor, propertyName, out var prefilterValue); + + if (productFound != expectedFound + || copyFound != expectedFound + || prefilterFound != expectedFound) + { + throw new InvalidOperationException( + $"Shape {shape}, element {elementIndex}, name '{name}': expected " + + $"found={expectedFound} but product={productFound}, " + + $"copy={copyFound}, prefilter={prefilterFound}."); + } + + if (expectedFound + && (!ReferenceEquals(productValue._parent, copyValue._parent) + || !ReferenceEquals(productValue._parent, prefilterValue._parent) + || productValue._cursor != copyValue._cursor + || productValue._cursor != prefilterValue._cursor)) + { + throw new InvalidOperationException( + $"Shape {shape}, element {elementIndex}, name '{name}': the variants " + + $"resolved different elements (product {productValue._cursor}, " + + $"copy {copyValue._cursor}, prefilter {prefilterValue._cursor})."); + } + } + + private void VerifyChecksums() + { + var productSum = TryGetProperty_Product(); + var copySum = TryGetProperty_CoreCopy(); + var prefilterSum = TryGetProperty_LengthPrefilter(); + + if (productSum != copySum || productSum != prefilterSum) + { + throw new InvalidOperationException( + $"Checksum mismatch for shape {Shape}: product {productSum}, " + + $"copy {copySum}, prefilter {prefilterSum}."); + } + } + + private static byte[] BuildPayload(PropertyNameShape shape) + { + // Every payload stays far below the 128 KiB single-chunk limit, so all name + // reads take the single-chunk fast path of ReadRawValue and the cross-chunk + // copy path cannot skew the comparison. + var json = new StringBuilder(64 * 1024); + json.Append("{\"data\":{\"products\":["); + + for (var i = 0; i < ItemCount; i++) + { + if (i > 0) + { + json.Append(','); + } + + switch (shape) + { + case PropertyNameShape.VariedLengths: + // Name byte lengths: 10, 2, 4, 5, 7, 6, 4, 11, 12 raw (7 unescaped), 3. + // Only __typename is 10 bytes, so a __typename probe length-rejects + // every other unescaped candidate. "tags" holds a container value so + // the scan's NumberOfRows skip runs; "escaped" parses with + // ValueIsEscaped and unescapes to "escaped". + json.Append("{\"__typename\":\"Product\""); + json.Append(",\"id\":\"prod-").Append(i).Append('"'); + json.Append(",\"name\":\"Product ").Append(i).Append('"'); + json.Append(",\"price\":").Append(i).Append(".99"); + json.Append(",\"inStock\":").Append(i % 3 == 0 ? "false" : "true"); + json.Append(",\"rating\":4.5"); + json.Append(",\"tags\":[\"red\",\"new\"]"); + json.Append(",\"description\":\"Plain description for item ").Append(i).Append('"'); + json.Append(",\"esc\\u0061ped\":\"value-").Append(i).Append('"'); + json.Append(",\"sku\":\"SKU-").Append(i).Append("\"}"); + break; + + case PropertyNameShape.UniformLength: + // All ten names are exactly 10 bytes, matching the __typename probe. + json.Append("{\"__typename\":\"Product\""); + json.Append(",\"identifier\":\"prod-").Append(i).Append('"'); + json.Append(",\"namefield0\":\"Product ").Append(i).Append('"'); + json.Append(",\"pricefield\":").Append(i).Append(".99"); + json.Append(",\"inventory0\":").Append(i % 3 == 0 ? "false" : "true"); + json.Append(",\"quantity00\":").Append(i % 50); + json.Append(",\"ratingval0\":4.5"); + json.Append(",\"descriptn0\":\"Plain description ").Append(i).Append('"'); + json.Append(",\"extrafeld0\":\"extra-").Append(i).Append('"'); + json.Append(",\"skunumber0\":\"SKU-").Append(i).Append("\"}"); + break; + + case PropertyNameShape.SingleProperty: + json.Append("{\"id\":\"prod-").Append(i).Append("\"}"); + break; + + default: + throw new InvalidOperationException($"Unknown shape {shape}."); + } + } + + json.Append("]}}"); + return Encoding.UTF8.GetBytes(json.ToString()); + } +} diff --git a/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SelectionChildMapBenchmark.cs b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SelectionChildMapBenchmark.cs new file mode 100644 index 00000000000..8a761ac6b95 --- /dev/null +++ b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SelectionChildMapBenchmark.cs @@ -0,0 +1,798 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using HotChocolate.Fusion.Execution.Nodes; +using HotChocolate.Fusion.Execution.Rewriters; +using HotChocolate.Fusion.Planning; +using HotChocolate.Language; +using HotChocolate.Types; +using Microsoft.Extensions.ObjectPool; +using SelectionSet = HotChocolate.Fusion.Execution.Nodes.SelectionSet; + +#nullable enable + +namespace HotChocolate.Fusion.Execution.Benchmarks; + +/// +/// Measures the per-property child selection set resolution of the merge completion +/// loops. Today ValueCompletion resolves the child +/// by response name for every property that misses the scalar fast path: +/// BuildResult calls the type-unaware ResultSelectionSet.TryGetChild +/// at ValueCompletion.cs lines 147 and 198 (once per property per merged row) and +/// TryCompleteObjectValue calls the type-aware overload at lines 1183 and +/// 1227 (once per property per object element). TryGetChild (ResultSelectionSet.cs +/// lines 83-121 and 129-176) does up to 7 ordinal string compares, or a dictionary +/// probe at 8 or more selections, and on a direct miss recurses through the inline +/// fragment bodies. For a batch of N rows with k such properties the identical +/// (response name to child) answers are recomputed N times k times per merge, and +/// leaf selections always resolve to null yet pay the full lookup. +/// +/// The optimized variant is the reconciled two-stage design. Stage 1 is a leaf +/// gate on the cached Selection.IsLeaf flag (Selection.cs line 85): leaf +/// named types never have a child set, so the lookup is skipped. Stage 2 is a +/// lazily built copy-on-write memo mapping a to an +/// array of child references indexed by +/// selection.Id - selectionSet.Id - 1, the dense id invariant that +/// CompositeObjectContext.cs line 48 already relies on. Memo entries are built by +/// calling the existing TryGetChild once per selection, so lookup semantics are +/// identical by construction, and the type-aware map is only used when the runtime +/// object type is reference-equal to the selection set's own type (the guard that +/// excludes the @interfaceObject divergence case). In the product the memo would +/// live on the plan-lifetime ResultSelectionSet itself; the benchmark keeps it in +/// a local wrapper because product code stays unchanged. +/// +/// Shapes: Small (5 direct fields, linear scan) and Large (9 direct fields, +/// dictionary) exercise the type-unaware BuildResult sites; Fragment (3 direct +/// fields plus 2 type-conditioned inline fragments) exercises the type-aware +/// TryCompleteObjectValue site, where fragment-nested fields miss the direct scan +/// and pay the recursive fragment walk today. Rows models the merged rows per +/// batch (1 is the single-object regression shape). Every property is modeled as +/// missing the scalar fast path, matching null-heavy or error-carrying rows; fast +/// path properties never reach this lookup in either variant. The cold variant +/// rebuilds the memo per invocation to expose the first-use build cost. +/// +[MemoryDiagnoser] +[Config(typeof(BenchmarkConfig))] +public class SelectionChildMapBenchmark : FusionBenchmarkBase +{ + /// + /// BenchmarkDotNet 0.15.8 has no RuntimeMoniker for the net11.0 preview host and + /// this project pins TargetFramework to net11.0, so out-of-process toolchains can + /// neither validate nor build a child process here. The job therefore runs in + /// process with the intended 3 warmup and 10 measurement iterations. + /// + private sealed class BenchmarkConfig : ManualConfig + { + public BenchmarkConfig() + => AddJob( + Job.Default + .WithWarmupCount(3) + .WithIterationCount(10) + .WithToolchain(InProcessEmitToolchain.Instance)); + } + + private const string OperationId = "123456789101112"; + + // The composite operation. Its compiled SelectionSets provide the real + // Selection/SelectionSet ids that feed the indexed map math. + private const string QuerySource = + """ + { + productById(id: "1") { + id + name + description + price + dimension { height width } + } + products(first: 10) { + nodes { + id + name + description + price + averageRating + title + estimatedDelivery + dimension { height width } + reviews { nodes { id body stars } } + } + } + searchContent(query: "benchmark") { + id + title + description + ... on Product { + name + price + dimension { height width } + } + ... on Article { + content + publishedAt + author { id displayName } + } + } + } + """; + + // 5 direct fields, no fragments: a linear-scan ResultSelectionSet, matching + // the productById selection set of the operation. Type-unaware BuildResult shape. + private const string SmallSource = + """ + { + id + name + description + price + dimension { height width } + } + """; + + // 9 direct fields, no fragments: a dictionary ResultSelectionSet, matching the + // products.nodes selection set of the operation. Type-unaware BuildResult shape. + private const string LargeSource = + """ + { + id + name + description + price + averageRating + title + estimatedDelivery + dimension { height width } + reviews { nodes { id body stars } } + } + """; + + // 3 direct fields plus 2 type-conditioned inline fragments, matching the + // searchContent selection set of the operation compiled for the Product type + // context. Type-aware TryCompleteObjectValue shape: name, price and dimension + // miss the direct scan and resolve through the fragment walk today. + private const string FragmentSource = + """ + { + id + title + description + ... on Product { + name + price + dimension { height width } + } + ... on Article { + content + publishedAt + author { id displayName } + } + } + """; + + [Params("Small", "Large", "Fragment")] + public string Shape = "Small"; + + [Params(1, 100)] + public int Rows; + + private Workload[] _workloads = null!; + private Workload _current = null!; + + public int Consumed; + + [GlobalSetup] + public void Setup() + { + var schema = CreateFusionSchema(); + + if (!schema.Types.TryGetType("Product", out var productType)) + { + throw new InvalidOperationException("Product type not found in the composed schema."); + } + + var documentRewriter = new DocumentRewriter(schema); + var operationDefinition = documentRewriter + .RewriteDocument(Utf8GraphQLParser.Parse(QuerySource)) + .GetOperation(operationName: null); + var compiler = new OperationCompiler( + schema, + new NoOpObjectPool>>()); + var operation = compiler.Compile(OperationId, OperationId, operationDefinition); + + var root = operation.RootSelectionSet; + var smallSet = GetChildSet(root, "productById"); + var largeSet = GetChildSet(GetChildSet(root, "products"), "nodes"); + + if (!root.TryGetSelection("searchContent", out var searchSelection)) + { + throw new InvalidOperationException("searchContent selection not found."); + } + + var fragmentSet = searchSelection.GetSelectionSet(productType) + ?? throw new InvalidOperationException( + "searchContent has no selection set for the Product type context."); + + // The type-aware map is keyed by the selection set and built for its own + // type context, so the workload's runtime type must be the exact instance + // the set was compiled for; otherwise the optimized variant would silently + // measure the fallback path. + if (!ReferenceEquals(fragmentSet.Type, productType)) + { + throw new InvalidOperationException( + "The Product-context selection set does not reference the schema's " + + "Product type instance."); + } + + // Guard the fragment workload against rewriter changes silently dropping + // the fragment-nested selections that make this shape expensive today. + RequireSelection(fragmentSet, "id"); + RequireSelection(fragmentSet, "title"); + RequireSelection(fragmentSet, "description"); + RequireSelection(fragmentSet, "name"); + RequireSelection(fragmentSet, "price"); + RequireSelection(fragmentSet, "dimension"); + + var smallResultSet = ResultSelectionSet.Create( + Utf8GraphQLParser.Syntax.ParseSelectionSet(SmallSource), + schema); + var largeResultSet = ResultSelectionSet.Create( + Utf8GraphQLParser.Syntax.ParseSelectionSet(LargeSource), + schema); + var fragmentResultSet = ResultSelectionSet.Create( + Utf8GraphQLParser.Syntax.ParseSelectionSet(FragmentSource), + schema); + + // The candidate claim is per lookup strategy, so the shapes must keep + // mapping to the expected strategies. + if (smallResultSet.UsesDictionaryLookup) + { + throw new InvalidOperationException("The 5-field shape no longer uses a linear scan."); + } + + if (!largeResultSet.UsesDictionaryLookup) + { + throw new InvalidOperationException( + "The 9-field shape no longer uses a dictionary lookup."); + } + + if (fragmentResultSet.UsesDictionaryLookup) + { + throw new InvalidOperationException( + "The fragment shape no longer uses a linear scan for its direct fields."); + } + + _workloads = + [ + new Workload + { + Name = "Small", + ResultSet = smallResultSet, + SelectionSet = smallSet, + ObjectType = null, + WarmCache = new ChildMapCache(smallResultSet), + ExpectedHitsPerRow = 1 + }, + new Workload + { + Name = "Large", + ResultSet = largeResultSet, + SelectionSet = largeSet, + ObjectType = null, + WarmCache = new ChildMapCache(largeResultSet), + ExpectedHitsPerRow = 2 + }, + new Workload + { + Name = "Fragment", + ResultSet = fragmentResultSet, + SelectionSet = fragmentSet, + ObjectType = productType, + WarmCache = new ChildMapCache(fragmentResultSet), + ExpectedHitsPerRow = 1 + } + ]; + + VerifyEquivalence(_workloads); + + _current = Shape switch + { + "Small" => _workloads[0], + "Large" => _workloads[1], + "Fragment" => _workloads[2], + _ => throw new InvalidOperationException($"Unknown shape '{Shape}'.") + }; + } + + /// + /// Current product behavior: one TryGetChild per property per row, exactly like + /// ValueCompletion.cs lines 198 (type-unaware, Small/Large shapes) and 1227 + /// (type-aware, Fragment shape), including the lookups for leaf selections + /// whose answer is always null. + /// + [Benchmark(Baseline = true)] + public int PerPropertyLookup_TryGetChild() + { + var workload = _current; + var rows = Rows; + var hits = 0; + + if (workload.ObjectType is { } objectType) + { + for (var row = 0; row < rows; row++) + { + hits += BaselineRowTypeAware( + workload.ResultSet, + workload.SelectionSet, + objectType); + } + } + else + { + for (var row = 0; row < rows; row++) + { + hits += BaselineRowTypeUnaware(workload.ResultSet, workload.SelectionSet); + } + } + + Consumed = hits; + return hits; + } + + /// + /// Candidate optimization in steady state: the leaf gate skips leaf selections + /// entirely and the remaining properties read a prebuilt child array indexed by + /// selection id, fetched once per row from the plan-lifetime memo. + /// + [Benchmark] + public int IndexedChildMap_Warm() + { + var workload = _current; + var rows = Rows; + var hits = 0; + + if (workload.ObjectType is { } objectType) + { + for (var row = 0; row < rows; row++) + { + hits += OptimizedRowTypeAware( + workload.WarmCache, + workload.ResultSet, + workload.SelectionSet, + objectType); + } + } + else + { + for (var row = 0; row < rows; row++) + { + hits += OptimizedRowTypeUnaware(workload.WarmCache, workload.SelectionSet); + } + } + + Consumed = hits; + return hits; + } + + /// + /// First-use regression shape: a fresh memo per invocation, so the map build + /// (one TryGetChild per selection plus the entry allocations) is paid inside + /// the measurement. At Rows = 1 this is the worst case for the candidate; at + /// Rows = 100 it shows the build amortizing within a single batch merge. + /// + [Benchmark] + public int IndexedChildMap_ColdFirstUse() + { + var workload = _current; + var cache = new ChildMapCache(workload.ResultSet); + var rows = Rows; + var hits = 0; + + if (workload.ObjectType is { } objectType) + { + for (var row = 0; row < rows; row++) + { + hits += OptimizedRowTypeAware( + cache, + workload.ResultSet, + workload.SelectionSet, + objectType); + } + } + else + { + for (var row = 0; row < rows; row++) + { + hits += OptimizedRowTypeUnaware(cache, workload.SelectionSet); + } + } + + Consumed = hits; + return hits; + } + + /// + /// Mirrors ValueCompletion.cs line 198: the type-unaware BuildResult site + /// resolves the child set for every property that misses the scalar fast path. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int BaselineRowTypeUnaware( + ResultSelectionSet resultSet, + SelectionSet selectionSet) + { + var selections = selectionSet.Selections; + var hits = 0; + + for (var i = 0; i < selections.Length; i++) + { + var childSet = resultSet.TryGetChild(selections[i].ResponseName); + + if (childSet is not null) + { + hits++; + } + } + + return hits; + } + + /// + /// Mirrors ValueCompletion.cs line 1227: the type-aware TryCompleteObjectValue + /// site resolves the child set with the runtime object type per property. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int BaselineRowTypeAware( + ResultSelectionSet resultSet, + SelectionSet selectionSet, + IComplexTypeDefinition objectType) + { + var selections = selectionSet.Selections; + var hits = 0; + + for (var i = 0; i < selections.Length; i++) + { + var childSet = resultSet.TryGetChild(selections[i].ResponseName, objectType); + + if (childSet is not null) + { + hits++; + } + } + + return hits; + } + + /// + /// Optimized type-unaware row: leaf gate plus the selection-id-indexed child + /// array, resolved lazily on the first property that needs it so all-leaf rows + /// never pay for the map fetch. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int OptimizedRowTypeUnaware( + ChildMapCache cache, + SelectionSet selectionSet) + { + var selections = selectionSet.Selections; + ResultSelectionSet?[]? children = null; + var hits = 0; + + for (var i = 0; i < selections.Length; i++) + { + var selection = selections[i]; + + // Stage 1 leaf gate: a leaf-named-type selection always maps to a + // null child set. + if (selection.IsLeaf) + { + continue; + } + + children ??= cache.GetTypeUnawareChildren(selectionSet); + + var childSet = children[selection.Id - selectionSet.Id - 1]; + + if (childSet is not null) + { + hits++; + } + } + + return hits; + } + + /// + /// Optimized type-aware row: same as the type-unaware variant plus the + /// divergence guard. The memo entry is built for the selection set's own type + /// context, so a diverging runtime type (the @interfaceObject opaque case) + /// falls back to the per-property lookup. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int OptimizedRowTypeAware( + ChildMapCache cache, + ResultSelectionSet resultSet, + SelectionSet selectionSet, + IComplexTypeDefinition objectType) + { + var selections = selectionSet.Selections; + ResultSelectionSet?[]? children = null; + var mapResolved = false; + var hits = 0; + + for (var i = 0; i < selections.Length; i++) + { + var selection = selections[i]; + + if (selection.IsLeaf) + { + continue; + } + + if (!mapResolved) + { + if (ReferenceEquals(selectionSet.Type, objectType)) + { + children = cache.GetTypeAwareChildren(selectionSet); + } + + mapResolved = true; + } + + var childSet = children is not null + ? children[selection.Id - selectionSet.Id - 1] + : resultSet.TryGetChild(selection.ResponseName, objectType); + + if (childSet is not null) + { + hits++; + } + } + + return hits; + } + + private void VerifyEquivalence(Workload[] workloads) + { + foreach (var workload in workloads) + { + var set = workload.SelectionSet; + var selections = set.Selections; + + // The indexed map relies on the dense id invariant that + // CompositeObjectContext.cs line 48 already uses in production. + for (var i = 0; i < selections.Length; i++) + { + if (selections[i].Id != set.Id + 1 + i) + { + throw new InvalidOperationException( + $"Shape {workload.Name}: selection id {selections[i].Id} at index {i} " + + $"breaks the dense id invariant for selection set {set.Id}."); + } + } + + // Element-wise reference equality between today's TryGetChild answer + // and the leaf gate plus indexed map answer, for every selection. + for (var i = 0; i < selections.Length; i++) + { + var selection = selections[i]; + + var baseline = workload.ObjectType is null + ? workload.ResultSet.TryGetChild(selection.ResponseName) + : workload.ResultSet.TryGetChild(selection.ResponseName, workload.ObjectType); + + ResultSelectionSet? optimized; + + if (selection.IsLeaf) + { + optimized = null; + } + else + { + var children = workload.ObjectType is null + ? workload.WarmCache.GetTypeUnawareChildren(set) + : workload.WarmCache.GetTypeAwareChildren(set); + optimized = children[selection.Id - set.Id - 1]; + } + + if (!ReferenceEquals(baseline, optimized)) + { + throw new InvalidOperationException( + $"Shape {workload.Name}: selection '{selection.ResponseName}' resolves " + + $"to {(baseline is null ? "null" : "a child")} via TryGetChild but " + + $"{(optimized is null ? "null" : "a child")} via the indexed map."); + } + } + + // The three row implementations must agree, and the hit count must + // match the expected shape so a probe typo cannot silently turn the + // workload into an all-miss loop. + var baselineHits = workload.ObjectType is { } objectType + ? BaselineRowTypeAware(workload.ResultSet, set, objectType) + : BaselineRowTypeUnaware(workload.ResultSet, set); + + var warmHits = workload.ObjectType is { } warmType + ? OptimizedRowTypeAware(workload.WarmCache, workload.ResultSet, set, warmType) + : OptimizedRowTypeUnaware(workload.WarmCache, set); + + var coldCache = new ChildMapCache(workload.ResultSet); + var coldHits = workload.ObjectType is { } coldType + ? OptimizedRowTypeAware(coldCache, workload.ResultSet, set, coldType) + : OptimizedRowTypeUnaware(coldCache, set); + + if (baselineHits != warmHits || baselineHits != coldHits) + { + throw new InvalidOperationException( + $"Shape {workload.Name}: hit counts diverge (baseline {baselineHits}, " + + $"warm {warmHits}, cold {coldHits})."); + } + + if (baselineHits != workload.ExpectedHitsPerRow) + { + throw new InvalidOperationException( + $"Shape {workload.Name}: expected {workload.ExpectedHitsPerRow} non-null " + + $"children per row but observed {baselineHits}."); + } + } + } + + private static SelectionSet GetChildSet(SelectionSet parent, string responseName) + { + if (!parent.TryGetSelection(responseName, out var selection)) + { + throw new InvalidOperationException( + $"Selection '{responseName}' not found in selection set {parent.Id}."); + } + + return selection.GetSelectionSet() + ?? throw new InvalidOperationException( + $"Selection '{responseName}' has no child selection set."); + } + + private static void RequireSelection(SelectionSet set, string responseName) + { + if (!set.TryGetSelection(responseName, out _)) + { + throw new InvalidOperationException( + $"Selection '{responseName}' not found in selection set {set.Id}; " + + "the compiled fragment shape no longer matches the workload."); + } + } + + private sealed class Workload + { + public required string Name { get; init; } + + public required ResultSelectionSet ResultSet { get; init; } + + public required SelectionSet SelectionSet { get; init; } + + /// + /// The runtime object type for the type-aware call sites, or null for the + /// type-unaware BuildResult sites. + /// + public required IComplexTypeDefinition? ObjectType { get; init; } + + public required ChildMapCache WarmCache { get; init; } + + public required int ExpectedHitsPerRow { get; init; } + } + + /// + /// Benchmark-local implementation of the candidate memo. In the product these + /// two entry arrays would be volatile fields on the plan-lifetime + /// ResultSelectionSet (one memo per TryGetChild overload family); the wrapper + /// exists only because the benchmark must not modify product code. Entries are + /// matched by selection set reference, built by calling the existing + /// TryGetChild once per selection, published copy-on-write, and capped at 8 + /// entries beyond which unseen selection sets are served without publishing. + /// + private sealed class ChildMapCache(ResultSelectionSet resultSelectionSet) + { + private const int MaxEntries = 8; + + private readonly ResultSelectionSet _resultSelectionSet = resultSelectionSet; + private Entry[] _typeAwareEntries = []; + private Entry[] _typeUnawareEntries = []; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ResultSelectionSet?[] GetTypeAwareChildren(SelectionSet selectionSet) + { + var entries = Volatile.Read(ref _typeAwareEntries); + + for (var i = 0; i < entries.Length; i++) + { + if (ReferenceEquals(entries[i].Set, selectionSet)) + { + return entries[i].Children; + } + } + + return BuildAndPublish(ref _typeAwareEntries, selectionSet, typeAware: true); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ResultSelectionSet?[] GetTypeUnawareChildren(SelectionSet selectionSet) + { + var entries = Volatile.Read(ref _typeUnawareEntries); + + for (var i = 0; i < entries.Length; i++) + { + if (ReferenceEquals(entries[i].Set, selectionSet)) + { + return entries[i].Children; + } + } + + return BuildAndPublish(ref _typeUnawareEntries, selectionSet, typeAware: false); + } + + private ResultSelectionSet?[] BuildAndPublish( + ref Entry[] entriesField, + SelectionSet selectionSet, + bool typeAware) + { + var selections = selectionSet.Selections; + var children = new ResultSelectionSet?[selections.Length]; + + // Built through the existing TryGetChild so direct-first order, + // fragment order and type filtering are identical by construction. + // The type-aware map uses the selection set's own type context, which + // is what the divergence guard requires at the call site. + for (var i = 0; i < selections.Length; i++) + { + children[i] = typeAware + ? _resultSelectionSet.TryGetChild( + selections[i].ResponseName, + selectionSet.Type) + : _resultSelectionSet.TryGetChild(selections[i].ResponseName); + } + + while (true) + { + var current = Volatile.Read(ref entriesField); + + // A racing builder may have published first; its content is + // identical, so return the published entry. + for (var i = 0; i < current.Length; i++) + { + if (ReferenceEquals(current[i].Set, selectionSet)) + { + return current[i].Children; + } + } + + if (current.Length >= MaxEntries) + { + return children; + } + + var next = new Entry[current.Length + 1]; + Array.Copy(current, next, current.Length); + next[current.Length] = new Entry(selectionSet, children); + + if (Interlocked.CompareExchange(ref entriesField, next, current) == current) + { + return children; + } + } + } + + private sealed class Entry(SelectionSet set, ResultSelectionSet?[] children) + { + public readonly SelectionSet Set = set; + public readonly ResultSelectionSet?[] Children = children; + } + } + + private sealed class NoOpObjectPool : ObjectPool where T : class, new() + { + public override T Get() => new(); + + public override void Return(T obj) + { + } + } +} diff --git a/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SelectionTypeShapeBenchmark.cs b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SelectionTypeShapeBenchmark.cs new file mode 100644 index 00000000000..770d6517cd3 --- /dev/null +++ b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SelectionTypeShapeBenchmark.cs @@ -0,0 +1,894 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text.Json; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using HotChocolate.Fusion.Execution.Nodes; +using HotChocolate.Fusion.Execution.Rewriters; +using HotChocolate.Fusion.Logging; +using HotChocolate.Fusion.Options; +using HotChocolate.Fusion.Planning; +using HotChocolate.Fusion.Types; +using HotChocolate.Language; +using HotChocolate.Types; +using Microsoft.Extensions.ObjectPool; + +#nullable enable + +namespace HotChocolate.Fusion.Execution.Benchmarks; + +/// +/// Measures the per-field type-shape derivation that ValueCompletion repeats for +/// every completion that misses the scalar fast path. Every call site passes +/// selection.Type, which re-dispatches through IOutputFieldDefinition.Type +/// on each read (Selection.cs line 100; call sites ValueCompletion.cs lines 153, 204, +/// 1189, 1233). Inside TryCompleteValue the code then pays type.Kind +/// (line 799), type.InnerType() (line 828), IsValueType(type) with a full +/// type.NamedType() walk (line 833, body at lines 766-782), and a second +/// switch (type.Kind) (line 867). The list arm re-derives the element shape per +/// invocation (TryCompleteList lines 923-930 via the file-local ElementType +/// extension at lines 1422-1429), the object arm walks type.NamedType() again +/// (lines 1088-1089), and the abstract arm walks it once more inside GetType +/// (line 1276). +/// +/// The candidate precomputes the shape once per at operation +/// compile time: the field type, a non-null flag, the type with one NonNull wrapper +/// removed, the unwrapped type kind, a value-type bit for the named type, and the +/// level-0 list element shape. The completion path then branches on plain field reads. +/// The enum arm keeps its defensive pattern match, matching the reviewed design. +/// +/// The baseline replays the exact product derivation sequence over real compiled +/// instances from a composed fusion schema; the optimized +/// variant reads a benchmark-local shape holder standing in for the new Selection +/// fields (same load pattern: one object dereference plus field reads). Work that is +/// identical in both variants (error building, target writes, source row reads and +/// abstract __typename resolution) is factored out of both loops. Scalar-typed events +/// with non-null sources model error-carrying merges, because without an error trie the +/// fast path at ValueCompletion.cs lines 132/183/1168/1212 bypasses +/// TryCompleteValue entirely; that fast path is untouched by the candidate. +/// +[MemoryDiagnoser] +[Config(typeof(BenchmarkConfig))] +public class SelectionTypeShapeBenchmark +{ + /// + /// BenchmarkDotNet 0.15.8 has no RuntimeMoniker for the net11.0 preview host and + /// this project pins TargetFramework to net11.0, so out-of-process toolchains can + /// neither validate nor build a child process here. The job therefore runs in + /// process with the intended 3 warmup and 10 measurement iterations. + /// + private sealed class BenchmarkConfig : ManualConfig + { + public BenchmarkConfig() + => AddJob( + Job.Default + .WithWarmupCount(3) + .WithIterationCount(10) + .WithToolchain(InProcessEmitToolchain.Instance)); + } + + private const int EventCount = 1000; + + // Distinct constants per derivation exit so different control-flow paths + // contribute distinguishable checksum amounts in both variants. + private const int NonNullViolationMarker = 101; + private const int ValueTypeNullMarker = 59; + private const int NullValueMarker = 23; + private const int ScalarMarker = 7; + private const int EnumMaskMarker = 11; + private const int ElementNonNullMarker = 17; + private const int ElementNullableMarker = 3; + + private const string MixedWorkload = "Mixed"; + private const string NullHeavyWorkload = "NullHeavy"; + private const string MonomorphicScalarWorkload = "MonomorphicScalar"; + private const string SingleEventWorkload = "SingleEvent"; + + private Selection[] _selections = null!; + private SelectionTypeShape[] _shapes = null!; + private Dictionary _workloads = null!; + private CompletionEvent[] _events = null!; + + public long Consumed; + + /// + /// Mixed cycles all thirteen selection shapes (non-null and nullable scalars, + /// enums, objects, lists, an interface) with an occasional null source, keeping + /// the dispatch sites polymorphic like a real payload. NullHeavy alternates null + /// and non-null sources over the nullable selections, stressing the + /// IsValueType walk. MonomorphicScalar replays one non-null scalar selection so + /// guarded devirtualization can do its best for the baseline, which bounds the + /// win honestly. SingleEvent is the N=1 regression shape. + /// + [Params(MixedWorkload, NullHeavyWorkload, MonomorphicScalarWorkload, SingleEventWorkload)] + public string Workload { get; set; } = MixedWorkload; + + [GlobalSetup] + public void Setup() + { + var schema = ComposeSchema(); + var operation = CompileOperation(schema); + + CollectSelections(schema, operation); + + _shapes = new SelectionTypeShape[_selections.Length]; + + for (var i = 0; i < _selections.Length; i++) + { + _shapes[i] = CreateShape(_selections[i]); + } + + VerifyKindCoverage(); + BuildWorkloads(); + VerifyShapes(); + VerifyChecksums(); + + _events = _workloads[Workload]; + } + + /// + /// Current product behavior: per completion event the type shape is derived from + /// selection.Type through the full interface-dispatch and type-test chain + /// of TryCompleteValue, TryCompleteList, TryCompleteObjectValue and GetType. + /// + [Benchmark(Baseline = true)] + public long DerivePerCompletion() + { + var sum = RunBaseline(_events); + Consumed = sum; + return sum; + } + + /// + /// Candidate optimization: the same decisions read a per-selection shape that was + /// computed once at operation compile time, replacing the dispatch chain with + /// field reads and a switch over the precomputed unwrapped kind. + /// + [Benchmark] + public long PrecomputedSelectionShape() + { + var sum = RunOptimized(_events); + Consumed = sum; + return sum; + } + + private long RunBaseline(CompletionEvent[] events) + { + var checksum = 0L; + var selections = _selections; + + for (var i = 0; i < events.Length; i++) + { + var completionEvent = events[i]; + var selection = selections[completionEvent.SelectionIndex]; + var sourceValueKind = completionEvent.SourceValueKind; + + // Call sites pass selection.Type (ValueCompletion.cs lines 153, 204, 1189, + // 1233), an interface re-dispatch through Field.Type per read + // (Selection.cs line 100). + var type = selection.Type; + + // TryCompleteValue lines 796-799. + var isNullOrUndefined = + sourceValueKind is JsonValueKind.Null or JsonValueKind.Undefined; + + if (type.Kind is TypeKind.NonNull) + { + if (isNullOrUndefined) + { + // Non-null violation. The error building at lines 803-825 is + // identical in both variants and factored out. + checksum += NonNullViolationMarker; + continue; + } + + // Line 828. + type = type.InnerType(); + } + + if (isNullOrUndefined) + { + // Line 833: IsValueType(type) walks type.NamedType() per call. + if (sourceValueKind is JsonValueKind.Null && IsValueTypeProduct(type)) + { + checksum += ValueTypeNullMarker; + continue; + } + + // Error lookup and SetNullValue (lines 850-862) are identical in both + // variants and factored out. + checksum += NullValueMarker; + continue; + } + + // Line 867: second interface dispatch on type.Kind. + switch (type.Kind) + { + case TypeKind.List: + { + // TryCompleteList lines 923-930: per-invocation element shape + // derivation through the file-local ElementType extension. + var elementType = ElementTypeProduct(type); + var elementTypeKind = elementType.Kind; + var isNonNull = elementTypeKind is TypeKind.NonNull; + + if (isNonNull) + { + elementTypeKind = + Unsafe.As(ref elementType).NullableType.Kind; + } + + checksum += (int)elementTypeKind + + (isNonNull ? ElementNonNullMarker : ElementNullableMarker) + + RuntimeHelpers.GetHashCode(elementType); + break; + } + + case TypeKind.Object: + { + // TryCompleteObjectValue(Selection, IType, ...) lines 1088-1089. + var namedType = type.NamedType(); + var objectType = + Unsafe.As(ref namedType); + checksum += RuntimeHelpers.GetHashCode(objectType); + break; + } + + case TypeKind.Interface or TypeKind.Union: + { + // TryCompleteAbstractValue enters GetType, whose line 1276 walks + // type.NamedType(). The __typename resolution that follows is + // identical in both variants and factored out. + var namedType = type.NamedType(); + checksum += RuntimeHelpers.GetHashCode(namedType); + break; + } + + case TypeKind.Scalar: + // Line 900: target.SetLeafValue(source) is identical in both + // variants and factored out. + checksum += ScalarMarker; + break; + + case TypeKind.Enum: + // CompleteEnumValue line 1067 re-tests the named type although the + // reviewed design keeps this pattern match on the slow path, so it + // appears identically in both variants. + if (selection.NamedType is FusionEnumTypeDefinition enumType) + { + checksum += RuntimeHelpers.GetHashCode(enumType); + } + else + { + checksum += EnumMaskMarker; + } + + break; + + default: + // Line 908. + throw new NotSupportedException($"The type {type} is not supported."); + } + } + + return checksum; + } + + private long RunOptimized(CompletionEvent[] events) + { + var checksum = 0L; + var shapes = _shapes; + + for (var i = 0; i < events.Length; i++) + { + var completionEvent = events[i]; + var shape = shapes[completionEvent.SelectionIndex]; + var sourceValueKind = completionEvent.SourceValueKind; + + var isNullOrUndefined = + sourceValueKind is JsonValueKind.Null or JsonValueKind.Undefined; + + // The precomputed flag replaces the type.Kind dispatch at line 799 and + // the InnerType() unwrap at line 828. + if (shape.IsNonNull && isNullOrUndefined) + { + checksum += NonNullViolationMarker; + continue; + } + + if (isNullOrUndefined) + { + // The precomputed bit replaces the IsValueType walk at line 833. + if (sourceValueKind is JsonValueKind.Null && shape.IsValueTypeNamedType) + { + checksum += ValueTypeNullMarker; + continue; + } + + checksum += NullValueMarker; + continue; + } + + // The precomputed kind replaces the type.Kind dispatch at line 867. + switch (shape.UnwrappedKind) + { + case TypeKind.List: + // The level-0 element shape replaces lines 923-930. + checksum += (int)shape.ElementKind + + (shape.ElementIsNonNull ? ElementNonNullMarker : ElementNullableMarker) + + RuntimeHelpers.GetHashCode(shape.ElementType!); + break; + + case TypeKind.Object: + // The cached named type replaces the walk at lines 1088-1089. + checksum += RuntimeHelpers.GetHashCode(shape.NamedType); + break; + + case TypeKind.Interface or TypeKind.Union: + // The cached named type replaces the walk at line 1276. + checksum += RuntimeHelpers.GetHashCode(shape.NamedType); + break; + + case TypeKind.Scalar: + checksum += ScalarMarker; + break; + + case TypeKind.Enum: + // The reviewed design keeps the defensive pattern match here. + if (shape.NamedType is FusionEnumTypeDefinition enumType) + { + checksum += RuntimeHelpers.GetHashCode(enumType); + } + else + { + checksum += EnumMaskMarker; + } + + break; + + default: + throw new NotSupportedException( + $"The type {shape.UnwrappedType} is not supported."); + } + } + + return checksum; + } + + // Byte-faithful copy of the private static ValueCompletion.IsValueType, + // src/HotChocolate/Fusion/src/Fusion.Execution/Execution/Results/ValueCompletion.cs + // lines 766-782. + private static bool IsValueTypeProduct(IType? type) + { + if (type is null) + { + return false; + } + + var namedType = type.NamedType(); + + return namedType switch + { + FusionObjectTypeDefinition { IsValueType: true } => true, + FusionInterfaceTypeDefinition { IsValueType: true } => true, + FusionUnionTypeDefinition { IsValueType: true } => true, + _ => false + }; + } + + // Byte-faithful copy of the file-local ElementType extension that line 923 + // resolves to, src/HotChocolate/Fusion/src/Fusion.Execution/Execution/Results/ + // ValueCompletion.cs lines 1422-1429. The file class is not reachable from + // outside its file, so the copy is required. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static IType ElementTypeProduct(IType type) + => type switch + { + ListType listType => listType.ElementType, + NonNullType { NullableType: ListType listType } => listType.ElementType, + _ => throw new ArgumentException($"The type '{type}' is not a list type.", nameof(type)) + }; + + // Mirrors the value-type bit the recommended design computes from the already + // cached Selection._namedType at construction time, using the same three-arm + // test as ValueCompletion.IsValueType (lines 775-781). + private static bool ComputeIsValueTypeBit(ITypeDefinition namedType) + => namedType switch + { + FusionObjectTypeDefinition { IsValueType: true } => true, + FusionInterfaceTypeDefinition { IsValueType: true } => true, + FusionUnionTypeDefinition { IsValueType: true } => true, + _ => false + }; + + // Computes the per-selection shape exactly once, standing in for the extra + // readonly fields the recommended design adds to the Selection constructor. + private static SelectionTypeShape CreateShape(Selection selection) + { + var type = selection.Type; + var isNonNull = type.Kind is TypeKind.NonNull; + var unwrappedType = isNonNull ? ((NonNullType)type).NullableType : type; + var unwrappedKind = unwrappedType.Kind; + var namedType = selection.NamedType; + var isValueTypeNamedType = ComputeIsValueTypeBit(namedType); + + IType? elementType = null; + var elementKind = default(TypeKind); + var elementIsNonNull = false; + + if (unwrappedKind is TypeKind.List) + { + elementType = ElementTypeProduct(unwrappedType); + elementKind = elementType.Kind; + elementIsNonNull = elementKind is TypeKind.NonNull; + + if (elementIsNonNull) + { + elementKind = ((NonNullType)elementType).NullableType.Kind; + } + } + + return new SelectionTypeShape( + type, + isNonNull, + unwrappedType, + unwrappedKind, + isValueTypeNamedType, + namedType, + elementType, + elementKind, + elementIsNonNull); + } + + private void CollectSelections(FusionSchemaDefinition schema, Operation operation) + { + var rootSet = operation.RootSelectionSet; + + if (!rootSet.TryGetSelection("product", out var productSelection)) + { + throw new InvalidOperationException( + "The compiled operation has no 'product' root selection."); + } + + var productType = schema.Types.GetType("Product"); + var productSet = operation.GetSelectionSet(productSelection, productType); + + string[] fieldNames = + [ + "id", "name", "description", "price", "status", "statusRequired", + "dimension", "packaging", "tags", "variants", "related", "media" + ]; + + var selections = new Selection[fieldNames.Length + 1]; + selections[0] = productSelection; + + for (var i = 0; i < fieldNames.Length; i++) + { + if (!productSet.TryGetSelection(fieldNames[i], out var selection)) + { + throw new InvalidOperationException( + $"The Product selection set has no '{fieldNames[i]}' selection."); + } + + selections[i + 1] = selection; + } + + _selections = selections; + } + + /// + /// Guards workload strength: the selection list must cover every switch arm of + /// TryCompleteValue plus non-null and nullable wrappers and both list element + /// nullabilities, otherwise the benchmark silently measures a weaker mix. + /// + private void VerifyKindCoverage() + { + var hasList = false; + var hasObject = false; + var hasAbstract = false; + var hasScalar = false; + var hasEnum = false; + var hasNonNull = false; + var hasNullable = false; + var hasNonNullElement = false; + var hasNullableElement = false; + + foreach (var shape in _shapes) + { + switch (shape.UnwrappedKind) + { + case TypeKind.List: + hasList = true; + hasNonNullElement |= shape.ElementIsNonNull; + hasNullableElement |= !shape.ElementIsNonNull; + break; + case TypeKind.Object: + hasObject = true; + break; + case TypeKind.Interface or TypeKind.Union: + hasAbstract = true; + break; + case TypeKind.Scalar: + hasScalar = true; + break; + case TypeKind.Enum: + hasEnum = true; + break; + } + + hasNonNull |= shape.IsNonNull; + hasNullable |= !shape.IsNonNull; + } + + if (!hasList || !hasObject || !hasAbstract || !hasScalar || !hasEnum + || !hasNonNull || !hasNullable || !hasNonNullElement || !hasNullableElement) + { + throw new InvalidOperationException( + "The compiled selections do not cover all TryCompleteValue shapes; " + + "the schema or the operation drifted."); + } + } + + private void BuildWorkloads() + { + var mixed = new CompletionEvent[EventCount]; + + for (var i = 0; i < EventCount; i++) + { + var selectionIndex = i % _selections.Length; + var kind = i % 9 == 8 + ? JsonValueKind.Null + : NonNullSourceKind(_shapes[selectionIndex]); + mixed[i] = new CompletionEvent(selectionIndex, kind); + } + + var nullableIndices = new List(); + + for (var i = 0; i < _shapes.Length; i++) + { + if (!_shapes[i].IsNonNull) + { + nullableIndices.Add(i); + } + } + + var nullHeavy = new CompletionEvent[EventCount]; + + for (var i = 0; i < EventCount; i++) + { + var selectionIndex = nullableIndices[i % nullableIndices.Count]; + var kind = i % 2 == 0 + ? JsonValueKind.Null + : NonNullSourceKind(_shapes[selectionIndex]); + nullHeavy[i] = new CompletionEvent(selectionIndex, kind); + } + + var nameIndex = IndexOfResponseName("name"); + var monomorphic = new CompletionEvent[EventCount]; + + for (var i = 0; i < EventCount; i++) + { + monomorphic[i] = new CompletionEvent(nameIndex, JsonValueKind.String); + } + + var dimensionIndex = IndexOfResponseName("dimension"); + CompletionEvent[] single = [new CompletionEvent(dimensionIndex, JsonValueKind.Object)]; + + _workloads = new Dictionary + { + [MixedWorkload] = mixed, + [NullHeavyWorkload] = nullHeavy, + [MonomorphicScalarWorkload] = monomorphic, + [SingleEventWorkload] = single + }; + } + + private int IndexOfResponseName(string responseName) + { + for (var i = 0; i < _selections.Length; i++) + { + if (_selections[i].ResponseName == responseName) + { + return i; + } + } + + throw new InvalidOperationException( + $"No selection with response name '{responseName}' was collected."); + } + + private static JsonValueKind NonNullSourceKind(SelectionTypeShape shape) + => shape.UnwrappedKind switch + { + TypeKind.List => JsonValueKind.Array, + TypeKind.Object or TypeKind.Interface or TypeKind.Union => JsonValueKind.Object, + _ => JsonValueKind.String + }; + + /// + /// Structural equivalence: every precomputed shape value must equal what the + /// product derivation computes from the same Selection, by reference for types + /// and by value for kinds and flags. + /// + private void VerifyShapes() + { + for (var i = 0; i < _selections.Length; i++) + { + var selection = _selections[i]; + var shape = _shapes[i]; + + var typeFirstRead = selection.Type; + var typeSecondRead = selection.Type; + + if (!ReferenceEquals(typeFirstRead, typeSecondRead)) + { + throw new InvalidOperationException( + $"Selection '{selection.ResponseName}': Field.Type is not identity " + + "stable across reads, so caching it would change behavior."); + } + + if (!ReferenceEquals(shape.Type, typeFirstRead)) + { + throw new InvalidOperationException( + $"Selection '{selection.ResponseName}': cached Type differs."); + } + + var isNonNull = typeFirstRead.Kind is TypeKind.NonNull; + + if (shape.IsNonNull != isNonNull) + { + throw new InvalidOperationException( + $"Selection '{selection.ResponseName}': IsNonNull flag differs."); + } + + var unwrapped = isNonNull ? typeFirstRead.InnerType() : typeFirstRead; + + if (!ReferenceEquals(shape.UnwrappedType, unwrapped) + || shape.UnwrappedKind != unwrapped.Kind) + { + throw new InvalidOperationException( + $"Selection '{selection.ResponseName}': unwrapped type or kind differs."); + } + + if (shape.IsValueTypeNamedType != IsValueTypeProduct(unwrapped)) + { + throw new InvalidOperationException( + $"Selection '{selection.ResponseName}': value-type bit differs."); + } + + if (!ReferenceEquals(shape.NamedType, selection.NamedType) + || !ReferenceEquals(selection.NamedType, unwrapped.NamedType())) + { + throw new InvalidOperationException( + $"Selection '{selection.ResponseName}': named type differs."); + } + + if (unwrapped.Kind is TypeKind.List) + { + var elementType = ElementTypeProduct(unwrapped); + var elementTypeKind = elementType.Kind; + var elementIsNonNull = elementTypeKind is TypeKind.NonNull; + + if (elementIsNonNull) + { + elementTypeKind = + Unsafe.As(ref elementType).NullableType.Kind; + } + + if (!ReferenceEquals(shape.ElementType, elementType) + || shape.ElementKind != elementTypeKind + || shape.ElementIsNonNull != elementIsNonNull) + { + throw new InvalidOperationException( + $"Selection '{selection.ResponseName}': element shape differs."); + } + } + else if (shape.ElementType is not null) + { + throw new InvalidOperationException( + $"Selection '{selection.ResponseName}': non-list selection carries " + + "an element shape."); + } + } + } + + /// + /// Checksum equivalence over every workload, not only the active parameter: + /// both variants must take identical control-flow paths and consume identical + /// type instances for every event. + /// + private void VerifyChecksums() + { + foreach (var (name, events) in _workloads) + { + var baseline = RunBaseline(events); + var optimized = RunOptimized(events); + + if (baseline != optimized) + { + throw new InvalidOperationException( + $"Checksum mismatch for workload '{name}': baseline {baseline} vs " + + $"optimized {optimized}."); + } + } + } + + private static Operation CompileOperation(FusionSchemaDefinition schema) + { + var document = Utf8GraphQLParser.Parse( + """ + { + product(id: "1") { + id + name + description + price + status + statusRequired + dimension { height width } + packaging { height width } + tags + variants { id name } + related { id title } + media { id title } + } + } + """); + + var documentRewriter = new DocumentRewriter(schema); + var operationDefinition = + documentRewriter.RewriteDocument(document).GetOperation(operationName: null); + + var pool = new NoOpObjectPool>>(); + var compiler = new OperationCompiler(schema, pool); + + return compiler.Compile("bench", "bench", operationDefinition); + } + + /// + /// Composes a single-source fusion schema whose Product type covers every shape + /// TryCompleteValue dispatches over: non-null and nullable scalars, enums, + /// objects, lists with non-null and nullable elements, and an interface. + /// Composition recipe follows FusionBenchmarkBase.CreateFusionSchema. + /// + private static FusionSchemaDefinition ComposeSchema() + { + List sourceSchemas = + [ + new SourceSchemaText( + "catalog", + """ + type Query { + product(id: ID!): Product + search(query: String!): [SearchResult]! + } + + interface SearchResult { + id: ID! + title: String! + } + + type Product implements SearchResult { + id: ID! + title: String! + name: String! + description: String + price: Float + status: ProductStatus + statusRequired: ProductStatus! + dimension: ProductDimension + packaging: ProductDimension! + tags: [String!] + variants: [Product] + related: SearchResult + media: [SearchResult!]! + } + + type Article implements SearchResult { + id: ID! + title: String! + } + + type ProductDimension { + height: Int! + width: Int! + } + + enum ProductStatus { + ACTIVE + DISCONTINUED + ARCHIVED + } + """) + ]; + + var compositionLog = new CompositionLog(); + var composerOptions = new SchemaComposerOptions(); + var composer = new SchemaComposer(sourceSchemas, composerOptions, compositionLog); + var result = composer.Compose(); + + if (!result.IsSuccess) + { + throw new InvalidOperationException(result.Errors[0].Message); + } + + return FusionSchemaDefinition.Create(result.Value.ToSyntaxNode()); + } + + private readonly struct CompletionEvent + { + public CompletionEvent(int selectionIndex, JsonValueKind sourceValueKind) + { + SelectionIndex = selectionIndex; + SourceValueKind = sourceValueKind; + } + + public readonly int SelectionIndex; + + public readonly JsonValueKind SourceValueKind; + } + + /// + /// Benchmark-local stand-in for the readonly fields the recommended design adds + /// to : the cached field type, the non-null flag, the type + /// with one NonNull wrapper removed, its kind, the value-type bit, the cached + /// named type, and the level-0 list element shape. A class keeps the optimized + /// read pattern identical to production: one object dereference plus field reads. + /// In the product the kind fields would be packed as bytes; field width does not + /// change the load cost measured here. + /// + private sealed class SelectionTypeShape + { + public SelectionTypeShape( + IType type, + bool isNonNull, + IType unwrappedType, + TypeKind unwrappedKind, + bool isValueTypeNamedType, + ITypeDefinition namedType, + IType? elementType, + TypeKind elementKind, + bool elementIsNonNull) + { + Type = type; + IsNonNull = isNonNull; + UnwrappedType = unwrappedType; + UnwrappedKind = unwrappedKind; + IsValueTypeNamedType = isValueTypeNamedType; + NamedType = namedType; + ElementType = elementType; + ElementKind = elementKind; + ElementIsNonNull = elementIsNonNull; + } + + public readonly IType Type; + + public readonly bool IsNonNull; + + public readonly IType UnwrappedType; + + public readonly TypeKind UnwrappedKind; + + public readonly bool IsValueTypeNamedType; + + public readonly ITypeDefinition NamedType; + + public readonly IType? ElementType; + + public readonly TypeKind ElementKind; + + public readonly bool ElementIsNonNull; + } + + private sealed class NoOpObjectPool : ObjectPool where T : class, new() + { + public override T Get() => new(); + + public override void Return(T obj) + { + } + } +} diff --git a/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SourceEnumeratorRowCarryBenchmark.cs b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SourceEnumeratorRowCarryBenchmark.cs new file mode 100644 index 00000000000..a2d6f47f05c --- /dev/null +++ b/src/HotChocolate/Fusion/benchmarks/Fusion.Execution.Benchmarks/SourceEnumeratorRowCarryBenchmark.cs @@ -0,0 +1,526 @@ +using System; +using System.Text; +using System.Text.Json; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using HotChocolate.Buffers; +using HotChocolate.Fusion.Text.Json; + +namespace HotChocolate.Fusion.Execution.Benchmarks; + +/// +/// Isolates the duplicate MetaDb row decodes of source object and array enumeration on the +/// FetchResultStore.SaveSafeResult -> ValueCompletion.BuildResult -> TryComplete* hot path. +/// +/// Today each visited property or element pays two identical row decodes: the loop body +/// snapshots the value via CreateSnapshot -> GetValueRow (SourceResultElement.cs line 534, +/// called from ValueCompletion.cs lines 129, 177, 976, 1165 and 1206) and the advancing +/// MoveNext decodes the same row again to compute the skip to the next value +/// (SourceResultElement.ObjectEnumerator.cs line 116, SourceResultElement.ArrayEnumerator.cs +/// line 105). Entering an object or array pays two further container row reads even when the +/// caller already holds the decoded row in a snapshot: the token-type guard +/// (SourceResultElement.cs lines 600 and 623) and GetEndIndex inside the enumerator +/// constructor (SourceResultDocument.Text.cs line 311, reached from ObjectEnumerator.cs +/// line 25 and ArrayEnumerator.cs line 25). SourceResultElementSnapshot.EnumerateArray and +/// EnumerateObject (SourceResultElementSnapshot.cs lines 469 and 477) delegate to the element +/// and pay both container reads despite holding the row. +/// +/// The baseline replays this sequence with the real product enumerators and CreateSnapshot. +/// The optimized variant is a benchmark-local copy of the recommended row-carrying design: +/// MoveNext decodes each value row once on arrival and caches it, the advancing MoveNext +/// computes the skip from the cached row, the property struct carries the row into the loop +/// body, and snapshot-built enumerators derive the end cursor from the cached container row +/// (endCursor = cursor + (NumberOfRows - 1), mirroring SourceResultDocument.Text.cs +/// lines 319-320) with zero row reads. Every row decode is the same two dependent loads +/// (SourceResultDocument.MetaDb.cs lines 184-196), so the variants differ only in decode +/// counts. Property name reads and selection matching are factored out of both variants +/// because the design leaves the name path unchanged. +/// +/// Shapes: Dense1000 is a 1000-element array of 7-property objects with nested containers +/// whose MetaDb spans multiple chunks (the likely cache-miss case). Tiny1000 (1-property +/// objects) and Empty1000 (empty objects) bound the enumerator and property struct growth +/// risk flagged in the verification. Single1 is the N=1 shape. +/// +[MemoryDiagnoser] +[Config(typeof(BenchmarkConfig))] +public class SourceEnumeratorRowCarryBenchmark +{ + /// + /// BenchmarkDotNet 0.15.8 has no RuntimeMoniker for the net11.0 preview host and + /// this project pins TargetFramework to net11.0, so out-of-process toolchains can + /// neither validate nor build a child process here. The job therefore runs in + /// process with the intended 3 warmup and 10 measurement iterations. + /// + private sealed class BenchmarkConfig : ManualConfig + { + public BenchmarkConfig() + => AddJob( + Job.Default + .WithWarmupCount(3) + .WithIterationCount(10) + .WithToolchain(InProcessEmitToolchain.Instance)); + } + + private const string DenseShape = "Dense1000"; + private const string TinyShape = "Tiny1000"; + private const string EmptyShape = "Empty1000"; + private const string SingleShape = "Single1"; + + private const long ChecksumSeed = 17L; + + private static readonly string[] s_shapes = [DenseShape, TinyShape, EmptyShape, SingleShape]; + + private MemoryArena _arena = null!; + private SourceResultDocument[] _documents = null!; + private SourceResultDocument _document = null!; + + [Params(DenseShape, TinyShape, EmptyShape, SingleShape)] + public string Shape { get; set; } = DenseShape; + + [GlobalSetup] + public void Setup() + { + _arena = new MemoryArena(); + _documents = new SourceResultDocument[s_shapes.Length]; + + for (var i = 0; i < s_shapes.Length; i++) + { + var json = BuildPayload(s_shapes[i]); + _documents[i] = SourceResultDocument.Parse(_arena, json, json.Length); + + // The equivalence check runs over every shape, not just the measured one. The + // checksum is an order-sensitive rolling hash over (cursor, token type, location, + // size) of every visited value, so equality proves both variants visit the same + // rows in the same order and decode identical content. + var baseline = BaselineWalk(_documents[i]); + var optimized = OptimizedWalk(_documents[i]); + + if (baseline != optimized) + { + throw new InvalidOperationException( + $"Shape '{s_shapes[i]}': baseline checksum {baseline} differs from " + + $"row-carrying checksum {optimized}."); + } + + if (baseline == ChecksumSeed) + { + throw new InvalidOperationException( + $"Shape '{s_shapes[i]}': the walk visited no values."); + } + } + + _document = _documents[Array.IndexOf(s_shapes, Shape)]; + } + + [GlobalCleanup] + public void Cleanup() + { + for (var i = 0; i < _documents.Length; i++) + { + _documents[i].Dispose(); + } + + _arena.Dispose(); + } + + /// + /// Current product enumeration sequence using only real product internals: the root + /// snapshot mirrors ValueCompletion.BuildResult line 60, EnumerateObject/EnumerateArray + /// pay the guard read plus the constructor GetEndIndex read, per value CreateSnapshot + /// decodes the row the enumerator's MoveNext decodes again to advance. + /// + [Benchmark(Baseline = true)] + public long CurrentEnumerationDoubleDecode() => BaselineWalk(_document); + + /// + /// Candidate optimization: benchmark-local row-carrying enumerators decode each value row + /// once on arrival, and snapshot-built construction performs zero container row reads. + /// + [Benchmark] + public long RowCarryingEnumerationSingleDecode() => OptimizedWalk(_document); + + private static long BaselineWalk(SourceResultDocument document) + => BaselineWalkValue(ChecksumSeed, document.Root.CreateSnapshot()); + + private static long OptimizedWalk(SourceResultDocument document) + => OptimizedWalkValue(ChecksumSeed, document.Root.CreateSnapshot()); + + private static long Accumulate(long checksum, SourceResultElementSnapshot value) + { + // The row is cached on the snapshot, so reading it here is free in both variants. + // Consuming cursor, token type, location and size keeps the decoded row live and + // makes the checksum order-sensitive and content-sensitive. + var row = value.GetValueRow(); + + unchecked + { + checksum = (checksum * 31) + value._cursor.Value; + checksum = (checksum * 31) + (int)row.TokenType; + checksum = (checksum * 31) + row.Location; + checksum = (checksum * 31) + row.SizeOrLength; + } + + return checksum; + } + + private static long BaselineWalkValue(long checksum, SourceResultElementSnapshot value) + { + checksum = Accumulate(checksum, value); + + // The kind dispatch mirrors the ValueCompletion loop bodies; it reads the cached + // snapshot row and costs the same in both variants. + var kind = value.ValueKind; + + if (kind is JsonValueKind.Object) + { + // Mirrors the property loops at ValueCompletion.cs lines 170-177 and 1199-1206: + // EnumerateObject pays the token-type guard plus GetEndIndex, each property pays + // CreateSnapshot's GetValueRow plus the advancing MoveNext's GetDbRow. + foreach (var property in value.EnumerateObject()) + { + checksum = BaselineWalkValue(checksum, property.Value.CreateSnapshot()); + } + } + else if (kind is JsonValueKind.Array) + { + // Mirrors the element loop at ValueCompletion.cs lines 958-976. + foreach (var element in value.EnumerateArray()) + { + checksum = BaselineWalkValue(checksum, element.CreateSnapshot()); + } + } + + return checksum; + } + + private static long OptimizedWalkValue(long checksum, SourceResultElementSnapshot value) + { + checksum = Accumulate(checksum, value); + + var kind = value.ValueKind; + + if (kind is JsonValueKind.Object) + { + foreach (var property in EnumerateObjectRowCarry(value)) + { + checksum = OptimizedWalkValue(checksum, property.CreateValueSnapshot()); + } + } + else if (kind is JsonValueKind.Array) + { + foreach (var elementSnapshot in EnumerateArrayRowCarry(value)) + { + checksum = OptimizedWalkValue(checksum, elementSnapshot); + } + } + + return checksum; + } + + // Stands in for the proposed SourceResultElementSnapshot.EnumerateObject overload: the + // guard checks the cached row's token type instead of re-reading it (the product version + // must throw exactly the exceptions SourceResultElement.cs lines 619-629 throws today) + // and the enumerator constructor performs zero row reads. + private static RowCarryObjectEnumerator EnumerateObjectRowCarry( + SourceResultElementSnapshot snapshot) + { + if (snapshot._parent is null || snapshot._row.TokenType != JsonTokenType.StartObject) + { + throw new InvalidOperationException(); + } + + return new RowCarryObjectEnumerator(snapshot._parent, snapshot._cursor, snapshot._row); + } + + // Stands in for the proposed SourceResultElementSnapshot.EnumerateArray overload + // (product version replicates the formatted message and Source marker of + // SourceResultElement.cs lines 596-611). + private static RowCarryArrayEnumerator EnumerateArrayRowCarry( + SourceResultElementSnapshot snapshot) + { + if (snapshot._parent is null || snapshot._row.TokenType != JsonTokenType.StartArray) + { + throw new InvalidOperationException(); + } + + return new RowCarryArrayEnumerator(snapshot._parent, snapshot._cursor, snapshot._row); + } + + /// + /// Benchmark-local copy of the row-carrying variant of + /// SourceResultElement.ObjectEnumerator (ObjectEnumerator.cs lines 20-133): identical + /// cursor arithmetic and bounds checks, but the row is decoded once on arrival and the + /// advance uses the cached row instead of re-reading it (line 116 today). Construction + /// derives the end cursor from the caller's cached container row instead of GetEndIndex. + /// + private struct RowCarryObjectEnumerator + { + private readonly SourceResultDocument _parent; + private readonly SourceResultDocument.Cursor _containerCursor; + private readonly SourceResultDocument.Cursor _endCursor; + private SourceResultDocument.Cursor _current; + private SourceResultDocument.DbRow _row; + private bool _hasStarted; + + internal RowCarryObjectEnumerator( + SourceResultDocument parent, + SourceResultDocument.Cursor containerCursor, + SourceResultDocument.DbRow containerRow) + { + _parent = parent; + _containerCursor = containerCursor; + + // GetEndIndex(cursor, includeEndElement: false) for a composite value is + // cursor + (NumberOfRows - 1) (SourceResultDocument.Text.cs lines 319-320), + // computable from the cached row without a MetaDb read. + _endCursor = containerCursor + (containerRow.NumberOfRows - 1); + + _current = default; + _row = default; + _hasStarted = false; + } + + public readonly RowCarryProperty Current + { + get + { + if (!_hasStarted) + { + return default; + } + + return new RowCarryProperty(_parent, _current, _row); + } + } + + public readonly RowCarryObjectEnumerator GetEnumerator() => this; + + public bool MoveNext() + { + if (!_hasStarted) + { + // First property: after StartObject comes PropertyName (+1), then Value (+1). + // Identical arithmetic and bounds check as ObjectEnumerator.cs lines 94-110; + // the only change is the arrival decode of the row the loop body will use. + var firstName = _containerCursor + 1; + var firstValue = firstName + 1; + + if (firstValue < _endCursor) + { + _current = firstValue; + _row = _parent.GetDbRow(firstValue); + _hasStarted = true; + return true; + } + + _current = _endCursor; + _hasStarted = false; + return false; + } + + // The skip past the current value uses the cached row (decoded on arrival) + // instead of the departure GetDbRow at ObjectEnumerator.cs line 116. + var afterCurrent = _row.IsSimpleValue ? _current + 1 : _current + _row.NumberOfRows; + var nextValue = afterCurrent + 1; + + if (nextValue < _endCursor) + { + _current = nextValue; + _row = _parent.GetDbRow(nextValue); + return true; + } + + _current = _endCursor; + _hasStarted = false; + return false; + } + } + + /// + /// Benchmark-local copy of the row-carrying variant of + /// SourceResultElement.ArrayEnumerator (ArrayEnumerator.cs lines 20-119) with the same + /// arrival-decode change as the object enumerator, exposing the cached row to the loop + /// body as a snapshot (the proposed internal CurrentSnapshot accessor). + /// + private struct RowCarryArrayEnumerator + { + private readonly SourceResultDocument _parent; + private readonly SourceResultDocument.Cursor _containerCursor; + private readonly SourceResultDocument.Cursor _endCursor; + private SourceResultDocument.Cursor _current; + private SourceResultDocument.DbRow _row; + private bool _hasStarted; + + internal RowCarryArrayEnumerator( + SourceResultDocument parent, + SourceResultDocument.Cursor containerCursor, + SourceResultDocument.DbRow containerRow) + { + _parent = parent; + _containerCursor = containerCursor; + _endCursor = containerCursor + (containerRow.NumberOfRows - 1); + + _current = default; + _row = default; + _hasStarted = false; + } + + public readonly SourceResultElementSnapshot Current + { + get + { + if (!_hasStarted) + { + return default; + } + + return new SourceResultElementSnapshot(_parent, _current, _row); + } + } + + public readonly RowCarryArrayEnumerator GetEnumerator() => this; + + public bool MoveNext() + { + if (!_hasStarted) + { + // Identical arithmetic and bounds check as ArrayEnumerator.cs lines 83-99. + var first = _containerCursor + 1; + + if (first < _endCursor) + { + _current = first; + _row = _parent.GetDbRow(first); + _hasStarted = true; + return true; + } + + _current = _endCursor; + _hasStarted = false; + return false; + } + + // Cached-row skip replacing the departure GetDbRow at ArrayEnumerator.cs line 105. + var next = _row.IsSimpleValue ? _current + 1 : _current + _row.NumberOfRows; + + if (next < _endCursor) + { + _current = next; + _row = _parent.GetDbRow(next); + return true; + } + + _current = _endCursor; + _hasStarted = false; + return false; + } + } + + /// + /// Benchmark-local stand-in for the grown SourceResultProperty of the recommended design: + /// it carries the decoded value row (SourceResultProperty.cs lines 10-20 plus a DbRow + /// field) and is copied per iteration through Current, so the flagged struct-growth cost + /// is part of the measurement. CreateValueSnapshot mirrors the null-parent behavior of + /// SourceResultElement.CreateSnapshot (SourceResultElement.cs lines 529-532). + /// + private readonly struct RowCarryProperty + { + private readonly SourceResultDocument _parent; + private readonly SourceResultDocument.Cursor _valueCursor; + private readonly SourceResultDocument.DbRow _valueRow; + + internal RowCarryProperty( + SourceResultDocument parent, + SourceResultDocument.Cursor valueCursor, + SourceResultDocument.DbRow valueRow) + { + _parent = parent; + _valueCursor = valueCursor; + _valueRow = valueRow; + } + + public SourceResultElementSnapshot CreateValueSnapshot() + { + if (_parent is null) + { + return default; + } + + return new SourceResultElementSnapshot(_parent, _valueCursor, _valueRow); + } + } + + private static byte[] BuildPayload(string shape) + { + var payload = new StringBuilder(); + payload.Append("{\"data\":["); + + switch (shape) + { + case DenseShape: + AppendDenseObjects(payload, 1000); + break; + + case TinyShape: + for (var i = 0; i < 1000; i++) + { + if (i > 0) + { + payload.Append(','); + } + + payload.Append("{\"a\":").Append(i).Append('}'); + } + + break; + + case EmptyShape: + for (var i = 0; i < 1000; i++) + { + if (i > 0) + { + payload.Append(','); + } + + payload.Append("{}"); + } + + break; + + case SingleShape: + AppendDenseObjects(payload, 1); + break; + + default: + throw new InvalidOperationException($"Unknown shape '{shape}'."); + } + + payload.Append("]}"); + return Encoding.UTF8.GetBytes(payload.ToString()); + } + + private static void AppendDenseObjects(StringBuilder payload, int count) + { + for (var i = 0; i < count; i++) + { + if (i > 0) + { + payload.Append(','); + } + + // 7 mixed-kind properties per object: number, string, number, bool, string + // array, nested object, null. The nested containers exercise the recursive + // enumerator construction paths in both variants. + payload.Append("{\"id\":").Append(i) + .Append(",\"name\":\"Product ").Append(i).Append('"') + .Append(",\"price\":").Append(i).Append(".25") + .Append(",\"inStock\":").Append(i % 2 == 0 ? "true" : "false") + .Append(",\"tags\":[\"new\",\"sale\",\"eco\"]") + .Append(",\"dimensions\":{\"width\":").Append(i % 50) + .Append(",\"height\":").Append(i % 30).Append('}') + .Append(",\"note\":null}"); + } + } +}