From d184a84bde2e36fc251fc5a688e000de4a1d3469 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:45:34 +0000 Subject: [PATCH] [JSC] The slow path of a run-time-length array allocation takes the butterfly from the size class of its inline path DFG and FTL allocate the butterfly of an array with a run-time length inline (SpeculativeJIT::emitAllocateButterfly, FTL allocateJSArray). They ask vm.auxiliarySpace() for sizeof(IndexingHeader) + 8 * length bytes. For a length of 0 or 1 that is size class 16. Their slow path, operationNewArrayWithSize, called JSArray::tryCreate. That applies the minimum capacity of the runtime: 5 elements (48 bytes) for length 0 and 3 elements (32 bytes) for length 1. So the slow path never ran LocalAllocator::allocateSlowCase on the allocator whose free list the inline path reads. Each collection empties that free list. After that, each allocation of such an array called the slow path again, until other code allocated a 16-byte auxiliary cell. Rest parameters with 0 or 1 arguments, new Array(n), slice and map of short arrays are on this path. operationNewArrayWithSizeAfterInlineAllocation is now the slow path of these inline attempts. With no butterfly and a contiguous structure, it takes the butterfly from the size class that the inline path asked for. Lengths of 2 and more get the same butterfly as before. The two DFG slow path generators for a run-time length call the operation by name. FTL allocateJSArray calls it when the vector length is not static. A static vector length already has the capacity of the runtime, so that arm keeps operationNewArrayWithSize. The inline code does not change. * JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js: Added. * JSTests/microbenchmarks/runtime-length-small-array-then-grow.js: Added. * Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h: * Source/JavaScriptCore/dfg/DFGOperations.cpp: * Source/JavaScriptCore/dfg/DFGOperations.h: * Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp: * Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp: --- .../runtime-length-small-array-then-grow.js | 62 ++++++++++ ...rray-slow-path-refills-inline-allocator.js | 115 ++++++++++++++++++ .../DFGCallArrayAllocatorSlowPathGenerator.h | 19 ++- Source/JavaScriptCore/dfg/DFGOperations.cpp | 53 ++++++++ Source/JavaScriptCore/dfg/DFGOperations.h | 1 + .../JavaScriptCore/dfg/DFGSpeculativeJIT.cpp | 6 +- Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp | 6 +- 7 files changed, 249 insertions(+), 13 deletions(-) create mode 100644 JSTests/microbenchmarks/runtime-length-small-array-then-grow.js create mode 100644 JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js diff --git a/JSTests/microbenchmarks/runtime-length-small-array-then-grow.js b/JSTests/microbenchmarks/runtime-length-small-array-then-grow.js new file mode 100644 index 0000000000000..5e329f0924d9d --- /dev/null +++ b/JSTests/microbenchmarks/runtime-length-small-array-then-grow.js @@ -0,0 +1,62 @@ +// Arrays with a run-time length of 0, 1 and 2, from a rest parameter and from new Array(length): never grown, grown by +// one push, grown by five pushes, and grown by one unshift. An array that optimized code allocates inline has room for +// its length and no more, so the first push or unshift after it pays for the growth. + +function rest(...values) { + return values; +} +noInline(rest); + +function restThenPush(...values) { + values.push(0); + return values; +} +noInline(restThenPush); + +function restThenFivePushes(...values) { + for (let i = 0; i < 5; ++i) + values.push(i); + return values; +} +noInline(restThenFivePushes); + +function restThenUnshift(first, ...values) { + values.unshift(first); + return values; +} +noInline(restThenUnshift); + +function make(length) { + return new Array(length); +} +noInline(make); + +function makeThenPush(length) { + const array = new Array(length); + array.push(0); + return array; +} +noInline(makeThenPush); + +function makeThenFivePushes(length) { + const array = new Array(length); + for (let i = 0; i < 5; ++i) + array.push(i); + return array; +} +noInline(makeThenFivePushes); + +const iterations = 30000; +let totalLength = 0; +for (let i = 0; i < iterations; ++i) { + totalLength += rest().length + rest(i).length + rest(i, i).length; + totalLength += restThenPush().length + restThenPush(i).length + restThenPush(i, i).length; + totalLength += restThenFivePushes().length + restThenFivePushes(i).length + restThenFivePushes(i, i).length; + totalLength += restThenUnshift(i).length + restThenUnshift(i, i).length + restThenUnshift(i, i, i).length; + totalLength += make(0).length + make(1).length + make(2).length; + totalLength += makeThenPush(0).length + makeThenPush(1).length + makeThenPush(2).length; + totalLength += makeThenFivePushes(0).length + makeThenFivePushes(1).length + makeThenFivePushes(2).length; +} + +if (totalLength !== 60 * iterations) + throw new Error("bad total length: " + totalLength); diff --git a/JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js b/JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js new file mode 100644 index 0000000000000..f6c2cb5fbfa7c --- /dev/null +++ b/JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js @@ -0,0 +1,115 @@ +//@ skip if not $jitTests +//@ runDefault("--useConcurrentJIT=false") +//@ runDefault("--useConcurrentJIT=false", "--useFTLJIT=false") +//@ runDefault("--useConcurrentJIT=false", "--forceGCSlowPaths=true") +//@ runDefault("--useConcurrentJIT=false", "--useFTLJIT=false", "--forceGCSlowPaths=true") + +// DFG and FTL allocate the storage of an array inline. For a length that is known only at run time, the inline path +// asks for room for exactly that many elements. When it finds no free cell of that size class, it calls a slow path. +// The slow path must take the storage from the same size class: that is what refills the free list that the inline path +// reads. If it takes a larger cell (the capacity that the runtime gives a short array: 5 elements for length 0, 3 for +// length 1), that free list stays empty and each later allocation calls the slow path again. +// +// Arrays that one function makes one after the other have their storage in adjacent cells. So the most frequent +// distance between two consecutive storages is the cell size of the size class that they come from. The test reads +// that distance for the arrays that DFG code made and for the arrays that FTL code made. --forceGCSlowPaths=true sends +// each allocation to the slow path, so those modes read the size class of the slow path alone. + +const interpreterOrBaseline = 0; +const dfg = 1; +const ftl = 2; +const tierNames = ["the interpreter or the Baseline JIT", "DFG", "FTL"]; +const topTier = $vm.useFTLJIT() ? ftl : dfg; + +// The function under test sets this to the tier that ran it. +let tier = interpreterOrBaseline; + +// The loop of a test ends when the top tier made this many arrays. +const wantedSamples = 1000; +// A tier below the top one is checked when it made at least this many. +const minimumSamples = 200; +const maximumCalls = 100 * testLoopCount; + +// A default constructor passes its arguments on with a spread, and DFG calls the Array constructor for that. +class DerivedArray extends Array { + constructor(length) + { + super(length); + } +} +function identity(value) { return value; } + +function mostFrequent(counts) { + let result; + let best = 0; + for (const [value, count] of counts) { + if (count > best) { + best = count; + result = value; + } + } + return result; +} + +// Each test has a function of its own, so that each one goes through DFG and then FTL. The name in its source keeps +// two tests of one expression from sharing code. +function test(name, parameters, expression, args, expectedDistance) { + const make = new Function(parameters, "/* " + name + " */ tier = $vm.ftlTrue() ? ftl : ($vm.dfgTrue() ? dfg : interpreterOrBaseline); return " + expression + ";"); + noInline(make); + + const distances = [null, new Map, new Map]; + const samples = [0, 0, 0]; + // Every array stays alive, so that no cell is used again while the test measures. + const arrays = [make(...args)]; + for (let i = 0; i < maximumCalls && samples[topTier] < wantedSamples; ++i) { + const array = make(...args); + if (tier !== interpreterOrBaseline) { + const distance = $vm.deltaBetweenButterflies(array, arrays[arrays.length - 1]); + distances[tier].set(distance, (distances[tier].get(distance) || 0) + 1); + samples[tier]++; + } + arrays.push(array); + } + + if (samples[topTier] < wantedSamples) + throw new Error(name + ": only " + samples[topTier] + " arrays came from " + tierNames[topTier]); + for (let measured = dfg; measured <= topTier; ++measured) { + if (samples[measured] < minimumSamples) + continue; + const distance = mostFrequent(distances[measured]); + if (distance !== expectedDistance) + throw new Error(name + " in " + tierNames[measured] + ": expected " + expectedDistance + " but got " + distance); + } +} + +if ($vm.useDFGJIT()) { + // Not literals: a literal shares its storage until a write, and an empty literal has no element type yet. + const noElements = [1, 2]; + noElements.length = 0; + const oneElement = [1, 2]; + oneElement.length = 1; + const twoElements = [1, 2, 3]; + twoElements.length = 2; + + // A length of 0 or 1 that is known only at run time: 8 bytes of header and at most one element of 8 bytes. + test("new Array(0)", "length", "new Array(length)", [0], 16); + test("new Array(1)", "length", "new Array(length)", [1], 16); + test("Array(1)", "length", "Array(length)", [1], 16); + test("new DerivedArray(0)", "length", "new DerivedArray(length)", [0], 16); + test("rest parameter, no arguments", "...values", "values", [], 16); + test("rest parameter, one argument", "...values", "values", [1], 16); + test("rest parameter after a named one, one argument", "first, ...values", "values", [1, 2], 16); + test("slice of no elements", "array", "array.slice()", [noElements], 16); + test("slice of one element", "array", "array.slice()", [oneElement], 16); + test("map of one element", "array", "array.map(identity)", [oneElement], 16); + test("two spreads, one element", "first, second", "[...first, ...second]", [oneElement, noElements], 16); + + // A run-time length of 2: the inline path and the runtime agree on 32 bytes. + test("new Array(2)", "length", "new Array(length)", [2], 32); + test("rest parameter, two arguments", "...values", "values", [1, 2], 32); + test("slice of two elements", "array", "array.slice()", [twoElements], 32); + + // A length that the compiler knows: the inline path gives the capacity of the runtime, and so does its slow path. + test("[value]", "value", "[value]", [1], 32); + test("[]", "", "[]", [], 48); +} diff --git a/Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h b/Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h index da9aaa1be1317..d8a1d0347527a 100644 --- a/Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h +++ b/Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h @@ -27,6 +27,7 @@ #if ENABLE(DFG_JIT) +#include "DFGOperations.h" #include "DFGSlowPathGenerator.h" #include "DFGSpeculativeJIT.h" #include @@ -67,14 +68,15 @@ class CallArrayAllocatorSlowPathGenerator final : public JumpingSlowPathGenerato Vector m_plans; }; +// The two generators below are the slow path of SpeculativeJIT::emitAllocateButterfly and of the JSArray cell that +// follows it. They own the operation: it must take the butterfly from the size class that the inline path asked for. class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public JumpingSlowPathGenerator { WTF_MAKE_SEQUESTERED_ARENA_ALLOCATED(CallArrayAllocatorWithVariableSizeSlowPathGenerator); public: CallArrayAllocatorWithVariableSizeSlowPathGenerator( - MacroAssembler::JumpList from, SpeculativeJIT* jit, P_JITOperation_GStZB function, + MacroAssembler::JumpList from, SpeculativeJIT* jit, GPRReg resultGPR, JITCompiler::LinkableConstant globalObject, RegisteredStructure contiguousStructure, RegisteredStructure arrayStorageStructure, GPRReg sizeGPR, GPRReg storageGPR) : JumpingSlowPathGenerator(from, jit) - , m_function(function) , m_contiguousStructure(contiguousStructure) , m_arrayStorageOrContiguousStructure(arrayStorageStructure) , m_resultGPR(resultGPR) @@ -100,9 +102,9 @@ class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public Jumping done.link(jit); } else jit->move(SpeculativeJIT::TrustedImmPtr(m_contiguousStructure), scratchGPR); - jit->setupArguments(m_globalObject, scratchGPR, m_sizeGPR, m_storageGPR); - jit->appendCall(m_function); - std::optional exception = jit->tryHandleOrGetExceptionUnderSilentSpill(m_plans, m_resultGPR); + jit->setupArguments(m_globalObject, scratchGPR, m_sizeGPR, m_storageGPR); + jit->appendCall(operationNewArrayWithSizeAfterInlineAllocation); + std::optional exception = jit->tryHandleOrGetExceptionUnderSilentSpill(m_plans, m_resultGPR); jit->setupResults(m_resultGPR); jit->silentFill(m_plans); @@ -112,7 +114,6 @@ class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public Jumping jumpTo(jit); } - P_JITOperation_GStZB m_function; RegisteredStructure m_contiguousStructure; RegisteredStructure m_arrayStorageOrContiguousStructure; GPRReg m_resultGPR; @@ -126,10 +127,9 @@ class CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator final WTF_MAKE_SEQUESTERED_ARENA_ALLOCATED(CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator); public: CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator( - MacroAssembler::JumpList from, SpeculativeJIT* jit, P_JITOperation_GStZB function, + MacroAssembler::JumpList from, SpeculativeJIT* jit, GPRReg resultGPR, JITCompiler::LinkableConstant globalObject, GPRReg structureGPR, GPRReg sizeGPR, GPRReg storageGPR) : JumpingSlowPathGenerator(from, jit) - , m_function(function) , m_resultGPR(resultGPR) , m_globalObject(globalObject) , m_structureGPR(structureGPR) @@ -143,11 +143,10 @@ class CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator final void generateInternal(SpeculativeJIT* jit) final { linkFrom(jit); - jit->callOperationWithSilentSpill(m_plans, m_function, m_resultGPR, m_globalObject, m_structureGPR, m_sizeGPR, m_storageGPR); + jit->callOperationWithSilentSpill(m_plans, operationNewArrayWithSizeAfterInlineAllocation, m_resultGPR, m_globalObject, m_structureGPR, m_sizeGPR, m_storageGPR); jumpTo(jit); } - P_JITOperation_GStZB m_function; GPRReg m_resultGPR; JITCompiler::LinkableConstant m_globalObject; GPRReg m_structureGPR; diff --git a/Source/JavaScriptCore/dfg/DFGOperations.cpp b/Source/JavaScriptCore/dfg/DFGOperations.cpp index 601ce5829698c..1365df917479b 100644 --- a/Source/JavaScriptCore/dfg/DFGOperations.cpp +++ b/Source/JavaScriptCore/dfg/DFGOperations.cpp @@ -2888,6 +2888,59 @@ JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSize, char*, (JSGlobalObject* glob OPERATION_RETURN(scope, std::bit_cast(result)); } +// An inline allocation of an array with a run-time length asks auxiliarySpace for sizeof(IndexingHeader) + +// length * sizeof(JSValue) bytes (SpeculativeJIT::emitAllocateButterfly, FTL allocateJSArray). This storage comes from +// that same size class, so that the slow path refills the free list that the inline path reads. JSArray::tryCreate +// gives a length of 0 or 1 the minimum capacity of the runtime instead, which is another size class: the inline path +// would then find its free list empty at each allocation. +static ALWAYS_INLINE JSArray* tryCreateArrayInSizeClassOfInlineAllocation(VM& vm, Structure* structure, unsigned length) +{ + ASSERT(!hasAnyArrayStorage(structure->indexingType())); + if (length > MAX_STORAGE_VECTOR_LENGTH) [[unlikely]] + return nullptr; + + unsigned vectorLength = Butterfly::availableContiguousVectorLength(structure, length); + Butterfly* butterfly = Butterfly::tryCreateUninitialized(vm, nullptr, 0, structure->outOfLineCapacity(), true, vectorLength * sizeof(EncodedJSValue)); + if (!butterfly) [[unlikely]] + return nullptr; + butterfly->setVectorLength(vectorLength); + butterfly->setPublicLength(length); + Butterfly::clearRange(structure->indexingType(), butterfly, 0, vectorLength); + return JSArray::createWithButterfly(vm, nullptr, structure, butterfly); +} + +// The slow path of an inline allocation of an array with a run-time length. Code that makes no inline attempt calls +// operationNewArrayWithSize. +JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSizeAfterInlineAllocation, char*, (JSGlobalObject* globalObject, Structure* arrayStructure, int32_t size, Butterfly* butterfly)) +{ + VM& vm = globalObject->vm(); + CallFrame* callFrame = DECLARE_CALL_FRAME(vm); + JITOperationPrologueCallFrameTracer tracer(vm, callFrame); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (size < 0) [[unlikely]] { + throwException(globalObject, scope, createRangeError(globalObject, ArrayInvalidLengthError)); + OPERATION_RETURN(scope, nullptr); + } + + JSArray* result; + if (butterfly) { + ASSERT(butterfly->publicLength() <= butterfly->vectorLength()); + result = JSArray::createWithButterfly(vm, nullptr, arrayStructure, butterfly); + } else { + // A length that is too large for contiguous storage comes with an ArrayStorage structure. The inline path made no attempt. + if (hasAnyArrayStorage(arrayStructure->indexingType())) [[unlikely]] + result = JSArray::tryCreate(vm, arrayStructure, size); + else + result = tryCreateArrayInSizeClassOfInlineAllocation(vm, arrayStructure, size); + if (!result) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope); + OPERATION_RETURN(scope, nullptr); + } + } + OPERATION_RETURN(scope, std::bit_cast(result)); +} + JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSizeAndHint, char*, (JSGlobalObject* globalObject, Structure* arrayStructure, int32_t size, int32_t vectorLengthHint, Butterfly* butterfly)) { VM& vm = globalObject->vm(); diff --git a/Source/JavaScriptCore/dfg/DFGOperations.h b/Source/JavaScriptCore/dfg/DFGOperations.h index cc29b5ab35710..30425f8237e0e 100644 --- a/Source/JavaScriptCore/dfg/DFGOperations.h +++ b/Source/JavaScriptCore/dfg/DFGOperations.h @@ -152,6 +152,7 @@ JSC_DECLARE_JIT_OPERATION(operationNewEmptyArray, char*, (VM*, Structure*)); static constexpr unsigned sortScratchSlotCount = 16; JSC_DECLARE_JIT_OPERATION(operationAcquireSortScratch, JSCell*, (VM*)); JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSize, char*, (JSGlobalObject*, Structure*, int32_t, Butterfly*)); +JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSizeAfterInlineAllocation, char*, (JSGlobalObject*, Structure*, int32_t, Butterfly*)); JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSizeAndHint, char*, (JSGlobalObject*, Structure*, int32_t, int32_t, Butterfly*)); JSC_DECLARE_JIT_OPERATION(operationNewInt8ArrayWithSize, char*, (JSGlobalObject*, Structure*, intptr_t, char*)); JSC_DECLARE_JIT_OPERATION(operationNewInt8ArrayWithOneArgument, char*, (JSGlobalObject*, EncodedJSValue)); diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp index ccee24b97d688..cb9ab5aeed929 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp @@ -9638,7 +9638,7 @@ void SpeculativeJIT::compileArraySlice(Node* node) mutatorFence(vm()); addSlowPathGenerator(makeUniqueWithoutFastMallocCheck( - slowCases, this, operationNewArrayWithSize, resultGPR, LinkableConstant::globalObject(*this, node), tempValue, sizeGPR, storageResultGPR)); + slowCases, this, resultGPR, LinkableConstant::globalObject(*this, node), tempValue, sizeGPR, storageResultGPR)); } GPRTemporary temp4(this); @@ -14310,6 +14310,8 @@ void SpeculativeJIT::compileObjectDefinePropertyFromFields(Node* node) noResult(node, UseChildrenCalledExplicitly); } +// The slow path of this allocation must take the butterfly from the size class of the byte size computed here, or the +// free list that this code reads is never refilled. operationNewArrayWithSizeAfterInlineAllocation does that. void SpeculativeJIT::emitAllocateButterfly(GPRReg storageResultGPR, GPRReg sizeGPR, GPRReg scratch1, GPRReg scratch2, GPRReg scratch3, JumpList& slowCases) { RELEASE_ASSERT(RegisterSet(storageResultGPR, sizeGPR, scratch1, scratch2, scratch3).numberOfSetGPRs() == 5); @@ -16669,7 +16671,7 @@ void SpeculativeJIT::compileAllocateNewArrayWithSize(Node* node, GPRReg resultGP mutatorFence(vm()); addSlowPathGenerator(makeUniqueWithoutFastMallocCheck( - slowCases, this, operationNewArrayWithSize, resultGPR, + slowCases, this, resultGPR, LinkableConstant::globalObject(*this, node), structure, shouldConvertLargeSizeToArrayStorage ? m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)) : structure, diff --git a/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp b/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp index 503b073506918..914f35832bf20 100644 --- a/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp +++ b/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp @@ -24324,10 +24324,14 @@ IGNORE_CLANG_WARNINGS_END LValue slowResultValue = nullptr; if (vectorLength == publicLength || (staticVectorLengthFromPublicLength && staticVectorLength && staticVectorLength.value() == staticVectorLengthFromPublicLength.value())) { + // The slow path must take the butterfly from the size class that the fast path above asked for. A static vector + // length has the capacity that operationNewArrayWithSize gives. A run-time one is the length itself. + bool fastPathAppliedRuntimeCapacity = !!staticVectorLength; slowResultValue = lazySlowPath( [=, &vm] (const Vector& locations) -> RefPtr { return createLazyCallGenerator(vm, - operationNewArrayWithSize, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject), + fastPathAppliedRuntimeCapacity ? operationNewArrayWithSize : operationNewArrayWithSizeAfterInlineAllocation, + locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject), locations[1].directGPR(), locations[2].directGPR(), locations[3].directGPR()); }, structureValue, publicLength, butterflyValue);