[JSC] The slow path of a run-time-length array allocation takes the butterfly from the size class of its inline path

JSTests/microbenchmarks/runtime-length-small-array-then-grow.js

@@ -0,0 +1,62 @@
+// Arrays with a run-time length of 0, 1 and 2, from a rest parameter and from new Array(length): never grown, grown by
+// one push, grown by five pushes, and grown by one unshift. An array that optimized code allocates inline has room for
+// its length and no more, so the first push or unshift after it pays for the growth.
+
+function rest(...values) {
+    return values;
+}
+noInline(rest);
+
+function restThenPush(...values) {
+    values.push(0);
+    return values;
+}
+noInline(restThenPush);
+
+function restThenFivePushes(...values) {
+    for (let i = 0; i < 5; ++i)
+        values.push(i);
+    return values;
+}
+noInline(restThenFivePushes);
+
+function restThenUnshift(first, ...values) {
+    values.unshift(first);
+    return values;
+}
+noInline(restThenUnshift);
+
+function make(length) {
+    return new Array(length);
+}
+noInline(make);
+
+function makeThenPush(length) {
+    const array = new Array(length);
+    array.push(0);
+    return array;
+}
+noInline(makeThenPush);
+
+function makeThenFivePushes(length) {
+    const array = new Array(length);
+    for (let i = 0; i < 5; ++i)
+        array.push(i);
+    return array;
+}
+noInline(makeThenFivePushes);
+
+const iterations = 30000;
+let totalLength = 0;
+for (let i = 0; i < iterations; ++i) {
+    totalLength += rest().length + rest(i).length + rest(i, i).length;
+    totalLength += restThenPush().length + restThenPush(i).length + restThenPush(i, i).length;
+    totalLength += restThenFivePushes().length + restThenFivePushes(i).length + restThenFivePushes(i, i).length;
+    totalLength += restThenUnshift(i).length + restThenUnshift(i, i).length + restThenUnshift(i, i, i).length;
+    totalLength += make(0).length + make(1).length + make(2).length;
+    totalLength += makeThenPush(0).length + makeThenPush(1).length + makeThenPush(2).length;
+    totalLength += makeThenFivePushes(0).length + makeThenFivePushes(1).length + makeThenFivePushes(2).length;
+}
+
+if (totalLength !== 60 * iterations)
+    throw new Error("bad total length: " + totalLength);

JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js

@@ -0,0 +1,115 @@
+//@ skip if not $jitTests
+//@ runDefault("--useConcurrentJIT=false")
+//@ runDefault("--useConcurrentJIT=false", "--useFTLJIT=false")
+//@ runDefault("--useConcurrentJIT=false", "--forceGCSlowPaths=true")
+//@ runDefault("--useConcurrentJIT=false", "--useFTLJIT=false", "--forceGCSlowPaths=true")
+
+// DFG and FTL allocate the storage of an array inline. For a length that is known only at run time, the inline path
+// asks for room for exactly that many elements. When it finds no free cell of that size class, it calls a slow path.
+// The slow path must take the storage from the same size class: that is what refills the free list that the inline path
+// reads. If it takes a larger cell (the capacity that the runtime gives a short array: 5 elements for length 0, 3 for
+// length 1), that free list stays empty and each later allocation calls the slow path again.
+//
+// Arrays that one function makes one after the other have their storage in adjacent cells. So the most frequent
+// distance between two consecutive storages is the cell size of the size class that they come from. The test reads
+// that distance for the arrays that DFG code made and for the arrays that FTL code made. --forceGCSlowPaths=true sends
+// each allocation to the slow path, so those modes read the size class of the slow path alone.
+
+const interpreterOrBaseline = 0;
+const dfg = 1;
+const ftl = 2;
+const tierNames = ["the interpreter or the Baseline JIT", "DFG", "FTL"];
+const topTier = $vm.useFTLJIT() ? ftl : dfg;
+
+// The function under test sets this to the tier that ran it.
+let tier = interpreterOrBaseline;
+
+// The loop of a test ends when the top tier made this many arrays.
+const wantedSamples = 1000;
+// A tier below the top one is checked when it made at least this many.
+const minimumSamples = 200;
+const maximumCalls = 100 * testLoopCount;
+
+// A default constructor passes its arguments on with a spread, and DFG calls the Array constructor for that.
+class DerivedArray extends Array {
+    constructor(length)
+    {
+        super(length);
+    }
+}
+function identity(value) { return value; }
+
+function mostFrequent(counts) {
+    let result;
+    let best = 0;
+    for (const [value, count] of counts) {
+        if (count > best) {
+            best = count;
+            result = value;
+        }
+    }
+    return result;
+}
+
+// Each test has a function of its own, so that each one goes through DFG and then FTL. The name in its source keeps
+// two tests of one expression from sharing code.
+function test(name, parameters, expression, args, expectedDistance) {
+    const make = new Function(parameters, "/* " + name + " */ tier = $vm.ftlTrue() ? ftl : ($vm.dfgTrue() ? dfg : interpreterOrBaseline); return " + expression + ";");
+    noInline(make);
+
+    const distances = [null, new Map, new Map];
+    const samples = [0, 0, 0];
+    // Every array stays alive, so that no cell is used again while the test measures.
+    const arrays = [make(...args)];
+    for (let i = 0; i < maximumCalls && samples[topTier] < wantedSamples; ++i) {
+        const array = make(...args);
+        if (tier !== interpreterOrBaseline) {
+            const distance = $vm.deltaBetweenButterflies(array, arrays[arrays.length - 1]);
+            distances[tier].set(distance, (distances[tier].get(distance) || 0) + 1);
+            samples[tier]++;
+        }
+        arrays.push(array);
+    }
+
+    if (samples[topTier] < wantedSamples)
+        throw new Error(name + ": only " + samples[topTier] + " arrays came from " + tierNames[topTier]);
+    for (let measured = dfg; measured <= topTier; ++measured) {
+        if (samples[measured] < minimumSamples)
+            continue;
+        const distance = mostFrequent(distances[measured]);
+        if (distance !== expectedDistance)
+            throw new Error(name + " in " + tierNames[measured] + ": expected " + expectedDistance + " but got " + distance);
+    }
+}
+
+if ($vm.useDFGJIT()) {
+    // Not literals: a literal shares its storage until a write, and an empty literal has no element type yet.
+    const noElements = [1, 2];
+    noElements.length = 0;
+    const oneElement = [1, 2];
+    oneElement.length = 1;
+    const twoElements = [1, 2, 3];
+    twoElements.length = 2;
+
+    // A length of 0 or 1 that is known only at run time: 8 bytes of header and at most one element of 8 bytes.
+    test("new Array(0)", "length", "new Array(length)", [0], 16);
+    test("new Array(1)", "length", "new Array(length)", [1], 16);
+    test("Array(1)", "length", "Array(length)", [1], 16);
+    test("new DerivedArray(0)", "length", "new DerivedArray(length)", [0], 16);
+    test("rest parameter, no arguments", "...values", "values", [], 16);
+    test("rest parameter, one argument", "...values", "values", [1], 16);
+    test("rest parameter after a named one, one argument", "first, ...values", "values", [1, 2], 16);
+    test("slice of no elements", "array", "array.slice()", [noElements], 16);
+    test("slice of one element", "array", "array.slice()", [oneElement], 16);
+    test("map of one element", "array", "array.map(identity)", [oneElement], 16);
+    test("two spreads, one element", "first, second", "[...first, ...second]", [oneElement, noElements], 16);
+
+    // A run-time length of 2: the inline path and the runtime agree on 32 bytes.
+    test("new Array(2)", "length", "new Array(length)", [2], 32);
+    test("rest parameter, two arguments", "...values", "values", [1, 2], 32);
+    test("slice of two elements", "array", "array.slice()", [twoElements], 32);
+
+    // A length that the compiler knows: the inline path gives the capacity of the runtime, and so does its slow path.
+    test("[value]", "value", "[value]", [1], 32);
+    test("[]", "", "[]", [], 48);
+}

Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h

@@ -27,6 +27,7 @@
 
 #if ENABLE(DFG_JIT)
 
+#include "DFGOperations.h"
 #include "DFGSlowPathGenerator.h"
 #include "DFGSpeculativeJIT.h"
 #include <wtf/Vector.h>
@@ -67,14 +68,15 @@ class CallArrayAllocatorSlowPathGenerator final : public JumpingSlowPathGenerato
     Vector<SilentRegisterSavePlan, 2> m_plans;
 };
 
+// The two generators below are the slow path of SpeculativeJIT::emitAllocateButterfly and of the JSArray cell that
+// follows it. They own the operation: it must take the butterfly from the size class that the inline path asked for.
 class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public JumpingSlowPathGenerator<MacroAssembler::JumpList> {
     WTF_MAKE_SEQUESTERED_ARENA_ALLOCATED(CallArrayAllocatorWithVariableSizeSlowPathGenerator);
 public:
     CallArrayAllocatorWithVariableSizeSlowPathGenerator(
-        MacroAssembler::JumpList from, SpeculativeJIT* jit, P_JITOperation_GStZB function,
+        MacroAssembler::JumpList from, SpeculativeJIT* jit,
         GPRReg resultGPR, JITCompiler::LinkableConstant globalObject, RegisteredStructure contiguousStructure, RegisteredStructure arrayStorageStructure, GPRReg sizeGPR, GPRReg storageGPR)
         : JumpingSlowPathGenerator<MacroAssembler::JumpList>(from, jit)
-        , m_function(function)
         , m_contiguousStructure(contiguousStructure)
         , m_arrayStorageOrContiguousStructure(arrayStorageStructure)
         , m_resultGPR(resultGPR)
@@ -100,9 +102,9 @@ class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public Jumping
             done.link(jit);
         } else
             jit->move(SpeculativeJIT::TrustedImmPtr(m_contiguousStructure), scratchGPR);
-        jit->setupArguments<decltype(m_function)>(m_globalObject, scratchGPR, m_sizeGPR, m_storageGPR);
-        jit->appendCall(m_function);
-        std::optional<GPRReg> exception = jit->tryHandleOrGetExceptionUnderSilentSpill<decltype(m_function)>(m_plans, m_resultGPR);
+        jit->setupArguments<P_JITOperation_GStZB>(m_globalObject, scratchGPR, m_sizeGPR, m_storageGPR);
+        jit->appendCall(operationNewArrayWithSizeAfterInlineAllocation);
+        std::optional<GPRReg> exception = jit->tryHandleOrGetExceptionUnderSilentSpill<P_JITOperation_GStZB>(m_plans, m_resultGPR);
         jit->setupResults(m_resultGPR);
         jit->silentFill(m_plans);
 
@@ -112,7 +114,6 @@ class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public Jumping
         jumpTo(jit);
     }
 
-    P_JITOperation_GStZB m_function;
     RegisteredStructure m_contiguousStructure;
     RegisteredStructure m_arrayStorageOrContiguousStructure;
     GPRReg m_resultGPR;
@@ -126,10 +127,9 @@ class CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator final
     WTF_MAKE_SEQUESTERED_ARENA_ALLOCATED(CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator);
 public:
     CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator(
-        MacroAssembler::JumpList from, SpeculativeJIT* jit, P_JITOperation_GStZB function,
+        MacroAssembler::JumpList from, SpeculativeJIT* jit,
         GPRReg resultGPR, JITCompiler::LinkableConstant globalObject, GPRReg structureGPR, GPRReg sizeGPR, GPRReg storageGPR)
         : JumpingSlowPathGenerator<MacroAssembler::JumpList>(from, jit)
-        , m_function(function)
         , m_resultGPR(resultGPR)
         , m_globalObject(globalObject)
         , m_structureGPR(structureGPR)
@@ -143,11 +143,10 @@ class CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator final
     void generateInternal(SpeculativeJIT* jit) final
     {
         linkFrom(jit);
-        jit->callOperationWithSilentSpill(m_plans, m_function, m_resultGPR, m_globalObject, m_structureGPR, m_sizeGPR, m_storageGPR);
+        jit->callOperationWithSilentSpill(m_plans, operationNewArrayWithSizeAfterInlineAllocation, m_resultGPR, m_globalObject, m_structureGPR, m_sizeGPR, m_storageGPR);
         jumpTo(jit);
     }
 
-    P_JITOperation_GStZB m_function;
     GPRReg m_resultGPR;
     JITCompiler::LinkableConstant m_globalObject;
     GPRReg m_structureGPR;

Source/JavaScriptCore/dfg/DFGOperations.cpp

@@ -2888,6 +2888,59 @@ JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSize, char*, (JSGlobalObject* glob
     OPERATION_RETURN(scope, std::bit_cast<char*>(result));
 }
 
+// An inline allocation of an array with a run-time length asks auxiliarySpace for sizeof(IndexingHeader) +
+// length * sizeof(JSValue) bytes (SpeculativeJIT::emitAllocateButterfly, FTL allocateJSArray). This storage comes from
+// that same size class, so that the slow path refills the free list that the inline path reads. JSArray::tryCreate
+// gives a length of 0 or 1 the minimum capacity of the runtime instead, which is another size class: the inline path
+// would then find its free list empty at each allocation.
+static ALWAYS_INLINE JSArray* tryCreateArrayInSizeClassOfInlineAllocation(VM& vm, Structure* structure, unsigned length)
+{
+    ASSERT(!hasAnyArrayStorage(structure->indexingType()));
+    if (length > MAX_STORAGE_VECTOR_LENGTH) [[unlikely]]
+        return nullptr;
+
+    unsigned vectorLength = Butterfly::availableContiguousVectorLength(structure, length);
+    Butterfly* butterfly = Butterfly::tryCreateUninitialized(vm, nullptr, 0, structure->outOfLineCapacity(), true, vectorLength * sizeof(EncodedJSValue));
+    if (!butterfly) [[unlikely]]
+        return nullptr;
+    butterfly->setVectorLength(vectorLength);
+    butterfly->setPublicLength(length);
+    Butterfly::clearRange(structure->indexingType(), butterfly, 0, vectorLength);
+    return JSArray::createWithButterfly(vm, nullptr, structure, butterfly);
+}
+
+// The slow path of an inline allocation of an array with a run-time length. Code that makes no inline attempt calls
+// operationNewArrayWithSize.
+JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSizeAfterInlineAllocation, char*, (JSGlobalObject* globalObject, Structure* arrayStructure, int32_t size, Butterfly* butterfly))
+{
+    VM& vm = globalObject->vm();
+    CallFrame* callFrame = DECLARE_CALL_FRAME(vm);
+    JITOperationPrologueCallFrameTracer tracer(vm, callFrame);
+    auto scope = DECLARE_THROW_SCOPE(vm);
+
+    if (size < 0) [[unlikely]] {
+        throwException(globalObject, scope, createRangeError(globalObject, ArrayInvalidLengthError));
+        OPERATION_RETURN(scope, nullptr);
+    }
+
+    JSArray* result;
+    if (butterfly) {
+        ASSERT(butterfly->publicLength() <= butterfly->vectorLength());
+        result = JSArray::createWithButterfly(vm, nullptr, arrayStructure, butterfly);
+    } else {
+        // A length that is too large for contiguous storage comes with an ArrayStorage structure. The inline path made no attempt.
+        if (hasAnyArrayStorage(arrayStructure->indexingType())) [[unlikely]]
+            result = JSArray::tryCreate(vm, arrayStructure, size);
+        else
+            result = tryCreateArrayInSizeClassOfInlineAllocation(vm, arrayStructure, size);
+        if (!result) [[unlikely]] {
+            throwOutOfMemoryError(globalObject, scope);
+            OPERATION_RETURN(scope, nullptr);
+        }
+    }
+    OPERATION_RETURN(scope, std::bit_cast<char*>(result));
+}
+
 JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSizeAndHint, char*, (JSGlobalObject* globalObject, Structure* arrayStructure, int32_t size, int32_t vectorLengthHint, Butterfly* butterfly))
 {
     VM& vm = globalObject->vm();

Source/JavaScriptCore/dfg/DFGOperations.h

@@ -152,6 +152,7 @@ JSC_DECLARE_JIT_OPERATION(operationNewEmptyArray, char*, (VM*, Structure*));
 static constexpr unsigned sortScratchSlotCount = 16;
 JSC_DECLARE_JIT_OPERATION(operationAcquireSortScratch, JSCell*, (VM*));
 JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSize, char*, (JSGlobalObject*, Structure*, int32_t, Butterfly*));
+JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSizeAfterInlineAllocation, char*, (JSGlobalObject*, Structure*, int32_t, Butterfly*));
 JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSizeAndHint, char*, (JSGlobalObject*, Structure*, int32_t, int32_t, Butterfly*));
 JSC_DECLARE_JIT_OPERATION(operationNewInt8ArrayWithSize, char*, (JSGlobalObject*, Structure*, intptr_t, char*));
 JSC_DECLARE_JIT_OPERATION(operationNewInt8ArrayWithOneArgument, char*, (JSGlobalObject*, EncodedJSValue));

Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp

@@ -9638,7 +9638,7 @@ void SpeculativeJIT::compileArraySlice(Node* node)
         mutatorFence(vm());
 
         addSlowPathGenerator(makeUniqueWithoutFastMallocCheck<CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator>(
-            slowCases, this, operationNewArrayWithSize, resultGPR, LinkableConstant::globalObject(*this, node), tempValue, sizeGPR, storageResultGPR));
+            slowCases, this, resultGPR, LinkableConstant::globalObject(*this, node), tempValue, sizeGPR, storageResultGPR));
     }
 
     GPRTemporary temp4(this);
@@ -14310,6 +14310,8 @@ void SpeculativeJIT::compileObjectDefinePropertyFromFields(Node* node)
     noResult(node, UseChildrenCalledExplicitly);
 }
 
+// The slow path of this allocation must take the butterfly from the size class of the byte size computed here, or the
+// free list that this code reads is never refilled. operationNewArrayWithSizeAfterInlineAllocation does that.
 void SpeculativeJIT::emitAllocateButterfly(GPRReg storageResultGPR, GPRReg sizeGPR, GPRReg scratch1, GPRReg scratch2, GPRReg scratch3, JumpList& slowCases)
 {
     RELEASE_ASSERT(RegisterSet(storageResultGPR, sizeGPR, scratch1, scratch2, scratch3).numberOfSetGPRs() == 5);
@@ -16669,7 +16671,7 @@ void SpeculativeJIT::compileAllocateNewArrayWithSize(Node* node, GPRReg resultGP
     mutatorFence(vm());
 
     addSlowPathGenerator(makeUniqueWithoutFastMallocCheck<CallArrayAllocatorWithVariableSizeSlowPathGenerator>(
-        slowCases, this, operationNewArrayWithSize, resultGPR,
+        slowCases, this, resultGPR,
         LinkableConstant::globalObject(*this, node),
         structure,
         shouldConvertLargeSizeToArrayStorage ? m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)) : structure,

Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp

@@ -24324,10 +24324,14 @@ IGNORE_CLANG_WARNINGS_END
         LValue slowResultValue = nullptr;
         if (vectorLength == publicLength
             || (staticVectorLengthFromPublicLength && staticVectorLength && staticVectorLength.value() == staticVectorLengthFromPublicLength.value())) {
+            // The slow path must take the butterfly from the size class that the fast path above asked for. A static vector
+            // length has the capacity that operationNewArrayWithSize gives. A run-time one is the length itself.
+            bool fastPathAppliedRuntimeCapacity = !!staticVectorLength;
             slowResultValue = lazySlowPath(
                 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
                     return createLazyCallGenerator(vm,
-                        operationNewArrayWithSize, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject),
+                        fastPathAppliedRuntimeCapacity ? operationNewArrayWithSize : operationNewArrayWithSizeAfterInlineAllocation,
+                        locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject),
                         locations[1].directGPR(), locations[2].directGPR(), locations[3].directGPR());
                 },
                 structureValue, publicLength, butterflyValue);