diff --git a/JSTests/microbenchmarks/runtime-length-small-array-then-grow.js b/JSTests/microbenchmarks/runtime-length-small-array-then-grow.js new file mode 100644 index 0000000000000..5e329f0924d9d --- /dev/null +++ b/JSTests/microbenchmarks/runtime-length-small-array-then-grow.js @@ -0,0 +1,62 @@ +// Arrays with a run-time length of 0, 1 and 2, from a rest parameter and from new Array(length): never grown, grown by +// one push, grown by five pushes, and grown by one unshift. An array that optimized code allocates inline has room for +// its length and no more, so the first push or unshift after it pays for the growth. + +function rest(...values) { + return values; +} +noInline(rest); + +function restThenPush(...values) { + values.push(0); + return values; +} +noInline(restThenPush); + +function restThenFivePushes(...values) { + for (let i = 0; i < 5; ++i) + values.push(i); + return values; +} +noInline(restThenFivePushes); + +function restThenUnshift(first, ...values) { + values.unshift(first); + return values; +} +noInline(restThenUnshift); + +function make(length) { + return new Array(length); +} +noInline(make); + +function makeThenPush(length) { + const array = new Array(length); + array.push(0); + return array; +} +noInline(makeThenPush); + +function makeThenFivePushes(length) { + const array = new Array(length); + for (let i = 0; i < 5; ++i) + array.push(i); + return array; +} +noInline(makeThenFivePushes); + +const iterations = 30000; +let totalLength = 0; +for (let i = 0; i < iterations; ++i) { + totalLength += rest().length + rest(i).length + rest(i, i).length; + totalLength += restThenPush().length + restThenPush(i).length + restThenPush(i, i).length; + totalLength += restThenFivePushes().length + restThenFivePushes(i).length + restThenFivePushes(i, i).length; + totalLength += restThenUnshift(i).length + restThenUnshift(i, i).length + restThenUnshift(i, i, i).length; + totalLength += make(0).length + make(1).length + make(2).length; + totalLength += makeThenPush(0).length + makeThenPush(1).length + makeThenPush(2).length; + totalLength += makeThenFivePushes(0).length + makeThenFivePushes(1).length + makeThenFivePushes(2).length; +} + +if (totalLength !== 60 * iterations) + throw new Error("bad total length: " + totalLength); diff --git a/JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js b/JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js new file mode 100644 index 0000000000000..f6c2cb5fbfa7c --- /dev/null +++ b/JSTests/stress/runtime-length-array-slow-path-refills-inline-allocator.js @@ -0,0 +1,115 @@ +//@ skip if not $jitTests +//@ runDefault("--useConcurrentJIT=false") +//@ runDefault("--useConcurrentJIT=false", "--useFTLJIT=false") +//@ runDefault("--useConcurrentJIT=false", "--forceGCSlowPaths=true") +//@ runDefault("--useConcurrentJIT=false", "--useFTLJIT=false", "--forceGCSlowPaths=true") + +// DFG and FTL allocate the storage of an array inline. For a length that is known only at run time, the inline path +// asks for room for exactly that many elements. When it finds no free cell of that size class, it calls a slow path. +// The slow path must take the storage from the same size class: that is what refills the free list that the inline path +// reads. If it takes a larger cell (the capacity that the runtime gives a short array: 5 elements for length 0, 3 for +// length 1), that free list stays empty and each later allocation calls the slow path again. +// +// Arrays that one function makes one after the other have their storage in adjacent cells. So the most frequent +// distance between two consecutive storages is the cell size of the size class that they come from. The test reads +// that distance for the arrays that DFG code made and for the arrays that FTL code made. --forceGCSlowPaths=true sends +// each allocation to the slow path, so those modes read the size class of the slow path alone. + +const interpreterOrBaseline = 0; +const dfg = 1; +const ftl = 2; +const tierNames = ["the interpreter or the Baseline JIT", "DFG", "FTL"]; +const topTier = $vm.useFTLJIT() ? ftl : dfg; + +// The function under test sets this to the tier that ran it. +let tier = interpreterOrBaseline; + +// The loop of a test ends when the top tier made this many arrays. +const wantedSamples = 1000; +// A tier below the top one is checked when it made at least this many. +const minimumSamples = 200; +const maximumCalls = 100 * testLoopCount; + +// A default constructor passes its arguments on with a spread, and DFG calls the Array constructor for that. +class DerivedArray extends Array { + constructor(length) + { + super(length); + } +} +function identity(value) { return value; } + +function mostFrequent(counts) { + let result; + let best = 0; + for (const [value, count] of counts) { + if (count > best) { + best = count; + result = value; + } + } + return result; +} + +// Each test has a function of its own, so that each one goes through DFG and then FTL. The name in its source keeps +// two tests of one expression from sharing code. +function test(name, parameters, expression, args, expectedDistance) { + const make = new Function(parameters, "/* " + name + " */ tier = $vm.ftlTrue() ? ftl : ($vm.dfgTrue() ? dfg : interpreterOrBaseline); return " + expression + ";"); + noInline(make); + + const distances = [null, new Map, new Map]; + const samples = [0, 0, 0]; + // Every array stays alive, so that no cell is used again while the test measures. + const arrays = [make(...args)]; + for (let i = 0; i < maximumCalls && samples[topTier] < wantedSamples; ++i) { + const array = make(...args); + if (tier !== interpreterOrBaseline) { + const distance = $vm.deltaBetweenButterflies(array, arrays[arrays.length - 1]); + distances[tier].set(distance, (distances[tier].get(distance) || 0) + 1); + samples[tier]++; + } + arrays.push(array); + } + + if (samples[topTier] < wantedSamples) + throw new Error(name + ": only " + samples[topTier] + " arrays came from " + tierNames[topTier]); + for (let measured = dfg; measured <= topTier; ++measured) { + if (samples[measured] < minimumSamples) + continue; + const distance = mostFrequent(distances[measured]); + if (distance !== expectedDistance) + throw new Error(name + " in " + tierNames[measured] + ": expected " + expectedDistance + " but got " + distance); + } +} + +if ($vm.useDFGJIT()) { + // Not literals: a literal shares its storage until a write, and an empty literal has no element type yet. + const noElements = [1, 2]; + noElements.length = 0; + const oneElement = [1, 2]; + oneElement.length = 1; + const twoElements = [1, 2, 3]; + twoElements.length = 2; + + // A length of 0 or 1 that is known only at run time: 8 bytes of header and at most one element of 8 bytes. + test("new Array(0)", "length", "new Array(length)", [0], 16); + test("new Array(1)", "length", "new Array(length)", [1], 16); + test("Array(1)", "length", "Array(length)", [1], 16); + test("new DerivedArray(0)", "length", "new DerivedArray(length)", [0], 16); + test("rest parameter, no arguments", "...values", "values", [], 16); + test("rest parameter, one argument", "...values", "values", [1], 16); + test("rest parameter after a named one, one argument", "first, ...values", "values", [1, 2], 16); + test("slice of no elements", "array", "array.slice()", [noElements], 16); + test("slice of one element", "array", "array.slice()", [oneElement], 16); + test("map of one element", "array", "array.map(identity)", [oneElement], 16); + test("two spreads, one element", "first, second", "[...first, ...second]", [oneElement, noElements], 16); + + // A run-time length of 2: the inline path and the runtime agree on 32 bytes. + test("new Array(2)", "length", "new Array(length)", [2], 32); + test("rest parameter, two arguments", "...values", "values", [1, 2], 32); + test("slice of two elements", "array", "array.slice()", [twoElements], 32); + + // A length that the compiler knows: the inline path gives the capacity of the runtime, and so does its slow path. + test("[value]", "value", "[value]", [1], 32); + test("[]", "", "[]", [], 48); +} diff --git a/Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h b/Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h index da9aaa1be1317..d8a1d0347527a 100644 --- a/Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h +++ b/Source/JavaScriptCore/dfg/DFGCallArrayAllocatorSlowPathGenerator.h @@ -27,6 +27,7 @@ #if ENABLE(DFG_JIT) +#include "DFGOperations.h" #include "DFGSlowPathGenerator.h" #include "DFGSpeculativeJIT.h" #include @@ -67,14 +68,15 @@ class CallArrayAllocatorSlowPathGenerator final : public JumpingSlowPathGenerato Vector m_plans; }; +// The two generators below are the slow path of SpeculativeJIT::emitAllocateButterfly and of the JSArray cell that +// follows it. They own the operation: it must take the butterfly from the size class that the inline path asked for. class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public JumpingSlowPathGenerator { WTF_MAKE_SEQUESTERED_ARENA_ALLOCATED(CallArrayAllocatorWithVariableSizeSlowPathGenerator); public: CallArrayAllocatorWithVariableSizeSlowPathGenerator( - MacroAssembler::JumpList from, SpeculativeJIT* jit, P_JITOperation_GStZB function, + MacroAssembler::JumpList from, SpeculativeJIT* jit, GPRReg resultGPR, JITCompiler::LinkableConstant globalObject, RegisteredStructure contiguousStructure, RegisteredStructure arrayStorageStructure, GPRReg sizeGPR, GPRReg storageGPR) : JumpingSlowPathGenerator(from, jit) - , m_function(function) , m_contiguousStructure(contiguousStructure) , m_arrayStorageOrContiguousStructure(arrayStorageStructure) , m_resultGPR(resultGPR) @@ -100,9 +102,9 @@ class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public Jumping done.link(jit); } else jit->move(SpeculativeJIT::TrustedImmPtr(m_contiguousStructure), scratchGPR); - jit->setupArguments(m_globalObject, scratchGPR, m_sizeGPR, m_storageGPR); - jit->appendCall(m_function); - std::optional exception = jit->tryHandleOrGetExceptionUnderSilentSpill(m_plans, m_resultGPR); + jit->setupArguments(m_globalObject, scratchGPR, m_sizeGPR, m_storageGPR); + jit->appendCall(operationNewArrayWithSizeAfterInlineAllocation); + std::optional exception = jit->tryHandleOrGetExceptionUnderSilentSpill(m_plans, m_resultGPR); jit->setupResults(m_resultGPR); jit->silentFill(m_plans); @@ -112,7 +114,6 @@ class CallArrayAllocatorWithVariableSizeSlowPathGenerator final : public Jumping jumpTo(jit); } - P_JITOperation_GStZB m_function; RegisteredStructure m_contiguousStructure; RegisteredStructure m_arrayStorageOrContiguousStructure; GPRReg m_resultGPR; @@ -126,10 +127,9 @@ class CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator final WTF_MAKE_SEQUESTERED_ARENA_ALLOCATED(CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator); public: CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator( - MacroAssembler::JumpList from, SpeculativeJIT* jit, P_JITOperation_GStZB function, + MacroAssembler::JumpList from, SpeculativeJIT* jit, GPRReg resultGPR, JITCompiler::LinkableConstant globalObject, GPRReg structureGPR, GPRReg sizeGPR, GPRReg storageGPR) : JumpingSlowPathGenerator(from, jit) - , m_function(function) , m_resultGPR(resultGPR) , m_globalObject(globalObject) , m_structureGPR(structureGPR) @@ -143,11 +143,10 @@ class CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator final void generateInternal(SpeculativeJIT* jit) final { linkFrom(jit); - jit->callOperationWithSilentSpill(m_plans, m_function, m_resultGPR, m_globalObject, m_structureGPR, m_sizeGPR, m_storageGPR); + jit->callOperationWithSilentSpill(m_plans, operationNewArrayWithSizeAfterInlineAllocation, m_resultGPR, m_globalObject, m_structureGPR, m_sizeGPR, m_storageGPR); jumpTo(jit); } - P_JITOperation_GStZB m_function; GPRReg m_resultGPR; JITCompiler::LinkableConstant m_globalObject; GPRReg m_structureGPR; diff --git a/Source/JavaScriptCore/dfg/DFGOperations.cpp b/Source/JavaScriptCore/dfg/DFGOperations.cpp index 601ce5829698c..1365df917479b 100644 --- a/Source/JavaScriptCore/dfg/DFGOperations.cpp +++ b/Source/JavaScriptCore/dfg/DFGOperations.cpp @@ -2888,6 +2888,59 @@ JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSize, char*, (JSGlobalObject* glob OPERATION_RETURN(scope, std::bit_cast(result)); } +// An inline allocation of an array with a run-time length asks auxiliarySpace for sizeof(IndexingHeader) + +// length * sizeof(JSValue) bytes (SpeculativeJIT::emitAllocateButterfly, FTL allocateJSArray). This storage comes from +// that same size class, so that the slow path refills the free list that the inline path reads. JSArray::tryCreate +// gives a length of 0 or 1 the minimum capacity of the runtime instead, which is another size class: the inline path +// would then find its free list empty at each allocation. +static ALWAYS_INLINE JSArray* tryCreateArrayInSizeClassOfInlineAllocation(VM& vm, Structure* structure, unsigned length) +{ + ASSERT(!hasAnyArrayStorage(structure->indexingType())); + if (length > MAX_STORAGE_VECTOR_LENGTH) [[unlikely]] + return nullptr; + + unsigned vectorLength = Butterfly::availableContiguousVectorLength(structure, length); + Butterfly* butterfly = Butterfly::tryCreateUninitialized(vm, nullptr, 0, structure->outOfLineCapacity(), true, vectorLength * sizeof(EncodedJSValue)); + if (!butterfly) [[unlikely]] + return nullptr; + butterfly->setVectorLength(vectorLength); + butterfly->setPublicLength(length); + Butterfly::clearRange(structure->indexingType(), butterfly, 0, vectorLength); + return JSArray::createWithButterfly(vm, nullptr, structure, butterfly); +} + +// The slow path of an inline allocation of an array with a run-time length. Code that makes no inline attempt calls +// operationNewArrayWithSize. +JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSizeAfterInlineAllocation, char*, (JSGlobalObject* globalObject, Structure* arrayStructure, int32_t size, Butterfly* butterfly)) +{ + VM& vm = globalObject->vm(); + CallFrame* callFrame = DECLARE_CALL_FRAME(vm); + JITOperationPrologueCallFrameTracer tracer(vm, callFrame); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (size < 0) [[unlikely]] { + throwException(globalObject, scope, createRangeError(globalObject, ArrayInvalidLengthError)); + OPERATION_RETURN(scope, nullptr); + } + + JSArray* result; + if (butterfly) { + ASSERT(butterfly->publicLength() <= butterfly->vectorLength()); + result = JSArray::createWithButterfly(vm, nullptr, arrayStructure, butterfly); + } else { + // A length that is too large for contiguous storage comes with an ArrayStorage structure. The inline path made no attempt. + if (hasAnyArrayStorage(arrayStructure->indexingType())) [[unlikely]] + result = JSArray::tryCreate(vm, arrayStructure, size); + else + result = tryCreateArrayInSizeClassOfInlineAllocation(vm, arrayStructure, size); + if (!result) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope); + OPERATION_RETURN(scope, nullptr); + } + } + OPERATION_RETURN(scope, std::bit_cast(result)); +} + JSC_DEFINE_JIT_OPERATION(operationNewArrayWithSizeAndHint, char*, (JSGlobalObject* globalObject, Structure* arrayStructure, int32_t size, int32_t vectorLengthHint, Butterfly* butterfly)) { VM& vm = globalObject->vm(); diff --git a/Source/JavaScriptCore/dfg/DFGOperations.h b/Source/JavaScriptCore/dfg/DFGOperations.h index cc29b5ab35710..30425f8237e0e 100644 --- a/Source/JavaScriptCore/dfg/DFGOperations.h +++ b/Source/JavaScriptCore/dfg/DFGOperations.h @@ -152,6 +152,7 @@ JSC_DECLARE_JIT_OPERATION(operationNewEmptyArray, char*, (VM*, Structure*)); static constexpr unsigned sortScratchSlotCount = 16; JSC_DECLARE_JIT_OPERATION(operationAcquireSortScratch, JSCell*, (VM*)); JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSize, char*, (JSGlobalObject*, Structure*, int32_t, Butterfly*)); +JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSizeAfterInlineAllocation, char*, (JSGlobalObject*, Structure*, int32_t, Butterfly*)); JSC_DECLARE_JIT_OPERATION(operationNewArrayWithSizeAndHint, char*, (JSGlobalObject*, Structure*, int32_t, int32_t, Butterfly*)); JSC_DECLARE_JIT_OPERATION(operationNewInt8ArrayWithSize, char*, (JSGlobalObject*, Structure*, intptr_t, char*)); JSC_DECLARE_JIT_OPERATION(operationNewInt8ArrayWithOneArgument, char*, (JSGlobalObject*, EncodedJSValue)); diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp index ccee24b97d688..cb9ab5aeed929 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp @@ -9638,7 +9638,7 @@ void SpeculativeJIT::compileArraySlice(Node* node) mutatorFence(vm()); addSlowPathGenerator(makeUniqueWithoutFastMallocCheck( - slowCases, this, operationNewArrayWithSize, resultGPR, LinkableConstant::globalObject(*this, node), tempValue, sizeGPR, storageResultGPR)); + slowCases, this, resultGPR, LinkableConstant::globalObject(*this, node), tempValue, sizeGPR, storageResultGPR)); } GPRTemporary temp4(this); @@ -14310,6 +14310,8 @@ void SpeculativeJIT::compileObjectDefinePropertyFromFields(Node* node) noResult(node, UseChildrenCalledExplicitly); } +// The slow path of this allocation must take the butterfly from the size class of the byte size computed here, or the +// free list that this code reads is never refilled. operationNewArrayWithSizeAfterInlineAllocation does that. void SpeculativeJIT::emitAllocateButterfly(GPRReg storageResultGPR, GPRReg sizeGPR, GPRReg scratch1, GPRReg scratch2, GPRReg scratch3, JumpList& slowCases) { RELEASE_ASSERT(RegisterSet(storageResultGPR, sizeGPR, scratch1, scratch2, scratch3).numberOfSetGPRs() == 5); @@ -16669,7 +16671,7 @@ void SpeculativeJIT::compileAllocateNewArrayWithSize(Node* node, GPRReg resultGP mutatorFence(vm()); addSlowPathGenerator(makeUniqueWithoutFastMallocCheck( - slowCases, this, operationNewArrayWithSize, resultGPR, + slowCases, this, resultGPR, LinkableConstant::globalObject(*this, node), structure, shouldConvertLargeSizeToArrayStorage ? m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)) : structure, diff --git a/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp b/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp index 503b073506918..914f35832bf20 100644 --- a/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp +++ b/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp @@ -24324,10 +24324,14 @@ IGNORE_CLANG_WARNINGS_END LValue slowResultValue = nullptr; if (vectorLength == publicLength || (staticVectorLengthFromPublicLength && staticVectorLength && staticVectorLength.value() == staticVectorLengthFromPublicLength.value())) { + // The slow path must take the butterfly from the size class that the fast path above asked for. A static vector + // length has the capacity that operationNewArrayWithSize gives. A run-time one is the length itself. + bool fastPathAppliedRuntimeCapacity = !!staticVectorLength; slowResultValue = lazySlowPath( [=, &vm] (const Vector& locations) -> RefPtr { return createLazyCallGenerator(vm, - operationNewArrayWithSize, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject), + fastPathAppliedRuntimeCapacity ? operationNewArrayWithSize : operationNewArrayWithSizeAfterInlineAllocation, + locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject), locations[1].directGPR(), locations[2].directGPR(), locations[3].directGPR()); }, structureValue, publicLength, butterflyValue);