From 12ae2b417e1194c2de8439311140c54f69bd0b7c Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 14:02:08 -0400 Subject: [PATCH 01/53] Order the K composite by stride in the operand where K is the inner group `_classify_labels` listed the contracted labels in `indA` order and the K `AxisGroup` kept it. The packers step through K in the composite's enumeration order, so that order decides how BOTH packs walk memory, and `indA` order is wrong for the upstream suite's `contract_scrambled` layout, C[a,e] = A[a,b,c,d] * B[d,c,b,e]: B's unit-stride axis `d` became the slowest K coordinate, so `pack_b!` jumped 96^2 elements per K step and touched a new cache line for every element it read. Measured on the suite case dim96_1_3_1_contract_scrambled (Float64, Cascade Lake ccqlin038, pinned, benchmark/probes/probe_stage_breakdown.jl): pack B was 92% of a 4.56 s contraction at 49 ns per B element (pack A: 1.3 ns/elem). With the K order sorted by B's strides the same contraction takes 0.60 s (3.6 -> 27.2 GF/s; StridedBLAS 13.0), pack B 1.2 ns/elem. Rule (`_order_contract_labels`, src/planning/labels.jl): stable-sort the K labels by |stride| in whichever operand has the smaller minimum K stride (K is that operand's inner group and has no contiguous free axis to hide behind); ties by the larger free extent, then A. Layouts whose K strides are already ascending in the chosen operand -- every gemm_ready/a_permuted/ b_permuted suite layout -- keep exactly the previous order (checked: dim48_1_3_1_b_permuted plans the same K group before and after). The swapped orientation reuses the same order with the maps exchanged. Tests: planner pins for scrambled/gemm_ready/b_permuted/both-inner/tie/ singleton/single-label/swapped cases, and a correctness sweep (all four dtypes, alpha/beta, conj, both orientations, `execute!` and the tilewise oracle) against the loop reference. Also adds three probes: probe_stage_breakdown.jl (an instrumented copy of `_execute_nest!` timing pack B / pack A / kernel+store, with an optional hand-made M-axis split), probe_case_ab.jl (interleaved StridedBLAS vs QuasiStrided timing of one suite-style case through tensorcontract!), and probe_msplit_prototype.jl (M enumeration-order variants for the TCCG intensli cases). Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_case_ab.jl | 63 +++++ benchmark/probes/probe_msplit_prototype.jl | 123 ++++++++++ benchmark/probes/probe_stage_breakdown.jl | 259 +++++++++++++++++++++ src/planning/labels.jl | 81 +++++++ src/planning/plan.jl | 14 +- test/planning/test_plan_contract.jl | 116 +++++++++ 6 files changed, 653 insertions(+), 3 deletions(-) create mode 100644 benchmark/probes/probe_case_ab.jl create mode 100644 benchmark/probes/probe_msplit_prototype.jl create mode 100644 benchmark/probes/probe_stage_breakdown.jl diff --git a/benchmark/probes/probe_case_ab.jl b/benchmark/probes/probe_case_ab.jl new file mode 100644 index 0000000..0f6fc8f --- /dev/null +++ b/benchmark/probes/probe_case_ab.jl @@ -0,0 +1,63 @@ +# Interleaved A/B timing of ONE upstream-suite-style contraction case through +# `TensorOperations.tensorcontract!`, StridedBLAS vs QuasiStridedBackend, +# built exactly as benchmark/bench_to_suite.jl builds it (plain dense arrays in +# the given label orders, `contract_indices` for pA/pB/pAB, alpha=1, beta=0). +# ABAB interleaving over `--rounds` rounds, `--reps` calls per round per +# backend, so a slow patch on the shared machine hits both backends alike. +# +# julia -t 1 --project=benchmark benchmark/probes/probe_case_ab.jl \ +# --expr "abcd bdef>acef" --dim 63 --dtype ComplexF64 --reps 2 --rounds 2 + +using TensorOperations +using TensorOperations: StridedBLAS +using QuasiStrided +using QuasiStrided: QuasiStridedBackend +using Random +using Statistics: median +using Printf +using LinearAlgebra + +include(joinpath(@__DIR__, "..", "harness.jl")) + +const EXPR = argopt("expr", "abcd bdef>acef") +const DIM = argopt("dim", 63) +const T = parse_dtypes(argopt("dtype", "ComplexF64"))[1] +const REPS = argopt("reps", 2) +const ROUNDS = argopt("rounds", 2) + +function parse_expr(expr::AbstractString) + lhs, rhs = split(expr, '>') + a, b = split(strip(lhs), ' ') + return collect(a), collect(b), collect(strip(rhs)) +end +LA, LB, LC = parse_expr(EXPR) +IA = Symbol.(LA); IB = Symbol.(LB); IC = Symbol.(LC) + +rng = MersenneTwister(42) +A = randn(rng, T, ntuple(_ -> DIM, length(LA))...) +B = randn(rng, T, ntuple(_ -> DIM, length(LB))...) +C = zeros(T, ntuple(_ -> DIM, length(LC))...) +pA, pB, pAB = TensorOperations.contract_indices(Tuple(IA), Tuple(IB), Tuple(IC)) +labels = unique(vcat(LA, LB, LC)) +nk = length(intersect(LA, LB)) +flops = 2.0 * DIM^(length(labels)) + +run!(backend) = TensorOperations.tensorcontract!(C, A, pA, false, B, pB, false, pAB, one(T), zero(T), backend) + +println("expr = $EXPR dim = $DIM T = $T flops = $flops bytes = $(sizeof(A) + sizeof(B) + sizeof(C))") +run!(StridedBLAS()); Cref = copy(C) +run!(QuasiStridedBackend()); println("max abs diff QS vs BLAS = ", maximum(abs, C .- Cref)) + +tb = Float64[]; tq = Float64[] +for round in 1:ROUNDS + for _ in 1:REPS + t0 = time_ns(); run!(StridedBLAS()); push!(tb, (time_ns() - t0) / 1e9) + end + for _ in 1:REPS + t0 = time_ns(); run!(QuasiStridedBackend()); push!(tq, (time_ns() - t0) / 1e9) + end + @printf("round %d: BLAS %s QS %s\n", round, join((@sprintf("%.3f", x) for x in tb[end-REPS+1:end]), ","), join((@sprintf("%.3f", x) for x in tq[end-REPS+1:end]), ",")) +end +@printf("StridedBLAS median %.4f s (min %.4f, max %.4f) %.2f GF/s\n", median(tb), minimum(tb), maximum(tb), flops / median(tb) / 1e9) +@printf("QuasiStrided median %.4f s (min %.4f, max %.4f) %.2f GF/s\n", median(tq), minimum(tq), maximum(tq), flops / median(tq) / 1e9) +@printf("QS/BLAS = %.3f\n", median(tq) / median(tb)) diff --git a/benchmark/probes/probe_msplit_prototype.jl b/benchmark/probes/probe_msplit_prototype.jl new file mode 100644 index 0000000..438ed34 --- /dev/null +++ b/benchmark/probes/probe_msplit_prototype.jl @@ -0,0 +1,123 @@ +# Prototype: does splitting A's unit-stride M axis into (inner, outer) and +# moving the inner part forward in the M enumeration fix the A-pack line +# amplification of the TCCG intensli cases, and at what store cost? +# +# Builds the plan normally, then rebuilds it with a hand-made M AxisGroup: +# :asis -> the planner's order (C-stride order, A's fastest axis last) +# :pos2 -> C's fastest axis first, then `li` of A's fastest axis, then the rest +# :pos1 -> `li` of A's fastest axis first, then the C order +# +# julia -t 1 --project=benchmark benchmark/probes/probe_msplit_prototype.jl --case intensli_7 --dim 24 --dtype Float64 --li 8 + +using QuasiStrided +using QuasiStrided: plan_contract, execute!, mr, nr, axis_length, AxisGroup, ContractPlan, + _default_kernel, _kernel_from_shape, kernel_shapes, complex_method +using StridedViews +using Random +using Statistics: median +using Printf +using LinearAlgebra + +include(joinpath(@__DIR__, "..", "harness.jl")) + +const KNOWN = Dict( + "intensli_6" => "abcde bf>dcfea", + "intensli_7" => "abcde df>ecbfa", + "intensli_8" => "abcde fb>dfcea", + "ccsd_t_3" => "abcd cefg>fgdabe", +) +const CASE = argopt("case", "intensli_7") +const EXPR = something(argval("expr"), get(KNOWN, CASE, nothing)) +const DIM = argopt("dim", 24) +const T = parse_dtypes(argopt("dtype", "Float64"))[1] +const REPS = argopt("reps", 7) +const LI = argopt("li", 8) + +function parse_expr(expr::AbstractString) + lhs, rhs = split(expr, '>') + a, b = split(strip(lhs), ' ') + return collect(a), collect(b), collect(strip(rhs)) +end +const LA, LB, LC = parse_expr(EXPR) +const LETTERS = unique(vcat(LA, LB, LC)) +label(c) = findfirst(==(c), LETTERS) +const indA = Tuple(label.(LA)); const indB = Tuple(label.(LB)); const indC = Tuple(label.(LC)) + +rng = MersenneTwister(1234) +A = randn(rng, T, ntuple(_ -> DIM, length(LA))...) +B = randn(rng, T, ntuple(_ -> DIM, length(LB))...) +C = zeros(T, ntuple(_ -> DIM, length(LC))...) +Av, Bv, Cv = StridedView(A), StridedView(B), StridedView(C) + +plan0 = plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false) +g = plan0.mgroup +println("case = $CASE dim = $DIM T = $T kernel = ", typeof(plan0.kernel).name.name, " MR=", mr(plan0.kernel), " blocking = ", plan0.blocking) +println("as-is M group: lengths=", g.lengths, " strides=", g.strides) +flops = 2.0 * axis_length(plan0.mgroup) * axis_length(plan0.ngroup) * axis_length(plan0.kgroup) + +# Split dim `du` (1-based, in the group's own order) into (li, L/li) and place +# the inner part at `pos`, the outer part where `du` was. +function split_group(g::AxisGroup{D, P}, du::Int, li::Int, pos::Int) where {D, P} + L = g.lengths[du] + @assert L % li == 0 + dims = [(g.lengths[d], ntuple(p -> g.strides[p][d], P)) for d in 1:D] + inner = (li, ntuple(p -> g.strides[p][du], P)) + outer = (L ÷ li, ntuple(p -> li * g.strides[p][du], P)) + dims[du] = outer + insert!(dims, pos, inner) + lengths = ntuple(i -> dims[i][1], D + 1) + strides = ntuple(p -> ntuple(i -> dims[i][2][p], D + 1), P) + return AxisGroup(lengths, strides) +end + +function with_mgroup(plan::ContractPlan, mg; kernel = plan.kernel) + return ContractPlan( + kernel, mg, plan.ngroup, plan.kgroup, plan.blocking, + plan.Astorage, plan.Abase, plan.Bstorage, plan.Bbase, plan.Cstorage, plan.Cbase, + plan.atransform, plan.btransform, plan.workspace + ) +end + +du = argmin([g.lengths[d] > 1 ? abs(g.strides[1][d]) : typemax(Int) for d in 1:length(g.lengths)]) +println("A's fastest M axis is group dim $du (A stride $(g.strides[1][du]), length $(g.lengths[du]))") + +variants = Pair{String, Any}[] +push!(variants, "asis" => plan0) +if du != 1 + g2 = split_group(g, du, LI, 2) + g1 = split_group(g, du, LI, 1) + push!(variants, "pos2(li=$LI)" => with_mgroup(plan0, g2)) + push!(variants, "pos1(li=$LI)" => with_mgroup(plan0, g1)) + # pos2 with the run-demoted kernel (what `_demote_for_run` would pick for run = L_1) + function demoted_shape(run, kernel) + best = nothing + for shape in kernel_shapes(T, complex_method(kernel)) + if run % shape[1] == 0 && (best === nothing || shape[1] > best[1]) + best = shape + end + end + return best + end + if T <: Real + best = demoted_shape(g.lengths[1], plan0.kernel) + if best !== nothing && best[1] != mr(plan0.kernel) + k2 = _kernel_from_shape(best, T, complex_method(plan0.kernel)) + p2 = plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false, kernel = k2) + push!(variants, "pos2(li=$LI)+MR$(best[1])" => with_mgroup(p2, g2)) + end + end +end + +execute!(plan0, one(T), zero(T)); Cref = copy(C) +for (name, p) in variants + println(" ", name, ": M group lengths=", p.mgroup.lengths, " strides=", p.mgroup.strides) +end +for round in 1:2 + for (name, p) in variants + fill!(C, zero(T)) + execute!(p, one(T), zero(T)) + @assert C ≈ Cref "$name wrong result" + t = median_time_s(() -> execute!(p, one(T), zero(T)); reps = REPS) + @printf("round %d %-22s %9.4f s %7.2f GF/s\n", round, name, t, flops / t / 1e9) + end +end diff --git a/benchmark/probes/probe_stage_breakdown.jl b/benchmark/probes/probe_stage_breakdown.jl new file mode 100644 index 0000000..1474619 --- /dev/null +++ b/benchmark/probes/probe_stage_breakdown.jl @@ -0,0 +1,259 @@ +# Stage breakdown of one contraction under `execute!`: an instrumented copy of +# `_execute_nest!` (src/execution/execute.jl) with `time_ns` brackets around +# the B pack, the A pack and the micro-kernel+store loop, so that a slow case +# can be attributed to the stage that eats the time. Also prints the plan's +# resolved axis groups (each label's stride in both operands) so the memory +# walk each pack performs can be read off directly. +# +# julia -t 1 --project=benchmark benchmark/probes/probe_stage_breakdown.jl \ +# --case scrambled_1_3_1 --dim 48 --dtype Float64 --reps 5 +# +# Cases are einsum-style letter strings `A B>C` (TensorOperationsBenchmarks' +# `expr` column of bench_to_suite.csv); `--expr "abcd dcbe>ae"` accepts any. +# Every axis gets extent `--dim`. Times are medians over `--reps` calls after +# one warm-up; the stage times come from the instrumented nest, the "execute!" +# row from the shipped driver (they should agree to within the timer overhead). + +using QuasiStrided +using QuasiStrided: plan_contract, execute!, mr, nr, axis_length, affine_ramp, + fill_offsets!, describe_block, descriptor_offset_range, checked_span_bounds, + unsafe_pack_a!, unsafe_pack_b!, packed_a_per_k, packed_b_per_k, + _ramp_slivers!, _ramp_descriptor, _ramp_offset_range, _classify_slivers!, + _axis_of, _sliver_panel, _pack_sliver!, unsafe_execute_micro_tile! +using StridedViews +using Random +using Statistics: median +using Printf +using LinearAlgebra + +include(joinpath(@__DIR__, "..", "harness.jl")) + +const KNOWN = Dict( + "scrambled_1_3_1" => "abcd dcbe>ae", + "gemm_ready_1_3_1" => "abcd bcde>ae", + "b_permuted_1_3_1" => "abcd becd>ae", + "a_permuted_2_2_2" => "abcd bdef>acef", + "b_permuted_2_2_2" => "abcd cedf>abef", + "gemm_ready_2_2_2" => "abcd cdef>abef", + "intensli_6" => "abcde bf>dcfea", + "intensli_7" => "abcde df>ecbfa", + "intensli_8" => "abcde fb>dfcea", + "ao2mo_1" => "ab acde>bcde", + "ccsd_t_3" => "abcd cefg>fgdabe", +) + +const CASE = argopt("case", "scrambled_1_3_1") +const EXPR = something(argval("expr"), get(KNOWN, CASE, nothing)) +EXPR === nothing && error("unknown --case $CASE and no --expr given") +const DIM = argopt("dim", 48) +const T = parse_dtypes(argopt("dtype", "Float64"))[1] +const REPS = argopt("reps", 5) + +function parse_expr(expr::AbstractString) + lhs, rhs = split(expr, '>') + a, b = split(strip(lhs), ' ') + return collect(a), collect(b), collect(strip(rhs)) +end + +const LA, LB, LC = parse_expr(EXPR) +const LETTERS = unique(vcat(LA, LB, LC)) +label(c) = findfirst(==(c), LETTERS) +const indA = Tuple(label.(LA)) +const indB = Tuple(label.(LB)) +const indC = Tuple(label.(LC)) + +rng = MersenneTwister(1234) +A = randn(rng, T, ntuple(_ -> DIM, length(LA))...) +B = randn(rng, T, ntuple(_ -> DIM, length(LB))...) +C = zeros(T, ntuple(_ -> DIM, length(LC))...) +Av, Bv, Cv = StridedView(A), StridedView(B), StridedView(C) + +plan = plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false) + +# Optional hand-made M split (see probe_msplit_prototype.jl): `--msplit-pos P +# --msplit-li L` splits A's fastest M axis into (L, rest) and moves the inner +# part to position P of the M enumeration. +const MSPLIT_POS = argopt("msplit-pos", 0) +const MSPLIT_LI = argopt("msplit-li", 8) +if MSPLIT_POS > 0 + using QuasiStrided: AxisGroup, ContractPlan + g = plan.mgroup + D = length(g.lengths) + du = argmin([g.lengths[d] > 1 ? abs(g.strides[1][d]) : typemax(Int) for d in 1:D]) + dims = [(g.lengths[d], ntuple(p -> g.strides[p][d], 2)) for d in 1:D] + inner = (MSPLIT_LI, dims[du][2]) + outer = (g.lengths[du] ÷ MSPLIT_LI, MSPLIT_LI .* dims[du][2]) + dims[du] = outer + insert!(dims, MSPLIT_POS, inner) + mg = AxisGroup(ntuple(i -> dims[i][1], D + 1), ntuple(p -> ntuple(i -> dims[i][2][p], D + 1), 2)) + global plan = ContractPlan( + plan.kernel, mg, plan.ngroup, plan.kgroup, plan.blocking, + plan.Astorage, plan.Abase, plan.Bstorage, plan.Bbase, plan.Cstorage, plan.Cbase, + plan.atransform, plan.btransform, plan.workspace + ) +end + +# --------------------------------------------------------------------------- +# Plan report +# --------------------------------------------------------------------------- +println("case = $CASE expr = $EXPR dim = $DIM T = $T") +println("A labels ", join(LA), " strides ", strides(Av)) +println("B labels ", join(LB), " strides ", strides(Bv)) +println("C labels ", join(LC), " strides ", strides(Cv)) +println("kernel = ", typeof(plan.kernel).name.name, " MR=", mr(plan.kernel), " NR=", nr(plan.kernel)) +println("blocking = ", plan.blocking) +println("swapped = ", plan.Astorage !== parent(Av)) +for (name, g, ops) in (("M", plan.mgroup, "(A,C)"), ("N", plan.ngroup, "(B,C)"), ("K", plan.kgroup, "(A,B)")) + println("$name group $ops: lengths=", g.lengths, " strides=", g.strides, " Q=", axis_length(g), " ramp=", affine_ramp(g)[1]) +end +flops = 2.0 * axis_length(plan.mgroup) * axis_length(plan.ngroup) * axis_length(plan.kgroup) +println("flops = ", flops, " bytes(A+B+C) = ", sizeof(A) + sizeof(B) + sizeof(C)) + +# --------------------------------------------------------------------------- +# Instrumented nest (copy of `_execute_nest!`, timers added, nothing else) +# --------------------------------------------------------------------------- +mutable struct StageTimes + packb::Int + packa::Int + kern::Int + total::Int +end + +function timed_nest!(plan, alphaT::T, betaT::T, st::StageTimes) where {T} + ws = plan.workspace + kernel = plan.kernel + MRk = mr(kernel) + NRk = nr(kernel) + Qm = axis_length(plan.mgroup) + Qn = axis_length(plan.ngroup) + Qk = axis_length(plan.kgroup) + mc_eff = plan.blocking.mc + kc_eff = plan.blocking.kc + nc_eff = plan.blocking.nc + MRp = packed_a_per_k(kernel) + NRp = packed_b_per_k(kernel) + atransform = plan.atransform + btransform = plan.btransform + (m_ramp, m_step) = affine_ramp(plan.mgroup) + (n_ramp, n_step) = affine_ramp(plan.ngroup) + (k_ramp, k_step) = affine_ramp(plan.kgroup) + lenA = length(plan.Astorage) + lenB = length(plan.Bstorage) + lenC = length(plan.Cstorage) + t_start = time_ns() + GC.@preserve ws begin + jc = 0 + while jc < Qn + nblock = min(nc_eff, Qn - jc) + n_slivers = cld(nblock, NRk) + (rng_nB, rng_nC) = if n_ramp + _ramp_slivers!(ws.n_desc_B, ws.n_desc_C, n_step[1], n_step[2], jc, nblock, NRk, n_slivers) + else + fill_offsets!((ws.n_buf_B, ws.n_buf_C), plan.ngroup, jc, nblock) + _classify_slivers!(ws.n_desc_B, ws.n_desc_C, ws.n_buf_B, ws.n_buf_C, nblock, NRk, n_slivers) + end + pc = 0 + firstpanel = true + while pc < Qk + kblock = min(kc_eff, Qk - pc) + dK_A, dK_B, rng_kA, rng_kB = if k_ramp + ( + _ramp_descriptor(k_step[1], pc, kblock), _ramp_descriptor(k_step[2], pc, kblock), + _ramp_offset_range(k_step[1], pc, kblock), _ramp_offset_range(k_step[2], pc, kblock), + ) + else + fill_offsets!((ws.k_buf_A, ws.k_buf_B), plan.kgroup, pc, kblock) + dA = describe_block(ws.k_buf_A, 0, kblock) + dB = describe_block(ws.k_buf_B, 0, kblock) + (dA, dB, descriptor_offset_range(dA, ws.k_buf_A, 0), descriptor_offset_range(dB, ws.k_buf_B, 0)) + end + colsA_k = _axis_of(dK_A, ws.k_buf_A, 0) + rowsB_k = _axis_of(dK_B, ws.k_buf_B, 0) + checked_span_bounds(plan.Bbase, rng_kB, rng_nB, lenB) + beta_eff = firstpanel ? betaT : one(T) + + t0 = time_ns() + for s in 0:(n_slivers - 1) + sfirst = s * NRk + colsB = _axis_of(ws.n_desc_B[s + 1], ws.n_buf_B, sfirst) + bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) + _pack_sliver!(unsafe_pack_b!, bpanel, plan.Bstorage, plan.Bbase, rowsB_k, colsB, kernel, btransform) + end + st.packb += time_ns() - t0 + + ic = 0 + while ic < Qm + mblock = min(mc_eff, Qm - ic) + m_slivers = cld(mblock, MRk) + (rng_mA, rng_mC) = if m_ramp + _ramp_slivers!(ws.m_desc_A, ws.m_desc_C, m_step[1], m_step[2], ic, mblock, MRk, m_slivers) + else + fill_offsets!((ws.m_buf_A, ws.m_buf_C), plan.mgroup, ic, mblock) + _classify_slivers!(ws.m_desc_A, ws.m_desc_C, ws.m_buf_A, ws.m_buf_C, mblock, MRk, m_slivers) + end + checked_span_bounds(plan.Abase, rng_mA, rng_kA, lenA) + checked_span_bounds(plan.Cbase, rng_mC, rng_nC, lenC) + + t0 = time_ns() + for r in 0:(m_slivers - 1) + rfirst = r * MRk + rowsA = _axis_of(ws.m_desc_A[r + 1], ws.m_buf_A, rfirst) + apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) + _pack_sliver!(unsafe_pack_a!, apanel, plan.Astorage, plan.Abase, rowsA, colsA_k, kernel, atransform) + end + t1 = time_ns() + st.packa += t1 - t0 + + for s in 0:(n_slivers - 1) + sfirst = s * NRk + colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst) + bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) + for r in 0:(m_slivers - 1) + rfirst = r * MRk + rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst) + apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) + unsafe_execute_micro_tile!(kernel, plan.Cstorage, plan.Cbase, rowsC, colsC, apanel, bpanel, kblock, alphaT, beta_eff) + end + end + st.kern += time_ns() - t1 + ic += mblock + end + firstpanel = false + pc += kblock + end + jc += nblock + end + end + st.total += time_ns() - t_start + return nothing +end + +# --------------------------------------------------------------------------- +# Timing +# --------------------------------------------------------------------------- +execute!(plan, one(T), zero(T)) # warm-up +Cref = copy(C) +t_exec = median_time_s(() -> execute!(plan, one(T), zero(T)); reps = REPS) + +st = StageTimes(0, 0, 0, 0) +timed_nest!(plan, one(T), zero(T), st) # warm-up +@assert C ≈ Cref +rows = StageTimes[] +for r in 1:REPS + s = StageTimes(0, 0, 0, 0) + timed_nest!(plan, one(T), zero(T), s) + push!(rows, s) +end +med(f) = median([f(s) for s in rows]) / 1.0e9 + +t_total = med(s -> s.total) +t_packb = med(s -> s.packb) +t_packa = med(s -> s.packa) +t_kern = med(s -> s.kern) +println() +@printf("execute! median %9.4f s %7.2f GF/s\n", t_exec, flops / t_exec / 1.0e9) +@printf("nest total %9.4f s %7.2f GF/s\n", t_total, flops / t_total / 1.0e9) +@printf(" pack B %9.4f s (%5.1f%%) %7.2f ns/elem of B read\n", t_packb, 100 * t_packb / t_total, 1.0e9 * t_packb / (length(B) * cld(axis_length(plan.mgroup), plan.blocking.mc) / cld(axis_length(plan.mgroup), plan.blocking.mc))) +@printf(" pack A %9.4f s (%5.1f%%) %7.2f ns/elem of A read (x%d N-blocks)\n", t_packa, 100 * t_packa / t_total, 1.0e9 * t_packa / (length(A) * cld(axis_length(plan.ngroup), plan.blocking.nc)), cld(axis_length(plan.ngroup), plan.blocking.nc)) +@printf(" kernel+store %9.4f s (%5.1f%%) %7.2f GF/s kernel-only\n", t_kern, 100 * t_kern / t_total, flops / t_kern / 1.0e9) +@printf(" other %9.4f s (%5.1f%%)\n", t_total - t_packb - t_packa - t_kern, 100 * (t_total - t_packb - t_packa - t_kern) / t_total) diff --git a/src/planning/labels.jl b/src/planning/labels.jl index 97b857b..bbc02e9 100644 --- a/src/planning/labels.jl +++ b/src/planning/labels.jl @@ -134,6 +134,87 @@ function _order_free_labels( return out end +# ---------------------------------------------------------------------------- +# Contracted-label order. `_classify_labels` lists the K labels in `indA` +# order, which is incidental to the memory walk: `_pack_panel!` steps through +# K in the composite's enumeration order (its OUTER loop, one K step per +# `mr`/`nr`-element sliver row), so the K order decides how BOTH packs walk +# their operand. When the K axes sit in a different relative order in A and B +# -- `C[a,e] = A[a,b,c,d] * B[d,c,b,e]`, the upstream suite's +# `contract_scrambled` layout -- no single order suits both, and `indA` order +# is the one that suits neither: B's unit-stride axis `d` becomes the SLOWEST K +# coordinate, so `pack_b!` jumps `96^2` elements per K step and touches a new +# cache line (and page) for every element it reads. Measured (dim 96, Float64, +# Cascade Lake, benchmark/probes/probe_stage_breakdown.jl): 49 ns per B +# element, 92% of a 4.6 s contraction, against 1.3 ns per A element of the +# same case. +# ---------------------------------------------------------------------------- + +# Smallest `abs(stride)` among the K labels' non-singleton axes in one operand +# (`typemax(Int)` when every K axis is a singleton, i.e. K does not walk that +# operand at all). +function _min_label_stride( + labels::Vector{Int}, ind::NTuple{N, Int}, v::StridedView + ) where {N} + st = Base.strides(v) + best = typemax(Int) + for l in labels + p = findfirst(==(l), ind)::Int + size(v, p) == 1 && continue + best = min(best, abs(st[p])) + end + return best +end + +# Stable sort of `labels` by `abs(stride)` of each label's axis in `v` (the +# same insertion sort as `_order_free_labels`, over a different operand). +function _sort_labels_by_stride( + labels::Vector{Int}, ind::NTuple{N, Int}, v::StridedView + ) where {N} + st = Base.strides(v) + key(l::Int) = abs(st[findfirst(==(l), ind)::Int]) + out = copy(labels) + @inbounds for i in 2:length(out) + x = out[i] + kx = key(x) + j = i - 1 + while j >= 1 && key(out[j]) > kx + out[j + 1] = out[j] + j -= 1 + end + out[j + 1] = x + end + return out +end + +# The K order: sort by `abs(stride)` in the operand in which K is the more +# "inner" group, i.e. has the smaller minimum K stride. That operand's pack has +# no contiguous free axis to hide behind (its sliver rows are far apart in +# memory), so its line locality has to come from consecutive K steps landing +# in the same line; the other operand's free axis supplies the locality +# instead (an `mr`-row unit-stride sliver per K step is already whole cache +# lines, whatever the K stride). Ties -- K inner in both, `A[k..,m] * B[k..,n]` +# with the K axes in different orders -- go to the operand with more elements +# to read (the larger free extent), then to A. Stable, so labels with equal +# strides keep `indA` order, and any layout whose K strides are already +# ascending in the chosen operand (every gemm_ready/a_permuted/b_permuted +# layout of the upstream suite) gets exactly the order it had before this rule +# existed. The order is a property of the pair (A, B) and is the same whether +# or not the M/N orientation is later swapped. +function _order_contract_labels( + klabels::Vector{Int}, + indA::NTuple{NA, Int}, A::StridedView, + indB::NTuple{NB, Int}, B::StridedView, + Qm::Int, Qn::Int + ) where {NA, NB} + length(klabels) <= 1 && return klabels + minA = _min_label_stride(klabels, indA, A) + minB = _min_label_stride(klabels, indB, B) + by_b = minB < minA || (minB == minA && Qn > Qm) + return by_b ? _sort_labels_by_stride(klabels, indB, B) : + _sort_labels_by_stride(klabels, indA, A) +end + # Element count of the leading unit-stride run when `labels` (already ordered # by `_order_free_labels`) is enumerated first-label-fastest into C: the first # non-singleton label must have C-stride exactly +1 (`_unit_stride_rows` is diff --git a/src/planning/plan.jl b/src/planning/plan.jl index 34edea2..888593a 100644 --- a/src/planning/plan.jl +++ b/src/planning/plan.jl @@ -81,7 +81,10 @@ labels inside the M composite (A's free labels) and the N composite (B's free labels) are each stable-sorted by `abs(stride)` of the label's axis *in `C`*, ascending, ties keeping the operand's own axis order -- so each composite is enumerated with `C`'s fastest axis fastest, whatever A's or B's layout is. The -K composite keeps `indA` order. Then, if the sorted M list does *not* begin +K composite is stable-sorted by `abs(stride)` in whichever of `A`/`B` has the +smaller minimum K stride (ties: the operand with the larger free extent, then +`A`), so the operand in which K is the inner group walks it contiguously (see +`_order_contract_labels`). Then, if the sorted M list does *not* begin with a unit-stride run of at least `mr(kernel)` elements in `C` while the sorted N list does, the operand roles are swapped: `B` feeds M and `A` feeds N, and the K maps, the storage/base fields and the conjugation transforms move with @@ -169,10 +172,15 @@ function plan_contract( mgroup = _build_pair_group(morder, indA, A, indC, C) # maps: (A, C) ngroup = _build_pair_group(norder, indB, B, indC, C) # maps: (B, C) - kgroup = _build_pair_group(klabels, indA, A, indB, B) # maps: (A, B) Qm = axis_length(mgroup) Qn = axis_length(ngroup) + + # The K order is decided by A's and B's layouts (`_order_contract_labels`, + # src/planning/labels.jl); it needs the free extents only as a tie-break. + korder = _order_contract_labels(klabels, indA, A, indB, B, Qm, Qn) + kgroup = _build_pair_group(korder, indA, A, indB, B) # maps: (A, B) + Qk = axis_length(kgroup) # Resolved here, not in the signature default: the demotion needs Qm, and @@ -192,7 +200,7 @@ function plan_contract( # its A/B arguments, and the packing transforms. The contraction is # unchanged: `*` commutes on `T` and `conj` is elementwise, so # `sum_k conj?(B[n,k]) * conj?(A[m,k])` is the same sum. - kgroup_swapped = _build_pair_group(klabels, indB, B, indA, A) # maps: (B, A) + kgroup_swapped = _build_pair_group(korder, indB, B, indA, A) # maps: (B, A) # Run-length demotion (see `_demote_for_run`): only for an auto-selected # kernel, keyed on the CHOSEN (post-swap) M orientation, i.e. N's own # run against the kernel it would actually run. diff --git a/test/planning/test_plan_contract.jl b/test/planning/test_plan_contract.jl index cb1bea2..bb4d95b 100644 --- a/test/planning/test_plan_contract.jl +++ b/test/planning/test_plan_contract.jl @@ -813,3 +813,119 @@ end @test isapprox(C, Cref; rtol = 200 * d * eps(real(T))) end end + +# ===================================================================== +# Contracted-label order (`_order_contract_labels`) +# ===================================================================== + +# Labels: a=1 b=2 c=3 d=4 e=5, distinct extents so a wrong order shows up as a +# wrong stride, not a coincidentally equal one. +const _KO_EXT = Dict(1 => 7, 2 => 3, 3 => 4, 4 => 5, 5 => 6) +_ko_array(::Type{T}, ind) where {T} = randn(T, Tuple(_ko_EXT_get(l) for l in ind)...) +_ko_EXT_get(l) = _KO_EXT[l] +_ko_stride(v, ind, l) = Base.strides(v)[findfirst(==(l), ind)] + +@testset "K order: sorted by stride in the operand where K is the inner group" begin + T = Float64 + A = _ko_array(T, (1, 2, 3, 4)) # A[a,b,c,d]: M = a unit-stride, K = (b,c,d) ascending + Av = StridedView(A) + C = zeros(T, _KO_EXT[1], _KO_EXT[5]) # C[a,e] + Cv = StridedView(C) + + # contract_scrambled: B[d,c,b,e]. K is B's inner group (min stride 1 < A's + # min K stride 7), so the order follows B: (d, c, b). + Bs = StridedView(_ko_array(T, (4, 3, 2, 5))) + plan = plan_contract(Cv, Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), (1, 5)) + @test plan.kgroup.lengths == (5, 4, 3) + @test plan.kgroup.strides[1] == Tuple(_ko_stride(Av, (1, 2, 3, 4), l) for l in (4, 3, 2)) + @test plan.kgroup.strides[2] == (1, 5, 20) + @test issorted(abs.(plan.kgroup.strides[2])) + + # gemm_ready B[b,c,d,e] and b_permuted B[b,e,c,d]: B is still the inner + # operand and its K strides are already ascending in indA order, so the + # order is exactly the pre-rule (indA) order. + for indB in ((2, 3, 4, 5), (2, 5, 3, 4)) + Bv = StridedView(_ko_array(T, indB)) + p = plan_contract(Cv, Av, (1, 2, 3, 4), Bv, indB, (1, 5)) + @test p.kgroup.lengths == (3, 4, 5) + @test p.kgroup.strides[1] == (7, 21, 84) + @test p.kgroup.strides[2] == Tuple(_ko_stride(Bv, indB, l) for l in (2, 3, 4)) + end + + # A transposed as well (A[d,c,b,a]): K is inner in BOTH, min strides tie + # at 1, extents tie-break (Qm = 7 > Qn = 6) -> A's order (d, c, b). + At = StridedView(_ko_array(T, (4, 3, 2, 1))) + Bg = StridedView(_ko_array(T, (2, 3, 4, 5))) + p = plan_contract(Cv, At, (4, 3, 2, 1), Bg, (2, 3, 4, 5), (1, 5)) + @test p.kgroup.lengths == (5, 4, 3) + @test p.kgroup.strides[1] == (1, 5, 20) + @test p.kgroup.strides[2] == (12, 3, 1) + # ... and B's order (b, c, d) once N is the larger free extent. + Bbig = StridedView(randn(T, 3, 4, 5, 9)) + Cbig = StridedView(zeros(T, 7, 9)) + p = plan_contract(Cbig, At, (4, 3, 2, 1), Bbig, (2, 3, 4, 5), (1, 5)) + @test p.kgroup.lengths == (3, 4, 5) + @test p.kgroup.strides[1] == (20, 5, 1) + @test p.kgroup.strides[2] == (1, 3, 12) + + # A singleton K axis does not count as "inner": A[a,b,c,d] with b of + # length 1 at unit stride in B (B[b,d,c,e]) -- B's smallest NON-singleton + # K stride is 1 (d) anyway here, so the order follows B, but the singleton + # `b` is placed by its (irrelevant) stride and never decides the operand. + A1 = StridedView(randn(T, 7, 1, 4, 5)) + B1 = StridedView(randn(T, 1, 5, 4, 6)) + p = plan_contract(Cv, A1, (1, 2, 3, 4), B1, (2, 4, 3, 5), (1, 5)) + @test p.kgroup.lengths[findfirst(==(1), p.kgroup.lengths)] == 1 + @test p.kgroup.strides[2][end] == 5 || p.kgroup.strides[2][end] == 1 # d or the singleton b last + @test issorted(abs.(p.kgroup.strides[2])) + + # Single K label: nothing to order. + p = plan_contract(StridedView(zeros(T, 3, 5)), StridedView(randn(T, 3, 4)), (1, 2), StridedView(randn(T, 4, 5)), (2, 3), (1, 3)) + @test p.kgroup.strides == ((3,), (1,)) + + # The swapped orientation (B feeds M) uses the same K order with the maps + # exchanged: C stored as (e, a) with an N run of 8 >= mr = 8 swaps. + Ct = zeros(T, 8, 7) # C[e,a] physically; labels (5, 1) + B8 = StridedView(randn(T, 5, 4, 3, 8)) # B[d,c,b,e], e = 8 + kernel = SIMDKernel(Val(8), Val(6), T) + pt = plan_contract(StridedView(Ct), Av, (1, 2, 3, 4), B8, (4, 3, 2, 5), (5, 1); kernel = kernel) + @test pt.Astorage === parent(B8) + @test pt.kgroup.lengths == (5, 4, 3) + @test pt.kgroup.strides[1] == (1, 5, 20) # B's map first + @test pt.kgroup.strides[2] == (84, 21, 7) # then A's +end + +@testset "K order: correctness on scrambled K (all dtypes, alpha/beta, conj, both orientations, oracle)" begin + Random.seed!(0x5C7A_0B1E) + for T in (Float64, Float32, ComplexF64, ComplexF32) + rtol = 500 * eps(real(T)) + alpha = T <: Complex ? T(1.3, -0.4) : T(1.3) + beta = T <: Complex ? T(0.7, 0.2) : T(0.7) + for (indA, indB, indC) in ( + ((1, 2, 3, 4), (4, 3, 2, 5), (1, 5)), # scrambled, C[a,e] + ((1, 2, 3, 4), (4, 3, 2, 5), (5, 1)), # scrambled, C[e,a] (swap for real T) + ((4, 3, 2, 1), (2, 3, 4, 5), (1, 5)), # A transposed, K inner in both + ((4, 3, 2, 1), (3, 2, 4, 5), (5, 1)), # both scrambled, C transposed + ), (conjA, conjB) in ((false, false), (true, false), (true, true)) + (T <: Real) && conjA && continue + A = _ko_array(T, indA) + B = _ko_array(T, indB) + Av = conjA ? StridedView(A, size(A), strides(A), 0, conj) : StridedView(A) + Bv = StridedView(B) + C = randn(T, Tuple(_KO_EXT[l] for l in indC)...) + Cstart = copy(C) + Cv = StridedView(C) + Cref = _lo_reference(Cstart, Av, indA, Bv, indB, indC; conjA, conjB, alpha, beta) + plan = plan_contract(Cv, Av, indA, Bv, indB, indC; conjA = conjA, conjB = conjB) + # The whole point: the K composite is ascending in the inner operand. + inner = plan.kgroup.strides[argmin(map(s -> minimum(abs, s), plan.kgroup.strides))] + @test issorted(abs.(inner)) + execute!(plan, alpha, beta) + @test isapprox(C, Cref; rtol = rtol) + copyto!(C, Cstart) + plan_tw = plan_contract(Cv, Av, indA, Bv, indB, indC; conjA = conjA, conjB = conjB) + execute_tilewise!(plan_tw, alpha, beta) + @test isapprox(C, Cref; rtol = rtol) + end + end +end From d24a24488cdda533ae2ab8d1d87159521a4f2eb1 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 14:18:24 -0400 Subject: [PATCH 02/53] Add the node verification job for the K-order rule and a --shape probe option benchmark/submit_scrambled_verify.sh: one job per node class (icelake, genoa, rome) running the bench_to_suite.jl subset the rule touches (every synthetic layout at dims 32 and 96, TCCG at 8/16/24) and, for the Rome-only intensli_7_dim16 slowdown (18x slower than intensli_6_dim16 on znver2, 2.6x on znver4, absent on Ice Lake and Cascade Lake), the stage breakdown of intensli_7/intensli_6 at dims 16 and 8 as planned and under two hand-made M enumeration orders. probe_stage_breakdown.jl gains `--shape MR,NR,W` to plan with another ISA's menu shape (e.g. the AVX2 Float64 tile 8,6,4) on this host. Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_stage_breakdown.jl | 12 +++- benchmark/submit_scrambled_verify.sh | 78 +++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100755 benchmark/submit_scrambled_verify.sh diff --git a/benchmark/probes/probe_stage_breakdown.jl b/benchmark/probes/probe_stage_breakdown.jl index 1474619..20c7814 100644 --- a/benchmark/probes/probe_stage_breakdown.jl +++ b/benchmark/probes/probe_stage_breakdown.jl @@ -68,7 +68,17 @@ B = randn(rng, T, ntuple(_ -> DIM, length(LB))...) C = zeros(T, ntuple(_ -> DIM, length(LC))...) Av, Bv, Cv = StridedView(A), StridedView(B), StridedView(C) -plan = plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false) +# `--shape MR,NR,W` names a menu kernel shape (e.g. `8,6,4`, the AVX2 Float64 +# shape) instead of the host default, to reason about another ISA's register +# tile on this host (its cache geometry is still this host's, of course). +const SHAPE = argval("shape") +using QuasiStrided: _kernel_from_shape +plan = if SHAPE === nothing + plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false) +else + shape = Tuple(parse.(Int, split(SHAPE, ',')))::NTuple{3, Int} + plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false, kernel = _kernel_from_shape(shape, T)) +end # Optional hand-made M split (see probe_msplit_prototype.jl): `--msplit-pos P # --msplit-li L` splits A's fastest M axis into (L, rest) and moves the inner diff --git a/benchmark/submit_scrambled_verify.sh b/benchmark/submit_scrambled_verify.sh new file mode 100755 index 0000000..cd61022 --- /dev/null +++ b/benchmark/submit_scrambled_verify.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Slurm job verifying branch perf/scrambled-strides on one node class, and +# collecting the stage breakdown of the Rome-only intensli_7_dim16 slowdown. +# Single core, plain sbatch (see submit_bench_to_suite.sh). From the repo +# root of the branch's checkout: +# +# sbatch --reservation=rocky8 --constraint=icelake benchmark/submit_scrambled_verify.sh +# sbatch --reservation=rocky8 --constraint=genoa benchmark/submit_scrambled_verify.sh +# sbatch --reservation=rocky8 --constraint=rome benchmark/submit_scrambled_verify.sh +# +# Part 1: bench_to_suite.jl on the subset the K-order rule touches -- every +# synthetic layout at leg dims 32 and 96 (dim{32,96}_1_3_1_contract_scrambled +# are the targets; gemm_ready/a_permuted/b_permuted/both_permuted plan the same +# K group as before and are the regression guard) plus TCCG at 8/16/24 +# (single-K-label cases: unchanged plans, regression guard; includes +# intensli_7_dim16). Compare against benchmark/results/to_suite-71120{88,89,90}-*. +# +# Part 2: benchmark/probes/probe_stage_breakdown.jl for intensli_7 and +# intensli_6 at dim 16 and 8 (Float64): pack B / pack A / kernel+store split, +# as planned (as-is) and with two hand-made M enumeration orders that move one +# cache line of A's unit-stride axis forward (`--msplit-pos 2/1 --msplit-li 8`). +# On Cascade Lake all of these run at 3-4 ns per A element; on Rome the suite +# has intensli_7_dim16 18x slower than intensli_6_dim16, and this says which +# stage and whether the enumeration order is the lever. +#SBATCH --job-name=qs-scrambled-verify +#SBATCH --partition=ccq +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=32G +#SBATCH --time=03:00:00 +#SBATCH --output=benchmark/results/slurm-%j.out + +set -euo pipefail +cd "${SLURM_SUBMIT_DIR:?SLURM_SUBMIT_DIR not set -- run this script via sbatch, not directly}" + +module load modules/2.5-beta1 2>/dev/null || true +module load julia/1.12.6 2>/dev/null || true +MODJULIA=$(command -v julia 2>/dev/null || true) +case "$MODJULIA" in /mnt/sw/*) ;; *) MODJULIA="" ;; esac # only a module-provided one +JULIA=${QS_JULIA:-${MODJULIA:-$(ls -d "$HOME"/.julia/juliaup/julia-1.12*/bin/julia 2>/dev/null | tail -1)}} +[ -x "${JULIA:-}" ] || { echo "no julia found; set QS_JULIA" >&2; exit 127; } +echo "julia = $JULIA" +echo "git = $(git rev-parse --short HEAD 2>/dev/null || echo '?') on $(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo '?')" + +export JULIA_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 +"$JULIA" --project=benchmark -e 'using Pkg; Pkg.instantiate()' + +OUTDIR="benchmark/results/scrambled-verify-${SLURM_JOB_ID}-$(hostname -s)" +mkdir -p "$OUTDIR" + +echo "=== part 1: suite subset ===" +"$JULIA" --project=benchmark benchmark/bench_to_suite.jl \ + --categories contract \ + --sources synthetic,tccg \ + --dtypes Float64,ComplexF64 \ + --synthetic-sizes 32,96 \ + --tccg-sizes 8,16,24 \ + --reps 21 --time-budget 5 \ + --outdir "$OUTDIR" + +echo "=== part 2: intensli stage breakdown (Float64) ===" +for dim in 16 8; do + for case in intensli_7 intensli_6; do + for split in "0 8" "2 8" "1 8"; do + set -- $split + echo "--- $case dim=$dim msplit-pos=$1 li=$2" + "$JULIA" --project=benchmark benchmark/probes/probe_stage_breakdown.jl \ + --case "$case" --dim "$dim" --dtype Float64 --reps 21 \ + --msplit-pos "$1" --msplit-li "$2" 2>&1 | grep -v "^WARNING" + done + done +done | tee "$OUTDIR/intensli_stage_breakdown.txt" + +echo "=== node check ===" +echo "SLURM_JOB_NODELIST = ${SLURM_JOB_NODELIST:-?}" +lscpu | grep -i "model name\|cache" || true +cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true From 30f8fe39488ca79446369d0066877d05cef9ba59 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 14:34:02 -0400 Subject: [PATCH 03/53] Real AVX-512 register tile MV = 4: (32,6,8) Float64 / (64,6,16) Float32 The large-GEMM plateau (QS 65-69 GF/s vs OpenBLAS ~101 at 3969^3 Float64 on Cascade Lake and Ice Lake; parity on Zen4/Zen2) is the (16,6,8) microkernel being FRONT-END bound on Intel: its K step is 2 A loads + 6 vbroadcastsd + 12 register FMAs + 5 loop-overhead instructions = 25 fused-domain uops per 6 FMA-cycles, above Skylake's 4-wide allocation. Measured with perf stat on L1-resident packed panels: 2.98 IPC, 23.8 flops/cycle (74% of 32). Doubling the A vectors per column halves the broadcasts and loop overhead per FMA: (32,6,8) is 38 instructions per 12 FMA-cycles and measures 29.1 flops/cycle (91%; OpenBLAS dgemm on the same core: 28.7). Zen4 is not affected because its 512-bit FMAs are double-pumped (12 cycles per K step, ~2 uops/cycle) and it dispatches 6-wide. - `_rule_mv`: MV = 4 for the real method on :avx512, 2 everywhere else (24 accumulators + 4 A + 2 scratch = 30 of 32 registers; AVX2's 16 hold exactly the MV = 2 tile). Complex methods keep MV = 2. - Menus gain (32,6,8) / (64,6,16); the MV = 2 shapes stay as the short-M step-down and `_demote_for_run` targets. - `_extent_shape`: the MV = 4 shape steps down to its MV = 2 sibling below one tall tile, and below two when the half tile pads strictly less (Qm in (MR, 3MR/2]); a padding comparison, not a throughput model. Engine, 3969^3 Float64, ccqlin038 (interleaved, same process): (16,6,8) 73.8 -> (32,6,8) 87.1 GF/s, OpenBLAS 105.7. bench_to_suite subset: dim63_2_2_2 all five layouts 1.22-1.31x, dim32_2_2_2 1.26-1.32x, dim24_2_2_2 1.14-1.30x; ComplexF64 unchanged (0.94-1.02x); no correctness mismatches; per execute! allocation unchanged (336 B, size-independent). benchmark/bench_large_gemm_peak.jl + submit_large_gemm_peak.sh: the cross-node validation (kernel-only ceiling per menu shape, engine per shape vs OpenBLAS, the step-down boundary, an (mc, kc) arm) for icelake / genoa / rome, plus the suite subset. The job also prints its CPU confinement and BLAS thread count before timing. Co-Authored-By: Claude Opus 5.5 --- benchmark/bench_large_gemm_peak.jl | 225 +++++++++++++++++++++++++ benchmark/submit_large_gemm_peak.sh | 80 +++++++++ src/planning/kernel_selection.jl | 110 ++++++++++-- test/planning/test_kernel_selection.jl | 73 +++++++- 4 files changed, 462 insertions(+), 26 deletions(-) create mode 100644 benchmark/bench_large_gemm_peak.jl create mode 100644 benchmark/submit_large_gemm_peak.sh diff --git a/benchmark/bench_large_gemm_peak.jl b/benchmark/bench_large_gemm_peak.jl new file mode 100644 index 0000000..46de611 --- /dev/null +++ b/benchmark/bench_large_gemm_peak.jl @@ -0,0 +1,225 @@ +# Large-GEMM peak: the engine's default real kernel against every real +# SIMDKernel menu shape and against OpenBLAS, on the compute-bound regime the +# 2026-09-26 AVX-512 register-tile change targets (`_rule_mv`, +# src/planning/kernel_selection.jl). +# +# julia --project=benchmark benchmark/bench_large_gemm_peak.jl [--outdir DIR] [--sizes 2048,3969] [--rounds 3] +# +# Four arms, Float64 and Float32, single core, interleaved rounds (median): +# +# kernel kernel-only ceiling per menu shape: `unsafe_execute_tile!` on one +# L1-resident packed A sliver and B sliver at the model's `kc`, +# i.e. flops/cycle of the register tile with nothing around it. +# engine `plan_contract`/`execute!` at n^3 for every menu shape (caller-named +# kernel, the engine's own blocking) and for the DEFAULT kernel, plus +# OpenBLAS `mul!` on the same operands. +# stepdown the short-M boundary of `_extent_shape`: M in a sweep around one +# and two tall tiles at K = N = 1024, default kernel vs the tall and +# half-height shapes named explicitly. +# blocking the default kernel at n^3 over a small (mc, kc) grid around the +# model row (an L1/2 vs 3L1/4 `kc` check; the model's `nc`). +# +# Writes bench_large_gemm_peak.csv and summary_large_gemm_peak.txt to --outdir +# (default benchmark/results/-). + +include(joinpath(@__DIR__, "harness.jl")) + +using QuasiStrided: SIMDKernel, plan_contract, execute!, default_blocking, target_profile, + _derived_shape, _default_kernel, kernel_shapes, packed_panel, unsafe_execute_tile!, + DestinationTile, AffineAxis, _modelled_blocking + +const OUTDIR = let d = argopt("outdir", "") + isempty(d) ? results_dir() : d +end +mkpath(OUTDIR) +const SIZES = parse_ints(argopt("sizes", "2048,3969")) +const ROUNDS = argopt("rounds", 3) +const REAL_DTYPES = (Float64, Float32) + +const CSV_PATH = joinpath(OUTDIR, "bench_large_gemm_peak.csv") +const SUMMARY_PATH = joinpath(OUTDIR, "summary_large_gemm_peak.txt") + +csv_io = open(CSV_PATH, "w") +println(csv_io, "arm,dtype,case,shape,M,K,N,mc,kc,nc,round,seconds,gflops") +function log_row(arm, T, case, shape, M, K, N, blk, round, t) + gf = 2.0 * M * K * N / t / 1.0e9 + println(csv_io, "$arm,$T,$case,$(shape),$M,$K,$N,$(blk === nothing ? "" : blk.mc),", + "$(blk === nothing ? "" : blk.kc),$(blk === nothing ? "" : blk.nc),$round,", + @sprintf("%.9f", t), ",", @sprintf("%.3f", gf)) + flush(csv_io) + return gf +end + +shapestr(sh) = "$(sh[1])x$(sh[2])/W$(sh[3])" +shape_of(k) = (mr(k), nr(k), lanewidth(k)) + +rows = NamedTuple[] +function record!(arm, T, case, shape, M, K, N, blk, round, t) + gf = log_row(arm, T, case, shape, M, K, N, blk, round, t) + push!(rows, (arm = arm, dtype = T, case = case, shape = shape, gf = gf)) + return nothing +end + +println("bench_large_gemm_peak.jl cpu = ", Sys.CPU_NAME, " profile = ", target_profile()) +for T in REAL_DTYPES + println(" ", T, ": derived shape ", _derived_shape(target_profile(), T), " default kernel ", + shapestr(shape_of(_default_kernel(T))), " blocking ", default_blocking(_default_kernel(T))) +end + +# --------------------------------------------------------------------------- +# kernel arm +# --------------------------------------------------------------------------- + +function kernel_hot!(kernel, C, apack, bpack, kc, reps) + MR = mr(kernel); NR = nr(kernel) + GC.@preserve apack bpack begin + ap = packed_panel(apack, 1, length(apack)); bp = packed_panel(bpack, 1, length(bpack)) + dest = DestinationTile(C, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) + for _ in 1:reps + unsafe_execute_tile!(kernel, dest, ap, bp, kc, one(eltype(C)), one(eltype(C))) + end + end + return nothing +end + +for T in REAL_DTYPES + fixtures = [] + for sh in kernel_shapes(T) + MR, NR, W = sh + kernel = SIMDKernel(Val(MR), Val(NR), T, Val(W)) + kc = default_blocking(kernel).kc + apack = rand(T, MR * kc); bpack = rand(T, NR * kc); C = zeros(T, MR * NR) + kernel_hot!(kernel, C, apack, bpack, kc, 10) + t1 = @elapsed kernel_hot!(kernel, C, apack, bpack, kc, 1000) + reps = max(100, round(Int, 0.5 / t1 * 1000)) + push!(fixtures, (sh, kernel, C, apack, bpack, kc, reps)) + end + for r in 1:ROUNDS, (sh, kernel, C, apack, bpack, kc, reps) in fixtures + t = @elapsed kernel_hot!(kernel, C, apack, bpack, kc, reps) + record!("kernel", T, "l1_panels", shapestr(sh), sh[1], kc * reps, sh[2], nothing, r, t) + end +end +println("kernel arm done") + +# --------------------------------------------------------------------------- +# engine arm +# --------------------------------------------------------------------------- + +for T in REAL_DTYPES, n in SIZES + rng = MersenneTwister(0x1a2b + n) + A = randn(rng, T, n, n); B = randn(rng, T, n, n); C = zeros(T, n, n) + plans = [] + for sh in kernel_shapes(T) + kernel = SIMDKernel(Val(sh[1]), Val(sh[2]), T, Val(sh[3])) + plan = plan_contract(StridedView(C), StridedView(A), (1, 2), StridedView(B), (2, 3), (1, 3); kernel = kernel) + execute!(plan, one(T), zero(T)) + push!(plans, (shapestr(sh), plan)) + end + pdef = plan_contract(StridedView(C), StridedView(A), (1, 2), StridedView(B), (2, 3), (1, 3)) + execute!(pdef, one(T), zero(T)) + mul!(C, A, B) + for r in 1:ROUNDS + for (name, plan) in plans + t = @elapsed execute!(plan, one(T), zero(T)) + record!("engine", T, "n$n", name, n, n, n, plan.blocking, r, t) + end + t = @elapsed execute!(pdef, one(T), zero(T)) + record!("engine", T, "n$n", "default:" * shapestr(shape_of(pdef.kernel)), n, n, n, pdef.blocking, r, t) + t = @elapsed mul!(C, A, B) + record!("engine", T, "n$n", "openblas", n, n, n, nothing, r, t) + end + @info "engine arm" T n +end + +# --------------------------------------------------------------------------- +# stepdown arm: M around one and two tall tiles, K = N = 1024 +# --------------------------------------------------------------------------- + +for T in REAL_DTYPES + tall = _derived_shape(target_profile(), T) + MR, NR, W = tall + half = (MR ÷ 2, NR, W) + half in kernel_shapes(T) || continue + K = N = 1024 + Ms = unique([MR ÷ 2, MR - 1, MR, MR + 1, MR + W, MR + 2W, MR + 2W + 1, 2MR - 1, 2MR, 2MR + W, 3MR + W]) + for M in Ms + rng = MersenneTwister(0x5d + M) + A = randn(rng, T, M, K); B = randn(rng, T, K, N); C = zeros(T, M, N) + plans = [] + for (label, sh) in (("tall", tall), ("half", half)) + kernel = SIMDKernel(Val(sh[1]), Val(sh[2]), T, Val(sh[3])) + plan = plan_contract(StridedView(C), StridedView(A), (1, 2), StridedView(B), (2, 3), (1, 3); kernel = kernel) + execute!(plan, one(T), zero(T)) + push!(plans, (label * ":" * shapestr(sh), plan)) + end + pdef = plan_contract(StridedView(C), StridedView(A), (1, 2), StridedView(B), (2, 3), (1, 3)) + execute!(pdef, one(T), zero(T)) + push!(plans, ("default:" * shapestr(shape_of(pdef.kernel)), pdef)) + reps = max(3, round(Int, 0.2 / (2.0 * M * K * N / 30.0e9))) + for r in 1:ROUNDS, (name, plan) in plans + t = @elapsed for _ in 1:reps; execute!(plan, one(T), zero(T)); end + record!("stepdown", T, "M$M", name, M, K * reps, N, plan.blocking, r, t) + end + end + @info "stepdown arm" T +end + +# --------------------------------------------------------------------------- +# blocking arm: default kernel, (mc, kc) grid around the model row +# --------------------------------------------------------------------------- + +for T in REAL_DTYPES + n = maximum(SIZES) + rng = MersenneTwister(0x77) + A = randn(rng, T, n, n); B = randn(rng, T, n, n); C = zeros(T, n, n) + kernel = _default_kernel(T) + MR = mr(kernel) + model = default_blocking(kernel) + kc34 = max(1, (target_profile().l1d.bytes * 3 ÷ 4) ÷ (nr(kernel) * sizeof(T))) + grid = unique([ + (model.mc, model.kc), (model.mc ÷ 2 ÷ MR * MR, model.kc), (model.mc * 2, model.kc), + (max(MR, model.mc * model.kc ÷ kc34 ÷ MR * MR), kc34), (model.mc, kc34), + (max(MR, model.mc ÷ 2 ÷ MR * MR), 2 * model.kc), + ]) + plans = [] + for (mc, kc) in grid + (mc >= MR && kc >= 1) || continue + plan = plan_contract(StridedView(C), StridedView(A), (1, 2), StridedView(B), (2, 3), (1, 3); kernel = kernel, mc = mc, kc = kc, nc = model.nc) + execute!(plan, one(T), zero(T)) + push!(plans, ("mc$(mc)_kc$(kc)", plan)) + end + for r in 1:ROUNDS, (name, plan) in plans + t = @elapsed execute!(plan, one(T), zero(T)) + record!("blocking", T, "n$n", name, n, n, n, plan.blocking, r, t) + end + @info "blocking arm" T +end +close(csv_io) + +# --------------------------------------------------------------------------- +# summary: medians per (arm, dtype, case, shape) +# --------------------------------------------------------------------------- + +open(SUMMARY_PATH, "w") do io + print_env_header(io, "bench_large_gemm_peak.jl") + println(io, "profile = ", target_profile()) + for T in REAL_DTYPES + println(io, T, ": derived shape ", _derived_shape(target_profile(), T), + " default kernel ", shapestr(shape_of(_default_kernel(T))), + " blocking ", default_blocking(_default_kernel(T))) + end + keys_ = unique([(r.arm, r.dtype, r.case, r.shape) for r in rows]) + lastarm = "" + for key in keys_ + arm, T, case, shape = key + if arm != lastarm + println(io, "\n== ", arm, " (median GF/s over ", ROUNDS, " rounds; min..max) ==") + lastarm = arm + end + v = [r.gf for r in rows if (r.arm, r.dtype, r.case, r.shape) == key] + println(io, " ", rpad(string(T), 8), rpad(case, 12), rpad(shape, 22), + @sprintf("%7.1f", median(v)), " (", @sprintf("%.1f", minimum(v)), "..", @sprintf("%.1f", maximum(v)), ")") + end +end +println(read(SUMMARY_PATH, String)) +println("Done. Results in ", OUTDIR) diff --git a/benchmark/submit_large_gemm_peak.sh b/benchmark/submit_large_gemm_peak.sh new file mode 100644 index 0000000..5de047e --- /dev/null +++ b/benchmark/submit_large_gemm_peak.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Slurm job validating the AVX-512 real register-tile change (`_rule_mv`, +# src/planning/kernel_selection.jl: MV = 4, i.e. (32,6,8) Float64 / (64,6,16) +# Float32, stepping down to the MV = 2 sibling for short M) on the other node +# classes. One job per microarchitecture, from the repo root: +# +# sbatch --reservation=rocky8 --constraint=icelake benchmark/submit_large_gemm_peak.sh +# sbatch --reservation=rocky8 --constraint=genoa benchmark/submit_large_gemm_peak.sh +# sbatch --reservation=rocky8 --constraint=rome benchmark/submit_large_gemm_peak.sh +# +# What it runs, single core, single thread: +# +# 1. benchmark/bench_large_gemm_peak.jl -- the engine's default kernel vs +# every real SIMDKernel menu shape vs OpenBLAS at large square GEMMs and +# at the short-M step-down boundary, Float64 and Float32, interleaved +# rounds; plus a kernel-only (L1-resident packed panels) ceiling per +# shape and an (mc, kc) arm for the shipped shape. On rome (AVX2, 16 +# registers) the derived shape is unchanged, (8,6,4)/(16,6,8); the run +# there is the regression check. +# 2. benchmark/bench_to_suite.jl on the large :contract/:network cases the +# change targets (synthetic 24/32/63, mps D = 256/512), Float64 and +# ComplexF64, so the before/after is on the same cases as the local +# ccqlin038 run. +# +# The job also prints the CPU set it was confined to and BLAS's thread count +# right before timing: the 2026-09-24 icelake run (worker6199) reported +# StridedBLAS rates up to 8x a single Ice Lake core's peak (797 GF/s at +# dim63_2_1_2, K = 63) while the QuasiStrided rows of the same run were +# ordinary, which is only possible if BLAS was not confined to one core there. +# +# See https://wiki.flatironinstitute.org/SCC/Software/Slurm. +#SBATCH --job-name=qs-large-gemm-peak +#SBATCH --partition=ccq +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=24G +#SBATCH --time=03:00:00 +#SBATCH --output=benchmark/results/slurm-%j.out + +set -euo pipefail +# Slurm runs a spooled copy of this script; SLURM_SUBMIT_DIR is the repo root. +cd "${SLURM_SUBMIT_DIR:?SLURM_SUBMIT_DIR not set -- run this script via sbatch, not directly}" + +# See submit_blocking_model.sh: on Rocky 8 nodes (--reservation=rocky8) the +# module system's binaries need a newer glibc than the node has, so fall back +# to $QS_JULIA or a juliaup-installed 1.12 on the shared home. +module load modules/2.5-beta1 2>/dev/null || true +module load julia/1.12.6 2>/dev/null || true +MODJULIA=$(command -v julia 2>/dev/null || true) +case "$MODJULIA" in /mnt/sw/*) ;; *) MODJULIA="" ;; esac # only a module-provided one +JULIA=${QS_JULIA:-${MODJULIA:-$(ls -d "$HOME"/.julia/juliaup/julia-1.12*/bin/julia 2>/dev/null | tail -1)}} +[ -x "${JULIA:-}" ] || { echo "no julia found; set QS_JULIA" >&2; exit 127; } +echo "julia = $JULIA" + +export JULIA_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 + +echo "=== confinement check ===" +echo "SLURM_JOB_NODELIST = ${SLURM_JOB_NODELIST:-?} SLURM_CPUS_ON_NODE = ${SLURM_CPUS_ON_NODE:-?} SLURM_CPUS_PER_TASK = ${SLURM_CPUS_PER_TASK:-?}" +echo "nproc (visible to this job) = $(nproc)" +taskset -cp $$ || true +lscpu | grep -i "model name\|^CPU(s)\|cache" || true + +"$JULIA" --project=benchmark -e 'using Pkg; Pkg.instantiate()' + +OUTDIR="benchmark/results/large-gemm-peak-${SLURM_JOB_ID}-$(hostname -s)" +mkdir -p "$OUTDIR" + +"$JULIA" --project=benchmark benchmark/bench_large_gemm_peak.jl --outdir "$OUTDIR" + +"$JULIA" --project=benchmark benchmark/bench_to_suite.jl \ + --categories contract,network \ + --sources synthetic --synthetic-sizes 24,32,63 \ + --topics mps --mps-bonddims 256,512 \ + --dtypes Float64,ComplexF64 \ + --reps 11 --time-budget 5 \ + --outdir "$OUTDIR/to_suite" + +echo "=== node check ===" +lscpu | grep -i "model name\|cache" || true diff --git a/src/planning/kernel_selection.jl b/src/planning/kernel_selection.jl index 5d4eeb4..4c57ea6 100644 --- a/src/planning/kernel_selection.jl +++ b/src/planning/kernel_selection.jl @@ -11,12 +11,16 @@ # * a kernel type (`_kernel_type`), built from a menu shape by # `_kernel_from_shape`; # * a shape resolved from the detected hardware (`_derived_shape`): an -# explicit override row, else the `MR = 2W` rule where it has been -# validated, else a conservative fitted shape (`_fitted_shape`); -# * two plan-time demotions: to the fitted shape when M cannot fill one -# register tile (`_default_kernel`; complex on `:avx512` goes to an -# FMAddSub shape instead, `_small_m_shape`), and to a menu shape that -# keeps every register sliver unit-stride in C (`_demote_for_run`). +# explicit override row, else the `MR = MV*W, NR = 6` rule where it has +# been validated (`MV = 4` for the real method on `:avx512`, `2` +# elsewhere; `_rule_mv`), else a conservative fitted shape +# (`_fitted_shape`); +# * three plan-time demotions: the real `MV = 4` shape steps down to its +# `MV = 2` sibling when M is short enough that the taller tile pads more +# (`_extent_shape`); to the fitted shape when M cannot fill one register +# tile (`_default_kernel`; complex on `:avx512` goes to an FMAddSub shape +# instead, `_small_m_shape`); and to a menu shape that keeps every +# register sliver unit-stride in C (`_demote_for_run`). # ---------------------------------------------------------------------------- # Methods, menus and kernel types @@ -57,8 +61,13 @@ _fallback_shape(::Type{T}) where {T} = (8, NR_DEFAULT, _default_lanewidth(real(T # ComplexF32's best 1m shape but does not beat planar there, so it stays # menu-only (reachable via an explicit `kernel = OneMMethod()`, never picked # automatically). -const KERNEL_SHAPES_F64 = ((8, 6, 4), (16, 6, 8)) -const KERNEL_SHAPES_F32 = ((8, 6, 8), (32, 6, 16), (16, 6, 8)) +# +# 2026-09-26: the `MV = 4` shapes `(32, 6, 8)` / `(64, 6, 16)` are the real +# AVX-512 rule shapes (see `_rule_mv`); the `MV = 2` shapes stay in the menus +# as their short-M step-down (`_extent_shape`) and as `_demote_for_run` +# targets. +const KERNEL_SHAPES_F64 = ((8, 6, 4), (16, 6, 8), (32, 6, 8)) +const KERNEL_SHAPES_F32 = ((8, 6, 8), (32, 6, 16), (16, 6, 8), (64, 6, 16)) const KERNEL_SHAPES_C64_PLANAR = ( (24, 3, 8), (16, 6, 8), (8, 8, 8), (4, 5, 4), (4, 6, 2), (2, 6, 2), ) @@ -227,7 +236,7 @@ _shape_override(::Val{:avx2}, ::Type{ComplexF32}, ::PlanarMethod) = (8, 5, 8) # complex `T`, so nothing selects 1m unless the caller names it. _shape_override(::Val{:avx2}, ::Type{ComplexF64}, ::OneMMethod) = (4, 6, 4) -# Where the `MR = 2W` rule is validated. Real: AVX-512 and AVX2 (on NEON it +# Where the `MR = MV*W` rule is validated. Real: AVX-512 and AVX2 (on NEON it # would pick MR = 4 on 128-bit lanes, not obviously better than the fallback). # Complex: AVX-512 only -- planar holds separate real and imaginary # accumulator planes, so on AVX2's 16 registers even `(MV, NR) = (1, 6)` @@ -237,9 +246,37 @@ _rule_applies(::Val{:avx512}, ::Union{PlanarMethod, OneMMethod}) = true _rule_applies(::Val{:avx2}, ::RealMethod) = true _rule_applies(::Val, ::ComplexMethod) = false -# `W` is one vector register's worth of REAL lanes, `MR = 2W`, `NR = NR_DEFAULT`. -_rule_shape(vb::Int, ::Type{T}) where {T} = - (2 * (vb ÷ sizeof(real(T))), NR_DEFAULT, vb ÷ sizeof(real(T))) +# `W` is one vector register's worth of REAL lanes, `MR = MV * W` with `MV` +# A vectors (and accumulator rows) per output column, `NR = NR_DEFAULT`. The +# default `MV = 2` is the rule as it was validated for every method on every +# ISA; `_derived_shape` passes `_rule_mv`. +_rule_shape(vb::Int, ::Type{T}, mv::Int = 2) where {T} = + (mv * (vb ÷ sizeof(real(T))), NR_DEFAULT, vb ÷ sizeof(real(T))) + +# A vectors per column in the rule shape. `4` for the real method on +# `:avx512`, `2` everywhere else. +# +# Why 4 on AVX-512 (measured 2026-09-26, ccqlin038 / Cascade Lake, Julia +# 1.12.7, `perf stat`): the `MV = 2` real kernel `(16, 6, 8)` is FRONT-END +# bound, not FMA bound. Its K step is 2 A loads + 6 `vbroadcastsd` + 12 +# register FMAs + 5 loop-overhead instructions = 25 fused-domain uops for 6 +# FMA-cycles, i.e. 4.2 uops/cycle against Skylake's 4-wide allocation; it +# runs at 2.98 IPC and 23.8 flops/cycle (74% of the 32-flop peak) even with +# both packed panels L1-resident. Doubling `MV` doubles the FMAs per B +# broadcast and per loop iteration: `(32, 6, 8)` is 38 instructions per 12 +# FMA-cycles and measures 29.1 flops/cycle (91%; OpenBLAS dgemm on the same +# core: 28.7). The register budget is what caps `MV` at 4: `MV*NR` +# accumulators + `MV` A vectors + 2 broadcast/scratch = 30 of AVX-512's 32 +# registers, while AVX2's 16 hold exactly the `MV = 2` tile (12 + 2 + 2). +# Float32 behaves identically (`(32, 6, 16)` 180 -> `(64, 6, 16)` 218 GF/s, +# kernel-only). Engine, 3969^3 Float64 (same session, interleaved): +# `(16, 6, 8)` 73.8 -> `(32, 6, 8)` 87.1 GF/s, OpenBLAS 105.7. +# +# Complex methods keep `MV = 2`: planar holds two accumulator planes (its +# `(16, 6, 8)` already spills, see `planar_register_pressure`), and 1m runs a +# real kernel of `2 MR` rows over its own menu. +_rule_mv(::Val{:avx512}, ::RealMethod) = 4 +_rule_mv(::Val, ::ComplexMethod) = 2 """ _derived_shape(profile::TargetProfile, T, method = _default_method(T)) -> (MR, NR, W) @@ -258,7 +295,7 @@ function _derived_shape(profile::TargetProfile, ::Type{T}, method) where {T} ovr === nothing || return ovr vb = profile.vector_bytes if _rule_applies(key, method) && vb > 0 && vb % sizeof(real(T)) == 0 - shape = _rule_shape(vb, T) + shape = _rule_shape(vb, T, _rule_mv(key, method)) shape in kernel_shapes(T, method) && return shape end return _fitted_shape(profile, T, method) @@ -335,18 +372,57 @@ _default_kernel(::Type{T}) where {T} = _kernel_for(target_profile(), T) # Extent-aware variant, used only when the caller did not name a kernel: a # contraction whose M extent cannot fill one register tile pads every -# micro-tile away, so demote to the fitted shape. Keys on padding waste, a -# countable quantity known at plan time, not on a cache estimate. +# micro-tile away, so demote to the fitted shape; before that, a real `MV = 4` +# shape steps down to its `MV = 2` sibling where the taller tile would pad +# more (`_extent_shape`). Both key on padding waste, a countable quantity +# known at plan time, not on a cache estimate. @noinline function _default_kernel(::Type{T}, Qm::Int, Qn::Int) where {T} profile = target_profile() - kernel = _kernel_for(profile, T) + method = _default_method(T) + kernel = _kernel_from_shape(_extent_shape(profile, T, method, Qm), T, method) (Qm > 0 && Qm < mr(kernel)) || return kernel small = _small_m_shape(Val(profile.isa), profile, T, Qm) small === nothing || return _kernel_from_shape(small, T, FMAddSubMethod()) - method = _default_method(T) return _kernel_from_shape(_fitted_shape(profile, T, method), T, method) end +""" + _extent_shape(profile, T, method, Qm) -> (MR, NR, W) + +The derived shape, or -- real method only, and only when the derived shape is +the `MV = 4` rule shape -- its `MV = 2` sibling `(MR ÷ 2, NR, W)` when `Qm` is +shorter than one tall tile, or shorter than two and padded to strictly fewer +rows by the half-height tile. Always a member of `kernel_shapes(T, method)`. + +The step-down rule is a padding comparison, not a throughput model: with +`MR = 32`, `Qm = 40` costs 64 padded rows against the half tile's 48 (1.33x +the work for a tile that is only ~1.2x faster per padded row, engine +measurement at large M), while `Qm = 56` pads to 64 either way and keeps the +tall tile. Above `2 MR` the tall tile's worst-case padding excess is `MR/2` +over `2 MR + 1` rows (< 1.2x) and it always wins, which is why the comparison +stops there. It never steps below `MV = 2`: an `MV = 1` real tile has half +the FMAs per broadcast and per loop iteration of the `MV = 2` tile and runs +at roughly half its rate, more than any padding it could save; `Qm < 2W` +still reaches the fitted shape through `_default_kernel`'s own demotion. +""" +_extent_shape(profile::TargetProfile, ::Type{T}, method, Qm::Int) where {T} = + _derived_shape(profile, T, method) + +function _extent_shape(profile::TargetProfile, ::Type{T}, method::RealMethod, Qm::Int) where {T} + shape = _derived_shape(profile, T, method) + MR, NR, W = shape + (Qm > 0 && MR == 4 * W && Qm < 2 * MR) || return shape + half = (2 * W, NR, W) + half in kernel_shapes(T, method) || return shape + # Below one tall tile the half tile is taken unconditionally, padding tie + # or not: keeping the tall shape there would hand `_default_kernel`'s + # `Qm < mr` demotion the FITTED shape, two steps down instead of one + # (measured at `Qm = 63`, Float32, K = N = 1024: fitted 58.6 GF/s against + # the half tile's 94.6 and the tall tile's 109.3). + Qm < MR && return half + return cld(Qm, half[1]) * half[1] < cld(Qm, MR) * MR ? half : shape +end + # Small-M demotion target for complex `T` on AVX-512: the native-width # (`W == lanes`) FMAddSub menu shape that pads `Qm` least (`cld(Qm, MR) * MR`), # ties by the larger `MR * NR` tile; `nothing` (keep the planar fitted shape) diff --git a/test/planning/test_kernel_selection.jl b/test/planning/test_kernel_selection.jl index 5d1474c..dc6ec18 100644 --- a/test/planning/test_kernel_selection.jl +++ b/test/planning/test_kernel_selection.jl @@ -33,15 +33,29 @@ using QuasiStrided: TargetProfile, CacheLevel, target_profile, cache_topology, @test default_blocking(Float32) === _fallback_blocking(Float32) end - @testset "derivation rule: MR = 2W, NR = NR_DEFAULT, NV = 12" begin - # NV = 12 everywhere keeps the rule safe on a 16-register AVX2 machine - # and on Julia 1.10, and it is also the measured optimum. - for (isakey, vb) in ((:avx512, 64), (:avx2, 32)), T in (Float64, Float32) + @testset "derivation rule: MR = MV*W, NR = NR_DEFAULT; MV = 4 on :avx512, 2 on :avx2" begin + # NV = 12 on AVX2 is what its 16 registers hold (12 + 2 A + 2 spare); + # AVX-512's 32 registers hold the MV = 4 tile (24 + 4 + 2), and the + # MV = 2 tile is front-end bound there (see `_rule_mv`). + for (isakey, vb, MV) in ((:avx512, 64, 4), (:avx2, 32, 2)), T in (Float64, Float32) MR, NR, W = _derived_shape(synthetic(isakey, vb), T) @test W == vb ÷ sizeof(T) - @test (MR, NR) == (2 * W, NR_DEFAULT) - @test (MR ÷ W) * NR == 12 + @test (MR, NR) == (MV * W, NR_DEFAULT) + @test (MR ÷ W) * NR == 6 * MV @test MR % W == 0 # the packing/kernel contract + @test QuasiStrided._rule_mv(Val(isakey), RealMethod()) == MV + end + # The multiplier is a REAL-method, AVX-512-only refinement: every + # complex method keeps the validated MV = 2 rule shape. + for m in (PlanarMethod(), OneMMethod(), QuasiStrided.FMAddSubMethod()) + @test QuasiStrided._rule_mv(Val(:avx512), m) == 2 + @test QuasiStrided._rule_mv(Val(:avx2), m) == 2 + end + # The default `mv` of `_rule_shape` is the unrefined rule. + for T in (Float64, Float32) + W = 64 ÷ sizeof(T) + @test QuasiStrided._rule_shape(64, T) === (2 * W, NR_DEFAULT, W) + @test QuasiStrided._rule_shape(64, T, 4) === (4 * W, NR_DEFAULT, W) end # On AVX2 the rule coincides with the fallback shape. @test _derived_shape(synthetic(:avx2, 32), Float64) === (8, 6, 4) @@ -72,8 +86,11 @@ using QuasiStrided: TargetProfile, CacheLevel, target_profile, cache_topology, @test _shape_override(Val(key), T) === nothing end W = 64 ÷ sizeof(T) - @test _derived_shape(synthetic(:avx512, 64), T) === (2 * W, NR_DEFAULT, W) + @test _derived_shape(synthetic(:avx512, 64), T) === (4 * W, NR_DEFAULT, W) @test _derived_shape(synthetic(:avx512, 64), T) in kernel_shapes(T) + # The MV = 2 sibling stays in the menu: it is the short-M + # step-down target and a `_demote_for_run` candidate. + @test (2 * W, NR_DEFAULT, W) in kernel_shapes(T) end end @@ -115,6 +132,43 @@ using QuasiStrided: TargetProfile, CacheLevel, target_profile, cache_topology, @test mr(_default_kernel(T, 0, 256)) == MR # empty must not demote end end + + @testset "_extent_shape: the MV = 4 real shape steps down to MV = 2 where it pads less" begin + ext = QuasiStrided._extent_shape + for T in (Float64, Float32) + p = synthetic(:avx512, 64) + tall = _derived_shape(p, T) + MR, NR, W = tall + @test MR == 4 * W + half = (2 * W, NR, W) + @test half in kernel_shapes(T) + @test ext(p, T, RealMethod(), 0) === tall # empty: no step-down + for Qm in (1, 2 * W - 1, 2 * W, MR - 1) # below one tall tile + @test ext(p, T, RealMethod(), Qm) === half + end + @test ext(p, T, RealMethod(), MR) === tall # exactly one tall tile + for Qm in (MR + 1, MR + W, MR + 2 * W) # (MR, 3MR/2]: half pads less + @test ext(p, T, RealMethod(), Qm) === half + end + for Qm in (MR + 2 * W + 1, 2 * MR - 1, 2 * MR, 2 * MR + 1, 3 * MR + W, 100 * MR) + @test ext(p, T, RealMethod(), Qm) === tall # equal padding or >= 2 MR + end + # Through `_default_kernel` on the live host: only when the host's + # derived shape IS the tall one does the step-down apply; below + # `2W` the fitted-shape demotion still wins. + hostshape = _derived_shape(target_profile(), T) + if hostshape === tall + @test (mr(_default_kernel(T, MR + W, 256)), nr(_default_kernel(T, MR + W, 256))) == (2 * W, NR) + @test mr(_default_kernel(T, 2 * W, 256)) == 2 * W + @test mr(_default_kernel(T, 2 * W - 1, 256)) == _fallback_shape(T)[1] + @test mr(_default_kernel(T, 2 * MR, 256)) == MR + end + # Never below MV = 2, and complex methods are untouched. + avx2 = synthetic(:avx2, 32) + @test ext(avx2, T, RealMethod(), 3) === _derived_shape(avx2, T) + @test ext(p, ComplexF64, PlanarMethod(), 3) === _derived_shape(p, ComplexF64, PlanarMethod()) + end + end end @@ -208,10 +262,11 @@ end @test _shape_override(Val(key), T) === nothing end # The REAL rule is separate: a real element type reaches the real - # `_derived_shape` method, unaffected by the complex one. + # `_derived_shape` method, unaffected by the complex one (and carries the + # real-only MV = 4 refinement). for T in (Float64, Float32) W = 64 ÷ sizeof(T) - @test _derived_shape(synthetic(:avx512, 64), T) === (2 * W, NR_DEFAULT, W) + @test _derived_shape(synthetic(:avx512, 64), T) === (4 * W, NR_DEFAULT, W) end end From 7b64f9d65187272fa6cacc3b743044b8b18110f9 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 14:35:36 -0400 Subject: [PATCH 04/53] Cut the per-call planning floor: cached defaults, static ranks, no dynamic dispatch The TensorOperations adapter paid a fixed ~1.1 us (Float64) / ~5.5 us (ComplexF64) per tensorcontract! call on tiny problems -- 4-8x / 13-23x StridedBLAS on the upstream suite's batched dim 4-16 cases, 8-15x on the tiny blas-equivalent ones. Profiled on ccqlin038 (Cascade Lake, Julia 1.12.7), the 8x8x8 GEMM: execute! was 0.23 / 0.41 us; the rest was planning. Root causes, in order of size: * Every plan re-derived the default kernel shape and blocking row from the TargetProfile through `Val(profile.isa)` -- a dynamic dispatch on a runtime Symbol, chained through the override row, the rule, the fitted shape, the small-M rule and the blocking model. _default_kernel(C64,8,8) cost 1.2 us and plan_contract called it TWICE (the swapped-orientation candidate, dead for complex T); default_blocking(kernel) another 0.3-0.4 us. Float64's were 75 + 95 ns. * The M/N/K label lists were Vector{Int}s (five allocations, ~370 B) and the composite ranks therefore runtime values, so the _plan_contract barrier was a dynamic dispatch that boxed every non-isbits argument (~800 B more) -- even for Float64, whose kernel Union alone would have union-split. * For a complex T the kernel Union spans the whole menu (ten members), so even with everything else concrete the barrier stayed a jl_apply_generic that missed the method cache far more often than not (profiled: ~15% of the call in subtype.c), and execute!(plan) was a second such dispatch over a Union-typed, boxed plan. * _qs_task_workspace's lazy-build branch was inferred as the abstract ContractWorkspace, making the adapter's own call into the planner dynamic too (three boxed StridedViews + tuples + executor per call). Changes: * src/planning/defaults.jl (new): ResolvedDefaults, cached per (profile, eltype) in one slot per supported eltype, keyed by === on the immutable TargetProfile (so a profile written after load, as forced_isa_runner.jl does, takes effect without a hook). _default_kernel and default_blocking(kernel) move here and read the cache; _default_shape returns the (shape, method) choice as plain values. * labels.jl / pair_group.jl: label lists are NTuple{D,Int} with D fixed by the label tuples' lengths (_group_ranks: NA = M+K, NB = N+K, NC = M+N); _build_pair_group is a single static method. Allocation-free. * plan.jl: plan_contract's body is _planned(f, ...) with a continuation applied inside the barrier; the kernel is resolved for an automatic choice through _with_menu_kernel (kernel_selection.jl), a generated if-ladder over the closed menu that calls the continuation with a CONCRETE kernel in every arm -- no menu-wide Union is ever dispatched on. Everything the barrier needs travels in one _PlanRequest. The swapped-orientation candidate is resolved only for real T. * tensoroperations.jl: the default-allocator tensorcontract! passes an executor as the continuation (plan built and run in one typed frame), and _qs_task_workspace is concretely typed on both branches. Semantics are unchanged: same kernel choice, same blocking, same swap and run-length demotion (pinned by the existing planning tests, updated only for the Vector -> Tuple representation). Measured (same host, medians of 15 x 2000 calls through tensorcontract!(..., QuasiStridedBackend()), pooled workspace): Float64 8x8x8 1.10 us / 1232 B -> ~0.5 us / 0 B (StridedBLAS 0.15) Float64 4x4 outer 0.79 us / 1072 B -> ~0.3 us / 0 B (StridedBLAS 0.22) ComplexF64 8x8x8 5.50 us / 2032 B -> 0.67 us / 0 B (StridedBLAS 0.43) ComplexF64 4x4 outer 5.06 us / 1776 B -> 0.43 us / 0 B (StridedBLAS 0.33) ComplexF64 16^3 6.90 us / 2032 B -> 1.98 us / 0 B (StridedBLAS 1.5-2.0) plan_contract itself: 0.62 -> ~0.22 us (Float64), 3.58 -> 0.24 us (ComplexF64). Full test suite green (137144 passed). Co-Authored-By: Claude Opus 5.5 --- benchmark/profile_buckets.jl | 9 +- src/QuasiStrided.jl | 2 + src/integrations/tensoroperations.jl | 54 +++++-- src/layout/pair_group.jl | 41 ++--- src/planning/blocking.jl | 26 ++-- src/planning/defaults.jl | 165 ++++++++++++++++++++ src/planning/kernel_selection.jl | 87 +++++++---- src/planning/labels.jl | 101 ++++++++---- src/planning/plan.jl | 199 ++++++++++++++++++------ test/planning/test_per_call_overhead.jl | 33 ++-- test/planning/test_plan_contract.jl | 75 ++++----- 11 files changed, 578 insertions(+), 214 deletions(-) create mode 100644 src/planning/defaults.jl diff --git a/benchmark/profile_buckets.jl b/benchmark/profile_buckets.jl index 7a5cb0f..7229083 100644 --- a/benchmark/profile_buckets.jl +++ b/benchmark/profile_buckets.jl @@ -39,13 +39,14 @@ const SPECIFIC_QS_BUCKETS = [ ( "planning", [ - "plan_contract", "_plan_contract", "_classify_labels", "_order_free_labels", - "_prefer_swap", "_default_kernel", "_kernel_from_shape", - "default_blocking", + "plan_contract", "_planned", "_plan_contract", "_plan_with_kernel", + "_plan_demoted", "_with_menu_kernel", "_classify_labels", "_order_free_labels", + "_prefer_swap", "_default_kernel", "_default_shape", "_resolved_defaults", + "_kernel_from_shape", "default_blocking", # These would land in "planning" through the ancestor walk anyway # (`plan_contract` is their only caller); naming them makes the # attribution explicit and cannot change any classification. - "_build_pair_group", "_pair_group_rank", "_pair_group_static", + "_build_pair_group", "_candidate_mrs", ], ), ("packing", ["pack_a!", "pack_b!", "_pack_panel!", "_pack_a_contiguous!", "_pack_sliver!", "_pack_emit", "_check_pack_a", "_check_pack_b"]), diff --git a/src/QuasiStrided.jl b/src/QuasiStrided.jl index 5e2e633..356991d 100644 --- a/src/QuasiStrided.jl +++ b/src/QuasiStrided.jl @@ -44,6 +44,8 @@ include("planning/labels.jl") include("planning/conjugation.jl") include("planning/kernel_selection.jl") include("planning/blocking.jl") +# The per-(profile, eltype) cache the two files above feed; it needs `Blocking`. +include("planning/defaults.jl") # `ContractPlan` holds a `ContractWorkspace`, so the workspace comes first. include("execution/workspace.jl") include("planning/plan.jl") diff --git a/src/integrations/tensoroperations.jl b/src/integrations/tensoroperations.jl index d6dc679..8d4c5de 100644 --- a/src/integrations/tensoroperations.jl +++ b/src/integrations/tensoroperations.jl @@ -73,10 +73,22 @@ end # grow. # Keying on the storage type rather than the packed type is what keeps a # `Float64` and a `ComplexF64` contraction from colliding on one workspace. +# +# Both returns carry the same concrete type assertion. The build branch's +# type is derived from `_default_kernel(T)`, a Union over `T`'s kernel menu, +# so without the assertion the two branches join to the abstract +# `ContractWorkspace` -- which made the adapter's call into `_planned` a +# dynamic dispatch that boxed all three `StridedView`s, the label tuples and +# the executor on every call (measured: 3x64 + 3x32 + 48 B, ccqlin038, Julia +# 1.12.7, ComplexF64 8x8x8). @inline function _qs_task_workspace(::Type{T}) where {T} pool = _qs_workspace_pool() ws = get(pool, T, nothing) ws === nothing || return ws::ContractWorkspace{T, Vector{real(T)}} + return _qs_build_task_workspace!(pool, T)::ContractWorkspace{T, Vector{real(T)}} +end + +@noinline function _qs_build_task_workspace!(pool::Dict{DataType, ContractWorkspace}, ::Type{T}) where {T} kernel = _default_kernel(T) new_ws = ContractWorkspace(T, kernel, default_blocking(kernel), false, TO.DefaultAllocator()) pool[T] = new_ws @@ -302,23 +314,41 @@ function TO.tensorcontract!( allocator::TO.DefaultAllocator = TO.DefaultAllocator() ) Cv, Av, Bv, indA, indB, indC, α′, β′ = _qs_prepare(C, A, pA, B, pB, pAB, α, β) - # `plan_contract` `reserve!`s the pooled workspace itself. oracle=false: - # the backend path never needs `execute_tilewise!`'s buffers. - plan = plan_contract( - Cv, Av, indA, Bv, indB, indC; - conjA = conjA, conjB = conjB, - workspace = _qs_task_workspace(eltype(C)), allocator = allocator, oracle = false + # `plan_contract`'s body with `execute!` as the continuation (`_planned`, + # src/planning/plan.jl): the plan is built and run inside the + # `_plan_contract` barrier, where its type is concrete, so + # `execute!(plan, ...)` is a static call and the plan never escapes this + # frame -- `plan = plan_contract(...); execute!(plan, ...)` would instead + # dispatch `execute!` at runtime on a Union-typed plan and box it (the + # kernel choice is a runtime value; for a complex `T` its Union spans the + # whole menu). Measured on ccqlin038 (Julia 1.12.7, 8x8x8, pooled + # workspace): 1232 B and 1.1 us per Float64 call, 2032 B and 5.5 us per + # ComplexF64 call before this and the planning-side changes it is paired + # with (statically sized label tuples, the per-(profile, eltype) defaults + # cache and the static kernel ladder); 0 B and ~0.5 / ~0.7 us after. + # `_planned` `reserve!`s the pooled workspace itself. oracle=false: the + # backend path never needs `execute_tilewise!`'s buffers. + _planned( + _QSExecute(α′, β′), Cv, Av, indA, Bv, indB, indC, + nothing, conjA, conjB, nothing, nothing, nothing, + _qs_task_workspace(eltype(C)), allocator, false ) - execute!(plan, α′, β′) return C end +# The continuation the method above hands `_planned`: `execute!` on the plan, +# with `alpha`/`beta` already converted to `eltype(C)`. A callable struct +# rather than a closure so the captured scalars are concretely typed fields. +struct _QSExecute{T} + alpha::T + beta::T +end +(e::_QSExecute)(plan::ContractPlan) = (execute!(plan, e.alpha, e.beta); nothing) + # NOT merged with the method above, although the two differ only in allocator -# handling. Merging them into one method over a dispatched `_qs_run!` helper -# would read better and save a duplicated 10-line signature, but costs an -# extra allocation per call (`Float64` and `ComplexF64`, both allocator -# regimes): the extra frame changes what escapes, so the -# `ContractPlan` stops being elided. Two methods, deliberately. +# handling: this one needs the plan back after `execute!` (to `release!` its +# allocator-owned workspace in `finally`), so it goes through `plan_contract` +# and cannot share the executor continuation. Two methods, deliberately. function TO.tensorcontract!( C::AbstractArray, A::AbstractArray, pA::Index2Tuple, conjA::Bool, diff --git a/src/layout/pair_group.jl b/src/layout/pair_group.jl index 82b136d..dd8eff4 100644 --- a/src/layout/pair_group.jl +++ b/src/layout/pair_group.jl @@ -3,38 +3,15 @@ # Build the two-map AxisGroup for one of M/N/K: (v1,v2) is (A,C)/(B,C)/(A,B). # Raises DimensionMismatch on a matched-label length mismatch. # -# `D = length(labels)` is a RUNTIME value (which labels are shared is a -# property of the label values, not of their tuple types), so `ntuple`s built -# directly from it are runtime-length -- inferred as `Tuple{Vararg{Int}}`, -# heap-boxed, and read back through a dynamic `getindex`, allocating per group -# even at `D == 1`. `D` is bounded above by `N1` (every label here occurs in -# `ind1`), which IS compile-time known, so the rank is resolved once through -# the unrolled `_pair_group_rank` ladder below and the body then runs at a -# literal `Val{D}` with statically sized tuples throughout. -function _build_pair_group( - labels::Vector{Int}, - ind1::NTuple{N1, Int}, v1::StridedView, - ind2::NTuple{N2, Int}, v2::StridedView - ) where {N1, N2} - return _pair_group_rank(Val(N1), labels, ind1, v1, ind2, v2) -end - -# Unrolled rank ladder: `length(labels) <= N1` always, so descending from -# `Val(N1)` reaches the matching literal in at most `N1 + 1` compares, each arm -# calling a concretely-typed `_pair_group_static`. A plain `Val(D)` on a -# runtime `D` would be a dynamic dispatch instead. -@inline function _pair_group_rank( - ::Val{K}, labels::Vector{Int}, ind1, v1, ind2, v2 - ) where {K} - length(labels) == K && return _pair_group_static(Val(K), labels, ind1, v1, ind2, v2) - return _pair_group_rank(Val(K - 1), labels, ind1, v1, ind2, v2) -end - -@inline _pair_group_rank(::Val{0}, labels::Vector{Int}, ind1, v1, ind2, v2) = - _pair_group_static(Val(0), labels, ind1, v1, ind2, v2) - -@inline function _pair_group_static( - ::Val{D}, labels::Vector{Int}, +# `labels` arrives as an `NTuple{D,Int}` whose length IS the composite's rank +# (`_classify_labels` in src/planning/labels.jl sizes it from the label +# tuples' lengths, see `_group_ranks` there), so `D` is a type parameter here +# and the group comes out as a concrete `AxisGroup{D,2}` with statically sized +# tuples throughout. A runtime-length list would instead be inferred as +# `Tuple{Vararg{Int}}`, heap-boxed, and read back through a dynamic +# `getindex`, allocating per group even at `D == 1`. +@inline function _build_pair_group( + labels::NTuple{D, Int}, ind1::NTuple{N1, Int}, v1::StridedView, ind2::NTuple{N2, Int}, v2::StridedView ) where {D, N1, N2} diff --git a/src/planning/blocking.jl b/src/planning/blocking.jl index 42fc02c..0e0997b 100644 --- a/src/planning/blocking.jl +++ b/src/planning/blocking.jl @@ -23,20 +23,9 @@ struct Blocking end end -""" - default_blocking(kernel) -> Blocking - -Cache-blocking factors for `kernel` on this host: an analytical model of the -detected cache geometry ([`target_profile`](@ref), `_modelled_blocking`) for -the real row at the engine's real register shape, scaled per complex method -by packed reals per element. With L1d or L2 undetected, fixed fallback -constants instead. Measured grids are wide plateaus (1.7-17% best-to-worst -over `bench_driver.jl`'s 36 points, 4-24% over 157) whose one cliff is small -`kc`, so no row is a sharp optimum. `plan_contract` rounds `mc`/`nc` to -`mr`/`nr` multiples and clamps all three to the contraction's extents. -""" -default_blocking(kernel) = - default_blocking(Val(target_profile().isa), scalartype(kernel), complex_method(kernel)) +# `default_blocking(kernel)`, the form `plan_contract` consumes, lives in +# src/planning/defaults.jl: it reads the real row below through the +# per-(profile, eltype) cache rather than re-deriving it on every plan. # Complex blocking is the real row divided by the packed reals per element of # each operand, so every method gets the same packed BYTE budget rather than @@ -72,7 +61,14 @@ default_blocking(::Val, ::Type{T}) where {T} = _fallback_blocking(T) # Every ISA takes the model; the `Val` key is kept so a future measured row # can override one ISA without touching the others. default_blocking(::Val, ::Type{T}) where {T <: Union{Float32, Float64}} = - something(_modelled_blocking(target_profile(), T), _fallback_blocking(T)) + _real_blocking_row(target_profile(), T) + +# The real row for `T` on `profile`: the cache model where the geometry is +# detected, the fallback constants otherwise. The one definition behind both +# `default_blocking(::Val, T)` above and the per-(profile, eltype) cache in +# src/planning/defaults.jl, so the two cannot disagree. +_real_blocking_row(profile::TargetProfile, ::Type{T}) where {T <: Union{Float32, Float64}} = + something(_modelled_blocking(profile, T), _fallback_blocking(T)) # --- analytical real row, from the detected cache geometry ------------------- # diff --git a/src/planning/defaults.jl b/src/planning/defaults.jl new file mode 100644 index 0000000..4d8dc20 --- /dev/null +++ b/src/planning/defaults.jl @@ -0,0 +1,165 @@ +# The engine's per-host defaults -- the register shape, the small-M demotion +# targets and the cache-blocking row for an element type -- resolved ONCE per +# (detected profile, element type) and cached, so that `plan_contract` reads +# them instead of re-deriving them on every call. +# +# Why a cache at all: everything in src/planning/kernel_selection.jl and +# src/planning/blocking.jl is keyed on `Val(profile.isa)`, and `profile.isa` +# is a runtime `Symbol`, so each of those lookups is a dynamic dispatch (a +# `Val` type built at runtime, then `jl_apply_generic`). Chained -- override +# row, rule, fitted shape, small-M rule, the blocking model, each behind its +# own `Val` -- they dominated the per-call floor. Measured on ccqlin038 +# (Cascade Lake, Julia 1.12.7) before this file existed: +# `_default_kernel(ComplexF64, 8, 8)` took 1.2 us and `plan_contract` called +# it twice, `default_blocking(kernel)` another 0.3-0.4 us; the same two for +# `Float64` were 75 ns and 95 ns. Against a whole 8x8x8 ComplexF64 +# contraction of 5.5 us (StridedBLAS: 0.4 us), planning was 3.6 us of it. +# With the cache the two reads are a slot load and a profile comparison. + +""" + ResolvedDefaults + +What `plan_contract` needs from the hardware profile for one element type, +resolved by [`_resolve_defaults`](@ref) and cached per (profile, eltype) by +[`_resolved_defaults`](@ref): the default register `shape` +(`_derived_shape`), the `fitted` shape the off-AVX-512 small-M demotion falls +to (`_fitted_shape`), the native-width FMAddSub `small_m` candidates of the +AVX-512 complex small-M rule (`_small_m_candidates`, empty where the rule does +not apply), and the unscaled `real_row` of the cache-blocking model for +`real(T)` (`_real_blocking_row`). `profile` is the profile these were derived +from and is the cache key: a cached entry serves exactly while it is `===` +the current [`target_profile`](@ref). Every field but the candidate list is +isbits, so reads are type-stable and allocation-free. +""" +struct ResolvedDefaults + profile::TargetProfile + shape::NTuple{3, Int} + fitted::NTuple{3, Int} + small_m::Vector{NTuple{3, Int}} + real_row::Blocking +end + +""" + _resolve_defaults(profile::TargetProfile, T) -> ResolvedDefaults + +Derive every [`ResolvedDefaults`](@ref) field for `T` on `profile` through +the pure selection functions of src/planning/kernel_selection.jl and +src/planning/blocking.jl. Uncached; `_resolved_defaults` is the cached entry. +""" +function _resolve_defaults(profile::TargetProfile, ::Type{T}) where {T} + method = _default_method(T) + # First, so an element type with no menu throws from `kernel_shapes` here, + # as `_kernel_for` did, rather than from a later field. + shape = _derived_shape(profile, T, method) + fitted = _fitted_shape(profile, T, method) + small_m = _small_m_candidates(Val(profile.isa), profile, T) + real_row = _real_blocking_row(profile, real(T)) + return ResolvedDefaults(profile, shape, fitted, small_m, real_row) +end + +# One slot per supported element type, so the lookup is a method dispatch on +# `T` rather than a `Dict` probe. `nothing` for any other `T`: those take the +# uncached path and fail in `_resolve_defaults` exactly as they did before +# (no shape menu for the type). A slot is filled lazily, on the first plan +# for its type after the profile last changed. +const _DEFAULTS_F64 = Ref{Union{Nothing, ResolvedDefaults}}(nothing) +const _DEFAULTS_F32 = Ref{Union{Nothing, ResolvedDefaults}}(nothing) +const _DEFAULTS_C64 = Ref{Union{Nothing, ResolvedDefaults}}(nothing) +const _DEFAULTS_C32 = Ref{Union{Nothing, ResolvedDefaults}}(nothing) +_defaults_slot(::Type{Float64}) = _DEFAULTS_F64 +_defaults_slot(::Type{Float32}) = _DEFAULTS_F32 +_defaults_slot(::Type{ComplexF64}) = _DEFAULTS_C64 +_defaults_slot(::Type{ComplexF32}) = _DEFAULTS_C32 +_defaults_slot(::Type) = nothing + +""" + _resolved_defaults(T) -> ResolvedDefaults + +The cached [`ResolvedDefaults`](@ref) of `T` for the current +[`target_profile`](@ref), (re)derived when the slot is empty or holds an +entry for a different profile -- which is how a profile installed after load +(`_init_target!`, or a test writing `_TARGET[]` directly, as +test/forced_isa_runner.jl does) takes effect without an explicit +invalidation hook. The comparison is `===` on the immutable `TargetProfile`, +i.e. field-wise, so two detections of the same host share an entry. + +Not locked: a racing refill computes the same immutable value and the slot +write is a single pointer store, so the worst case is redundant work. +""" +@inline function _resolved_defaults(::Type{T}) where {T} + slot = _defaults_slot(T) + profile = target_profile() + slot === nothing && return _resolve_defaults(profile, T) + cached = slot[] + (cached !== nothing && cached.profile === profile) && return cached + return _refill_defaults!(slot, profile, T) +end + +@noinline function _refill_defaults!(slot, profile::TargetProfile, ::Type{T}) where {T} + fresh = _resolve_defaults(profile, T) + slot[] = fresh + return fresh +end + +# ---------------------------------------------------------------------------- +# The engine's default kernel, and plan-time demotion +# ---------------------------------------------------------------------------- + +# `_kernel_for(target_profile(), T)`, read through the cache. +_default_kernel(::Type{T}) where {T} = + _kernel_from_shape(_resolved_defaults(T).shape, T, _default_method(T)) + +# Extent-aware choice, used only when the caller did not name a kernel: a +# contraction whose M extent cannot fill one register tile pads every +# micro-tile away, so demote to the fitted shape. Keys on padding waste, a +# countable quantity known at plan time, not on a cache estimate. `shape[1]` +# is `mr` of the kernel `_kernel_from_shape` builds at `shape`. +# +# Returns the `(shape, method)` pair as plain values rather than the kernel: +# `plan_contract` consumes it through `_with_menu_kernel`, so the choice is +# made without ever holding a menu-wide kernel Union (ten members for +# ComplexF64), which its function barrier would otherwise have to dispatch on +# at runtime. `_default_kernel` below builds the kernel from the same pair. +@inline function _default_shape(::Type{T}, Qm::Int, Qn::Int) where {T} + d = _resolved_defaults(T) + method = _default_method(T) + (Qm > 0 && Qm < d.shape[1]) || return (d.shape, method) + small = _small_m_shape(d.small_m, Qm) + small === nothing || return (small, FMAddSubMethod()) + return (d.fitted, method) +end + +# `@noinline` deliberately: the return type is the Union of every menu kernel +# of `T`, and inlining that Union into a caller would only widen the caller's +# inference for no gain. +@noinline function _default_kernel(::Type{T}, Qm::Int, Qn::Int) where {T} + shape, method = _default_shape(T, Qm, Qn) + return _kernel_from_shape(shape, T, method) +end + +# ---------------------------------------------------------------------------- +# default_blocking(kernel) +# ---------------------------------------------------------------------------- + +""" + default_blocking(kernel) -> Blocking + +Cache-blocking factors for `kernel` on this host: an analytical model of the +detected cache geometry ([`target_profile`](@ref), `_modelled_blocking`) for +the real row at the engine's real register shape, scaled per complex method +by packed reals per element. With L1d or L2 undetected, fixed fallback +constants instead. Measured grids are wide plateaus (1.7-17% best-to-worst +over `bench_driver.jl`'s 36 points, 4-24% over 157) whose one cliff is small +`kc`, so no row is a sharp optimum. `plan_contract` rounds `mc`/`nc` to +`mr`/`nr` multiples and clamps all three to the contraction's extents. + +Identical to `default_blocking(Val(target_profile().isa), scalartype(kernel), +complex_method(kernel))`, read through the per-(profile, eltype) cache: the +real row for `real(scalartype(kernel))` comes from [`_resolved_defaults`](@ref) +and is scaled here by the kernel's own method (`_scale_blocking`, the identity +for a real kernel). +""" +function default_blocking(kernel) + T = scalartype(kernel) + return _scale_blocking(_resolved_defaults(T).real_row, complex_method(kernel)) +end diff --git a/src/planning/kernel_selection.jl b/src/planning/kernel_selection.jl index 5d4eeb4..74b9a06 100644 --- a/src/planning/kernel_selection.jl +++ b/src/planning/kernel_selection.jl @@ -14,9 +14,17 @@ # explicit override row, else the `MR = 2W` rule where it has been # validated, else a conservative fitted shape (`_fitted_shape`); # * two plan-time demotions: to the fitted shape when M cannot fill one -# register tile (`_default_kernel`; complex on `:avx512` goes to an -# FMAddSub shape instead, `_small_m_shape`), and to a menu shape that -# keeps every register sliver unit-stride in C (`_demote_for_run`). +# register tile (`_default_kernel`, src/planning/defaults.jl; complex on +# `:avx512` goes to an FMAddSub shape instead, `_small_m_shape`), and to +# a menu shape that keeps every register sliver unit-stride in C +# (`_demote_for_run`). +# +# Everything here is a pure function of a `TargetProfile` and `T`. The +# engine's per-call entry points (`_default_kernel`, `default_blocking(kernel)`) +# live in src/planning/defaults.jl, which resolves these once per (profile, +# `T`) and caches the result: the `Val(profile.isa)` dispatch below is a +# dynamic one (the ISA is a runtime `Symbol`), too slow to repeat on every +# `plan_contract`. # ---------------------------------------------------------------------------- # Methods, menus and kernel types @@ -156,6 +164,32 @@ _kernel_from_shape(shape::Tuple{Int, Int, Int}, ::Type{T}) where {T} = return ex end +""" + _with_menu_kernel(g, shape, T, method, args...) -> g(kernel, args...) + +`_kernel_from_shape`'s continuation-passing twin: build the `method` kernel +for `T` at `shape` and hand it to `g` together with `args...`, so that `g` +runs on a CONCRETE kernel type in every arm of the unrolled menu ladder. +`_kernel_from_shape`'s return type is the Union of the whole menu, which a +consumer has to dispatch on at runtime once the Union exceeds four members +(ComplexF64's does: six planar plus four FMAddSub shapes); here the dispatch +stays a chain of literal tuple compares and every `g` call is static. Same +menu, same constructor and the same two throws as `_kernel_from_shape`. This +is how `plan_contract` reaches its function barrier when the caller named no +kernel; see `_plan_with_kernel` in src/planning/plan.jl for the measurement. +""" +@generated function _with_menu_kernel( + g::G, shape::Tuple{Int, Int, Int}, ::Type{T}, method::M, args... + ) where {G, T, M} + K = _kernel_type(M.instance, T) + K === nothing && return :(_throw_no_kernel(shape, T, method)) + ex = :(_throw_shape_not_in_menu(shape, T, method)) + for (MR, NR, W) in reverse(kernel_shapes(T, M.instance)) + ex = :(shape === ($MR, $NR, $W) ? g($K(Val($MR), Val($NR), T, Val($W)), args...) : $ex) + end + return ex +end + @noinline _throw_no_kernel(shape, ::Type{T}, method) where {T} = throw( ArgumentError( "no microkernel is available for $T at shape $shape under $(method). " * @@ -325,33 +359,26 @@ function _fitted_shape(profile::TargetProfile, ::Type{T}, method::PlanarMethod) end # ---------------------------------------------------------------------------- -# The engine's default kernel, and plan-time demotion +# The default kernel for a profile, and the small-M demotion rule # ---------------------------------------------------------------------------- +# The default kernel for `T` on `profile`, uncached: what `_default_kernel(T)` +# (src/planning/defaults.jl) resolves for the detected host, and the form the +# tests use with synthetic profiles. @noinline _kernel_for(profile::TargetProfile, ::Type{T}) where {T} = _kernel_from_shape(_derived_shape(profile, T), T, _default_method(T)) -_default_kernel(::Type{T}) where {T} = _kernel_for(target_profile(), T) - -# Extent-aware variant, used only when the caller did not name a kernel: a -# contraction whose M extent cannot fill one register tile pads every -# micro-tile away, so demote to the fitted shape. Keys on padding waste, a -# countable quantity known at plan time, not on a cache estimate. -@noinline function _default_kernel(::Type{T}, Qm::Int, Qn::Int) where {T} - profile = target_profile() - kernel = _kernel_for(profile, T) - (Qm > 0 && Qm < mr(kernel)) || return kernel - small = _small_m_shape(Val(profile.isa), profile, T, Qm) - small === nothing || return _kernel_from_shape(small, T, FMAddSubMethod()) - method = _default_method(T) - return _kernel_from_shape(_fitted_shape(profile, T, method), T, method) -end - # Small-M demotion target for complex `T` on AVX-512: the native-width # (`W == lanes`) FMAddSub menu shape that pads `Qm` least (`cld(Qm, MR) * MR`), # ties by the larger `MR * NR` tile; `nothing` (keep the planar fitted shape) -# everywhere else. Reached only through the demotion above, i.e. when `Qm` is -# below the planar override's `MR` (24 for ComplexF64, 48 for ComplexF32). +# everywhere else. Reached only through `_default_kernel`'s extent demotion +# (src/planning/defaults.jl), i.e. when `Qm` is below the planar override's +# `MR` (24 for ComplexF64, 48 for ComplexF32). +# +# Split in two so the host-dependent half can be cached: `_small_m_candidates` +# is the native-width slice of the FMAddSub menu (a property of the profile +# and `T` alone, empty wherever the rule does not apply), and +# `_small_m_shape(candidates, Qm)` is the pure pick among them. # # Why: the planar fitted shape is the LARGEST fitting tile, `(16,6,8)` / # `(32,6,16)` on AVX-512 -- the planar shape measured 38-41% worst there (see @@ -380,13 +407,16 @@ end # 45-79% before). Not extended to `:avx2` (where # the planar override has `MR = 4`, so demotion needs `Qm < 4`; unmeasured) # or to other ISAs. -_small_m_shape(::Val, ::TargetProfile, ::Type, ::Int) = nothing -function _small_m_shape(::Val{:avx512}, profile::TargetProfile, ::Type{T}, Qm::Int) where {T <: Complex} +_small_m_candidates(::Val, ::TargetProfile, ::Type) = NTuple{3, Int}[] +function _small_m_candidates(::Val{:avx512}, profile::TargetProfile, ::Type{T}) where {T <: Complex} lanes = profile.vector_bytes ÷ sizeof(real(T)) + return [shape for shape in kernel_shapes(T, FMAddSubMethod()) if shape[3] == lanes] +end + +function _small_m_shape(candidates::Vector{NTuple{3, Int}}, Qm::Int) best = nothing - for shape in kernel_shapes(T, FMAddSubMethod()) - MR, NR, W = shape - W == lanes || continue + for shape in candidates + MR, NR, _ = shape key = (-(cld(Qm, MR) * MR), MR * NR) if best === nothing || key > best[1] best = (key, shape) @@ -395,6 +425,9 @@ function _small_m_shape(::Val{:avx512}, profile::TargetProfile, ::Type{T}, Qm::I return best === nothing ? nothing : best[2] end +_small_m_shape(key::Val, profile::TargetProfile, ::Type{T}, Qm::Int) where {T} = + _small_m_shape(_small_m_candidates(key, profile, T), Qm) + # Run-length-aware demotion, applied to whichever operand the M/N swap decision # chose to feed M. The vectorized store needs EVERY register sliver unit-stride # in C; given a leading unit-stride run of diff --git a/src/planning/labels.jl b/src/planning/labels.jl index 97b857b..a90d580 100644 --- a/src/planning/labels.jl +++ b/src/planning/labels.jl @@ -1,5 +1,15 @@ # Label planning: classify every label into M/N/K, order the free labels # by their stride in C, and decide the M/N operand orientation. +# +# Every label list here is a statically sized `NTuple{D,Int}`, never a +# `Vector{Int}`: the three composite RANKS are fixed by the label tuples' +# lengths alone (`_group_ranks`), so the lists can be built, sorted and handed +# to `_build_pair_group` without one heap allocation, and each `AxisGroup{D,2}` +# comes out concretely typed. Measured (ccqlin038, Julia 1.12.7, 8x8x8 real +# GEMM through the TensorOperations adapter): the previous `Vector`-based +# planner allocated five vectors (~370 B) per call, and the value-dependent +# rank made the `_plan_contract` barrier a dynamic dispatch that boxed every +# non-isbits argument (~800 B more) -- together roughly half of a 1.1 us call. # Membership test against a statically-sized label tuple, used instead of a # `Set`: the label tuples have a compile-time-known LENGTH (the `NA`/`NB`/`NC` @@ -10,8 +20,41 @@ # of the membership question. @inline _label_in(lbl::Int, t::NTuple{N, Int}) where {N} = any(==(lbl), t) +# The M/N/K composite ranks, from the label tuples' lengths alone. Once +# `_classify_labels`' validation has passed, the labels of A, B and C are each +# partitioned as A = M ∪ K, B = N ∪ K, C = M ∪ N (every label is in exactly two +# operands), so +# +# NA = |M| + |K|, NB = |N| + |K|, NC = |M| + |N| +# +# and the three counts follow by solving. Compile-time constants: `NA`/`NB`/ +# `NC` are type parameters, so the label tuples below get literal lengths and +# every `AxisGroup` a concrete rank. On an input the validation REJECTS these +# formulas may be meaningless (even negative); they are only ever evaluated +# after validation has passed, and `_classify_labels` still asserts them +# against the counts it actually found. +@inline _group_ranks(NA::Int, NB::Int, NC::Int) = + ((NA + NC - NB) ÷ 2, (NB + NC - NA) ÷ 2, (NA + NB - NC) ÷ 2) + +@noinline _throw_rank_mismatch(which::Symbol, found::Int, derived::Int) = throw( + ArgumentError( + "internal error: $which label count $found does not match the rank $derived " * + "derived from the label tuple lengths" + ) +) + +# Append `x` as element `n` of the statically sized `out` (1-based), leaving +# it unchanged when `n` is past the end: `_classify_labels` fills the lists as +# it validates, and the length guard keeps an input that is about to be +# rejected from indexing past a literal-length tuple before the rejection +# fires. +@inline function _push_label(out::NTuple{D, Int}, n::Int, x::Int) where {D} + return n <= D ? _tupleset(out, n, x) : out +end + # Classify every label in indA ∪ indB ∪ indC into M/N/K. Returns -# (mlabels, nlabels, klabels) in indA/indB appearance order. Per (inA,inB,inC): +# (mlabels, nlabels, klabels) as `NTuple`s in indA/indB appearance order, with +# lengths fixed by `_group_ranks`. Per (inA,inB,inC): # (T,F,T)->M (F,T,T)->N (T,T,F)->K everything else -> ArgumentError # (labeled in C only, present in all three, or dangling in just A or B). function _classify_labels( @@ -25,10 +68,10 @@ function _classify_labels( allunique(indC) || throw(ArgumentError("indC has a repeated label (diagonal), not supported: $indC")) - # Sized once to their worst case and trimmed at the end, rather than grown - # by `push!`: `NA`/`NB` are compile-time bounds on the M+K and N counts. - mlabels = Vector{Int}(undef, NA) - klabels = Vector{Int}(undef, NA) + DM, DN, DK = _group_ranks(NA, NB, NC) + mlabels = ntuple(_ -> 0, Val(max(DM, 0))) + nlabels = ntuple(_ -> 0, Val(max(DN, 0))) + klabels = ntuple(_ -> 0, Val(max(DK, 0))) nm = 0 nk = 0 for lbl in indA @@ -43,10 +86,10 @@ function _classify_labels( ) elseif inB && !inC nk += 1 - @inbounds klabels[nk] = lbl + klabels = _push_label(klabels, nk, lbl) elseif !inB && inC nm += 1 - @inbounds mlabels[nm] = lbl + mlabels = _push_label(mlabels, nm, lbl) else throw( ArgumentError( @@ -57,7 +100,6 @@ function _classify_labels( end end - nlabels = Vector{Int}(undef, NB) nn = 0 for lbl in indB inA = _label_in(lbl, indA) @@ -68,7 +110,7 @@ function _classify_labels( continue # already classified as K, above. elseif !inA && inC nn += 1 - @inbounds nlabels[nn] = lbl + nlabels = _push_label(nlabels, nn, lbl) else throw( ArgumentError( @@ -86,9 +128,11 @@ function _classify_labels( throw(ArgumentError("label $lbl appears in indC but not in indA or indB")) end - resize!(mlabels, nm) - resize!(nlabels, nn) - resize!(klabels, nk) + # Unreachable once the three loops above have passed (see `_group_ranks`); + # kept so a wrong rank can never silently truncate a label list. + nm == DM || _throw_rank_mismatch(:M, nm, DM) + nn == DN || _throw_rank_mismatch(:N, nn, DN) + nk == DK || _throw_rank_mismatch(:K, nk, DK) return mlabels, nlabels, klabels end @@ -109,27 +153,26 @@ end # comes back unchanged. Every label must occur in `indC` (the M/N lists from # `_classify_labels` do by construction; K labels never come here). # Insertion sort rather than `sortperm` + permuted copy: the latter allocates -# the key vector, the permutation and the result (three `Vector`s where one is -# needed), and these lists have at most `ndims(C)` entries, so an O(n^2) sort -# with n <= 6 is not a cost. Strict `>` in the shift test keeps it STABLE, -# which is the contract (ties keep input order). A fresh vector is returned: -# sorting `labels` in place would mutate `_classify_labels`'s output, which -# callers (and test/planning/test_plan_contract.jl's label-order pinning) read afterwards. +# the key vector, the permutation and the result, and these lists have at most +# `ndims(C)` entries, so an O(n^2) sort with n <= 6 is not a cost. Strict `>` +# in the shift test keeps it STABLE, which is the contract (ties keep input +# order). Works on the tuple by value through `_tupleset`, so the caller's +# `labels` is untouched and nothing is allocated. function _order_free_labels( - labels::Vector{Int}, indC::NTuple{NC, Int}, C::StridedView - ) where {NC} + labels::NTuple{D, Int}, indC::NTuple{NC, Int}, C::StridedView + ) where {D, NC} st = Base.strides(C) key(l::Int) = abs(st[findfirst(==(l), indC)::Int]) - out = copy(labels) - @inbounds for i in 2:length(out) + out = labels + @inbounds for i in 2:D x = out[i] kx = key(x) j = i - 1 while j >= 1 && key(out[j]) > kx - out[j + 1] = out[j] + out = _tupleset(out, j + 1, out[j]) j -= 1 end - out[j + 1] = x + out = _tupleset(out, j + 1, x) end return out end @@ -142,8 +185,8 @@ end # skipped (their coordinate never advances, whatever their stride says). # Returns 1 when no run starts, 0 if an empty axis is met first. function _leading_unit_run( - labels::Vector{Int}, indC::NTuple{NC, Int}, C::StridedView - ) where {NC} + labels::NTuple{D, Int}, indC::NTuple{NC, Int}, C::StridedView + ) where {D, NC} st = Base.strides(C) run = 1 for l in labels @@ -186,9 +229,9 @@ end # Label-list form, for callers that have `morder`/`norder` but not their run # lengths; derives the same two run lengths `plan_contract` computes once. function _prefer_swap( - morder::Vector{Int}, norder::Vector{Int}, indC::NTuple{NC, Int}, C::StridedView, - mr_asis::Int, mr_swapped::Int = mr_asis - ) where {NC} + morder::NTuple{DM, Int}, norder::NTuple{DN, Int}, indC::NTuple{NC, Int}, + C::StridedView, mr_asis::Int, mr_swapped::Int = mr_asis + ) where {DM, DN, NC} return _prefer_swap( _leading_unit_run(morder, indC, C), _leading_unit_run(norder, indC, C), mr_asis, mr_swapped diff --git a/src/planning/plan.jl b/src/planning/plan.jl index 34edea2..eac8ef0 100644 --- a/src/planning/plan.jl +++ b/src/planning/plan.jl @@ -130,6 +130,70 @@ function plan_contract( allocator = TO.DefaultAllocator(), oracle::Bool = true ) where {NA, NB, NC} + return _planned( + identity, C, A, indA, B, indB, indC, + kernel, conjA, conjB, mc, kc, nc, workspace, allocator, oracle + ) +end + +# Everything `_plan_contract` needs that is already concretely typed BEFORE +# the kernel is known: the continuation, the three groups (their ranks are +# fixed by the label tuples' lengths, so `GM`/`GN`/`GK` are concrete), the +# operand storage/base pairs, the leading unit-stride run of the chosen M +# composite (for `_demote_for_run`), the blocking overrides and the +# workspace/allocator choice. One value, so the kernel-resolution ladder +# (`_plan_with_kernel`) forwards a single argument, and so the swapped and +# as-is orientations differ only in how it is filled. +# +# `T` is a phantom parameter (the storage element type, `eltype(C)`), carried +# so the barrier can check the kernel against it without re-deriving it. +struct _PlanRequest{ + T, F, GM <: AxisGroup, GN <: AxisGroup, GK <: AxisGroup, SA, SB, SC, + WS <: Union{Nothing, ContractWorkspace}, AL, + } + f::F + mgroup::GM + ngroup::GN + kgroup::GK + Astorage::SA + Abase::Int + Bstorage::SB + Bbase::Int + Cstorage::SC + Cbase::Int + run::Int + mc::Union{Int, Nothing} + kc::Union{Int, Nothing} + nc::Union{Int, Nothing} + workspace::WS + allocator::AL + oracle::Bool +end + +@inline function _plan_request( + ::Type{T}, f::F, mgroup::GM, ngroup::GN, kgroup::GK, + Astorage::SA, Abase::Int, Bstorage::SB, Bbase::Int, Cstorage::SC, Cbase::Int, + run::Int, mc, kc, nc, workspace::WS, allocator::AL, oracle::Bool + ) where {T, F, GM, GN, GK, SA, SB, SC, WS, AL} + return _PlanRequest{T, F, GM, GN, GK, SA, SB, SC, WS, AL}( + f, mgroup, ngroup, kgroup, Astorage, Abase, Bstorage, Bbase, Cstorage, Cbase, + run, mc, kc, nc, workspace, allocator, oracle + ) +end + +# `plan_contract`'s body, with a continuation: `f(plan)` is applied INSIDE the +# `_plan_contract` barrier, where the plan's type is concrete, and its result +# returned. `plan_contract` passes `identity`; the TensorOperations adapter +# (src/integrations/tensoroperations.jl) passes an executor, so that the plan +# is built and consumed in one concretely typed frame. Positional throughout: +# this is the hot path, and the keyword handling is `plan_contract`'s job. +function _planned( + f::F, C::StridedView, A::StridedView, indA::NTuple{NA, Int}, + B::StridedView, indB::NTuple{NB, Int}, indC::NTuple{NC, Int}, + kernel, conjA::Bool, conjB::Bool, + mc::Union{Int, Nothing}, kc::Union{Int, Nothing}, nc::Union{Int, Nothing}, + workspace::Union{Nothing, ContractWorkspace}, allocator, oracle::Bool + ) where {F, NA, NB, NC} T = eltype(C) eltype(A) === T || throw(ArgumentError("eltype(A) = $(eltype(A)) does not match eltype(C) = $T")) @@ -152,6 +216,9 @@ function plan_contract( atransform = _qs_isconj(A, conjA) ? conj : identity btransform = _qs_isconj(B, conjB) ? conj : identity + # Statically sized label tuples (src/planning/labels.jl): the three + # composite ranks follow from `NA`/`NB`/`NC`, so the groups built below + # are concretely typed and nothing here allocates. mlabels, nlabels, klabels = _classify_labels(indA, indB, indC) # C's layout, not A's/B's, decides the order within each composite. @@ -175,62 +242,106 @@ function plan_contract( Qn = axis_length(ngroup) Qk = axis_length(kgroup) - # Resolved here, not in the signature default: the demotion needs Qm, and - # Qm depends on the orientation, so both candidates are resolved. - # Nothing between the eltype checks and here reads `kernel`. - kernel_asis = kernel === nothing ? _default_kernel(T, Qm, Qn) : kernel - kernel_swapped = kernel === nothing ? _default_kernel(T, Qn, Qm) : kernel + # The register width each orientation would run at (`_candidate_mrs`): + # only this much of the kernel choice is needed here, for the swap + # decision. The kernel itself is resolved by `_plan_with_kernel`, after + # the orientation is fixed -- and resolved as a `(shape, method)` value, + # not a kernel object, so no menu-wide Union is ever held in this frame. + mr_asis, mr_swapped = _candidate_mrs(T, kernel, Qm, Qn) # The swap is for real element types only. Extending it to complex # kernels, which now also have a vectorized store, is a deliberately # deferred, unmeasured follow-up. Real kernels (`SIMDKernel` and # `ScalarKernel`) keep it. Uses the precomputed `run_m`/`run_n` directly. - if T <: Real && _prefer_swap(run_m, run_n, mr(kernel_asis), mr(kernel_swapped)) + if T <: Real && _prefer_swap(run_m, run_n, mr_asis, mr_swapped) # B takes the M role and A the N role. Everything operand-bound moves # together: the groups (each already carries its own C map), the K # group's two maps, the storage/base pair `_plan_contract` reads off - # its A/B arguments, and the packing transforms. The contraction is - # unchanged: `*` commutes on `T` and `conj` is elementwise, so - # `sum_k conj?(B[n,k]) * conj?(A[m,k])` is the same sum. + # the request's A/B fields, the run length the demotion is keyed on + # (N's own, i.e. the CHOSEN M orientation's) and the packing + # transforms. The contraction is unchanged: `*` commutes on `T` and + # `conj` is elementwise, so `sum_k conj?(B[n,k]) * conj?(A[m,k])` is + # the same sum. kgroup_swapped = _build_pair_group(klabels, indB, B, indA, A) # maps: (B, A) - # Run-length demotion (see `_demote_for_run`): only for an auto-selected - # kernel, keyed on the CHOSEN (post-swap) M orientation, i.e. N's own - # run against the kernel it would actually run. - kernel_final = kernel === nothing ? - _demote_for_run(T, kernel_swapped, run_n, Qn, Qk) : - kernel_swapped - return _plan_contract( - C, B, A, indC, ngroup, mgroup, kgroup_swapped, Qn, Qm, Qk, - kernel_final, btransform, atransform, mc, kc, nc, workspace, allocator, oracle + req_swapped = _plan_request( + T, f, ngroup, mgroup, kgroup_swapped, + parent(B), offset(B), parent(A), offset(A), parent(C), offset(C), + run_n, mc, kc, nc, workspace, allocator, oracle ) + return _plan_with_kernel(kernel, btransform, atransform, req_swapped) end - kernel_final = kernel === nothing ? - _demote_for_run(T, kernel_asis, run_m, Qm, Qk) : - kernel_asis - return _plan_contract( - C, A, B, indC, mgroup, ngroup, kgroup, Qm, Qn, Qk, - kernel_final, atransform, btransform, mc, kc, nc, workspace, allocator, oracle + req = _plan_request( + T, f, mgroup, ngroup, kgroup, + parent(A), offset(A), parent(B), offset(B), parent(C), offset(C), + run_m, mc, kc, nc, workspace, allocator, oracle + ) + return _plan_with_kernel(kernel, atransform, btransform, req) +end + +# `mr` of the kernel each orientation would run: the as-is one at `(Qm, Qn)` +# and the swapped one at `(Qn, Qm)`. A caller-named kernel runs either way; +# an automatic one is `_default_shape`'s pick, whose two candidates differ +# only when the small-M demotion applies to one orientation. The swapped +# candidate is resolved for a real `T` only, the only `T` that can swap. +@inline _candidate_mrs(::Type{T}, kernel, Qm::Int, Qn::Int) where {T} = (mr(kernel), mr(kernel)) +@inline function _candidate_mrs(::Type{T}, ::Nothing, Qm::Int, Qn::Int) where {T} + mr_asis = _default_shape(T, Qm, Qn)[1][1] + mr_swapped = T <: Real ? _default_shape(T, Qn, Qm)[1][1] : mr_asis + return mr_asis, mr_swapped +end + +# Kernel resolution, the last step before the barrier. A caller-named kernel +# goes straight through, never demoted. An automatic one is chosen as a +# `(shape, method)` value (`_default_shape`) and built INSIDE +# `_with_menu_kernel`'s unrolled ladder, so that `_plan_demoted` -- and +# through it `_demote_for_run` and `_plan_contract` -- always runs on a +# concrete kernel type and every call below this line is static. +# +# Why not simply `_plan_contract(_default_kernel(T, Qm, Qn), ...)`: that +# value's type is the Union of `T`'s whole menu, ten members for ComplexF64, +# past the four Julia union-splits, so the barrier call became a +# `jl_apply_generic` that boxed the request and -- for reasons this file does +# not claim to understand -- missed the method cache more often than not. +# Measured on ccqlin038 (Julia 1.12.7, 8x8x8 ComplexF64, pooled workspace): +# `plan_contract` 0.78 us and 704 B/call with the Union-typed call; the same +# tail with a concrete kernel 0.09 us and 0 B. Float64 was already static +# (a two-member Union) and is unchanged by this. +@inline _plan_with_kernel(kernel, atransform, btransform, req::_PlanRequest) = + _plan_contract(kernel, atransform, btransform, req) +@inline function _plan_with_kernel(::Nothing, atransform, btransform, req::_PlanRequest{T}) where {T} + shape, method = _default_shape(T, axis_length(req.mgroup), axis_length(req.ngroup)) + return _with_menu_kernel(_plan_demoted, shape, T, method, atransform, btransform, req) +end + +# Run-length demotion (`_demote_for_run`) of an automatically chosen kernel, +# keyed on the chosen M orientation's own run (`req.run`), then the barrier. +@inline function _plan_demoted(kernel, atransform, btransform, req::_PlanRequest{T}) where {T} + kernel_final = _demote_for_run( + T, kernel, req.run, axis_length(req.mgroup), axis_length(req.kgroup) ) + return _plan_contract(kernel_final, atransform, btransform, req) end -# Function barrier: the small Unions from `_default_kernel` and from the -# `conj`/`identity` transforms die here, so `ContractPlan`'s `Kern`, `TA` and -# `TB` are concrete and `execute!` sees no abstract type. `TA`/`TB` each get -# their own bound parameter for the same reason `K` does. +# Function barrier: the `conj`/`identity` transform Unions die here (and, on +# the real path, `_demote_for_run`'s small kernel Union), so `ContractPlan`'s +# `Kern`, `TA` and `TB` are concrete and `execute!` sees no abstract type. +# `TA`/`TB` each get their own bound parameter for the same reason `K` does. +# The continuation `req.f` runs in here rather than on the returned plan: +# `f(plan)` sees a concrete plan type, so an executor passed as `f` needs no +# further dispatch and no boxed plan. function _plan_contract( - C::StridedView, A::StridedView, B::StridedView, indC::NTuple{NC, Int}, - mgroup, ngroup, kgroup, Qm::Int, Qn::Int, Qk::Int, - kernel::K, atransform::TA, btransform::TB, - mc::Union{Int, Nothing}, kc::Union{Int, Nothing}, - nc::Union{Int, Nothing}, workspace::Union{Nothing, ContractWorkspace}, - allocator, oracle::Bool - ) where {NC, K, TA, TB} - T = eltype(C) + kernel::K, atransform::TA, btransform::TB, req::_PlanRequest{T} + ) where {K, TA, TB, T} scalartype(kernel) === T || throw(ArgumentError("kernel scalar type $(scalartype(kernel)) does not match eltype(C) = $T")) + Qm = axis_length(req.mgroup) + Qn = axis_length(req.ngroup) + Qk = axis_length(req.kgroup) + defaults = default_blocking(kernel) # Blocking's own constructor validates all three >= 1. + mc, kc, nc = req.mc, req.kc, req.nc requested = Blocking( mc === nothing ? defaults.mc : mc, kc === nothing ? defaults.kc : kc, @@ -252,19 +363,13 @@ function _plan_contract( blocking = Blocking(mc_eff, kc_eff, nc_eff) - Astorage = parent(A) - Abase = offset(A) - Bstorage = parent(B) - Bbase = offset(B) - Cstorage = parent(C) - Cbase = offset(C) - # Buffers are `undef`-initialized, not zeroed; see `ContractWorkspace`. - ws = _resolve_workspace(T, workspace, kernel, blocking, oracle, allocator) + ws = _resolve_workspace(T, req.workspace, kernel, blocking, req.oracle, req.allocator) - return ContractPlan( - kernel, mgroup, ngroup, kgroup, blocking, - Astorage, Abase, Bstorage, Bbase, Cstorage, Cbase, + plan = ContractPlan( + kernel, req.mgroup, req.ngroup, req.kgroup, blocking, + req.Astorage, req.Abase, req.Bstorage, req.Bbase, req.Cstorage, req.Cbase, atransform, btransform, ws ) + return req.f(plan) end diff --git a/test/planning/test_per_call_overhead.jl b/test/planning/test_per_call_overhead.jl index 252ddbe..36aa2ce 100644 --- a/test/planning/test_per_call_overhead.jl +++ b/test/planning/test_per_call_overhead.jl @@ -55,10 +55,16 @@ end for (indA, indB, indC) in cases got = QS._classify_labels(indA, indB, indC) want = _ref_classify(indA, indB, indC) - @test got == want + # Statically sized tuples, one per composite, with the ranks fixed by + # the label tuples' lengths (`_group_ranks`); same labels, same order. + @test map(collect, got) == want + @test all(l -> l isa Tuple{Vararg{Int}}, got) + @test isconcretetype(typeof(got)) # Trimmed to the real counts, not left at their `NA`/`NB` worst case. @test length(got[1]) + length(got[3]) == length(indA) @test length(got[2]) + length(got[3]) == length(indB) + @test (length(got[1]), length(got[2]), length(got[3])) == + QS._group_ranks(length(indA), length(indB), length(indC)) end # Rejections. @@ -77,20 +83,20 @@ end indC = ntuple(identity, nd) st = Base.strides(C) labels = shuffle(collect(indC))[1:rand(1:nd)] - got = QS._order_free_labels(labels, indC, C) + got = QS._order_free_labels(Tuple(labels), indC, C) want = labels[sortperm([abs(st[l]) for l in labels]; alg = Base.Sort.DEFAULT_STABLE)] - @test got == want - # Never mutates its input (planning/test_plan_contract.jl's label-order pinning reads - # `_classify_labels`'s output after ordering it). - @test length(got) == length(labels) + @test collect(got) == want + # Same statically sized tuple type in as out (planning/test_plan_contract.jl's + # label-order pinning reads `_classify_labels`'s output after ordering it). + @test got isa NTuple{length(labels), Int} end # Ties keep input order, explicitly: a size-1 axis of a square array gives # equal |stride| only when the strides really are equal, so build the tie # by hand with a 1-element axis. C = StridedView(randn(1, 1, 3)) - @test QS._order_free_labels([2, 1], (1, 2, 3), C) == [2, 1] - @test QS._order_free_labels([1, 2], (1, 2, 3), C) == [1, 2] + @test QS._order_free_labels((2, 1), (1, 2, 3), C) === (2, 1) + @test QS._order_free_labels((1, 2), (1, 2, 3), C) === (1, 2) end @testset "per-call floor: _build_pair_group matches a direct construction" begin @@ -111,7 +117,9 @@ end ) v2 = StridedView(randn(dims2)) - g = QS._build_pair_group(labels, ind1, v1, ind2, v2) + # The label tuple's length IS the rank (`_group_ranks`), so the group + # type is concrete without any runtime rank ladder. + g = QS._build_pair_group(Tuple(labels), ind1, v1, ind2, v2) s1 = Base.strides(v1) s2 = Base.strides(v2) p1 = ntuple(d -> findfirst(==(labels[d]), ind1)::Int, shared) @@ -121,15 +129,18 @@ end @test g.strides[2] == ntuple(d -> s2[p2[d]], shared) @test g isa AxisGroup{shared, 2} @test isconcretetype(typeof(g)) + @test Base.return_types( + QS._build_pair_group, (typeof(Tuple(labels)), typeof(ind1), typeof(v1), typeof(ind2), typeof(v2)) + ) == [AxisGroup{shared, 2}] end # A matched label whose lengths disagree is still a DimensionMismatch. v1 = StridedView(randn(3, 4)) v2 = StridedView(randn(5, 4)) - @test_throws DimensionMismatch QS._build_pair_group([1], (1, 2), v1, (1, 2), v2) + @test_throws DimensionMismatch QS._build_pair_group((1,), (1, 2), v1, (1, 2), v2) # Rank zero is reachable and well formed (an outer product's K group). - g0 = QS._build_pair_group(Int[], (1, 2), v1, (1, 2), v2) + g0 = QS._build_pair_group((), (1, 2), v1, (1, 2), v2) @test g0 isa AxisGroup{0, 2} @test axis_length(g0) == 1 end diff --git a/test/planning/test_plan_contract.jl b/test/planning/test_plan_contract.jl index cb1bea2..c3f00fb 100644 --- a/test/planning/test_plan_contract.jl +++ b/test/planning/test_plan_contract.jl @@ -258,28 +258,29 @@ end @testset "label order: _order_free_labels sorts by |C-stride|, stably" begin # C strides (12, 1, 60, 3) at indC positions carrying labels 10,20,30,40. + # Label lists are statically sized tuples (src/planning/labels.jl), so the + # sort returns a tuple of the same length, compared with `===`. Cv = _lo_view((5, 3, 2, 4), (12, 1, 60, 3)) indC = (10, 20, 30, 40) - labels = [10, 20, 30, 40] - @test _lo_order(labels, indC, Cv) == [20, 40, 10, 30] - @test labels == [10, 20, 30, 40] # input untouched - @test _lo_order([40, 10], indC, Cv) == [40, 10] # a subset: only its own members - @test _lo_order([30, 20], indC, Cv) == [20, 30] + labels = (10, 20, 30, 40) + @test _lo_order(labels, indC, Cv) === (20, 40, 10, 30) + @test _lo_order((40, 10), indC, Cv) === (40, 10) # a subset: only its own members + @test _lo_order((30, 20), indC, Cv) === (20, 30) # Ties keep input order, whichever way the input is given. Ct = _lo_view((2, 3, 4), (1, 1, 1)) - @test _lo_order([7, 8, 9], (7, 8, 9), Ct) == [7, 8, 9] - @test _lo_order([9, 7, 8], (7, 8, 9), Ct) == [9, 7, 8] + @test _lo_order((7, 8, 9), (7, 8, 9), Ct) === (7, 8, 9) + @test _lo_order((9, 7, 8), (7, 8, 9), Ct) === (9, 7, 8) # Negative strides sort by magnitude. Cn = _lo_view((3, 4), (-1, 4)) - @test _lo_order([20, 10], (10, 20), Cn) == [10, 20] + @test _lo_order((20, 10), (10, 20), Cn) === (10, 20) Cn2 = _lo_view((3, 4), (4, -1)) - @test _lo_order([10, 20], (10, 20), Cn2) == [20, 10] + @test _lo_order((10, 20), (10, 20), Cn2) === (20, 10) # Degenerate lengths. - @test _lo_order(Int[], indC, Cv) == Int[] - @test _lo_order([30], indC, Cv) == [30] + @test _lo_order((), indC, Cv) === () + @test _lo_order((30,), indC, Cv) === (30,) end @testset "label order: _leading_unit_run / _prefer_swap" begin @@ -288,38 +289,38 @@ end indC = (1, 2, 3, 4, 5, 6) # a,b,c,i,j,k a, b, c, i, j, k = indC - @test _lo_run([a, i, j], indC, C6) == d # ccsd_t_1's sorted M: run stops at i - @test _lo_run([a, b, k], indC, C6) == d^2 # ccsd_t_3's sorted N: b is C-adjacent to a - @test _lo_run([a, b, c, i, j, k], indC, C6) == d^6 - @test _lo_run([b, i, j], indC, C6) == 1 # no unit-stride head - @test _lo_run([a, c, b], indC, C6) == d # sorted order is the caller's job - @test _lo_run(Int[], indC, C6) == 1 + @test _lo_run((a, i, j), indC, C6) == d # ccsd_t_1's sorted M: run stops at i + @test _lo_run((a, b, k), indC, C6) == d^2 # ccsd_t_3's sorted N: b is C-adjacent to a + @test _lo_run((a, b, c, i, j, k), indC, C6) == d^6 + @test _lo_run((b, i, j), indC, C6) == 1 # no unit-stride head + @test _lo_run((a, c, b), indC, C6) == d # sorted order is the caller's job + @test _lo_run((), indC, C6) == 1 # A descending contiguous axis is NOT a unit-stride run (`_unit_stride_rows` # is `stride == 1`). - @test _lo_run([10, 20], (10, 20), _lo_view((3, 4), (-1, 3))) == 1 + @test _lo_run((10, 20), (10, 20), _lo_view((3, 4), (-1, 3))) == 1 # Singleton axes are skipped whatever their stride; an empty axis ends it. - @test _lo_run([10, 20], (10, 20), _lo_view((1, 6), (5, 1))) == 6 - @test _lo_run([10, 20], (10, 20), _lo_view((6, 1), (1, 17))) == 6 - @test _lo_run([10, 20], (10, 20), _lo_view((0, 6), (1, 1))) == 0 + @test _lo_run((10, 20), (10, 20), _lo_view((1, 6), (5, 1))) == 6 + @test _lo_run((10, 20), (10, 20), _lo_view((6, 1), (1, 17))) == 6 + @test _lo_run((10, 20), (10, 20), _lo_view((0, 6), (1, 1))) == 0 # The swap rule on the ccsd_t shapes at dim 4, with `mr` played by hand. - # ccsd_t_2: sorted M = [b,i,j] (run 1), sorted N = [a,c,k] (run 4). - @test _lo_swap([b, i, j], [a, c, k], indC, C6, 4) # 4-wide kernel: swap - @test !_lo_swap([b, i, j], [a, c, k], indC, C6, 8) # 8-wide: 4 < 8, do not swap - # ccsd_t_3: sorted N = [a,b,k] (run 16): the adjacent label extends it. - @test _lo_swap([c, i, j], [a, b, k], indC, C6, 8) - @test _lo_swap([c, i, j], [a, b, k], indC, C6, 16) - @test !_lo_swap([c, i, j], [a, b, k], indC, C6, 32) + # ccsd_t_2: sorted M = (b,i,j) (run 1), sorted N = (a,c,k) (run 4). + @test _lo_swap((b, i, j), (a, c, k), indC, C6, 4) # 4-wide kernel: swap + @test !_lo_swap((b, i, j), (a, c, k), indC, C6, 8) # 8-wide: 4 < 8, do not swap + # ccsd_t_3: sorted N = (a,b,k) (run 16): the adjacent label extends it. + @test _lo_swap((c, i, j), (a, b, k), indC, C6, 8) + @test _lo_swap((c, i, j), (a, b, k), indC, C6, 16) + @test !_lo_swap((c, i, j), (a, b, k), indC, C6, 32) # ccsd_t_1: M already has the run; never swap, whatever mr says. - @test !_lo_swap([a, i, j], [b, c, k], indC, C6, 4) - @test !_lo_swap([a, i, j], [b, c, k], indC, C6, 8) + @test !_lo_swap((a, i, j), (b, c, k), indC, C6, 4) + @test !_lo_swap((a, i, j), (b, c, k), indC, C6, 8) # Two-mr form: each orientation is judged against the kernel it would run. - @test _lo_swap([b, i, j], [a, c, k], indC, C6, 8, 4) - @test !_lo_swap([b, i, j], [a, c, k], indC, C6, 8, 8) - @test !_lo_swap([a, i, j], [b, c, k], indC, C6, 4, 4) + @test _lo_swap((b, i, j), (a, c, k), indC, C6, 8, 4) + @test !_lo_swap((b, i, j), (a, c, k), indC, C6, 8, 8) + @test !_lo_swap((a, i, j), (b, c, k), indC, C6, 4, 4) # Nothing to swap onto. - @test !_lo_swap([b, i, j], Int[], indC, C6, 1) + @test !_lo_swap((b, i, j), (), indC, C6, 1) end @testset "label order: the two _prefer_swap methods agree" begin @@ -332,8 +333,8 @@ end indC = (1, 2, 3, 4, 5, 6) a, b, c, i, j, k = indC for (morder, norder, mr_asis, mr_swapped) in ( - ([b, i, j], [a, c, k], 4, 4), ([b, i, j], [a, c, k], 8, 8), - ([c, i, j], [a, b, k], 8, 16), ([a, i, j], [b, c, k], 4, 8), + ((b, i, j), (a, c, k), 4, 4), ((b, i, j), (a, c, k), 8, 8), + ((c, i, j), (a, b, k), 8, 16), ((a, i, j), (b, c, k), 4, 8), ) run_m = _lo_run(morder, indC, C6) run_n = _lo_run(norder, indC, C6) @@ -378,7 +379,7 @@ end strides = ntuple(_ -> rand(rng, (-3, -1, 1, 2, 3, 7)), D) Cr = _lo_view(lens, strides) nlabels = rand(rng, 1:D) - order = Random.shuffle(rng, collect(1:D))[1:nlabels] + order = Tuple(Random.shuffle(rng, collect(1:D))[1:nlabels]) Qm = prod(lens[l] for l in order) Qm == 1 && continue ntested += 1 From dea949fbbb67fc90a3961c28e0c95d432c7f4b18 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 19:20:45 -0400 Subject: [PATCH 05/53] Vectorized unit-stride store for the FMAddSub complex kernel The FMAddSub kernel is what the AVX-512 small-M demotion selects for every complex contraction with Qm < 24, so its store sits on the per-call floor of exactly the tiny problems where the floor matters. Its only store was the scalar interleaved reader (1m's), ~3 ns per element through _axpby_tile!: 46% of an 8x8x8 ComplexF64 execute! (0.19 of 0.41 us). The accumulator is already in Complex{R}'s native [re, im, ...] order, so the fast path needs no interleave shuffle: one W-real accumulator vector is W/2 consecutive destination elements, and alpha*R + beta*C is computed lane-wise with addsub / fmaddsub on the same Base Complex expression trees _axpby_tile! reaches (bitwise equal on the vectorized lanes; the straddling row tail goes through _axpby_at!). Gated by _complex_vector_eligible, the same predicate as planar's vector store; everything else keeps the scalar reader. Tests: new test/microkernels/test_fmaddsub_store_fastpath.jl (values bitwise vs an independent reference and within tolerance vs the scalar reader over every menu shape and edge extent, alpha/beta contracts, guard band, allocation-free); the existing reader test now uses scattered rows so it still exercises the scalar path. Full suite green (181764 passed). Measured on ccqlin038 (Cascade Lake, Julia 1.12.7), interleaved, two rounds, whole tensorcontract!(..., QuasiStridedBackend()) calls, scalar-only store vs this one (benchmark/probes/probe_call_floor.jl, medians of 15 x 2000): ComplexF64 8x8x8 0.68 -> 0.53 us (StridedBLAS 0.43) ComplexF64 16x16x16 1.97 -> 1.35 us (StridedBLAS 1.46) ComplexF64 dim-4 outer 1.45 -> 0.84 us (StridedBLAS 1.00) Also adds the two probes used for this work: probe_call_floor.jl (public API, runs against any checkout's benchmark env, for A/B) and probe_call_breakdown.jl (prefix / workspace / planning / execute! split). Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_call_breakdown.jl | 65 ++++++ benchmark/probes/probe_call_floor.jl | 56 +++++ src/microkernels/fmaddsub.jl | 195 +++++++++++++++- test/microkernels/test_fmaddsub_kernel.jl | 12 +- .../test_fmaddsub_store_fastpath.jl | 214 ++++++++++++++++++ test/runtests.jl | 2 + 6 files changed, 536 insertions(+), 8 deletions(-) create mode 100644 benchmark/probes/probe_call_breakdown.jl create mode 100644 benchmark/probes/probe_call_floor.jl create mode 100644 test/microkernels/test_fmaddsub_store_fastpath.jl diff --git a/benchmark/probes/probe_call_breakdown.jl b/benchmark/probes/probe_call_breakdown.jl new file mode 100644 index 0000000..b6941af --- /dev/null +++ b/benchmark/probes/probe_call_breakdown.jl @@ -0,0 +1,65 @@ +# Where the per-call floor of `tensorcontract!(..., QuasiStridedBackend())` +# goes on a tiny problem: the adapter's prefix (`_qs_prepare`), the pooled +# workspace lookup, planning alone (`_planned` with `identity` as the +# continuation, i.e. `plan_contract`'s body), `execute!` alone on a prebuilt +# plan, and the whole call. Internal API: tracks the current checkout only. +# +# JULIA_NUM_THREADS=1 taskset -c 4-7 \ +# julia --project=benchmark benchmark/probes/probe_call_breakdown.jl + +using TensorOperations, Statistics, Printf, StridedViews +using QuasiStrided +const TO = TensorOperations +const QS = QuasiStrided + +const SAMPLES = 15 +const CALLS = 2000 + +function timeit(f, args...) + for _ in 1:10 + f(args...) + end + bytes = @allocated f(args...) + ts = map(1:SAMPLES) do _ + t = time_ns() + for _ in 1:CALLS + f(args...) + end + (time_ns() - t) / CALLS + end + return median(ts), minimum(ts), bytes +end + +full(C, A, pA, B, pB, pAB) = + TO.tensorcontract!(C, A, pA, false, B, pB, false, pAB, true, false, QuasiStridedBackend()) +prep(C, A, pA, B, pB, pAB) = QS._qs_prepare(C, A, pA, B, pB, pAB, true, false) +wsget(::AbstractArray{T}) where {T} = QS._qs_task_workspace(T) +function plan_only(C, A, pA, B, pB, pAB) + Cv, Av, Bv, iA, iB, iC, _, _ = QS._qs_prepare(C, A, pA, B, pB, pAB, true, false) + return QS._planned( + _ -> nothing, Cv, Av, iA, Bv, iB, iC, nothing, false, false, + nothing, nothing, nothing, QS._qs_task_workspace(eltype(C)), TO.DefaultAllocator(), false + ) +end +exec_only(plan, a, b) = QS.execute!(plan, a, b) + +const CASES = ( + ("8x8x8 gemm", (8, 8), (8, 8), (8, 8), ((1,), (2,)), ((1,), (2,)), ((1, 2), ())), + ("d4 outer", (4, 4), (4, 4), (4, 4, 4, 4), ((1, 2), ()), ((), (1, 2)), ((1, 2, 3, 4), ())), +) + +for T in (Float64, ComplexF64), (name, sa, sb, sc, pA, pB, pAB) in CASES + A, B, C = rand(T, sa), rand(T, sb), zeros(T, sc) + Cv, Av, Bv, iA, iB, iC, _, _ = prep(C, A, pA, B, pB, pAB) + plan = QS.plan_contract(Cv, Av, iA, Bv, iB, iC; workspace = QS._qs_task_workspace(T)) + for (what, f, args) in ( + ("full", full, (C, A, pA, B, pB, pAB)), + ("prepare", prep, (C, A, pA, B, pB, pAB)), + ("workspace", wsget, (C,)), + ("prep+plan", plan_only, (C, A, pA, B, pB, pAB)), + ("execute!", exec_only, (plan, one(T), zero(T))), + ) + med, lo, b = timeit(f, args...) + @printf("%-10s\t%-10s\t%-10s\t%7.1f ns\t(min %7.1f)\t%4d B\n", T, name, what, med, lo, b) + end +end diff --git a/benchmark/probes/probe_call_floor.jl b/benchmark/probes/probe_call_floor.jl new file mode 100644 index 0000000..f6c32b8 --- /dev/null +++ b/benchmark/probes/probe_call_floor.jl @@ -0,0 +1,56 @@ +# Per-call floor of `tensorcontract!` on tiny problems: QuasiStridedBackend vs +# StridedBLAS, through TensorOperations' public entry point with the default +# allocator (so QuasiStrided uses its pooled task workspace). +# +# JULIA_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 taskset -c 4-7 \ +# julia --project=benchmark benchmark/probes/probe_call_floor.jl [label] +# +# Prints one line per (case, backend): median ns/call over SAMPLES samples of +# CALLS calls each, the [min, max] of those samples, and bytes/call. Whatever +# QuasiStrided the `--project` environment resolves is what is measured, so an +# A/B is two runs of this script from two checkouts, interleaved. + +using TensorOperations, Statistics, Printf, LinearAlgebra +using QuasiStrided +const TO = TensorOperations + +BLAS.set_num_threads(1) + +const SAMPLES = 15 +const CALLS = 2000 + +# (name, sizes of A, B, C, pA, pB, pAB) +const CASES = ( + ("8x8x8 gemm", (8, 8), (8, 8), (8, 8), ((1,), (2,)), ((1,), (2,)), ((1, 2), ())), + ("16^3 gemm", (16, 16), (16, 16), (16, 16), ((1,), (2,)), ((1,), (2,)), ((1, 2), ())), + # dim-4 outer product C[i,j,k,l] = A[i,j] * B[k,l] + ("d4 outer", (4, 4), (4, 4), (4, 4, 4, 4), ((1, 2), ()), ((), (1, 2)), ((1, 2, 3, 4), ())), +) + +function ncalls(C, A, pA, B, pB, pAB, backend, n) + for _ in 1:n + TO.tensorcontract!(C, A, pA, false, B, pB, false, pAB, true, false, backend) + end + return C +end + +function measure(C, A, pA, B, pB, pAB, backend) + ncalls(C, A, pA, B, pB, pAB, backend, 10) + bytes = @allocated ncalls(C, A, pA, B, pB, pAB, backend, 100) + ts = map(1:SAMPLES) do _ + t = time_ns() + ncalls(C, A, pA, B, pB, pAB, backend, CALLS) + (time_ns() - t) / CALLS + end + return median(ts), minimum(ts), maximum(ts), bytes / 100 +end + +label = isempty(ARGS) ? "" : ARGS[1] +for T in (Float64, ComplexF64), (name, sa, sb, sc, pA, pB, pAB) in CASES + A, B, C = rand(T, sa), rand(T, sb), zeros(T, sc) + for (bname, backend) in (("QS", QuasiStridedBackend()), ("BLAS", TO.StridedBLAS())) + med, lo, hi, b = measure(C, A, pA, B, pB, pAB, backend) + @printf("%s\t%-10s\t%-10s\t%-4s\t%8.1f ns\t[%7.1f, %7.1f]\t%6.0f B\n", + label, T, name, bname, med, lo, hi, b) + end +end diff --git a/src/microkernels/fmaddsub.jl b/src/microkernels/fmaddsub.jl index 182fe53..72c4f84 100644 --- a/src/microkernels/fmaddsub.jl +++ b/src/microkernels/fmaddsub.jl @@ -369,10 +369,8 @@ end # The interleaved tile reader: 1m's `_store_tile_onem!` (src/microkernels/ # onem.jl) verbatim but for the kernel type -- the accumulator layout is the # same, so is the reader, and the test suite checks the two agree bitwise on -# one accumulator. Scattered/scalar only, like 1m's; there is no unit-stride -# vector store counterpart to planar's `_store_tile_planar_vector!` yet (the -# accumulator is already in native `Complex{T}` order, so one would need no -# interleave shuffle -- a deliberately deferred follow-up). +# one accumulator. The scalar fallback of `store_tile!` below, for every +# destination `_store_tile_fmaddsub_vector!` is ineligible for. @generated function _store_tile_fmaddsub!( destination::QSTile, acc::NTuple{NV, Vec{W, R}}, alpha::T, beta::T, kernel::FMAddSubKernel{MR, NR, T, W}, @@ -424,13 +422,195 @@ end end end +# ---------------------------------------------------------------------------- +# The vectorized store for a unit-stride destination +# ---------------------------------------------------------------------------- +# +# The accumulator is already in `Complex{R}`'s native `[re, im, re, im, ...]` +# order, so unlike planar's `_store_tile_planar_vector!` no interleave shuffle +# is needed: one `W`-real accumulator vector IS `W÷2` consecutive destination +# elements, and `C = alpha*R + beta*C` is computed lane-wise with the two +# primitives the K loop already uses. Per full block: +# +# beta == 0: alpha*r = addsub(ar*r, ai*swap(r)) +# beta == 1: muladd(alpha, r, C) = fmaddsub(ar, r, fmaddsub(ai, swap(r), C)) +# otherwise: as beta == 1 with C := addsub(br*C, bi*swap(C)) +# +# where `swap` is `_swap_pairs`, `addsub(p, q)` is `p - q` in even (real) lanes +# and `p + q` in odd (imaginary) lanes, and `fmaddsub(x, y, c)` is `x*y - c` / +# `x*y + c` fused. These are exactly Base's `Complex` expression trees as +# `_axpby_tile!` (src/microkernels/interface.jl) reaches them -- `*` is +# `(zr*wr - zi*wi, zr*wi + zi*wr)`, two separately rounded products, and +# `muladd(z, w, x)` is `(muladd(zr, wr, -muladd(zi, wi, -xr)), muladd(zr, wi, +# muladd(zi, wr, xi)))`, fused at both levels -- so on the lanes it vectorizes +# the fast path is bitwise identical to a fused transcription of those trees +# (test/microkernels/test_fmaddsub_store_fastpath.jl pins this with `isequal`), +# and agrees with the scalar fallback exactly at `beta == 0` and `beta == 1` +# and to within LLVM's `muladd` contraction choice otherwise, as planar's does. +# +# Why it exists: the scalar reader above was 46% of an 8x8x8 ComplexF64 +# `execute!` (0.19 of 0.41 us, ~3 ns per element through `_axpby_tile!`; +# ccqlin038, Julia 1.12.7, profile of `execute!` alone) -- and this kernel is +# the one the AVX-512 small-M demotion selects for every complex contraction +# with `Qm < 24`, so that store is on the per-call floor of exactly the small +# problems where the floor is what matters. Measured, same host and probe +# (medians of 15 x 2000 calls, scalar store then this one, back to back, not +# interleaved), `execute!` on a dense ComplexF64 GEMM at the `8x8/W8` shape: +# 8x8x8 0.41 -> 0.27 us, 16x16x16 1.64 -> 1.00 us, 4x4 outer product 0.20 -> +# 0.15 us; through the TensorOperations adapter the 16x16x16 call went from +# 1.33x StridedBLAS's time to 0.93x. Re-measured interleaved (two rounds, +# benchmark/probes/probe_call_floor.jl, scalar-only store vs this one, whole +# `tensorcontract!` calls): 8x8x8 0.68 -> 0.53 us, 16x16x16 1.97 -> 1.35 us, +# dim-4 outer product 1.45 -> 0.84 us. + +# `p - q` in even lanes, `p + q` in odd lanes, both separately rounded: Base's +# unfused `Complex` `*` on interleaved data. +@generated function _addsub(p::Vec{N, R}, q::Vec{N, R}) where {N, R} + iseven(N) || return :(throw(ArgumentError("_addsub: expected even N, got $N"))) + idx = ntuple(k -> iseven(k - 1) ? k - 1 : N + k - 1, N) + return :(Base.@_inline_meta; shufflevector(p - q, p + q, Val($idx))) +end + +# One full `W÷2`-row block of one column: `at` is the ZERO-based index of the +# block's first REAL in the reinterpreted storage, i.e. twice the complex +# element index. `ar`/`ai`/`br`/`bi` are pre-broadcast once per `store_tile!` +# call; `beta` itself is passed only so the two `iszero`/`isone` tests are the +# identical tests `_axpby_tile!` makes on the identical value. +@inline function _fmaddsub_store_block!( + sp::Ptr{R}, at::Int, r::Vec{W, R}, + ar::Vec{W, R}, ai::Vec{W, R}, br::Vec{W, R}, bi::Vec{W, R}, + beta::Complex{R} + ) where {R, W} + s = _swap_pairs(r) + if iszero(beta) + new = _addsub(ar * r, ai * s) + else + old = vload(Vec{W, R}, sp + sizeof(R) * at) + x = isone(beta) ? old : _addsub(br * old, bi * _swap_pairs(old)) + new = _fmaddsub(ar, r, _fmaddsub(ai, s, x)) + end + vstore(new, sp + sizeof(R) * at) + return nothing +end + +# Vectorized FMAddSub store. The same `@generated` unroll as planar's +# `_store_tile_planar_vector!` -- every `acc[...]` a literal tuple index +# (Cliff B), the same full-block vs. row-tail split, the same `j < n` column +# guard -- with a "row block" being one accumulator vector of `W÷2` complex +# rows. `rows::AffineAxis` is pinned in the signature so an ineligible tile is +# a MethodError rather than a wrong answer; the caller checks +# `_complex_vector_eligible` first. The scalar row tail goes through +# `storage` itself via `_axpby_at!`, character-identical to the fallback's +# arithmetic. GC.@preserve: the raw `Ptr{R}` is derived from `storage` and +# every dereference happens inside the preserve block. +@generated function _store_tile_fmaddsub_vector!( + destination::QSTile{S, <:AffineAxis}, acc::NTuple{NV, Vec{W, R}}, + alpha::T, beta::T, kernel::FMAddSubKernel{MR, NR, T, W}, + m::Int, n::Int + ) where {S, MR, NR, T, W, R, NV} + R === real(T) || + throw( + ArgumentError( + "_store_tile_fmaddsub_vector!: accumulator lane type $R does not match " * + "real($T) = $(real(T))" + ) + ) + # The bitcast below is only sound on rank-1 dense storage of exactly `T`. + S <: DenseVector{T} || + throw( + ArgumentError( + "_store_tile_fmaddsub_vector!: destination storage $S is not a DenseVector{$T}" + ) + ) + iseven(W) || + throw(ArgumentError("_store_tile_fmaddsub_vector!: requires an even W, got $W")) + MV = (2 * MR) ÷ W + MV * NR == NV || + throw( + ArgumentError( + "_store_tile_fmaddsub_vector!: accumulator length $NV does not match " * + "(2*mr÷W)*nr = $(MV * NR) for MR=$MR, NR=$NR, W=$W" + ) + ) + HW = W ÷ 2 + + blocks = Any[] + for j in 0:(NR - 1) + vblocks = Any[] + for v in 0:(MV - 1) + idx = v + MV * j + 1 + push!( + vblocks, quote + vec = acc[$idx] + if $((v + 1) * HW) <= m + # Rows v*HW .. v*HW+HW-1 all inside [0, m): one W-real + # load/store at that block's first real. + _fmaddsub_store_block!( + sp, 2 * (colbase + $(v * HW)), vec, ar, ai, br, bi, beta + ) + elseif $(v * HW) < m + # Row tail: this vector straddles m, so store the + # valid rows one at a time through the SAME + # `_axpby_at!` arithmetic the scalar reader uses. + for u in 0:$(HW - 1) + i = $(v * HW) + u + i < m || break + _axpby_at!( + storage, colbase + i + 1, alpha, + Complex(vec[2 * u + 1], vec[2 * u + 2]), beta + ) + end + end + end + ) + end + push!( + blocks, quote + if $j < n + colbase = rowbase0 + axis_offset(cols, $j) # zero-based element address of (i=0, j) + $(vblocks...) + end + end + ) + end + + return quote + storage = destination.storage + cols = destination.cols + # zero-based element address at (i=0, j=0)'s row contribution; rows are + # unit-stride, so row `i` is `rowbase0 + i`. + rowbase0 = destination.base + destination.rows.base + ar = Vec{$W, $R}(real(alpha)) + ai = Vec{$W, $R}(imag(alpha)) + br = Vec{$W, $R}(real(beta)) + bi = Vec{$W, $R}(imag(beta)) + GC.@preserve storage begin + sp = reinterpret(Ptr{$R}, pointer(storage)) + @inbounds begin + $(blocks...) + end + end + return destination + end +end + """ store_tile!(destination::QSTile, acc, alpha::T, beta::T, kernel::FMAddSubKernel) -> destination Same contract as every other kernel's `store_tile!`: `C = alpha*R + beta*C` over the valid rectangle only, `alpha == 0` never reads `acc`, `beta == 0` never reads old `C`, padding lanes are never read, an empty destination is a -no-op. Reads the interleaved accumulator exactly as 1m's does. +no-op. + +Two paths, chosen by `_complex_vector_eligible` (src/microkernels/planar.jl), +the same gate as planar's store: a unit-stride `AffineAxis` row axis into +rank-1 dense `Complex` storage on a shipped ISA takes +`_store_tile_fmaddsub_vector!`, one `W`-real load (only when `beta != 0`) +and store per full `W÷2`-row block with the arithmetic done in the +accumulator's own interleaved layout; everything else takes the scalar +reader `_store_tile_fmaddsub!`, which reads the interleaved accumulator +exactly as 1m's does. The two agree bitwise at `beta == 0` and `beta == 1`; +compare the general-`beta` case with a tolerance, as for planar. """ function store_tile!( destination::QSTile, acc::NTuple{NV, Vec{W, R}}, @@ -438,5 +618,10 @@ function store_tile!( ) where {MR, NR, T, W, R, NV} m, n = _store_prologue!(destination, alpha, beta) (m == 0 || n == 0) && return destination + + if _complex_vector_eligible(destination, T) + return _store_tile_fmaddsub_vector!(destination, acc, alpha, beta, kernel, m, n) + end + return _store_tile_fmaddsub!(destination, acc, alpha, beta, kernel, m, n) end diff --git a/test/microkernels/test_fmaddsub_kernel.jl b/test/microkernels/test_fmaddsub_kernel.jl index df44b0b..15637e1 100644 --- a/test/microkernels/test_fmaddsub_kernel.jl +++ b/test/microkernels/test_fmaddsub_kernel.jl @@ -375,17 +375,23 @@ const _QSF = QuasiStrided v, j = (idx - 1) % MV, (idx - 1) ÷ MV Vec{W, R}(ntuple(l -> R(1000 * j + v * W + l - 1), W)) end + # Scattered rows (same physical layout as unit stride): the + # gate of the vectorized fast path excludes them, so this is + # the scalar READER under test, as the testset name says. The + # fast path has its own file, + # test/microkernels/test_fmaddsub_store_fastpath.jl. + rows_scattered = ScatterAxis(collect(0:(MR - 1)), MR) for (alpha, beta) in ((one(T), zero(T)), (T(2, -1), T(0.5, 0.25))) s1 = [T(i, -i) for i in 1:(MR * NR)] s2 = copy(s1) - d1 = DestinationTile(s1, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) - d2 = DestinationTile(s2, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) + d1 = DestinationTile(s1, 0, rows_scattered, AffineAxis(0, MR, NR)) + d2 = DestinationTile(s2, 0, rows_scattered, AffineAxis(0, MR, NR)) store_tile!(d1, acc, alpha, beta, kf) store_tile!(d2, acc, alpha, beta, km) @test isequal(s1, s2) end storage = zeros(T, MR * NR) - dst = DestinationTile(storage, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) + dst = DestinationTile(storage, 0, rows_scattered, AffineAxis(0, MR, NR)) store_tile!(dst, acc, one(T), zero(T), kf) for j in 0:(NR - 1), i in 0:(MR - 1) @test storage[i + j * MR + 1] == diff --git a/test/microkernels/test_fmaddsub_store_fastpath.jl b/test/microkernels/test_fmaddsub_store_fastpath.jl new file mode 100644 index 0000000..eabfa51 --- /dev/null +++ b/test/microkernels/test_fmaddsub_store_fastpath.jl @@ -0,0 +1,214 @@ +# The vectorized FMAddSub store fast path (src/microkernels/fmaddsub.jl, +# `_store_tile_fmaddsub_vector!`): the interleaved-layout counterpart of the +# planar one, pinned the same four ways test_planar_store_fastpath.jl pins +# that -- values (bitwise against the independent reference on the lanes it +# vectorizes, tolerance against the scalar reader everywhere), the alpha/beta +# contract on the fast path specifically, the gate, and ISA portability. The +# reference `ref_axpby`, the `barrier` it is built on and `STORE_FASTPATH_ON` +# are test_planar_store_fastpath.jl's, which runtests.jl includes into this +# scope first: the arithmetic contract is `_axpby_tile!`'s, shared by every +# complex kernel, so one transcription of it serves both fast paths. + +using Test +using Random +using QuasiStrided +using QuasiStrided: FMAddSubKernel, AffineAxis, ScatterAxis, DestinationTile, store_tile!, + KERNEL_SHAPES_C64_FMADDSUB, KERNEL_SHAPES_C32_FMADDSUB +using SIMD: Vec + +const _QSFS = QuasiStrided + +# Complex row `i` of column `j` in an FMAddSub/1m accumulator: vector +# `v = i ÷ (W÷2)`, tuple index `v + MV*j + 1`, lanes `2u+1`/`2u+2` (1-based) +# with `u = i % (W÷2)`. Re-derived from the layout claim in `zero_accumulator`'s +# docstring, not from the store's own code. +function fsfp_lanes(acc, MR::Int, NR::Int, W::Int, i::Int, j::Int) + HW = W ÷ 2 + MV = (2 * MR) ÷ W + v, u = i ÷ HW, i % HW + vec = acc[v + MV * j + 1] + return vec[2 * u + 1], vec[2 * u + 2] +end + +function fsfp_acc(::Type{T}, MR::Int, NR::Int, W::Int, seed::Int) where {T} + R = real(T) + rng = MersenneTwister(seed) + NV = ((2 * MR) ÷ W) * NR + return ntuple(_ -> Vec{W, R}(ntuple(_ -> R(2 * rand(rng) - 1), W)), NV) +end + +fsfp_cold(::Type{T}, len::Int, seed::Int) where {T} = + ( + rng = MersenneTwister(seed); + T[Complex(real(T)(2 * rand(rng) - 1), real(T)(2 * rand(rng) - 1)) for _ in 1:len] +) + +fsfp_ab(::Type{T}) where {T} = ( + (one(T), zero(T)), + (T(-0.5, 0.25), zero(T)), + (one(T), one(T)), + (T(2, -1), one(T)), + (T(2.5, -1), T(-1.75, 0.5)), + (one(T), T(2, 0)), + (T(1, 1), T(0, -3)), +) + +const FSFP_MENUS = ( + (ComplexF64, KERNEL_SHAPES_C64_FMADDSUB), + (ComplexF32, KERNEL_SHAPES_C32_FMADDSUB), +) + +fsfp_kernel(::Type{T}, MR::Int, NR::Int, W::Int) where {T} = + FMAddSubKernel(Val(MR), Val(NR), T, Val(W)) + +# --------------------------------------------------------------------------- +# 1. Values +# --------------------------------------------------------------------------- + +@testset "fmaddsub store fast path: values ($T)" for (T, menu) in FSFP_MENUS + R = real(T) + reltol = 8 * eps(R) # as for planar: one ULP measured, headroom for LLVM + + @testset "($MR,$NR,$W)" for (MR, NR, W) in menu + HW = W ÷ 2 + # The full tile, one element, one row/column, one short of full, an + # extent that is not a multiple of HW (a straddling block exists) and + # one that is exactly a multiple of HW below MR (a block is skipped). + extents = unique( + ( + (MR, NR), (1, 1), (MR, 1), (1, NR), (MR - 1, NR), + (max(1, MR ÷ 2 + 1), max(1, NR - 1)), (max(1, MR ÷ HW - 1) * HW, NR), + (max(1, MR - HW), NR), + ) + ) + for (m, n) in extents + (m <= 0 || n <= 0) && continue + acc = fsfp_acc(T, MR, NR, W, MR * 97 + NR * 13 + m * 7 + n) + cold = fsfp_cold(T, m * n, MR * 31 + m * 5 + n) + + for (alpha, beta) in fsfp_ab(T) + fast = copy(cold) + dfast = DestinationTile(fast, 0, AffineAxis(0, 1, m), AffineAxis(0, m, n)) + @test _QSFS._complex_vector_eligible(dfast, T) == STORE_FASTPATH_ON + store_tile!(dfast, acc, alpha, beta, fsfp_kernel(T, MR, NR, W)) + + # Same physical layout, scattered rows: the gate excludes it, + # so this is the scalar reader `_store_tile_fmaddsub!`. + scal = copy(cold) + dscal = DestinationTile( + scal, 0, ScatterAxis(collect(0:(m - 1)), m), AffineAxis(0, m, n) + ) + @test !_QSFS._complex_vector_eligible(dscal, T) + store_tile!(dscal, acc, alpha, beta, fsfp_kernel(T, MR, NR, W)) + + for j in 0:(n - 1), i in 0:(m - 1) + rr, ri = fsfp_lanes(acc, MR, NR, W, i, j) + want = ref_axpby(alpha, rr, ri, beta, cold[i + j * m + 1]) + got = fast[i + j * m + 1] + # (1a) bitwise where the fast path vectorizes: rows in a + # full HW-block; the straddling block's rows take the + # scalar tail, which inherits LLVM's contraction choice. + in_full_block = (i < (m ÷ HW) * HW) + if in_full_block && STORE_FASTPATH_ON + @test isequal(want, got) + end + # (1b) tolerance everywhere, against the scalar reader ... + ref = scal[i + j * m + 1] + @test abs(got - ref) <= reltol * max(abs(ref), one(R)) + # ... and against the independent reference. + @test abs(got - want) <= reltol * max(abs(want), one(R)) + # Exact agreement with the scalar reader where Base's tree + # has no contraction freedom: beta == 0 (unfused `*`). + if iszero(beta) + @test isequal(got, ref) + end + end + end + end + end +end + +# --------------------------------------------------------------------------- +# 2. Contract preservation on the fast path +# --------------------------------------------------------------------------- + +@testset "fmaddsub store fast path: contracts ($T)" for (T, menu) in FSFP_MENUS + R = real(T) + MR, NR, W = first(menu) + k = fsfp_kernel(T, MR, NR, W) + HW = W ÷ 2 + acc = fsfp_acc(T, MR, NR, W, 5) + + @testset "beta == 0 never reads old C (a full, vectorized block)" begin + poison = fill(T(NaN, Inf), MR * NR) + d = DestinationTile(poison, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) + @test _QSFS._complex_vector_eligible(d, T) == STORE_FASTPATH_ON + store_tile!(d, acc, T(2, -1), zero(T), k) + @test all(isfinite, poison) + end + + @testset "alpha == 0 never reads acc" begin + nanacc = ntuple(_ -> Vec{W, R}(R(NaN)), Val(((2 * MR) ÷ W) * NR)) + target = fill(T(2, 3), MR * NR) + d = DestinationTile(target, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) + store_tile!(d, nanacc, zero(T), T(2, 0), k) + @test all(==(T(4, 6)), target) + end + + @testset "nothing outside the valid rectangle is touched" begin + # A guard band around an (m, n) rectangle inside a larger dense array; + # rows unit-stride, so the fast path fires on the full blocks. + m, n = MR - 1, max(1, NR - 1) + ld = MR + 3 + storage = fill(T(-7, 7), ld * (NR + 2)) + base = 1 + ld # one row down, one column in + d = DestinationTile(storage, base, AffineAxis(0, 1, m), AffineAxis(0, ld, n)) + @test _QSFS._complex_vector_eligible(d, T) == STORE_FASTPATH_ON + store_tile!(d, acc, one(T), zero(T), k) + for idx in 0:(length(storage) - 1) + i, j = (idx - base) % ld, (idx - base) ÷ ld + inside = idx >= base && 0 <= i < m && 0 <= j < n + if inside + rr, ri = fsfp_lanes(acc, MR, NR, W, i, j) + @test storage[idx + 1] == Complex(rr, ri) + else + @test storage[idx + 1] == T(-7, 7) + end + end + end + + @testset "beta is applied exactly once" begin + cold = fsfp_cold(T, MR * NR, 11) + got = copy(cold) + d = DestinationTile(got, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) + alpha, beta = T(1.5, -0.5), T(-2, 0.25) + store_tile!(d, acc, alpha, beta, k) + for j in 0:(NR - 1), i in 0:(MR - 1) + rr, ri = fsfp_lanes(acc, MR, NR, W, i, j) + want = alpha * Complex(rr, ri) + beta * cold[i + j * MR + 1] + @test abs(got[i + j * MR + 1] - want) <= 8 * eps(R) * max(abs(want), one(R)) + end + end + + @testset "empty destination is a no-op" begin + ed = DestinationTile(T[], 0, AffineAxis(0, 1, 0), AffineAxis(0, 0, 0)) + @test store_tile!(ed, acc, one(T), zero(T), k) === ed + end +end + +# --------------------------------------------------------------------------- +# 3. Allocation-free, every shape +# --------------------------------------------------------------------------- + +@testset "fmaddsub store fast path: allocation-free ($T)" for (T, menu) in FSFP_MENUS + for (MR, NR, W) in menu + k = fsfp_kernel(T, MR, NR, W) + acc = fsfp_acc(T, MR, NR, W, 3) + storage = fsfp_cold(T, MR * NR, 4) + d = DestinationTile(storage, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) + for (alpha, beta) in fsfp_ab(T) + store_tile!(d, acc, alpha, beta, k) + @test (@allocated store_tile!(d, acc, alpha, beta, k)) == 0 + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index cafc495..3b6cf69 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -39,6 +39,8 @@ using QuasiStrided: AxisGroup, axis_length, offsets, fill_offsets!, BlockDescrip include("microkernels/test_planar_store_fastpath.jl") include("microkernels/test_onem_kernel.jl") include("microkernels/test_fmaddsub_kernel.jl") + # After test_planar_store_fastpath.jl: reuses its `ref_axpby` reference. + include("microkernels/test_fmaddsub_store_fastpath.jl") include("planning/test_kernel_selection.jl") include("planning/test_plan_contract.jl") From 50bb10bec248867e7a4b086c76d53298fdf67eed Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 19:22:42 -0400 Subject: [PATCH 06/53] Choose the K order by a cost model of the two packs, not by min K stride 12ae2b4 sorted K by stride in whichever operand had the smaller minimum K stride. That regresses TRG's plaquette contraction, A(24^5)[x,k1,y,z,k2] * B(24^3)[k2,w,k1]: B holds the unit-stride K axis, so the rule sorted by B and made A's fastest K step 24^4 elements (2.6 MB) -- a page-crossing, unprefetchable walk over the 8M-element operand -- to save refetches of a 110 KB B that lives in L2 whatever the order. Measured on ccqlin038 (Cascade Lake, pinned, benchmark/probes/probe_network_korder.jl, which now times indA order / 12ae2b4's rule / this model interleaved): trg_plaquette chi=24 Float64, pairwise: 12ae2b4 rule 1.69x slower than indA order; cost model keeps indA order (0.996x). Network: 1.23-1.29x vs 1.02x. trg_plaquette chi=32 ComplexF64, pairwise: 1.31x vs 0.996x; network 1.10x vs 1.00x. `_order_contract_labels` now picks among indA order, A-sorted and B-sorted by `_k_order_cost` summed over both operands: n * walk * amplification, where walk = 3 when the order's fastest K axis steps more than a page, and amplification = elements per line along the operand's smallest-stride axis u when u is a K axis that is not the fastest, the slivers are not whole lines, and the lines in flight before u advances exceed the core's L2 share (`_l2_core_bytes`, from the detected cache profile). Ties keep indA order. Which suite cases change (benchmark/probes/probe_korder_changes.jl, now with --sources, run at the benchmarked leg dims): only contract_scrambled layouts -- dim16/24 {0_2_3,3_2_3}, dim32 {2_2_2,1_3_1,0_2_3}, dim63/96 {2_2_2,1_3_1}. dim96_1_3_1_contract_scrambled plans the same K group as 12ae2b4 (0.60 s, was 4.56 s). No TCCG, batched, gemm_ready, a/b_permuted or both_permuted case changes; small scrambled cases whose operands fit L2 (dim8, dim16_1_3_1) keep indA order. Tests: the planner pins now drive `_order_contract_labels` with an explicit L2 size (flip at l2 = 0, keep at 1 MB), pin the TRG shape to indA order, check the plan-level flip and swapped orientation at an L2-exceeding size, and run the correctness sweep at that size so the reordered path is the one executed. Full suite green. Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_korder_changes.jl | 56 +++++++ benchmark/probes/probe_network_korder.jl | 123 +++++++++++++++ src/planning/labels.jl | 181 ++++++++++++++++++----- src/planning/plan.jl | 13 +- test/planning/test_plan_contract.jl | 143 +++++++++--------- 5 files changed, 406 insertions(+), 110 deletions(-) create mode 100644 benchmark/probes/probe_korder_changes.jl create mode 100644 benchmark/probes/probe_network_korder.jl diff --git a/benchmark/probes/probe_korder_changes.jl b/benchmark/probes/probe_korder_changes.jl new file mode 100644 index 0000000..d599051 --- /dev/null +++ b/benchmark/probes/probe_korder_changes.jl @@ -0,0 +1,56 @@ +# Which upstream-suite :contract cases does `_order_contract_labels` +# (src/planning/labels.jl) actually reorder? For every synthetic/TCCG/batched +# case at one small leg dim, plan it and compare the K composite against the +# pre-rule order (`indA` order, i.e. `_classify_labels`' output). Cases whose +# K group is unchanged run the identical code path before and after the rule +# and cannot have moved in a before/after benchmark except by noise. +# +# The cost model reads the operand sizes (L2 fit, page-sized walks), so run it +# at the leg dims actually benchmarked. +# +# julia --project=benchmark benchmark/probes/probe_korder_changes.jl \ +# [--dim 4] [--sources synthetic,tccg,batched] + +using TensorOperations +using TensorOperationsBenchmarks +using TensorOperationsBenchmarks: ContractSpec, BatchedContractSpec +using QuasiStrided +using QuasiStrided: plan_contract, _classify_labels, _build_pair_group +using StridedViews +using Random + +include(joinpath(@__DIR__, "..", "harness.jl")) + +const TOB = TensorOperationsBenchmarks +const DIM = argopt("dim", 4) +const SOURCES = split(argopt("sources", "synthetic,tccg,batched"), ',') + +gens = Dict("synthetic" => TOB._synthetic_contract_cases, "tccg" => TOB._tccg_cases, "batched" => TOB._batched_contract_cases) +cases = reduce(vcat, [gens[s]((DIM,)) for s in SOURCES]) +changed = String[] +unchanged = 0 +for case in cases + spec = case.spec + spec isa Union{ContractSpec, BatchedContractSpec} || continue + dimsA = ntuple(i -> spec.dims[spec.IA[i]], length(spec.IA)) + dimsB = ntuple(i -> spec.dims[spec.IB[i]], length(spec.IB)) + dimsC = ntuple(i -> spec.dims[spec.IC[i]], length(spec.IC)) + A = StridedView(Array{Float64}(undef, dimsA...)) # planning never reads values + B = StridedView(Array{Float64}(undef, dimsB...)) + C = StridedView(Array{Float64}(undef, dimsC...)) + pA, pB, pAB = TensorOperations.contract_indices(spec.IA, spec.IB, spec.IC) + indA, indB, indC = QuasiStrided._qs_labels(pA, pB, pAB) + plan = plan_contract(C, A, indA, B, indB, indC; oracle = false) + _, _, klabels = _classify_labels(indA, indB, indC) + old = _build_pair_group(klabels, indA, A, indB, B) # pre-rule K group, (A, B) maps + swapped = plan.Astorage === parent(B) + new = plan.kgroup + newstrides = swapped ? (new.strides[2], new.strides[1]) : new.strides + if (new.lengths, newstrides) == (old.lengths, old.strides) + global unchanged += 1 + else + push!(changed, "$(case.id) K old=$(klabels) A/B strides $(old.strides) -> new A/B strides $(newstrides)") + end +end +println("dim = $DIM: $(length(cases)) cases, $(unchanged) with an unchanged K group, $(length(changed)) reordered:") +foreach(println, changed) diff --git a/benchmark/probes/probe_network_korder.jl b/benchmark/probes/probe_network_korder.jl new file mode 100644 index 0000000..527c0c1 --- /dev/null +++ b/benchmark/probes/probe_network_korder.jl @@ -0,0 +1,123 @@ +# Which pairwise contractions inside an upstream :network case does the K +# order rule (`_order_contract_labels`, src/planning/labels.jl) change, and +# what does each change cost? Runs the case through `ncon` with +# QuasiStridedBackend, logging every K-order decision (labels, both operands' +# K strides and sizes, the chosen operand), then times the whole network with +# the rule ON and OFF (interleaved), and each changed pairwise contraction on +# its own, rebuilt from the logged shapes. +# +# julia -t 1 --project=benchmark benchmark/probes/probe_network_korder.jl \ +# --case mps_1site --size 32 --dtype ComplexF64 --reps 21 +# +# The rule is switched by redefining `_order_contract_labels` here with a +# mode flag -- a probe-only hack, never to be copied into src/. + +using TensorOperations +using TensorOperations: StridedBLAS +using TensorOperationsBenchmarks +using TensorOperationsBenchmarks: ArrayProvider, randtensor, NetworkSpec +using QuasiStrided +using QuasiStrided: QuasiStridedBackend, _choose_k_order, _k_order_cost +using StridedViews +using Random +using Statistics: median +using Printf +using LinearAlgebra + +include(joinpath(@__DIR__, "..", "harness.jl")) + +const TOB = TensorOperationsBenchmarks +const CASE = argopt("case", "mps_1site") +const SIZE = argopt("size", 32) +const T = parse_dtypes(argopt("dtype", "ComplexF64"))[1] +const REPS = argopt("reps", 21) + +const MODE = Ref(:new) # :new (cost model), :old (indA order), :simple (12ae2b4's rule) + +# 12ae2b4's rule: sort by stride in the operand with the smaller minimum K stride. +function simple_rule(klabels, indA, A, indB, B, Qm, Qn) + minst(ind, v) = minimum((abs(Base.strides(v)[findfirst(==(l), ind)]) for l in klabels if size(v, findfirst(==(l), ind)) > 1); init = typemax(Int)) + mA = minst(indA, A); mB = minst(indB, B) + by_b = mB < mA || (mB == mA && Qn > Qm) + return by_b ? QuasiStrided._sort_labels_by_stride(klabels, indB, B) : QuasiStrided._sort_labels_by_stride(klabels, indA, A) +end +const LOG = Vector{Any}() + +function QuasiStrided._order_contract_labels( + klabels::Vector{Int}, indA::NTuple{NA, Int}, A::StridedView, morder::Vector{Int}, + indB::NTuple{NB, Int}, B::StridedView, norder::Vector{Int}, Qm::Int, Qn::Int, + l2bytes::Int = QuasiStrided._l2_core_bytes() + ) where {NA, NB} + length(klabels) <= 1 && return klabels + new = _choose_k_order(klabels, indA, A, morder, indB, B, norder, Qm, Qn, l2bytes) + stA = Base.strides(A); stB = Base.strides(B) + kA = Tuple(stA[findfirst(==(l), indA)] for l in klabels) + kB = Tuple(stB[findfirst(==(l), indB)] for l in klabels) + kL = Tuple(size(A, findfirst(==(l), indA)) for l in klabels) + costs = Tuple((_k_order_cost(o, indA, A, morder, Qm, l2bytes), _k_order_cost(o, indB, B, norder, Qn, l2bytes)) for o in (klabels, new)) + simple = simple_rule(klabels, indA, A, indB, B, Qm, Qn) + push!(LOG, (; klabels = copy(klabels), kL, kA, kB, sizeA = size(A), stA, indA, sizeB = size(B), stB, indB, Qm, Qn, costs, changed = new != klabels || simple != klabels, new, simple)) + return MODE[] === :new ? new : MODE[] === :simple ? simple : klabels +end + +gens = Dict("mps_1site" => TOB._mps_cases, "mps_2site" => TOB._mps_cases, "trg_plaquette" => TOB._trg_cases, "ctmrg_corner" => TOB._ctmrg_cases) +cases = gens[CASE]((SIZE,)) +case = cases[findfirst(c -> startswith(c.id, CASE), cases)] +spec = case.spec::NetworkSpec +provider = ArrayProvider{T}() +tensors = map(spec.indexlists) do il + dims = ntuple(i -> spec.dims[abs(il[i])], length(il)) + randtensor(provider, il, dims, T) +end +run(backend) = TensorOperations.ncon(tensors, spec.indexlists, spec.conjlist; order = spec.order, output = spec.output, backend = backend) + +println("case = $(case.id) T = $T indexlists = $(spec.indexlists) order = $(spec.order)") +empty!(LOG); MODE[] = :new +Cnew = run(QuasiStridedBackend()) +decisions = unique(e -> (e.klabels, e.kA, e.kB, e.indA, e.indB, e.sizeA, e.sizeB), LOG) +println("pairwise contractions with >1 K label: $(length(decisions)) distinct (of $(length(LOG)) planned)") +for e in decisions + println(" K=", e.klabels, " lengths=", e.kL, " A: size=", e.sizeA, " ind=", e.indA, " Kstrides=", e.kA, " B: size=", e.sizeB, " ind=", e.indB, " Kstrides=", e.kB, + " Qm=", e.Qm, " Qn=", e.Qn, " cost(A,B) old=", e.costs[1], " new=", e.costs[2], e.changed ? " CHANGED: new=$(e.new) simple=$(e.simple)" : " (unchanged)") +end +MODE[] = :old +Cold = run(QuasiStridedBackend()) +Cblas = run(StridedBLAS()) +println("max|new-old| = ", maximum(abs, Cnew .- Cold), " max|new-blas| = ", maximum(abs, Cnew .- Cblas)) + +# Whole network, interleaved. +function tmed(f) + f() + ts = [(t0 = time_ns(); f(); (time_ns() - t0) / 1e9) for _ in 1:REPS] + return median(ts) +end +for round in 1:2 + MODE[] = :old; told = tmed(() -> run(QuasiStridedBackend())) + MODE[] = :new; tnew = tmed(() -> run(QuasiStridedBackend())) + MODE[] = :simple; tsim = tmed(() -> run(QuasiStridedBackend())) + @printf("round %d network: simple rule %.3e s (simple/old %.3f)\n", round, tsim, tsim / told) + tblas = tmed(() -> run(StridedBLAS())) + @printf("round %d network: QS old %.3e s QS new %.3e s (new/old %.3f) BLAS %.3e s\n", round, told, tnew, tnew / told, tblas) +end + +# Each CHANGED pairwise contraction on its own, rebuilt dense at the logged +# shapes (the network's intermediates are dense, so this is the same layout). +for e in decisions + e.changed || continue + A = randn(T, e.sizeA...); B = randn(T, e.sizeB...) + mlab = [l for l in e.indA if !(l in e.klabels)] + nlab = [l for l in e.indB if !(l in e.klabels)] + indC = Tuple(vcat(mlab, nlab)) + C = zeros(T, Tuple(l in e.indA ? size(A, findfirst(==(l), e.indA)) : size(B, findfirst(==(l), e.indB)) for l in indC)...) + Av, Bv, Cv = StridedView(A), StridedView(B), StridedView(C) + f() = QuasiStrided.contract!(Cv, one(T), Av, e.indA, Bv, e.indB, zero(T), indC) + MODE[] = :old; told = tmed(f) + MODE[] = :new; tnew = tmed(f) + MODE[] = :old; told2 = tmed(f) + MODE[] = :new; tnew2 = tmed(f) + MODE[] = :simple; tsim = tmed(f); MODE[] = :simple; tsim2 = tmed(f) + @printf(" pairwise K=%s: simple %.3e,%.3e s simple/old %.3f\n", e.klabels, tsim, tsim2, min(tsim, tsim2) / min(told, told2)) + flops = 2.0 * prod(size(C)) * prod(e.kL) + @printf(" pairwise K=%s A%s B%s: old %.3e,%.3e s new %.3e,%.3e s new/old %.3f (%.1f GF/s new)\n", + e.klabels, e.sizeA, e.sizeB, told, told2, tnew, tnew2, min(tnew, tnew2) / min(told, told2), flops / min(tnew, tnew2) / 1e9) +end diff --git a/src/planning/labels.jl b/src/planning/labels.jl index bbc02e9..8df2980 100644 --- a/src/planning/labels.jl +++ b/src/planning/labels.jl @@ -150,22 +150,6 @@ end # same case. # ---------------------------------------------------------------------------- -# Smallest `abs(stride)` among the K labels' non-singleton axes in one operand -# (`typemax(Int)` when every K axis is a singleton, i.e. K does not walk that -# operand at all). -function _min_label_stride( - labels::Vector{Int}, ind::NTuple{N, Int}, v::StridedView - ) where {N} - st = Base.strides(v) - best = typemax(Int) - for l in labels - p = findfirst(==(l), ind)::Int - size(v, p) == 1 && continue - best = min(best, abs(st[p])) - end - return best -end - # Stable sort of `labels` by `abs(stride)` of each label's axis in `v` (the # same insertion sort as `_order_free_labels`, over a different operand). function _sort_labels_by_stride( @@ -187,32 +171,153 @@ function _sort_labels_by_stride( return out end -# The K order: sort by `abs(stride)` in the operand in which K is the more -# "inner" group, i.e. has the smaller minimum K stride. That operand's pack has -# no contiguous free axis to hide behind (its sliver rows are far apart in -# memory), so its line locality has to come from consecutive K steps landing -# in the same line; the other operand's free axis supplies the locality -# instead (an `mr`-row unit-stride sliver per K step is already whole cache -# lines, whatever the K stride). Ties -- K inner in both, `A[k..,m] * B[k..,n]` -# with the K axes in different orders -- go to the operand with more elements -# to read (the larger free extent), then to A. Stable, so labels with equal -# strides keep `indA` order, and any layout whose K strides are already -# ascending in the chosen operand (every gemm_ready/a_permuted/b_permuted -# layout of the upstream suite) gets exactly the order it had before this rule -# existed. The order is a property of the pair (A, B) and is the same whether -# or not the M/N orientation is later swapped. +# The K order is chosen among three candidates -- `indA` order (the historical +# one), sorted by A's strides, sorted by B's strides -- by a cost model of the +# two packs, `_k_order_cost`. Ties keep `indA` order, so any layout the model +# is indifferent about (every gemm_ready/a_permuted/b_permuted layout of the +# upstream suite, and every single-K-label contraction) plans exactly what it +# did before the rule existed. The order is a property of the pair (A, B) and +# is the same whether or not the M/N orientation is later swapped. +# +# The model, per operand (`n` elements, `sizeof(T)` bytes each): +# +# cost = n * walk * amplification +# +# * `walk` is 1 when the fastest K axis of the order steps the operand by at +# most a page (`_K_WALK_FAR_BYTES`) and `_K_WALK_FAR_PENALTY` beyond that: +# a pack whose consecutive K steps land pages apart is a chain of demand +# misses no prefetcher follows (measured 3.3 ns per element against +# 0.9-1.3 for the same sliver shape at a 768 B step, dim96_1_3_1). +# * `amplification` is the number of times a cache line of the operand is +# fetched. It is 1 when the operand's slivers are whole lines (its free +# group's leading axis is unit-stride and at least a line long), or when +# the operand's smallest-stride axis `u` is a K axis that consecutive K +# steps advance (it is the fastest K axis), or when the lines touched +# before `u` advances -- (product of the faster K extents) x (free extent) +# lines -- still fit the core's L2. Otherwise every line is fetched once +# per element it holds along `u`, i.e. `min(L_u, line_bytes / stride_u)` +# times (8 for a unit-stride Float64 axis). +# +# Why not "sort by the operand with the unit-stride K axis": that operand may +# be small enough to live in L2 whatever the order (TRG's rank-3 factor at +# chi = 24 is 110 KB) while the other loses a 192 B walk for a 2.6 MB one -- +# measured 1.71x slower on that contraction. The model keeps `indA` order +# there (A: 8M elements x walk 1 vs x walk 3) and still flips +# `contract_scrambled` (B: 85M elements x walk 3 x amplification 8 before). + +# Fastest-K byte stride beyond which the pack is latency-bound, and its cost +# relative to a prefetchable walk. +const _K_WALK_FAR_BYTES = 4096 +const _K_WALK_FAR_PENALTY = 3 + +# Cache-line size the model assumes; every measured host has 64 B lines. +const _K_LINE_BYTES = 64 + +# The core's private L2 share (see `_modelled_blocking`, src/planning/blocking.jl), +# or 1 MB when undetected -- between the 512 KB (Zen 2) and 1.25 MB (Ice Lake) +# of the machines measured, so an unknown host errs neither way. +function _l2_core_bytes() + profile = target_profile() + l2 = profile.l2 + l2.bytes > 0 || return 1 << 20 + smt = max(1, profile.l1d.sharing) + return l2.bytes ÷ max(1, l2.sharing ÷ smt) +end + +# Cost of packing one operand under K order `korder`. `free` is the operand's +# free-label list in the order the M/N composite enumerates it (C-stride +# order, `_order_free_labels`), whose FIRST label decides whether a register +# sliver is whole lines. `Qfree` is the free extent. +function _k_order_cost( + korder::Vector{Int}, ind::NTuple{N, Int}, v::StridedView, + free::Vector{Int}, Qfree::Int, l2bytes::Int + ) where {N} + st = Base.strides(v) + S = sizeof(eltype(v)) + line_elems = max(1, _K_LINE_BYTES ÷ S) + pos(l::Int) = findfirst(==(l), ind)::Int + n = length(v) + + # Fastest non-singleton K axis of this order. + kfast = 0 + for l in korder + if size(v, pos(l)) > 1 + kfast = l + break + end + end + kfast == 0 && return 0 # K is all singletons here: nothing walks. + walk = abs(st[pos(kfast)]) * S > _K_WALK_FAR_BYTES ? _K_WALK_FAR_PENALTY : 1 + + # Whole-line slivers: the free composite's leading (C-fastest) axis is + # unit-stride here and at least a line long. + wholeline = false + for l in free + p = pos(l) + size(v, p) == 1 && continue + wholeline = st[p] == 1 && size(v, p) >= line_elems + break + end + wholeline && return n * walk + + # The operand's smallest-stride K axis `u`, and how many of its elements + # share a line. + u = 0 + su = typemax(Int) + for l in korder + p = pos(l) + size(v, p) == 1 && continue + if abs(st[p]) < su + su = abs(st[p]) + u = l + end + end + share = su * S >= _K_LINE_BYTES ? 1 : min(size(v, pos(u)), _K_LINE_BYTES ÷ (su * S)) + (share == 1 || u == kfast) && return n * walk + + # Lines touched before `u` advances: one per element (scattered slivers) + # over the faster K extents and the whole free extent. + faster = 1 + for l in korder + l == u && break + faster *= size(v, pos(l)) + end + footprint = Int128(faster) * Int128(Qfree) * Int128(_K_LINE_BYTES) + return footprint <= l2bytes ? n * walk : n * walk * share +end + +# Entry point `plan_contract` calls; the decision itself is `_choose_k_order` +# (kept apart so a probe can wrap the entry point and log/toggle decisions +# without duplicating the model -- benchmark/probes/probe_network_korder.jl). function _order_contract_labels( klabels::Vector{Int}, - indA::NTuple{NA, Int}, A::StridedView, - indB::NTuple{NB, Int}, B::StridedView, - Qm::Int, Qn::Int + indA::NTuple{NA, Int}, A::StridedView, morder::Vector{Int}, + indB::NTuple{NB, Int}, B::StridedView, norder::Vector{Int}, + Qm::Int, Qn::Int, l2bytes::Int = _l2_core_bytes() + ) where {NA, NB} + return _choose_k_order(klabels, indA, A, morder, indB, B, norder, Qm, Qn, l2bytes) +end + +function _choose_k_order( + klabels::Vector{Int}, + indA::NTuple{NA, Int}, A::StridedView, morder::Vector{Int}, + indB::NTuple{NB, Int}, B::StridedView, norder::Vector{Int}, + Qm::Int, Qn::Int, l2bytes::Int ) where {NA, NB} length(klabels) <= 1 && return klabels - minA = _min_label_stride(klabels, indA, A) - minB = _min_label_stride(klabels, indB, B) - by_b = minB < minA || (minB == minA && Qn > Qm) - return by_b ? _sort_labels_by_stride(klabels, indB, B) : - _sort_labels_by_stride(klabels, indA, A) + cost(order) = _k_order_cost(order, indA, A, morder, Qm, l2bytes) + + _k_order_cost(order, indB, B, norder, Qn, l2bytes) + best = klabels + bestcost = cost(klabels) + for cand in (_sort_labels_by_stride(klabels, indA, A), _sort_labels_by_stride(klabels, indB, B)) + cand == best && continue + c = cost(cand) + if c < bestcost + best = cand + bestcost = c + end + end + return best end # Element count of the leading unit-stride run when `labels` (already ordered diff --git a/src/planning/plan.jl b/src/planning/plan.jl index 888593a..cfece3d 100644 --- a/src/planning/plan.jl +++ b/src/planning/plan.jl @@ -81,9 +81,9 @@ labels inside the M composite (A's free labels) and the N composite (B's free labels) are each stable-sorted by `abs(stride)` of the label's axis *in `C`*, ascending, ties keeping the operand's own axis order -- so each composite is enumerated with `C`'s fastest axis fastest, whatever A's or B's layout is. The -K composite is stable-sorted by `abs(stride)` in whichever of `A`/`B` has the -smaller minimum K stride (ties: the operand with the larger free extent, then -`A`), so the operand in which K is the inner group walks it contiguously (see +K composite keeps `indA` order unless a cost model of the two packs (page- +crossing K walks, cache lines refetched once the lines in flight exceed L2) +prefers the order sorted by `abs(stride)` in `A` or in `B` (see `_order_contract_labels`). Then, if the sorted M list does *not* begin with a unit-stride run of at least `mr(kernel)` elements in `C` while the sorted N list does, the operand roles are swapped: `B` feeds M and `A` feeds N, and @@ -176,9 +176,10 @@ function plan_contract( Qm = axis_length(mgroup) Qn = axis_length(ngroup) - # The K order is decided by A's and B's layouts (`_order_contract_labels`, - # src/planning/labels.jl); it needs the free extents only as a tie-break. - korder = _order_contract_labels(klabels, indA, A, indB, B, Qm, Qn) + # The K order is decided by a cost model of the two packs + # (`_order_contract_labels`, src/planning/labels.jl), which needs each + # operand's free-label order (whole-line slivers or not) and free extent. + korder = _order_contract_labels(klabels, indA, A, morder, indB, B, norder, Qm, Qn) kgroup = _build_pair_group(korder, indA, A, indB, B) # maps: (A, B) Qk = axis_length(kgroup) diff --git a/test/planning/test_plan_contract.jl b/test/planning/test_plan_contract.jl index bb4d95b..069f3ec 100644 --- a/test/planning/test_plan_contract.jl +++ b/test/planning/test_plan_contract.jl @@ -815,111 +815,122 @@ end end # ===================================================================== -# Contracted-label order (`_order_contract_labels`) +# Contracted-label order (`_order_contract_labels`, a cost model of the packs) # ===================================================================== # Labels: a=1 b=2 c=3 d=4 e=5, distinct extents so a wrong order shows up as a -# wrong stride, not a coincidentally equal one. -const _KO_EXT = Dict(1 => 7, 2 => 3, 3 => 4, 4 => 5, 5 => 6) -_ko_array(::Type{T}, ind) where {T} = randn(T, Tuple(_ko_EXT_get(l) for l in ind)...) -_ko_EXT_get(l) = _KO_EXT[l] +# wrong stride, not a coincidentally equal one. `_KO_EXT` is small (every +# operand fits any L2); `_KO_BIG` makes B's lines-touched-before-`d`-advances +# footprint b*c*e*64 B = 4 MB, beyond every measured core's L2 share. +const _KO_EXT = Dict(1 => 9, 2 => 3, 3 => 4, 4 => 5, 5 => 6) +const _KO_BIG = Dict(1 => 9, 2 => 40, 3 => 40, 4 => 5, 5 => 40) +_ko_array(::Type{T}, ind, ext = _KO_EXT) where {T} = randn(T, Tuple(ext[l] for l in ind)...) _ko_stride(v, ind, l) = Base.strides(v)[findfirst(==(l), ind)] -@testset "K order: sorted by stride in the operand where K is the inner group" begin +# The K order `plan_contract` would pass on, with an explicit L2 size. +function _ko_order(Av, indA, Bv, indB, Cv, indC; l2bytes) + mlabels, nlabels, klabels = QuasiStrided._classify_labels(indA, indB, indC) + morder = QuasiStrided._order_free_labels(mlabels, indC, Cv) + norder = QuasiStrided._order_free_labels(nlabels, indC, Cv) + ext(ls) = prod((size(Cv, findfirst(==(l), indC)) for l in ls); init = 1) + return QuasiStrided._order_contract_labels( + klabels, indA, Av, morder, indB, Bv, norder, ext(mlabels), ext(nlabels), l2bytes + ) +end + +@testset "K order: cost model picks among indA / A-sorted / B-sorted" begin T = Float64 - A = _ko_array(T, (1, 2, 3, 4)) # A[a,b,c,d]: M = a unit-stride, K = (b,c,d) ascending - Av = StridedView(A) - C = zeros(T, _KO_EXT[1], _KO_EXT[5]) # C[a,e] - Cv = StridedView(C) + Av = StridedView(_ko_array(T, (1, 2, 3, 4))) # A[a,b,c,d]: a unit-stride, 9 >= a line + Cv = StridedView(zeros(T, _KO_EXT[1], _KO_EXT[5])) - # contract_scrambled: B[d,c,b,e]. K is B's inner group (min stride 1 < A's - # min K stride 7), so the order follows B: (d, c, b). + # contract_scrambled, B[d,c,b,e]: in indA order B's unit-stride K axis d is + # the slowest K coordinate, so each B line is refetched per d element + # (x5) once the lines in flight exceed L2 -- forced here with l2bytes = 0. Bs = StridedView(_ko_array(T, (4, 3, 2, 5))) - plan = plan_contract(Cv, Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), (1, 5)) - @test plan.kgroup.lengths == (5, 4, 3) - @test plan.kgroup.strides[1] == Tuple(_ko_stride(Av, (1, 2, 3, 4), l) for l in (4, 3, 2)) - @test plan.kgroup.strides[2] == (1, 5, 20) - @test issorted(abs.(plan.kgroup.strides[2])) + @test _ko_order(Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), Cv, (1, 5); l2bytes = 0) == [4, 3, 2] + # ... and when they fit L2 the model is indifferent: indA order is kept. + @test _ko_order(Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), Cv, (1, 5); l2bytes = 1 << 20) == [2, 3, 4] - # gemm_ready B[b,c,d,e] and b_permuted B[b,e,c,d]: B is still the inner - # operand and its K strides are already ascending in indA order, so the - # order is exactly the pre-rule (indA) order. + # gemm_ready B[b,c,d,e] and b_permuted B[b,e,c,d]: B-sorted == indA order. for indB in ((2, 3, 4, 5), (2, 5, 3, 4)) Bv = StridedView(_ko_array(T, indB)) - p = plan_contract(Cv, Av, (1, 2, 3, 4), Bv, indB, (1, 5)) - @test p.kgroup.lengths == (3, 4, 5) - @test p.kgroup.strides[1] == (7, 21, 84) - @test p.kgroup.strides[2] == Tuple(_ko_stride(Bv, indB, l) for l in (2, 3, 4)) + @test _ko_order(Av, (1, 2, 3, 4), Bv, indB, Cv, (1, 5); l2bytes = 0) == [2, 3, 4] end - # A transposed as well (A[d,c,b,a]): K is inner in BOTH, min strides tie - # at 1, extents tie-break (Qm = 7 > Qn = 6) -> A's order (d, c, b). - At = StridedView(_ko_array(T, (4, 3, 2, 1))) - Bg = StridedView(_ko_array(T, (2, 3, 4, 5))) - p = plan_contract(Cv, At, (4, 3, 2, 1), Bg, (2, 3, 4, 5), (1, 5)) - @test p.kgroup.lengths == (5, 4, 3) - @test p.kgroup.strides[1] == (1, 5, 20) - @test p.kgroup.strides[2] == (12, 3, 1) - # ... and B's order (b, c, d) once N is the larger free extent. - Bbig = StridedView(randn(T, 3, 4, 5, 9)) - Cbig = StridedView(zeros(T, 7, 9)) - p = plan_contract(Cbig, At, (4, 3, 2, 1), Bbig, (2, 3, 4, 5), (1, 5)) - @test p.kgroup.lengths == (3, 4, 5) - @test p.kgroup.strides[1] == (20, 5, 1) - @test p.kgroup.strides[2] == (1, 3, 12) - - # A singleton K axis does not count as "inner": A[a,b,c,d] with b of - # length 1 at unit stride in B (B[b,d,c,e]) -- B's smallest NON-singleton - # K stride is 1 (d) anyway here, so the order follows B, but the singleton - # `b` is placed by its (irrelevant) stride and never decides the operand. - A1 = StridedView(randn(T, 7, 1, 4, 5)) - B1 = StridedView(randn(T, 1, 5, 4, 6)) - p = plan_contract(Cv, A1, (1, 2, 3, 4), B1, (2, 4, 3, 5), (1, 5)) - @test p.kgroup.lengths[findfirst(==(1), p.kgroup.lengths)] == 1 - @test p.kgroup.strides[2][end] == 5 || p.kgroup.strides[2][end] == 1 # d or the singleton b last - @test issorted(abs.(p.kgroup.strides[2])) + # TRG-shaped: B's K axis 5 is unit-stride, but sorting by B would make A's + # fastest K step 8^4 elements (32 KB, a page-crossing latency-bound walk) + # to save refetches of a 4 KB B -- indA order stays (measured: B-sorted is + # 1.69x slower on the chi = 24 TRG contraction). Labels: A(1,2,3,4,5), + # B(5,6,2), K = (2, 5), C(1,3,4,6). + At = StridedView(randn(T, 8, 8, 8, 8, 8)) + Bt = StridedView(randn(T, 8, 8, 8)) + Ct = StridedView(zeros(T, 8, 8, 8, 8)) + for l2bytes in (0, 1 << 20) + @test _ko_order(At, (1, 2, 3, 4, 5), Bt, (5, 6, 2), Ct, (1, 3, 4, 6); l2bytes) == [2, 5] + end # Single K label: nothing to order. p = plan_contract(StridedView(zeros(T, 3, 5)), StridedView(randn(T, 3, 4)), (1, 2), StridedView(randn(T, 4, 5)), (2, 3), (1, 3)) @test p.kgroup.strides == ((3,), (1,)) + # A singleton K axis never decides: it is skipped as `kfast` and as `u`. + A1 = StridedView(randn(T, 9, 1, 4, 5)) + B1 = StridedView(randn(T, 1, 5, 4, 6)) + o = _ko_order(A1, (1, 2, 3, 4), B1, (2, 4, 3, 5), Cv, (1, 5); l2bytes = 0) + @test sort(o) == [2, 3, 4] + @test filter(!=(2), o) == [4, 3] # B-sorted among the non-singletons +end + +@testset "K order: plan_contract flips contract_scrambled at an L2-exceeding size" begin + T = Float64 + ext = _KO_BIG + Av = StridedView(_ko_array(T, (1, 2, 3, 4), ext)) + Bs = StridedView(_ko_array(T, (4, 3, 2, 5), ext)) # B[d,c,b,e] + Cv = StridedView(zeros(T, ext[1], ext[5])) + plan = plan_contract(Cv, Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), (1, 5)) + @test plan.kgroup.lengths == (5, 40, 40) + @test plan.kgroup.strides[2] == (1, 5, 200) + @test plan.kgroup.strides[1] == Tuple(_ko_stride(Av, (1, 2, 3, 4), l) for l in (4, 3, 2)) + + # gemm_ready at the same size: exactly the pre-rule (indA) order. + Bg = StridedView(_ko_array(T, (2, 3, 4, 5), ext)) + p = plan_contract(Cv, Av, (1, 2, 3, 4), Bg, (2, 3, 4, 5), (1, 5)) + @test p.kgroup.lengths == (40, 40, 5) + @test p.kgroup.strides[1] == (9, 360, 14400) - # The swapped orientation (B feeds M) uses the same K order with the maps - # exchanged: C stored as (e, a) with an N run of 8 >= mr = 8 swaps. - Ct = zeros(T, 8, 7) # C[e,a] physically; labels (5, 1) - B8 = StridedView(randn(T, 5, 4, 3, 8)) # B[d,c,b,e], e = 8 + # The swapped orientation (B feeds M: C stored as (e, a), an N run of + # 40 >= mr) uses the same K order with the maps exchanged. + Ct = StridedView(zeros(T, ext[5], ext[1])) kernel = SIMDKernel(Val(8), Val(6), T) - pt = plan_contract(StridedView(Ct), Av, (1, 2, 3, 4), B8, (4, 3, 2, 5), (5, 1); kernel = kernel) - @test pt.Astorage === parent(B8) - @test pt.kgroup.lengths == (5, 4, 3) - @test pt.kgroup.strides[1] == (1, 5, 20) # B's map first - @test pt.kgroup.strides[2] == (84, 21, 7) # then A's + pt = plan_contract(Ct, Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), (5, 1); kernel = kernel) + @test pt.Astorage === parent(Bs) + @test pt.kgroup.lengths == (5, 40, 40) + @test pt.kgroup.strides[1] == (1, 5, 200) # B's map first + @test pt.kgroup.strides[2] == (14400, 360, 9) # then A's end @testset "K order: correctness on scrambled K (all dtypes, alpha/beta, conj, both orientations, oracle)" begin Random.seed!(0x5C7A_0B1E) + ext = _KO_BIG for T in (Float64, Float32, ComplexF64, ComplexF32) rtol = 500 * eps(real(T)) alpha = T <: Complex ? T(1.3, -0.4) : T(1.3) beta = T <: Complex ? T(0.7, 0.2) : T(0.7) for (indA, indB, indC) in ( - ((1, 2, 3, 4), (4, 3, 2, 5), (1, 5)), # scrambled, C[a,e] + ((1, 2, 3, 4), (4, 3, 2, 5), (1, 5)), # scrambled, C[a,e] (reordered K) ((1, 2, 3, 4), (4, 3, 2, 5), (5, 1)), # scrambled, C[e,a] (swap for real T) ((4, 3, 2, 1), (2, 3, 4, 5), (1, 5)), # A transposed, K inner in both ((4, 3, 2, 1), (3, 2, 4, 5), (5, 1)), # both scrambled, C transposed ), (conjA, conjB) in ((false, false), (true, false), (true, true)) (T <: Real) && conjA && continue - A = _ko_array(T, indA) - B = _ko_array(T, indB) + A = _ko_array(T, indA, ext) + B = _ko_array(T, indB, ext) Av = conjA ? StridedView(A, size(A), strides(A), 0, conj) : StridedView(A) Bv = StridedView(B) - C = randn(T, Tuple(_KO_EXT[l] for l in indC)...) + C = randn(T, Tuple(ext[l] for l in indC)...) Cstart = copy(C) Cv = StridedView(C) Cref = _lo_reference(Cstart, Av, indA, Bv, indB, indC; conjA, conjB, alpha, beta) plan = plan_contract(Cv, Av, indA, Bv, indB, indC; conjA = conjA, conjB = conjB) - # The whole point: the K composite is ascending in the inner operand. - inner = plan.kgroup.strides[argmin(map(s -> minimum(abs, s), plan.kgroup.strides))] - @test issorted(abs.(inner)) execute!(plan, alpha, beta) @test isapprox(C, Cref; rtol = rtol) copyto!(C, Cstart) From 91505b2cc5fc5e5a35be9463dee815fbfda45e57 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 19:29:55 -0400 Subject: [PATCH 07/53] Verification job: cover the cost model's cases and discriminate the intensli_7 cause submit_scrambled_verify.sh part 1 now runs every synthetic size the K-order cost model reorders (16/32/63/96) and the :network cases (TRG is the shape it must keep; its decision reads the core's L2 share, 512 KB on Rome). Part 2 runs intensli_7/intensli_6 stage breakdowns under variants that separate the candidate causes of the Rome-only intensli_7_dim16 slowdown: dims 12/24 (non-power-of-two controls), --pad 1 (dim-16 extents with no power-of-two stride), --nothp 1 (4 KB pages only), --msplit-pos 2 (pack A reuses each A line from L1), and dim 32 with and without padding. probe_stage_breakdown.jl gains --pad P (A and C stored with the first axis padded to dim+P in the parent) and --nothp 1 (A and C in an anonymous mmap with MADV_NOHUGEPAGE). On ccqlin038 (Cascade Lake, AVX2 tile --shape 8,6,4) neither moves intensli_7_dim16 much (pack A 2.84 -> 2.87 ns/elem padded, 3.52 without THP), so a large effect on Rome would localise the cause. Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_stage_breakdown.jl | 36 ++++++++++++- benchmark/submit_scrambled_verify.sh | 65 ++++++++++++++--------- 2 files changed, 75 insertions(+), 26 deletions(-) diff --git a/benchmark/probes/probe_stage_breakdown.jl b/benchmark/probes/probe_stage_breakdown.jl index 20c7814..bc50fe6 100644 --- a/benchmark/probes/probe_stage_breakdown.jl +++ b/benchmark/probes/probe_stage_breakdown.jl @@ -62,10 +62,42 @@ const indA = Tuple(label.(LA)) const indB = Tuple(label.(LB)) const indC = Tuple(label.(LC)) +# `--nothp 1` backs A and C by 4 KB pages only (anonymous mmap + +# madvise(MADV_NOHUGEPAGE)), to emulate a node whose transparent huge pages +# are off (or not granted) on a host where they are on for Julia's arrays. +const NOTHP = argopt("nothp", 0) == 1 +function smallpage_array(::Type{T}, dims::Dims) where {T} + n = prod(dims) * sizeof(T) + len = cld(n, 4096) * 4096 + PROT_RW, MAP_PRIVATE_ANON, MADV_NOHUGEPAGE = Cint(3), Cint(0x22), Cint(15) + p = ccall(:mmap, Ptr{Cvoid}, (Ptr{Cvoid}, Csize_t, Cint, Cint, Cint, Int64), C_NULL, len, PROT_RW, MAP_PRIVATE_ANON, -1, 0) + p == Ptr{Cvoid}(-1) && error("mmap failed") + ccall(:madvise, Cint, (Ptr{Cvoid}, Csize_t, Cint), p, len, MADV_NOHUGEPAGE) == 0 || error("madvise failed") + return unsafe_wrap(Array, Ptr{T}(p), dims; own = false) # leaked on purpose: probe lifetime +end rng = MersenneTwister(1234) A = randn(rng, T, ntuple(_ -> DIM, length(LA))...) B = randn(rng, T, ntuple(_ -> DIM, length(LB))...) C = zeros(T, ntuple(_ -> DIM, length(LC))...) +if NOTHP + A = copyto!(smallpage_array(T, size(A)), A) + C = fill!(smallpage_array(T, size(C)), zero(T)) +end +# `--pad P` stores A (and C) with the first axis padded to `DIM + P` in the +# parent, so no stride is a power of two (a power-of-two `--dim` otherwise +# puts every axis at a 2^k stride: cache-set / DRAM-bank aliasing test). +const PAD = argopt("pad", 0) +function padded(X::Array{T}) where {T} + pdims = (size(X, 1) + PAD, Base.tail(size(X))...) + P = NOTHP ? fill!(smallpage_array(T, pdims), zero(T)) : zeros(T, pdims) + V = view(P, 1:size(X, 1), ntuple(_ -> Colon(), ndims(X) - 1)...) + copyto!(V, X) + return V +end +if PAD > 0 + A = padded(A) + C = padded(C) +end Av, Bv, Cv = StridedView(A), StridedView(B), StridedView(C) # `--shape MR,NR,W` names a menu kernel shape (e.g. `8,6,4`, the AVX2 Float64 @@ -106,7 +138,7 @@ end # --------------------------------------------------------------------------- # Plan report # --------------------------------------------------------------------------- -println("case = $CASE expr = $EXPR dim = $DIM T = $T") +println("case = $CASE expr = $EXPR dim = $DIM T = $T nothp = $NOTHP pad = $PAD") println("A labels ", join(LA), " strides ", strides(Av)) println("B labels ", join(LB), " strides ", strides(Bv)) println("C labels ", join(LC), " strides ", strides(Cv)) @@ -117,7 +149,7 @@ for (name, g, ops) in (("M", plan.mgroup, "(A,C)"), ("N", plan.ngroup, "(B,C)"), println("$name group $ops: lengths=", g.lengths, " strides=", g.strides, " Q=", axis_length(g), " ramp=", affine_ramp(g)[1]) end flops = 2.0 * axis_length(plan.mgroup) * axis_length(plan.ngroup) * axis_length(plan.kgroup) -println("flops = ", flops, " bytes(A+B+C) = ", sizeof(A) + sizeof(B) + sizeof(C)) +println("flops = ", flops, " bytes(A+B+C) = ", (length(A) + length(B) + length(C)) * sizeof(T)) # --------------------------------------------------------------------------- # Instrumented nest (copy of `_execute_nest!`, timers added, nothing else) diff --git a/benchmark/submit_scrambled_verify.sh b/benchmark/submit_scrambled_verify.sh index cd61022..1086727 100755 --- a/benchmark/submit_scrambled_verify.sh +++ b/benchmark/submit_scrambled_verify.sh @@ -8,27 +8,34 @@ # sbatch --reservation=rocky8 --constraint=genoa benchmark/submit_scrambled_verify.sh # sbatch --reservation=rocky8 --constraint=rome benchmark/submit_scrambled_verify.sh # -# Part 1: bench_to_suite.jl on the subset the K-order rule touches -- every -# synthetic layout at leg dims 32 and 96 (dim{32,96}_1_3_1_contract_scrambled -# are the targets; gemm_ready/a_permuted/b_permuted/both_permuted plan the same -# K group as before and are the regression guard) plus TCCG at 8/16/24 -# (single-K-label cases: unchanged plans, regression guard; includes -# intensli_7_dim16). Compare against benchmark/results/to_suite-71120{88,89,90}-*. +# Part 1: bench_to_suite.jl on everything the K-order cost model can touch -- +# every synthetic layout at leg dims 16/32/63/96 (the contract_scrambled +# layouts are the only ones it reorders: dim16 0_2_3/3_2_3, dim32 +# 2_2_2/1_3_1/0_2_3, dim63/96 2_2_2/1_3_1; benchmark/probes/ +# probe_korder_changes.jl), TCCG at 8/16/24 (unchanged plans, regression +# guard; includes intensli_7_dim16) and the :network cases (TRG is the shape +# the model must NOT flip; the decision reads the core's L2 share, 512 KB on +# Rome). Compare against benchmark/results/to_suite-71120{88,89,90}-*. # # Part 2: benchmark/probes/probe_stage_breakdown.jl for intensli_7 and -# intensli_6 at dim 16 and 8 (Float64): pack B / pack A / kernel+store split, -# as planned (as-is) and with two hand-made M enumeration orders that move one -# cache line of A's unit-stride axis forward (`--msplit-pos 2/1 --msplit-li 8`). -# On Cascade Lake all of these run at 3-4 ns per A element; on Rome the suite -# has intensli_7_dim16 18x slower than intensli_6_dim16, and this says which -# stage and whether the enumeration order is the lever. +# intensli_6 (Float64): pack B / pack A / kernel+store split. The suite has +# intensli_7_dim16 18x slower than intensli_6_dim16 on Rome (2.6x on Genoa, +# nothing on Intel), and intensli_7 at dims 12 and 24 fine on all three, so +# the variants discriminate between causes: +# dim 12 / 24 non-power-of-two controls (dim 24 has 8x the pages of 16) +# --pad 1 dim 16 extents, A/C first axis padded to 17: no stride is +# a power of two (cache-set / DRAM-bank aliasing) +# --nothp 1 A/C on 4 KB pages only (TLB reach; THP state printed below) +# --msplit-pos 2 8 elements of A's unit-stride axis enumerated right after +# C's leading run: pack A reuses each line from L1 instead of +# after 4096 M rows (line locality / DRAM traffic) #SBATCH --job-name=qs-scrambled-verify #SBATCH --partition=ccq #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=1 #SBATCH --mem=32G -#SBATCH --time=03:00:00 +#SBATCH --time=05:00:00 #SBATCH --output=benchmark/results/slurm-%j.out set -euo pipefail @@ -51,26 +58,36 @@ mkdir -p "$OUTDIR" echo "=== part 1: suite subset ===" "$JULIA" --project=benchmark benchmark/bench_to_suite.jl \ - --categories contract \ + --categories contract,network \ --sources synthetic,tccg \ --dtypes Float64,ComplexF64 \ - --synthetic-sizes 32,96 \ + --synthetic-sizes 16,32,63,96 \ --tccg-sizes 8,16,24 \ + --mps-bonddims 128,256 --ctmrg-chis 32,64 --trg-chis 16,32 \ --reps 21 --time-budget 5 \ --outdir "$OUTDIR" echo "=== part 2: intensli stage breakdown (Float64) ===" -for dim in 16 8; do +stage() { + echo "--- $*" + "$JULIA" --project=benchmark benchmark/probes/probe_stage_breakdown.jl \ + --dtype Float64 --reps 21 "$@" 2>&1 | grep -v "^WARNING" +} +{ for case in intensli_7 intensli_6; do - for split in "0 8" "2 8" "1 8"; do - set -- $split - echo "--- $case dim=$dim msplit-pos=$1 li=$2" - "$JULIA" --project=benchmark benchmark/probes/probe_stage_breakdown.jl \ - --case "$case" --dim "$dim" --dtype Float64 --reps 21 \ - --msplit-pos "$1" --msplit-li "$2" 2>&1 | grep -v "^WARNING" - done + stage --case "$case" --dim 16 + stage --case "$case" --dim 16 --pad 1 + stage --case "$case" --dim 16 --nothp 1 + stage --case "$case" --dim 16 --pad 1 --nothp 1 + stage --case "$case" --dim 16 --msplit-pos 2 --msplit-li 8 + stage --case "$case" --dim 12 + stage --case "$case" --dim 24 done -done | tee "$OUTDIR/intensli_stage_breakdown.txt" + # Pack-A walk in isolation at the Rome tile on other shapes of the same + # expression: dim 32 (power of two again, 8x the data of dim 16). + stage --case intensli_7 --dim 32 + stage --case intensli_7 --dim 32 --pad 1 +} | tee "$OUTDIR/intensli_stage_breakdown.txt" echo "=== node check ===" echo "SLURM_JOB_NODELIST = ${SLURM_JOB_NODELIST:-?}" From 1e49597f671992caeea23ee79cbc0c619eddd413 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 19:43:33 -0400 Subject: [PATCH 08/53] intensli_7: locate the slowdown in pack A's 4 KB-aligned gather; probe it The Rome-only intensli_7_dim16 slowdown has a twin on ccqlin038 (Cascade Lake): intensli_7 at dims 24 and 32, C[e,c,b,f,a] = A[a,b,c,d,e] * B[d,f]. The M composite follows C (e,c,b,a), so a pack-A sliver gathers 8 values of e (A stride dim^4) over kc steps of d (A stride dim^3), one element per cache line, and a line's other elements are needed only a full M pass later. When those strides are multiples of 4 KB the whole sliver panel aliases in set-indexed structures and pack A collapses to about one memory latency per element. Padding every axis of A (no 4 KB-multiple stride, same walk) removes it: ccqlin038, --shape 8,6,4, pinned (probe_stage_breakdown.jl): intensli_7 dim24 0.421 s, pack A 52.3 ns/elem intensli_7 dim24 --pad 1 0.082 s, pack A 6.3 ns/elem pack-A walk alone (probe_pack_order.jl, ns/elem, aligned -> padded): dim16 2.83 -> 2.47, dim20 5.34 -> 5.21, dim24 46.3 -> 6.95, dim28 11.4 -> 9.8, dim32 52.0 -> 10.1 dim20 and dim28 (strides not 4 KB multiples) never collapse. Which aligned dims trip depends on the machine (dim16 on Rome, 24/32 here), and neither THP (--nothp 1: 2.84 -> 3.52 ns/elem at dim16) nor walking the block K-outer (helps dim24 46 -> 21, hurts dim20/28) is the lever. probe_pack_order.jl times the pack-A walk alone (precomputed offsets, plain loads) in the shipped sliver order and K-outer order; probe_stage_breakdown --pad now pads every axis (padding only the first left dim^3*(dim+1)*8 a 4 KB multiple at dim 32). The verification job runs both at dims 16/20/24/28/32 so Rome and Genoa confirm or refute the aliasing diagnosis. Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_pack_order.jl | 114 ++++++++++++++++++++++ benchmark/probes/probe_stage_breakdown.jl | 11 ++- benchmark/submit_scrambled_verify.sh | 17 +++- 3 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 benchmark/probes/probe_pack_order.jl diff --git a/benchmark/probes/probe_pack_order.jl b/benchmark/probes/probe_pack_order.jl new file mode 100644 index 0000000..dc3ca9a --- /dev/null +++ b/benchmark/probes/probe_pack_order.jl @@ -0,0 +1,114 @@ +# Pack-A walk in isolation: gather every A element of a contraction into +# `mr`-row sliver panels, mc rows x kc K steps at a time, exactly as +# `_execute_nest!` visits them, with the leaf loop in two orders: +# +# sliver for r in slivers, for p in 0:kc-1, for t in 0:mr-1 (shipped: +# one sliver's whole panel, then the next sliver) +# kouter for p in 0:kc-1, for r in slivers, for t in 0:mr-1 (each K step +# across all slivers of the block before the next K step) +# +# When the sliver rows' and the K steps' byte strides in A are multiples of +# 4 KB (intensli_7 at dims 16/24/32: A strides dim^3, dim^4 elements), every +# load of one sliver's panel lands in the same L1 set; `kouter` interleaves +# slivers from other C-order coordinates (other sets). Plain scalar loads +# through precomputed offsets -- no QuasiStrided packer -- so only the walk +# differs. +# +# julia -t 1 --project=benchmark benchmark/probes/probe_pack_order.jl \ +# --case intensli_7 --dim 24 [--shape 8,6,4] [--pad 1] [--reps 7] + +using QuasiStrided +using QuasiStrided: plan_contract, mr, axis_length, _kernel_from_shape +using StridedViews +using Statistics: median +using Printf + +include(joinpath(@__DIR__, "..", "harness.jl")) + +const KNOWN = Dict("intensli_6" => "abcde bf>dcfea", "intensli_7" => "abcde df>ecbfa", "intensli_8" => "abcde fb>dfcea") +const CASE = argopt("case", "intensli_7") +const EXPR = something(argval("expr"), get(KNOWN, CASE, nothing)) +const DIM = argopt("dim", 16) +const PAD = argopt("pad", 0) +const REPS = argopt("reps", 7) +const SHAPE = Tuple(parse.(Int, split(argopt("shape", "8,6,4"), ',')))::NTuple{3, Int} +const T = Float64 + +lhs, rhs = split(EXPR, '>') +la, lb = split(strip(lhs), ' ') +letters = unique(vcat(collect(la), collect(lb), collect(rhs))) +lab(s) = Tuple(findfirst(==(c), letters) for c in s) +indA, indB, indC = lab(la), lab(lb), lab(rhs) + +# `--pad P` pads EVERY axis of A's parent to DIM + P, so no stride is a +# multiple of 4 KB (padding only the first axis leaves dim^3 * (dim+P) * 8 +# a 4 KB multiple at dim 32). +Apar = randn(T, ntuple(_ -> DIM + PAD, length(la))...) +A = view(Apar, ntuple(_ -> 1:DIM, length(la))...) +B = randn(T, ntuple(_ -> DIM, length(lb))...) +C = zeros(T, ntuple(_ -> DIM, length(rhs))...) +plan = plan_contract(StridedView(C), StridedView(A), indA, StridedView(B), indB, indC; + oracle = false, kernel = _kernel_from_shape(SHAPE, T)) +pointer(plan.Astorage) == pointer(Apar) || error("swapped plan: A does not feed M") +const Ast = plan.Astorage # the flat storage the engine indexes (0-based offsets) + +# Offsets of every M row and K step into A's parent (0-based), in the plan's +# enumeration order (first axis fastest). +function offsets(lengths, strides) + out = Int[0] + for (L, s) in zip(lengths, strides) + out = vec([o + i * s for o in out, i in 0:(L - 1)]) + end + return out +end +mo = offsets(plan.mgroup.lengths, plan.mgroup.strides[1]) +ko = offsets(plan.kgroup.lengths, plan.kgroup.strides[1]) +base = plan.Abase +MR = mr(plan.kernel) +mc, kc = plan.blocking.mc, plan.blocking.kc +println("case = $CASE dim = $DIM pad = $PAD MR = $MR mc = $mc kc = $kc") +println("A strides (bytes) = ", strides(A) .* sizeof(T), " M strides(A) = ", plan.mgroup.strides[1], " K strides(A) = ", plan.kgroup.strides[1]) + +function pack_all!(buf, Apar, mo, ko, base, MR, mc, kc, ::Val{ORDER}) where {ORDER} + Qm, Qk = length(mo), length(ko) + acc = 0.0 + @inbounds for pc in 0:kc:(Qk - 1) + kb = min(kc, Qk - pc) + for ic in 0:mc:(Qm - 1) + mb = min(mc, Qm - ic) + ns = cld(mb, MR) + if ORDER === :sliver + for r in 0:(ns - 1), p in 0:(kb - 1), t in 0:(MR - 1) + i = r * MR + t + i < mb || continue + buf[r * MR * kb + p * MR + t + 1] = Apar[base + mo[ic + i + 1] + ko[pc + p + 1]] + end + else + for p in 0:(kb - 1), r in 0:(ns - 1), t in 0:(MR - 1) + i = r * MR + t + i < mb || continue + buf[r * MR * kb + p * MR + t + 1] = Apar[base + mo[ic + i + 1] + ko[pc + p + 1]] + end + end + acc += buf[1] # keep the block's pack live + end + end + return acc +end + +buf = zeros(T, (mc + MR) * kc) +n = length(mo) * length(ko) +res = Dict{Symbol, Vector{Float64}}(:sliver => Float64[], :kouter => Float64[]) +for o in (:sliver, :kouter) + pack_all!(buf, Ast, mo, ko, base + 1, MR, mc, kc, Val(o)) # warm-up/compile +end +# `base` (plan.Abase) is 0-based into the parent; Julia indexing is 1-based. +for rep in 1:REPS, o in (:sliver, :kouter) # interleaved + t0 = time_ns() + pack_all!(buf, Ast, mo, ko, base + 1, MR, mc, kc, Val(o)) + push!(res[o], (time_ns() - t0) / n) +end +for o in (:sliver, :kouter) + v = res[o] + @printf(" %-7s %6.2f ns/elem (min %.2f, max %.2f)\n", o, median(v), minimum(v), maximum(v)) +end diff --git a/benchmark/probes/probe_stage_breakdown.jl b/benchmark/probes/probe_stage_breakdown.jl index bc50fe6..ecec113 100644 --- a/benchmark/probes/probe_stage_breakdown.jl +++ b/benchmark/probes/probe_stage_breakdown.jl @@ -83,14 +83,15 @@ if NOTHP A = copyto!(smallpage_array(T, size(A)), A) C = fill!(smallpage_array(T, size(C)), zero(T)) end -# `--pad P` stores A (and C) with the first axis padded to `DIM + P` in the -# parent, so no stride is a power of two (a power-of-two `--dim` otherwise -# puts every axis at a 2^k stride: cache-set / DRAM-bank aliasing test). +# `--pad P` stores A (and C) in a parent with EVERY axis padded to `DIM + P`, +# so no stride is a multiple of 4 KB (at a power-of-two or 24 `--dim` the +# plain layout puts A's outer strides at 4 KB multiples: cache-set / slice / +# DRAM-bank aliasing test; padding one axis leaves the outer strides aligned). const PAD = argopt("pad", 0) function padded(X::Array{T}) where {T} - pdims = (size(X, 1) + PAD, Base.tail(size(X))...) + pdims = size(X) .+ PAD P = NOTHP ? fill!(smallpage_array(T, pdims), zero(T)) : zeros(T, pdims) - V = view(P, 1:size(X, 1), ntuple(_ -> Colon(), ndims(X) - 1)...) + V = view(P, map(n -> 1:n, size(X))...) copyto!(V, X) return V end diff --git a/benchmark/submit_scrambled_verify.sh b/benchmark/submit_scrambled_verify.sh index 1086727..9f76072 100755 --- a/benchmark/submit_scrambled_verify.sh +++ b/benchmark/submit_scrambled_verify.sh @@ -23,8 +23,11 @@ # nothing on Intel), and intensli_7 at dims 12 and 24 fine on all three, so # the variants discriminate between causes: # dim 12 / 24 non-power-of-two controls (dim 24 has 8x the pages of 16) -# --pad 1 dim 16 extents, A/C first axis padded to 17: no stride is -# a power of two (cache-set / DRAM-bank aliasing) +# --pad 1 dim 16 extents, every A/C axis padded to 17 in the parent: +# no stride is a multiple of 4 KB (set aliasing). On +# ccqlin038 this is THE lever: intensli_7_dim24 pack A 52 -> +# 6.3 ns/elem, 0.42 -> 0.08 s (dims 24 and 32 alias there, +# 16 does not -- the trigger dims are machine-specific) # --nothp 1 A/C on 4 KB pages only (TLB reach; THP state printed below) # --msplit-pos 2 8 elements of A's unit-stride axis enumerated right after # C's leading run: pack A reuses each line from L1 instead of @@ -83,12 +86,18 @@ stage() { stage --case "$case" --dim 12 stage --case "$case" --dim 24 done - # Pack-A walk in isolation at the Rome tile on other shapes of the same - # expression: dim 32 (power of two again, 8x the data of dim 16). stage --case intensli_7 --dim 32 stage --case intensli_7 --dim 32 --pad 1 } | tee "$OUTDIR/intensli_stage_breakdown.txt" +echo "=== part 3: pack-A walk alone, aligned vs padded strides, sliver vs K-outer order ===" +for dim in 16 20 24 28 32; do + for pad in 0 1; do + "$JULIA" --project=benchmark benchmark/probes/probe_pack_order.jl \ + --case intensli_7 --dim "$dim" --pad "$pad" --reps 5 2>&1 | grep -v "^WARNING" + done +done | tee "$OUTDIR/intensli_pack_order.txt" + echo "=== node check ===" echo "SLURM_JOB_NODELIST = ${SLURM_JOB_NODELIST:-?}" lscpu | grep -i "model name\|cache" || true From 285e2829aa4de61478cd55437a1f3f4dbb4eeea1 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 12:51:09 -0400 Subject: [PATCH 09/53] Port the K-order cost model to the NTuple label lists perf/scrambled-strides wrote `_order_contract_labels` / `_choose_k_order` / `_k_order_cost` against Vector{Int} klabels/morder/norder; perf/call-overhead made those lists statically sized tuples so planning allocates nothing and the group ranks are compile-time constants. After the merge the planner called the Vector methods with tuples. This ports the model without changing a single decision: * `_sort_labels_by_stride` is `_order_free_labels` (the same stable, allocation-free insertion sort) over another operand. * `_choose_k_order` gathers each operand's facts once, in `klabels` order (`_KOperand`: K extents, |strides|, element count/size, whole-line slivers), and represents the three candidates (indA order, A-sorted, B-sorted) as permutation tuples of 1:DK, so costing one is plain tuple indexing instead of a `findfirst` per axis visited. Same candidates, same comparison order, same strict `<` (ties keep indA order). The result is an NTuple{DK,Int}: the K group type stays concrete. * Zero or one K label returns at a compile-time branch before anything is computed; when both sorted candidates equal indA order (every gemm_ready/a_permuted/b_permuted layout) it returns before any cost is evaluated or the L2 share read. * `_k_order_cost` tests `share == 1` as `2*su*S > 64` (equivalent: `u` is never a singleton) so its runtime division is paid only on the return that uses the value. * The core's L2 share is resolved once per (profile, eltype) into `ResolvedDefaults.l2_core` (`_l2_core_bytes(profile)`, read from the detected cache profile) and read lazily via `_l2_core_bytes(T)`; the label-keyed `_k_order_cost` and `_l2_core_bytes()` stay for probes. Checked: every K-order decision for the upstream suite (all :contract sources at synthetic dims 4..96, TCCG 8/16/24, batched 4..64, four dtypes, planning only; plus every pairwise contraction of the :network mps/ctmrg/ trg cases <= 2 GiB, Float64/ComplexF64) is identical to perf/scrambled-strides, K group strides included (958 decisions; only contract_scrambled layouts reorder: dim16/24 {0_2_3,3_2_3}, dim32 {2_2_2,1_3_1,0_2_3}, dim48 {2_2_2,1_3_1,0_2_3}, dim63/96 {2_2_2,1_3_1}; TRG and every gemm_ready/permuted/TCCG/batched case keep indA order). A new test replays the scrambled branch's label-keyed model verbatim against the port on 500 random layouts (187 of them reorder), and another pins the result as an inferred, allocation-free NTuple. Per call (ccqlin038, pinned, interleaved against perf/call-overhead, probe_call_floor.jl, medians of 4 runs): 0 B/call everywhere; Float64 8x8x8 523 vs 511 ns, d4 outer 535 vs 542; ComplexF64 8x8x8 517 vs 540, d4 outer 848 vs 855 (all single-K: no model work). Multi-K tiny calls pay the model: plan_contract +15-20 ns for a 2-label gemm_ready d4 case, +55-85 ns for a 3-label scrambled d4 case (of ~0.3 us planning, ~1.3-1.9 us per call). probe_network_korder.jl's override follows the tuple signature. Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_network_korder.jl | 14 +- src/planning/defaults.jl | 10 +- src/planning/labels.jl | 246 ++++++++++++++++------- test/planning/test_plan_contract.jl | 114 ++++++++++- 4 files changed, 291 insertions(+), 93 deletions(-) diff --git a/benchmark/probes/probe_network_korder.jl b/benchmark/probes/probe_network_korder.jl index 527c0c1..e838f04 100644 --- a/benchmark/probes/probe_network_korder.jl +++ b/benchmark/probes/probe_network_korder.jl @@ -43,12 +43,14 @@ function simple_rule(klabels, indA, A, indB, B, Qm, Qn) end const LOG = Vector{Any}() +# Replaces the entry point without an explicit `l2bytes` (the one +# `plan_contract` calls); the label lists are NTuples (src/planning/labels.jl). function QuasiStrided._order_contract_labels( - klabels::Vector{Int}, indA::NTuple{NA, Int}, A::StridedView, morder::Vector{Int}, - indB::NTuple{NB, Int}, B::StridedView, norder::Vector{Int}, Qm::Int, Qn::Int, - l2bytes::Int = QuasiStrided._l2_core_bytes() - ) where {NA, NB} - length(klabels) <= 1 && return klabels + klabels::NTuple{DK, Int}, indA::NTuple{NA, Int}, A::StridedView, morder::NTuple{DM, Int}, + indB::NTuple{NB, Int}, B::StridedView, norder::NTuple{DN, Int}, Qm::Int, Qn::Int + ) where {DK, NA, NB, DM, DN} + DK <= 1 && return klabels + l2bytes = QuasiStrided._l2_core_bytes() new = _choose_k_order(klabels, indA, A, morder, indB, B, norder, Qm, Qn, l2bytes) stA = Base.strides(A); stB = Base.strides(B) kA = Tuple(stA[findfirst(==(l), indA)] for l in klabels) @@ -56,7 +58,7 @@ function QuasiStrided._order_contract_labels( kL = Tuple(size(A, findfirst(==(l), indA)) for l in klabels) costs = Tuple((_k_order_cost(o, indA, A, morder, Qm, l2bytes), _k_order_cost(o, indB, B, norder, Qn, l2bytes)) for o in (klabels, new)) simple = simple_rule(klabels, indA, A, indB, B, Qm, Qn) - push!(LOG, (; klabels = copy(klabels), kL, kA, kB, sizeA = size(A), stA, indA, sizeB = size(B), stB, indB, Qm, Qn, costs, changed = new != klabels || simple != klabels, new, simple)) + push!(LOG, (; klabels, kL, kA, kB, sizeA = size(A), stA, indA, sizeB = size(B), stB, indB, Qm, Qn, costs, changed = new != klabels || simple != klabels, new, simple)) return MODE[] === :new ? new : MODE[] === :simple ? simple : klabels end diff --git a/src/planning/defaults.jl b/src/planning/defaults.jl index 4d8dc20..35fc89e 100644 --- a/src/planning/defaults.jl +++ b/src/planning/defaults.jl @@ -25,8 +25,10 @@ resolved by [`_resolve_defaults`](@ref) and cached per (profile, eltype) by (`_derived_shape`), the `fitted` shape the off-AVX-512 small-M demotion falls to (`_fitted_shape`), the native-width FMAddSub `small_m` candidates of the AVX-512 complex small-M rule (`_small_m_candidates`, empty where the rule does -not apply), and the unscaled `real_row` of the cache-blocking model for -`real(T)` (`_real_blocking_row`). `profile` is the profile these were derived +not apply), the unscaled `real_row` of the cache-blocking model for +`real(T)` (`_real_blocking_row`), and the core's private L2 share `l2_core` +that the K-order cost model reads (`_l2_core_bytes`, src/planning/labels.jl). +`profile` is the profile these were derived from and is the cache key: a cached entry serves exactly while it is `===` the current [`target_profile`](@ref). Every field but the candidate list is isbits, so reads are type-stable and allocation-free. @@ -37,6 +39,7 @@ struct ResolvedDefaults fitted::NTuple{3, Int} small_m::Vector{NTuple{3, Int}} real_row::Blocking + l2_core::Int end """ @@ -54,7 +57,8 @@ function _resolve_defaults(profile::TargetProfile, ::Type{T}) where {T} fitted = _fitted_shape(profile, T, method) small_m = _small_m_candidates(Val(profile.isa), profile, T) real_row = _real_blocking_row(profile, real(T)) - return ResolvedDefaults(profile, shape, fitted, small_m, real_row) + l2_core = _l2_core_bytes(profile) + return ResolvedDefaults(profile, shape, fitted, small_m, real_row, l2_core) end # One slot per supported element type, so the lookup is a method dispatch on diff --git a/src/planning/labels.jl b/src/planning/labels.jl index a4a66cc..428510e 100644 --- a/src/planning/labels.jl +++ b/src/planning/labels.jl @@ -193,26 +193,13 @@ end # same case. # ---------------------------------------------------------------------------- -# Stable sort of `labels` by `abs(stride)` of each label's axis in `v` (the -# same insertion sort as `_order_free_labels`, over a different operand). -function _sort_labels_by_stride( - labels::Vector{Int}, ind::NTuple{N, Int}, v::StridedView - ) where {N} - st = Base.strides(v) - key(l::Int) = abs(st[findfirst(==(l), ind)::Int]) - out = copy(labels) - @inbounds for i in 2:length(out) - x = out[i] - kx = key(x) - j = i - 1 - while j >= 1 && key(out[j]) > kx - out[j + 1] = out[j] - j -= 1 - end - out[j + 1] = x - end - return out -end +# Stable sort of `labels` by `abs(stride)` of each label's axis in `v`: the +# same allocation-free insertion sort as `_order_free_labels`, over a +# different operand, so it is that function under a name that does not say +# "free" -- the result is an `NTuple{D,Int}` of the input's static length. +@inline _sort_labels_by_stride( + labels::NTuple{D, Int}, ind::NTuple{N, Int}, v::StridedView +) where {D, N} = _order_free_labels(labels, ind, v) # The K order is chosen among three candidates -- `indA` order (the historical # one), sorted by A's strides, sorted by B's strides -- by a cost model of the @@ -258,101 +245,204 @@ const _K_LINE_BYTES = 64 # The core's private L2 share (see `_modelled_blocking`, src/planning/blocking.jl), # or 1 MB when undetected -- between the 512 KB (Zen 2) and 1.25 MB (Ice Lake) -# of the machines measured, so an unknown host errs neither way. -function _l2_core_bytes() - profile = target_profile() +# of the machines measured, so an unknown host errs neither way. Read from the +# detected cache profile (`target_profile()`, filled by `_init_target!` at +# load). The planner does not call this per plan: the value is resolved once +# per (profile, eltype) into `ResolvedDefaults.l2_core` (src/planning/ +# defaults.jl) and read from there (`_l2_core_bytes(T)`). +function _l2_core_bytes(profile::TargetProfile) l2 = profile.l2 l2.bytes > 0 || return 1 << 20 smt = max(1, profile.l1d.sharing) return l2.bytes ÷ max(1, l2.sharing ÷ smt) end +_l2_core_bytes() = _l2_core_bytes(target_profile()) -# Cost of packing one operand under K order `korder`. `free` is the operand's -# free-label list in the order the M/N composite enumerates it (C-stride -# order, `_order_free_labels`), whose FIRST label decides whether a register -# sliver is whole lines. `Qfree` is the free extent. -function _k_order_cost( - korder::Vector{Int}, ind::NTuple{N, Int}, v::StridedView, - free::Vector{Int}, Qfree::Int, l2bytes::Int - ) where {N} +# Cached form for element type `T`: a field of the per-(profile, eltype) +# defaults, so no division is redone per plan; an element type without a +# defaults slot (no kernel menu, it fails later in planning) reads the +# profile directly. +@inline _l2_core_bytes(::Type{T}) where {T} = + _defaults_slot(T) === nothing ? _l2_core_bytes() : _resolved_defaults(T).l2_core + +# What the cost model reads of one operand, gathered ONCE per plan in +# `klabels` order (`_k_operand`): each K label's extent `len[i]` and +# |stride| `st[i]` in the operand, the element count `n`, the element size +# `S`, and whether its register slivers are whole lines (`wholeline`: the +# free composite's leading -- C-fastest, first non-singleton of `free` -- +# axis is unit-stride here and at least a line long). An order is then a +# permutation `perm` of `1:DK` (`klabels[perm[j]]` is its j-th label), and +# costing a candidate is plain tuple indexing: no `findfirst` over the +# operand's labels per axis visited, as a label-keyed evaluation would do six +# times over (three candidates, two operands). +struct _KOperand{DK} + len::NTuple{DK, Int} + st::NTuple{DK, Int} + n::Int + S::Int + wholeline::Bool +end + +@inline function _k_operand( + klabels::NTuple{DK, Int}, ind::NTuple{N, Int}, v::StridedView, + free::NTuple{DF, Int} + ) where {DK, N, DF} st = Base.strides(v) + sz = size(v) S = sizeof(eltype(v)) - line_elems = max(1, _K_LINE_BYTES ÷ S) pos(l::Int) = findfirst(==(l), ind)::Int - n = length(v) + ps = map(pos, klabels) + len = map(p -> sz[p], ps) + kst = map(p -> abs(st[p]), ps) + line_elems = max(1, _K_LINE_BYTES ÷ S) + wholeline = false + for l in free + p = pos(l) + sz[p] == 1 && continue + wholeline = st[p] == 1 && sz[p] >= line_elems + break + end + return _KOperand{DK}(len, kst, length(v), S, wholeline) +end +# Stable insertion sort of the permutation `perm` by `key[perm[j]]`, +# ascending: with `perm = 1:DK` this is `_sort_labels_by_stride` on +# `klabels` expressed as positions (same comparisons, same tie rule), so +# `map(i -> klabels[i], _sort_perm(1:DK, op.st))` is the operand-sorted order. +@inline function _sort_perm(perm::NTuple{D, Int}, key::NTuple{D, Int}) where {D} + out = perm + @inbounds for i in 2:D + x = out[i] + kx = key[x] + j = i - 1 + while j >= 1 && key[out[j]] > kx + out = _tupleset(out, j + 1, out[j]) + j -= 1 + end + out = _tupleset(out, j + 1, x) + end + return out +end + +# Cost of packing one operand under the K order `perm` (positions into the +# `klabels` the operand facts were gathered in; see `_KOperand`). `Qfree` is +# the operand's free extent. +function _k_order_cost( + perm::NTuple{DK, Int}, op::_KOperand{DK}, Qfree::Int, l2bytes::Int + ) where {DK} + len, st, S = op.len, op.st, op.S # Fastest non-singleton K axis of this order. kfast = 0 - for l in korder - if size(v, pos(l)) > 1 - kfast = l + @inbounds for i in perm + if len[i] > 1 + kfast = i break end end kfast == 0 && return 0 # K is all singletons here: nothing walks. - walk = abs(st[pos(kfast)]) * S > _K_WALK_FAR_BYTES ? _K_WALK_FAR_PENALTY : 1 + walk = @inbounds(st[kfast]) * S > _K_WALK_FAR_BYTES ? _K_WALK_FAR_PENALTY : 1 - # Whole-line slivers: the free composite's leading (C-fastest) axis is - # unit-stride here and at least a line long. - wholeline = false - for l in free - p = pos(l) - size(v, p) == 1 && continue - wholeline = st[p] == 1 && size(v, p) >= line_elems - break - end - wholeline && return n * walk + # Whole-line slivers: every line is fetched once. + op.wholeline && return op.n * walk - # The operand's smallest-stride K axis `u`, and how many of its elements - # share a line. + # The operand's smallest-stride K axis `u` (first in this order among + # equal strides), and how many of its elements share a line. u = 0 su = typemax(Int) - for l in korder - p = pos(l) - size(v, p) == 1 && continue - if abs(st[p]) < su - su = abs(st[p]) - u = l + @inbounds for i in perm + len[i] == 1 && continue + if st[i] < su + su = st[i] + u = i end end - share = su * S >= _K_LINE_BYTES ? 1 : min(size(v, pos(u)), _K_LINE_BYTES ÷ (su * S)) - (share == 1 || u == kfast) && return n * walk + # share = su*S >= line ? 1 : min(len[u], line ÷ (su*S)). As `u` is not a + # singleton (len[u] >= 2; an empty axis makes n = 0 and every return 0), + # share == 1 exactly when su*S > line/2, which is tested first so the + # runtime division is only paid on the one return that uses its value. + suS = su * S + (2 * suS > _K_LINE_BYTES || u == kfast) && return op.n * walk # Lines touched before `u` advances: one per element (scattered slivers) # over the faster K extents and the whole free extent. faster = 1 - for l in korder - l == u && break - faster *= size(v, pos(l)) + @inbounds for i in perm + i == u && break + faster *= len[i] end footprint = Int128(faster) * Int128(Qfree) * Int128(_K_LINE_BYTES) - return footprint <= l2bytes ? n * walk : n * walk * share + footprint <= l2bytes && return op.n * walk + share = min(@inbounds(len[u]), _K_LINE_BYTES ÷ suS) + return op.n * walk * share +end + +# Label-keyed form: the cost of packing the operand `v` (labels `ind`) under +# the K order `korder`, where `free` is the operand's free-label list in the +# order the M/N composite enumerates it (C-stride order, `_order_free_labels`) +# -- its FIRST non-singleton label decides whether a register sliver is whole +# lines -- and `Qfree` the free extent. For probes and tests; the planner +# costs its candidates through the positional form above. +function _k_order_cost( + korder::NTuple{DK, Int}, ind::NTuple{N, Int}, v::StridedView, + free::NTuple{DF, Int}, Qfree::Int, l2bytes::Int + ) where {DK, N, DF} + return _k_order_cost(ntuple(identity, Val(DK)), _k_operand(korder, ind, v, free), Qfree, l2bytes) end # Entry point `plan_contract` calls; the decision itself is `_choose_k_order` # (kept apart so a probe can wrap the entry point and log/toggle decisions # without duplicating the model -- benchmark/probes/probe_network_korder.jl). +# Without an explicit `l2bytes` the core's L2 share (`_l2_core_bytes`) is +# used (`_choose_k_order` also takes `nothing` for it, resolved lazily). +# +# Tuple port (the label lists are `NTuple`s, see the top of this file): the +# model takes `klabels`, `morder` and `norder` as statically sized tuples, +# represents every candidate order as a permutation tuple of `1:DK`, and +# returns an `NTuple{DK,Int}` of the input's static length, so the K group's +# rank stays a compile-time constant and nothing allocates. Zero or one K +# label returns at a compile-time branch, before the model or the L2 lookup, +# so a plain GEMM pays nothing for it. +@inline function _order_contract_labels( + klabels::NTuple{DK, Int}, + indA::NTuple{NA, Int}, A::StridedView, morder::NTuple{DM, Int}, + indB::NTuple{NB, Int}, B::StridedView, norder::NTuple{DN, Int}, + Qm::Int, Qn::Int + ) where {DK, NA, NB, DM, DN} + DK <= 1 && return klabels + return _choose_k_order(klabels, indA, A, morder, indB, B, norder, Qm, Qn, nothing) +end function _order_contract_labels( - klabels::Vector{Int}, - indA::NTuple{NA, Int}, A::StridedView, morder::Vector{Int}, - indB::NTuple{NB, Int}, B::StridedView, norder::Vector{Int}, - Qm::Int, Qn::Int, l2bytes::Int = _l2_core_bytes() - ) where {NA, NB} + klabels::NTuple{DK, Int}, + indA::NTuple{NA, Int}, A::StridedView, morder::NTuple{DM, Int}, + indB::NTuple{NB, Int}, B::StridedView, norder::NTuple{DN, Int}, + Qm::Int, Qn::Int, l2bytes::Int + ) where {DK, NA, NB, DM, DN} return _choose_k_order(klabels, indA, A, morder, indB, B, norder, Qm, Qn, l2bytes) end function _choose_k_order( - klabels::Vector{Int}, - indA::NTuple{NA, Int}, A::StridedView, morder::Vector{Int}, - indB::NTuple{NB, Int}, B::StridedView, norder::Vector{Int}, - Qm::Int, Qn::Int, l2bytes::Int - ) where {NA, NB} - length(klabels) <= 1 && return klabels - cost(order) = _k_order_cost(order, indA, A, morder, Qm, l2bytes) + - _k_order_cost(order, indB, B, norder, Qn, l2bytes) - best = klabels - bestcost = cost(klabels) - for cand in (_sort_labels_by_stride(klabels, indA, A), _sort_labels_by_stride(klabels, indB, B)) + klabels::NTuple{DK, Int}, + indA::NTuple{NA, Int}, A::StridedView, morder::NTuple{DM, Int}, + indB::NTuple{NB, Int}, B::StridedView, norder::NTuple{DN, Int}, + Qm::Int, Qn::Int, l2bytes::Union{Int, Nothing} + ) where {DK, NA, NB, DM, DN} + DK <= 1 && return klabels + opA = _k_operand(klabels, indA, A, morder) + opB = _k_operand(klabels, indB, B, norder) + id = ntuple(identity, Val(DK)) + permA = _sort_perm(id, opA.st) # == _sort_labels_by_stride(klabels, indA, A) + permB = _sort_perm(id, opB.st) # == _sort_labels_by_stride(klabels, indB, B) + # Both sorts already give indA order (every gemm_ready/a_permuted/ + # b_permuted layout): the loop below would skip both candidates as equal + # to `best`, so the answer is `klabels` without evaluating a cost. + permA == id && permB == id && return klabels + # `nothing`: the core's L2 share, looked up only now that a cost is needed + # (a field of the cached defaults, `_l2_core_bytes(T)`). + l2 = l2bytes === nothing ? _l2_core_bytes(eltype(A)) : l2bytes + cost(perm) = _k_order_cost(perm, opA, Qm, l2) + _k_order_cost(perm, opB, Qn, l2) + best = id + bestcost = cost(id) + for cand in (permA, permB) cand == best && continue c = cost(cand) if c < bestcost @@ -360,7 +450,7 @@ function _choose_k_order( bestcost = c end end - return best + return map(i -> @inbounds(klabels[i]), best) end # Element count of the leading unit-stride run when `labels` (already ordered diff --git a/test/planning/test_plan_contract.jl b/test/planning/test_plan_contract.jl index 561cb9f..4c849f1 100644 --- a/test/planning/test_plan_contract.jl +++ b/test/planning/test_plan_contract.jl @@ -848,14 +848,14 @@ end # the slowest K coordinate, so each B line is refetched per d element # (x5) once the lines in flight exceed L2 -- forced here with l2bytes = 0. Bs = StridedView(_ko_array(T, (4, 3, 2, 5))) - @test _ko_order(Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), Cv, (1, 5); l2bytes = 0) == [4, 3, 2] + @test _ko_order(Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), Cv, (1, 5); l2bytes = 0) == (4, 3, 2) # ... and when they fit L2 the model is indifferent: indA order is kept. - @test _ko_order(Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), Cv, (1, 5); l2bytes = 1 << 20) == [2, 3, 4] + @test _ko_order(Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), Cv, (1, 5); l2bytes = 1 << 20) == (2, 3, 4) # gemm_ready B[b,c,d,e] and b_permuted B[b,e,c,d]: B-sorted == indA order. for indB in ((2, 3, 4, 5), (2, 5, 3, 4)) Bv = StridedView(_ko_array(T, indB)) - @test _ko_order(Av, (1, 2, 3, 4), Bv, indB, Cv, (1, 5); l2bytes = 0) == [2, 3, 4] + @test _ko_order(Av, (1, 2, 3, 4), Bv, indB, Cv, (1, 5); l2bytes = 0) == (2, 3, 4) end # TRG-shaped: B's K axis 5 is unit-stride, but sorting by B would make A's @@ -867,7 +867,7 @@ end Bt = StridedView(randn(T, 8, 8, 8)) Ct = StridedView(zeros(T, 8, 8, 8, 8)) for l2bytes in (0, 1 << 20) - @test _ko_order(At, (1, 2, 3, 4, 5), Bt, (5, 6, 2), Ct, (1, 3, 4, 6); l2bytes) == [2, 5] + @test _ko_order(At, (1, 2, 3, 4, 5), Bt, (5, 6, 2), Ct, (1, 3, 4, 6); l2bytes) == (2, 5) end # Single K label: nothing to order. @@ -877,8 +877,110 @@ end A1 = StridedView(randn(T, 9, 1, 4, 5)) B1 = StridedView(randn(T, 1, 5, 4, 6)) o = _ko_order(A1, (1, 2, 3, 4), B1, (2, 4, 3, 5), Cv, (1, 5); l2bytes = 0) - @test sort(o) == [2, 3, 4] - @test filter(!=(2), o) == [4, 3] # B-sorted among the non-singletons + @test sort(collect(o)) == [2, 3, 4] + @test filter(!=(2), o) == (4, 3) # B-sorted among the non-singletons +end + +@testset "K order: static-length tuple, inferred, allocation-free" begin + T = Float64 + ext = _KO_BIG + Av = StridedView(_ko_array(T, (1, 2, 3, 4), ext)) + Bs = StridedView(_ko_array(T, (4, 3, 2, 5), ext)) + Cv = StridedView(zeros(T, ext[1], ext[5])) + klabels, morder, norder = (2, 3, 4), (1,), (5,) + f(l2) = QuasiStrided._order_contract_labels(klabels, (1, 2, 3, 4), Av, morder, (4, 3, 2, 5), Bs, norder, ext[1], ext[5], l2) + g() = QuasiStrided._order_contract_labels(klabels, (1, 2, 3, 4), Av, morder, (4, 3, 2, 5), Bs, norder, ext[1], ext[5]) + @test @inferred(f(0)) === (4, 3, 2) + @test @inferred(g()) isa NTuple{3, Int} + f(0); g() + @test (@allocated f(0)) == 0 + @test (@allocated g()) == 0 + # Zero and one K label: returned as given, without the model. + h1() = QuasiStrided._order_contract_labels((2,), (1, 2), Av, (1,), (2, 5), Bs, (5,), 9, 40) + h0() = QuasiStrided._order_contract_labels((), (1,), Av, (1,), (5,), Bs, (5,), 9, 40) + @test @inferred(h1()) === (2,) + @test @inferred(h0()) === () +end + +# The cost model as first written on perf/scrambled-strides (label-keyed, +# `Vector` lists there), kept verbatim as the reference the planner's +# positional, tuple-based form (`_KOperand`, src/planning/labels.jl) must +# reproduce decision for decision. +function _ko_reference_cost(korder, ind, v, free, Qfree, l2bytes) + st = Base.strides(v) + S = sizeof(eltype(v)) + line_elems = max(1, 64 ÷ S) + pos(l) = findfirst(==(l), ind)::Int + n = length(v) + kfast = 0 + for l in korder + if size(v, pos(l)) > 1 + kfast = l + break + end + end + kfast == 0 && return 0 + walk = abs(st[pos(kfast)]) * S > 4096 ? 3 : 1 + wholeline = false + for l in free + p = pos(l) + size(v, p) == 1 && continue + wholeline = st[p] == 1 && size(v, p) >= line_elems + break + end + wholeline && return n * walk + u = 0 + su = typemax(Int) + for l in korder + p = pos(l) + size(v, p) == 1 && continue + if abs(st[p]) < su + su = abs(st[p]) + u = l + end + end + share = su * S >= 64 ? 1 : min(size(v, pos(u)), 64 ÷ (su * S)) + (share == 1 || u == kfast) && return n * walk + faster = 1 + for l in korder + l == u && break + faster *= size(v, pos(l)) + end + footprint = Int128(faster) * Int128(Qfree) * Int128(64) + return footprint <= l2bytes ? n * walk : n * walk * share +end +_ko_reference_sort(labels, ind, v) = + Tuple(sort(collect(labels); by = l -> abs(Base.strides(v)[findfirst(==(l), ind)]))) # `sort` is stable + +@testset "K order: tuple port reproduces the reference cost model" begin + Random.seed!(0x0C05_7000) + for trial in 1:500 + T = rand((Float64, Float32, ComplexF64)) + nk = rand(2:4) + ext = Dict(l => rand((1, 2, 3, 5, 8, 17, 40)) for l in 1:(nk + 2)) + klabels = Tuple(2:(nk + 1)) + # Operands stored in a random axis order, then viewed in another one, + # so the label order, the storage order and the stride order all differ. + function operand(labels) + store = Random.shuffle(collect(labels)) + v = StridedView(randn(T, Tuple(ext[l] for l in store)...)) + perm = Tuple(Random.randperm(length(store))) + return permutedims(v, perm), Tuple(store[collect(perm)]) + end + Av, indA = operand((1, klabels...)) + Bv, indB = operand((klabels..., nk + 2)) + l2 = rand((0, 256, 4096, 1 << 20)) + Qm, Qn = ext[1], ext[nk + 2] + cost(o) = _ko_reference_cost(o, indA, Av, (1,), Qm, l2) + _ko_reference_cost(o, indB, Bv, (nk + 2,), Qn, l2) + best = klabels + for cand in (_ko_reference_sort(klabels, indA, Av), _ko_reference_sort(klabels, indB, Bv)) + cand != best && cost(cand) < cost(best) && (best = cand) + end + @test QuasiStrided._order_contract_labels(klabels, indA, Av, (1,), indB, Bv, (nk + 2,), Qm, Qn, l2) === best + for o in (klabels, reverse(klabels)) + @test QuasiStrided._k_order_cost(o, indA, Av, (1,), Qm, l2) == _ko_reference_cost(o, indA, Av, (1,), Qm, l2) + end + end end @testset "K order: plan_contract flips contract_scrambled at an L2-exceeding size" begin From 785de1e98eecf135f1ced19441c43bceb15bd5c0 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sat, 26 Sep 2026 11:33:05 -0400 Subject: [PATCH 10/53] Inline vector tile stores; move tile/pack prologue throws out of line The fixed cost of one micro-tile dominates small contractions, and two things inflated it: the real and planar `store_tile!` were not inlined into the tile loop, and every per-tile/per-pack prologue carried inline `throw(ArgumentError("...$m..."))` checks whose string interpolation pulled formatting code into the hot function. * `store_tile!` for the real (SIMD) and planar kernels, and `unsafe_pack_a!`/`unsafe_pack_b!`, are now `@inline`. The FMAddSub store stays out of line: its store was scalar only (~64 `_axpby_tile!` calls at `(8,8)`) and inlining it measured slower (ComplexF64 16^3, tiles 1210 -> 1596 ns). * The prologue checks call `@noinline` helpers (`_throw_tile_extent`, `_throw_negative_kc`, src/microkernels/interface.jl) instead of throwing inline; a GUARDRAIL comment there keeps it that way. Measured on ccqlin038 (Cascade Lake, Julia 1.12.7): the 16x6 Float64 tile's fixed cost went from ~37 to ~26 ns. Co-Authored-By: Claude Opus 5.5 --- src/microkernels/fmaddsub.jl | 8 +++++- src/microkernels/interface.jl | 29 +++++++++++++++----- src/microkernels/onem.jl | 3 ++- src/microkernels/planar.jl | 10 +++++-- src/microkernels/simd.jl | 17 ++++++++++-- src/packing/pack.jl | 50 ++++++++++++++++++++--------------- 6 files changed, 83 insertions(+), 34 deletions(-) diff --git a/src/microkernels/fmaddsub.jl b/src/microkernels/fmaddsub.jl index 182fe53..7827d77 100644 --- a/src/microkernels/fmaddsub.jl +++ b/src/microkernels/fmaddsub.jl @@ -355,7 +355,7 @@ function Base.accumulate( packed_a::PA, packed_b::PB, kc::Int ) where {MR, NR, T, W, R, NA, PA, PB} kc == 0 && return acc - kc > 0 || throw(ArgumentError("accumulate requires kc >= 0, got kc = $kc")) + kc > 0 || _throw_negative_kc(:accumulate, kc) @inbounds for p in 0:(kc - 1) acc = _accumulate_step_fmaddsub(kernel, acc, packed_a, packed_b, p) end @@ -431,6 +431,12 @@ Same contract as every other kernel's `store_tile!`: `C = alpha*R + beta*C` over the valid rectangle only, `alpha == 0` never reads `acc`, `beta == 0` never reads old `C`, padding lanes are never read, an empty destination is a no-op. Reads the interleaved accumulator exactly as 1m's does. + +Deliberately NOT `@inline`, unlike the real and planar `store_tile!`: this +kernel has only a scalar (per-element) store, ~64 `_axpby_tile!` calls at +`(8,8)`, and inlining it was measured to cost time rather than save it +(ComplexF64 16^3, tiles 1210 -> 1596 ns), presumably from the code growth, +while the accumulator spill it would avoid is small next to the scalar stores. """ function store_tile!( destination::QSTile, acc::NTuple{NV, Vec{W, R}}, diff --git a/src/microkernels/interface.jl b/src/microkernels/interface.jl index 05f072a..07e02fd 100644 --- a/src/microkernels/interface.jl +++ b/src/microkernels/interface.jl @@ -155,12 +155,13 @@ pack_b!( pack_b!(packed, source, kernel.descriptor, transform) # Same forwarding for the bounds-check-skipping siblings (src/packing/pack.jl), with -# the same per-argument type parameters for the same reason. -unsafe_pack_a!( +# the same per-argument type parameters for the same reason. `@inline`, like +# the functions they forward to: these are the driver's per-sliver entry points. +@inline unsafe_pack_a!( packed::V, source::QSTile, kernel::K, transform::F ) where {V, MR, NR, T, K <: DescriptorKernel{MR, NR, T}, F} = unsafe_pack_a!(packed, source, kernel.descriptor, transform) -unsafe_pack_b!( +@inline unsafe_pack_b!( packed::V, source::QSTile, kernel::K, transform::F ) where {V, MR, NR, T, K <: DescriptorKernel{MR, NR, T}, F} = unsafe_pack_b!(packed, source, kernel.descriptor, transform) @@ -249,6 +250,22 @@ end ) ) +# GUARDRAIL: every throw reachable from the per-tile prologue lives behind +# one of these `@noinline` helpers, never as an inline +# `throw(ArgumentError("...$m..."))`. An inline interpolated message pulls +# `print_to_string` -- and with it a GC frame and ~1 KB of stack -- into the +# hot tile function, which is entered once per micro-tile (measured 2026-09-26, +# ccqlin038: the K=1 outer-product suite cases spend ~37 ns per 16x6 Float64 +# tile of pure fixed cost, of which this frame setup and the out-of-line store +# call below are the two avoidable parts). +@noinline _throw_tile_extent(which::Symbol, got::Int, limit::Int) = throw( + ArgumentError( + "destination valid $which extent $got exceeds $(which === :row ? "mr" : "nr")(kernel) = $limit" + ) +) +@noinline _throw_negative_kc(where::Symbol, kc::Int) = + throw(ArgumentError("$where requires kc >= 0, got kc = $kc")) + # `execute_tile!`'s validation sequence, in this order: # destination extent vs. kernel shape, `kc >= 0`, the alpha/beta converts, the # empty short-circuit, storage bounds BEFORE any `@inbounds` path, @@ -275,9 +292,9 @@ end ) where {MR, NR, T, K <: DescriptorKernel{MR, NR, T}, PA, PB, BOUNDS} m = nrows(destination) n = ncols(destination) - m <= MR || throw(ArgumentError("destination valid row extent $m exceeds mr(kernel) = $MR")) - n <= NR || throw(ArgumentError("destination valid column extent $n exceeds nr(kernel) = $NR")) - kc >= 0 || throw(ArgumentError("execute_tile! requires kc >= 0, got kc = $kc")) + m <= MR || _throw_tile_extent(:row, m, MR) + n <= NR || _throw_tile_extent(:column, n, NR) + kc >= 0 || _throw_negative_kc(:execute_tile!, kc) alphaT = convert(T, alpha) betaT = convert(T, beta) diff --git a/src/microkernels/onem.jl b/src/microkernels/onem.jl index 67000f8..fbda6e5 100644 --- a/src/microkernels/onem.jl +++ b/src/microkernels/onem.jl @@ -282,7 +282,8 @@ even real rows, those two halves are always **adjacent lanes of the same acc[v + MV*j + 1][2u+2])` with `MV = 2MR÷W`. No shuffle, no second load. Scattered/scalar path only, delegating to the generic `_axpby_tile!` -(src/microkernels/interface.jl). +(src/microkernels/interface.jl). Not `@inline`, for the reason given at the +FMAddSub `store_tile!` (src/microkernels/fmaddsub.jl): scalar-only store. """ function store_tile!( destination::QSTile, acc::NTuple{NV, Vec{W, R}}, diff --git a/src/microkernels/planar.jl b/src/microkernels/planar.jl index 6845d2e..be8b50b 100644 --- a/src/microkernels/planar.jl +++ b/src/microkernels/planar.jl @@ -239,7 +239,7 @@ function Base.accumulate( packed_a::PA, packed_b::PB, kc::Int ) where {MR, NR, T, W, R, NA, PA, PB} kc == 0 && return acc - kc > 0 || throw(ArgumentError("accumulate requires kc >= 0, got kc = $kc")) + kc > 0 || _throw_negative_kc(:accumulate, kc) @inbounds for p in 0:(kc - 1) acc = _accumulate_step_planar(kernel, acc, packed_a, packed_b, p) end @@ -570,6 +570,7 @@ end end return quote + Base.@_inline_meta storage = destination.storage cols = destination.cols # zero-based element address at (i=0, j=0)'s row contribution; rows are @@ -619,8 +620,13 @@ in the general-`beta` regime -- the fallback itself is not bit-reproducible against ITSELF across call sites there (LLVM inconsistently fuses Base's `muladd(::Complex,::Complex,::Complex)`; see the divergence measurement above). Compare the general-`beta` case with a tolerance, never `==`. + +`@inline`, with the vectorized store body, for the reason given at the real +`store_tile!` (src/microkernels/simd.jl): an out-of-line store spills the +whole two-plane accumulator to the stack on every micro-tile. The scalar +fallback stays out of line, as the real path's does. """ -function store_tile!( +@inline function store_tile!( destination::QSTile, acc::NTuple{NA, Vec{W, R}}, alpha::T, beta::T, kernel::PlanarKernel{MR, NR, T, W} ) where {MR, NR, T, W, R, NA} diff --git a/src/microkernels/simd.jl b/src/microkernels/simd.jl index 87fdd0a..0db3b8f 100644 --- a/src/microkernels/simd.jl +++ b/src/microkernels/simd.jl @@ -138,7 +138,7 @@ such as a `Vector` or unit-range `view`. `packed_a` may also be an packed_a::PA, packed_b::PB, kc::Int ) where {MR, NR, T, W, NV, PA, PB} kc == 0 && return acc - kc > 0 || throw(ArgumentError("accumulate requires kc >= 0, got kc = $kc")) + kc > 0 || _throw_negative_kc(:accumulate, kc) @inbounds for p in 0:(kc - 1) acc = _accumulate_step(kernel, acc, packed_a, packed_b, p) end @@ -269,6 +269,7 @@ end ) end return quote + Base.@_inline_meta storage = destination.storage cols = destination.cols # zero-based address at (i=0, j=0)'s row contribution; rows are @@ -291,8 +292,20 @@ dense storage (`Vector`, `Memory`, ...) get vector load/store for whole valid rectangle); see [`_vector_store_eligible`](@ref) for exactly which storage qualifies. Otherwise falls back to the scalar path, one lane at a time. Empty destination is a no-op. + +`@inline`, and so is the vectorized store body: left to the inliner's cost +model this call was made out of line, which forces the whole accumulator (12 +zmm at `(16,6,8)`) through the stack -- a spill after the K loop and a reload +inside the store -- on every micro-tile. Inlined, the accumulator stays in +registers from the last FMA to the `vstore`. Measured 2026-09-26 (ccqlin038, +Cascade Lake, together with moving the prologue's throws out of line): the +fixed per-tile cost of a 16x6 Float64 tile in `execute!` fell from ~37 to +~26 ns, and a 128^3 matmul's tile phase from 56 to 47 us. The scattered store +stays out of line: it is scalar per element (~160 ns per full tile), so the +spill it forces is noise, while inlining it grows the tile function by ~4000 +instructions. """ -function store_tile!( +@inline function store_tile!( destination::QSTile, acc::NTuple{NV, Vec{W, T}}, alpha::T, beta::T, kernel::SIMDKernel{MR, NR, T, W} ) where {MR, NR, T, W, NV} diff --git a/src/packing/pack.jl b/src/packing/pack.jl index 3d3de4e..3fac5b0 100644 --- a/src/packing/pack.jl +++ b/src/packing/pack.jl @@ -44,23 +44,33 @@ end @inline _check_pack_a(packed::V, source::QSTile, kernel::K) where {V, K} = _check_pack_a(packed, source, kernel, Val(true)) +# The throws live behind `@noinline` helpers so that the packers -- entered +# once per sliver -- carry no inline string formatting (and hence no GC frame); +# see `_throw_tile_extent` in src/microkernels/interface.jl for the reason. +@noinline _throw_pack_extent(which::Symbol, name::Symbol, got::Int, limit::Int) = throw( + ArgumentError( + "$(which)!: source $(name) count $got must satisfy 0 <= $got <= $(which === :pack_a ? "mr" : "nr")(kernel)=$limit" + ) +) +@noinline _throw_pack_kc(which::Symbol, kc::Int) = + throw(ArgumentError("$(which)!: source K extent (kc) must be nonnegative, got $kc")) +@noinline _throw_pack_short(which::Symbol, got::Int, need::Int, kc::Int) = throw( + DimensionMismatch( + "$(which)!: packed buffer has length $got, " * + "need at least packed_$(which === :pack_a ? "a" : "b")_length(kernel, kc=$kc) = $need" + ) +) + @inline function _check_pack_a( packed::V, source::QSTile, kernel::K, ::Val{BOUNDS} ) where {V, K, BOUNDS} MR = mr(kernel) m = nrows(source) kc = ncols(source) - (0 <= m <= MR) || - throw(ArgumentError("pack_a!: source row count m=$m must satisfy 0 <= m <= mr(kernel)=$MR")) - kc >= 0 || throw(ArgumentError("pack_a!: source column count (kc) must be nonnegative, got $kc")) + (0 <= m <= MR) || _throw_pack_extent(:pack_a, :row, m, MR) + kc >= 0 || _throw_pack_kc(:pack_a, kc) needed = packed_a_length(kernel, kc) - length(packed) >= needed || - throw( - DimensionMismatch( - "pack_a!: packed buffer has length $(length(packed)), " * - "need at least packed_a_length(kernel, kc=$kc) = $needed" - ) - ) + length(packed) >= needed || _throw_pack_short(:pack_a, length(packed), needed, kc) kc == 0 && return (m, 0) BOUNDS && checked_tile_storage_bounds(source) return (m, kc) @@ -75,17 +85,10 @@ end NR = nr(kernel) kc = nrows(source) n = ncols(source) - kc >= 0 || throw(ArgumentError("pack_b!: source row count (kc) must be nonnegative, got $kc")) - (0 <= n <= NR) || - throw(ArgumentError("pack_b!: source column count n=$n must satisfy 0 <= n <= nr(kernel)=$NR")) + kc >= 0 || _throw_pack_kc(:pack_b, kc) + (0 <= n <= NR) || _throw_pack_extent(:pack_b, :column, n, NR) needed = packed_b_length(kernel, kc) - length(packed) >= needed || - throw( - DimensionMismatch( - "pack_b!: packed buffer has length $(length(packed)), " * - "need at least packed_b_length(kernel, kc=$kc) = $needed" - ) - ) + length(packed) >= needed || _throw_pack_short(:pack_b, length(packed), needed, kc) kc == 0 && return (n, 0) BOUNDS && checked_tile_storage_bounds(source) return (n, kc) @@ -145,8 +148,11 @@ any of them -- an exactly equivalent test, because the block's slivers partition its offset buffer and all of them share the same K axis, so the block's offset range is the union of the slivers' and the check only ever looks at range extremes. + +`@inline` (as is `unsafe_pack_b!`): the driver calls these once per sliver, +and out of line each call marshals the `QSTile` through the stack. """ -function unsafe_pack_a!( +@inline function unsafe_pack_a!( packed::V, source::QSTile, kernel::Descriptor{MR, NR, T2, FA, FB}, transform::F ) where {V, MR, NR, T2, FA, FB, F} @@ -178,7 +184,7 @@ B-side counterpart of [`unsafe_pack_a!`](@ref), with the same precondition (the caller has validated every address `source` can read) and the same single caller in this package. """ -function unsafe_pack_b!( +@inline function unsafe_pack_b!( packed::V, source::QSTile, kernel::Descriptor{MR, NR, T2, FA, FB}, transform::F ) where {V, MR, NR, T2, FA, FB, F} From 279f50f981920d8982ee840e1db70075c79c5535 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 10:45:15 -0400 Subject: [PATCH 11/53] Read B in place (UnpackedBView) for small-M plans with unit-stride K The register kernels consume B one broadcast scalar at a time, so B needs no layout to be read unpacked. `_execute_nest!` now skips the B pack and hands the kernel an `UnpackedBView` (per N sliver: NR column base offsets plus the panel's K axis; padding columns alias the last valid one, so every address is inside the hoisted B bounds check) whenever `_use_unpacked_b` says so: an eligible kernel (SIMD, planar, FMAddSub -- not 1m or scalar), Qm <= 256, and a K composite that is an affine ramp with B step +-1. The kernels read B through `_b_step_load` / `_b_step_load2`, whose default methods are the packed-panel formulas (dispatch on the panel type, so the packed paths compile to the same code as before). Results are bitwise identical to the packed path. Why: at small M the O(nr*kc) scalar gather per (jc, pc) block is not repaid by M-sliver reuse. Measured on ccqlin038 (Cascade Lake, Julia 1.12.7), `execute!` only, interleaved medians, packed -> unpacked (OpenBLAS mul!): Float64 16^3 482 -> 361 ns (171) 32^3 1.93 -> 1.36 us (0.72) 64^3 9.8 -> 8.4 us (5.8) 128^3 64.6 -> 55.0 us (62.4) ComplexF64 16^3 1.56 -> 1.65 us (1.35, within noise) 32^3 6.9 -> 5.8 us (6.3) 64^3 32.8 -> 29.1 us (33.7) 128^3 242 -> 215 us (219) Strided-K B (B stored N-fastest) stays packed: 1.2-2.1x losses there. `_UNPACKED_B_MODE` (:auto/:always/:never) overrides the rule for tests and benchmarks. Co-Authored-By: Claude Opus 5.5 --- src/QuasiStrided.jl | 6 +- src/execution/execute.jl | 57 +++--- src/execution/unpackedb.jl | 180 +++++++++++++++++++ src/microkernels/fmaddsub.jl | 13 +- src/microkernels/planar.jl | 14 +- src/microkernels/simd.jl | 12 +- test/execution/test_unpackedb.jl | 287 +++++++++++++++++++++++++++++++ test/runtests.jl | 2 + 8 files changed, 537 insertions(+), 34 deletions(-) create mode 100644 src/execution/unpackedb.jl create mode 100644 test/execution/test_unpackedb.jl diff --git a/src/QuasiStrided.jl b/src/QuasiStrided.jl index 5e2e633..5ac5f8e 100644 --- a/src/QuasiStrided.jl +++ b/src/QuasiStrided.jl @@ -49,13 +49,15 @@ include("execution/workspace.jl") include("planning/plan.jl") # --- Execution: the five-loop nest, the tile-by-tile oracle, the opt-in -# unpacked direct path for small contractions, and the opt-in half-packed -# path (B packed, one-tile A read in place) --- +# unpacked direct path for small contractions, the opt-in half-packed +# path (B packed, one-tile A read in place), and the automatically selected +# unpacked-B path (A packed, B read in place) --- include("execution/macrokernel.jl") include("execution/execute.jl") include("execution/oracle.jl") include("execution/direct.jl") include("execution/halfpack.jl") +include("execution/unpackedb.jl") # --- Integrations --- include("integrations/tensoroperations.jl") diff --git a/src/execution/execute.jl b/src/execution/execute.jl index 1d7f049..6b9aaac 100644 --- a/src/execution/execute.jl +++ b/src/execution/execute.jl @@ -126,6 +126,11 @@ function _execute_nest!( lenB = length(plan.Bstorage) lenC = length(plan.Cstorage) + # Whether B is read in place by the microkernel instead of packed + # (src/execution/unpackedb.jl). Decided once per call; the B pack below + # is skipped and loops 2/1 run through `_micro_tiles_unpacked_b!`. + unpacked_b = _use_unpacked_b(plan) + # --- loop 5: jc over N in steps of nc_eff --- jc = 0 while jc < Qn @@ -179,15 +184,18 @@ function _execute_nest!( beta_eff = firstpanel ? betaT : one(T) - # Pack the whole B panel for this (jc, pc): every N-sliver. - for s in 0:(n_slivers - 1) - sfirst = s * NRk - colsB = _axis_of(ws.n_desc_B[s + 1], ws.n_buf_B, sfirst) - bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) - _pack_sliver!( - unsafe_pack_b!, bpanel, plan.Bstorage, plan.Bbase, rowsB_k, colsB, - kernel, btransform - ) + # Pack the whole B panel for this (jc, pc): every N-sliver -- + # unless the kernel reads B in place for this plan. + if !unpacked_b + for s in 0:(n_slivers - 1) + sfirst = s * NRk + colsB = _axis_of(ws.n_desc_B[s + 1], ws.n_buf_B, sfirst) + bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) + _pack_sliver!( + unsafe_pack_b!, bpanel, plan.Bstorage, plan.Bbase, rowsB_k, colsB, + kernel, btransform + ) + end end # --- loop 3: ic over M in steps of mc_eff --- @@ -233,18 +241,25 @@ function _execute_nest!( end # --- loop 2: jr over N-slivers; loop 1: ir over M-slivers --- - for s in 0:(n_slivers - 1) - sfirst = s * NRk - colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst) - bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) - for r in 0:(m_slivers - 1) - rfirst = r * MRk - rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst) - apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) - unsafe_execute_micro_tile!( - kernel, plan.Cstorage, plan.Cbase, rowsC, colsC, - apanel, bpanel, kblock, alphaT, beta_eff - ) + if unpacked_b + _micro_tiles_unpacked_b!( + kernel, plan, ws, rowsB_k, m_slivers, n_slivers, + MRk, NRk, MRp, kblock, alphaT, beta_eff + ) + else + for s in 0:(n_slivers - 1) + sfirst = s * NRk + colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst) + bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) + for r in 0:(m_slivers - 1) + rfirst = r * MRk + rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst) + apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) + unsafe_execute_micro_tile!( + kernel, plan.Cstorage, plan.Cbase, rowsC, colsC, + apanel, bpanel, kblock, alphaT, beta_eff + ) + end end end diff --git a/src/execution/unpackedb.jl b/src/execution/unpackedb.jl new file mode 100644 index 0000000..0251d40 --- /dev/null +++ b/src/execution/unpackedb.jl @@ -0,0 +1,180 @@ +# "Unpacked B": run the microkernel against B read in place from its own +# storage instead of from a packed panel. The mirror image of half-packing +# (src/execution/halfpack.jl), but far more general: the kernels only ever +# consume B one scalar (or one complex scalar) at a time and broadcast it, so +# B needs no layout at all to be read unpacked -- any affine or scattered K +# axis, any column offsets, any `AbstractVector` storage. Packing B buys +# locality (one contiguous stream instead of `nr` strided ones) and padding +# columns; it costs an O(N*K) scalar gather loop per (jc, pc) block, which at +# small M is comparable to the whole microkernel time (measured below). +# +# Selected automatically by `_use_unpacked_b` (see there for the measured +# rule) for every kernel that reads B through `_b_step_load`/`_b_step_load2` +# -- `SIMDKernel`, `PlanarKernel`, `FMAddSubKernel` -- and never for +# `OneMKernel` (its inner real kernel walks the planar B panel as `2kc` real +# steps, a layout only a packed panel has) or `ScalarKernel` (reference +# kernel, addresses the panel directly). + +""" + UnpackedBView{S,K<:Axis,NR,F}(storage, colbase::NTuple{NR,Int}, ksteps::K, per_k::Int, transform::F) + +Stands in for a packed B micro-panel of `NR` logical columns when B is read in +place. Column `j` at K step `p` is the storage element at zero-based address +`colbase[j+1] + axis_offset(ksteps, p)`; `colbase` already includes the +operand's base offset. Padding columns of a tail sliver (`j >= n`) ALIAS the +sliver's last valid column, so every address the kernel reads is one the +hoisted B bounds check (`_execute_nest!`, check 1) has validated -- the kernel +computes garbage for those columns and the store discards it (`j < n`), exactly +as with a zero-padded packed panel, only without the zeros. + +`transform` is the plan's `btransform` (`identity` or `conj`), applied to each +loaded element, so a conjugated complex B needs no packing either. `per_k` is +`packed_b_per_k(kernel)`: `length` is the packed-equivalent real count, what +`_execute_tile_prologue!` compares against `packed_b_length(kernel, kc)`, not +the storage length. Borrowed: `ksteps` may be a `PtrScatterAxis` into a +workspace buffer, which the caller keeps alive. Not bounds-checked; see +`_micro_tiles_unpacked_b!`. +""" +struct UnpackedBView{S, K <: Axis, NR, F} + storage::S + colbase::NTuple{NR, Int} + ksteps::K + per_k::Int + transform::F +end + +Base.length(u::UnpackedBView) = u.per_k * axis_length(u.ksteps) + +# One element of B, transformed. `@inbounds`: the caller has validated the +# whole B panel region once (see `_micro_tiles_unpacked_b!`). Linear indexing, +# so any `AbstractVector` storage works -- `Memory`, `Vector`, or a wrapper. +@inline function _unpacked_b_element(u::UnpackedBView{S, K, NR, F}, j::Int, p::Int) where {S, K, NR, F} + @inbounds z = u.storage[u.colbase[j + 1] + axis_offset(u.ksteps, p) + 1] + return u.transform(z) +end + +# Real kernels (`_accumulate_step`, src/microkernels/simd.jl). +@inline _b_step_load(u::UnpackedBView, kernel, j::Int, p::Int) = _unpacked_b_element(u, j, p) + +# Complex kernels (`_accumulate_step_planar`/`_accumulate_step_fmaddsub`): the +# planar `(re, im)` pair of one complex element. +@inline function _b_step_load2(u::UnpackedBView, kernel, j::Int, p::Int) + z = _unpacked_b_element(u, j, p) + return (real(z), imag(z)) +end + +# Which kernels can take an `UnpackedBView`: exactly those whose K step reads +# B through `_b_step_load`/`_b_step_load2`. Folds to a constant per plan type. +@inline _unpacked_b_kernel_eligible(::SIMDKernel) = true +@inline _unpacked_b_kernel_eligible(::PlanarKernel) = true +@inline _unpacked_b_kernel_eligible(::FMAddSubKernel) = true +@inline _unpacked_b_kernel_eligible(::Any) = false + +# Test/benchmark override of the automatic rule: `:auto` (the rule below), +# `:always`, `:never`. Read once per `execute!`. +const _UNPACKED_B_MODE = Ref{Symbol}(:auto) + +""" + _use_unpacked_b(plan::ContractPlan) -> Bool + +Whether this `execute!` reads B in place (`UnpackedBView`) rather than packing +it. Requires an eligible kernel (`_unpacked_b_kernel_eligible`); then, under +`_UNPACKED_B_MODE[] === :auto`, the rule is + + Qm <= _UNPACKED_B_MMAX && the K composite is an affine ramp whose B step is +-1 + +i.e. every B column the kernel streams is contiguous along K. Packing a B +sliver costs an O(nr*kc) scalar gather per (jc, pc) block that only the M +slivers reusing it repay, while reading `nr` contiguous columns in place costs +the kernel nothing measurable. With a LARGE K stride in B (B stored N-fastest, +or a scattered K), each K step touches its own cache line and, for a +power-of-two stride, only a few L1 sets: that is the case packing exists for, +and it stays packed. + +Measured 2026-09-26, ccqlin038 (Cascade Lake, Julia 1.12.7), `:always` vs +`:never` on the same tree, back-to-back pairs, `execute!` time ratio +unpacked/packed: + + K unit-stride in B (C[a,c] = A[a,b] B[b,c] and the rank-4 gemm_ready cases): + Float64 16^3 0.70 32^3 0.70 64^3 0.79 128^3 0.90 256^3 0.93 512^3 1.02 + dim8_1_3_1 (8x512x8) 0.48 dim6_2_2_2 (36^3) 0.87 dim8_2_1_2 (64x8x64) 0.91 + ComplexF64 16^3 1.00 32^3 0.86 64^3 0.93 128^3 0.91 256^3 0.96 512^3 1.01 + M=1 gemv (1x64x512 .. 1x256x4096, FMAddSub 8x8) 0.55-0.65 dim8_1_3_1 0.68 + K strided in B (B[c,b], stride N): + Float64 16^3 0.71 64^3 0.94 256^3 1.20 ComplexF64 16^3 1.12 64^3 1.05 256^3 2.12 + K = 1 (outer product, rank-0 K group, step 0 -- excluded by the rule): + Float64 ~1.00 ComplexF64 1.03-1.08 + +So the rule takes the unit-stride column and stops at M = 256, the last size +with a measured win on both dtypes; 512 is a wash. The strided-K wins at +small K are left on the table deliberately: the same rule would have to +model L1 set conflicts to avoid the 1.2-2.1x losses above. +""" +@inline function _use_unpacked_b(plan::ContractPlan) + _unpacked_b_kernel_eligible(plan.kernel) || return false + mode = _UNPACKED_B_MODE[] + mode === :always && return true + mode === :never && return false + axis_length(plan.mgroup) <= _UNPACKED_B_MMAX || return false + (k_ramp, k_step) = affine_ramp(plan.kgroup) + return k_ramp && abs(k_step[2]) == 1 +end + +# Largest M extent at which B is read in place under `:auto`; see +# `_use_unpacked_b`. +const _UNPACKED_B_MMAX = 256 + +# Build the view for N sliver `s` of the current (jc, pc) block. `d` is the +# sliver's B descriptor (`ws.n_desc_B[s+1]`), `buf`/`sfirst` locate its +# offsets in `ws.n_buf_B` when it is irregular. Padding columns alias the last +# valid one (see `UnpackedBView`). `NR` comes from the kernel's type, so the +# tuple is statically sized. +@inline function _unpacked_b_view( + kernel::DescriptorKernel{MR, NR, T}, storage::S, base::Int, + d::BlockDescriptor, buf::Vector{Int}, sfirst::Int, ksteps::K, transform::F + ) where {MR, NR, T, S, K <: Axis, F} + last = d.count - 1 + colbase = ntuple(Val(NR)) do j1 + jj = min(j1 - 1, last) + off = d.regular ? d.base + jj * d.stride : (@inbounds buf[sfirst + jj + 1]) + base + off + end + return UnpackedBView(storage, colbase, ksteps, packed_b_per_k(kernel), transform) +end + +# Loops 2/1 of `_execute_nest!` (src/execution/execute.jl) with every B sliver +# read in place: one `UnpackedBView` per N sliver, then the M slivers against +# it. Function barrier over the K axis type (the `_axis_of` Union dies here, +# per the guardrail at the top of src/execution/macrokernel.jl), so each view +# is concretely typed and the micro-tile calls specialize on it. +# +# Bounds: the addresses a view can read are `Bbase + colbase[j] + koffset(p)` +# for the sliver's own columns (padding aliases a valid one) and this panel's +# K offsets -- a subset of the `(rng_kB, rng_nB)` rectangle the caller has +# already passed through `checked_span_bounds` (check 1 of 3), which is why +# the element loads are `@inbounds`. C is covered by the caller's check 3. +@inline function _micro_tiles_unpacked_b!( + kernel::K, plan::ContractPlan, ws, rowsB_k::KA, m_slivers::Int, n_slivers::Int, + MRk::Int, NRk::Int, MRp::Int, kblock::Int, alphaT, beta_eff + ) where {K, KA <: Axis} + Bstorage = plan.Bstorage + Bbase = plan.Bbase + btransform = plan.btransform + for s in 0:(n_slivers - 1) + sfirst = s * NRk + colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst) + bview = _unpacked_b_view( + kernel, Bstorage, Bbase, ws.n_desc_B[s + 1], ws.n_buf_B, sfirst, rowsB_k, btransform + ) + for r in 0:(m_slivers - 1) + rfirst = r * MRk + rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst) + apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) + unsafe_execute_micro_tile!( + kernel, plan.Cstorage, plan.Cbase, rowsC, colsC, + apanel, bview, kblock, alphaT, beta_eff + ) + end + end + return nothing +end diff --git a/src/microkernels/fmaddsub.jl b/src/microkernels/fmaddsub.jl index 7827d77..9aa6aff 100644 --- a/src/microkernels/fmaddsub.jl +++ b/src/microkernels/fmaddsub.jl @@ -310,14 +310,11 @@ end load_b = Any[] for j in 0:(NR - 1) - push!( - load_b, - :($(brv[j + 1]) = Vec{$W, $R}(panel_load(packed_b, packed_b_plane_offset(kernel, 0, $j, p)))) - ) - push!( - load_b, - :($(biv[j + 1]) = Vec{$W, $R}(panel_load(packed_b, packed_b_plane_offset(kernel, 1, $j, p)))) - ) + # `_b_step_load2` (src/microkernels/planar.jl): the planar `(re, im)` + # pair, from a packed panel or an `UnpackedBView`. + push!(load_b, :((br_s, bi_s) = _b_step_load2(packed_b, kernel, $j, p))) + push!(load_b, :($(brv[j + 1]) = Vec{$W, $R}(br_s))) + push!(load_b, :($(biv[j + 1]) = Vec{$W, $R}(bi_s))) end acc_exprs = Vector{Any}(undef, NA) diff --git a/src/microkernels/planar.jl b/src/microkernels/planar.jl index be8b50b..447d5ac 100644 --- a/src/microkernels/planar.jl +++ b/src/microkernels/planar.jl @@ -124,6 +124,17 @@ function zero_accumulator(kernel::PlanarKernel{MR, NR, T, W}) where {MR, NR, T, return ntuple(_ -> z, Val(2 * (MR ÷ W) * NR)) end +# The complex counterpart of `_b_step_load` (src/microkernels/simd.jl): B's +# column `j` at K step `p` as its `(re, im)` pair of reals. The default reads +# the two planes of a `PlanarFormat` panel; `UnpackedBView` +# (src/execution/unpackedb.jl) reads one `Complex` element of B's own storage +# instead. Shared by the planar and fmaddsub kernels, both of which take B as +# planar scalars. +@inline _b_step_load2(packed_b::PB, kernel, j::Int, p::Int) where {PB} = ( + panel_load(packed_b, packed_b_plane_offset(kernel, 0, j, p)), + panel_load(packed_b, packed_b_plane_offset(kernel, 1, j, p)), +) + # Fully unrolled, closure-free K-step body, mirroring `_accumulate_step` in # src/microkernels/simd.jl: per logical K step, MV A vector loads per plane, NR B # scalar loads per plane, 4*MV*NR FMAs, literal tuple indices throughout @@ -188,8 +199,7 @@ end load_b = Any[] for j in 0:(NR - 1) - push!(load_b, :($(brv[j + 1]) = panel_load(packed_b, packed_b_plane_offset(kernel, 0, $j, p)))) - push!(load_b, :($(biv[j + 1]) = panel_load(packed_b, packed_b_plane_offset(kernel, 1, $j, p)))) + push!(load_b, :(($(brv[j + 1]), $(biv[j + 1])) = _b_step_load2(packed_b, kernel, $j, p))) end acc_exprs = Vector{Any}(undef, NA) diff --git a/src/microkernels/simd.jl b/src/microkernels/simd.jl index 0db3b8f..afffb91 100644 --- a/src/microkernels/simd.jl +++ b/src/microkernels/simd.jl @@ -78,6 +78,16 @@ end # no runtime branch on any path. @inline _a_step_offset(::PA, kernel, i::Int, p::Int) where {PA} = packed_a_offset(kernel, i, p) +# B's column `j` at K step `p`, as the scalar `_accumulate_step` broadcasts. +# The default -- a packed panel -- is the packed formula, so the packed paths +# compile to the same code as before this indirection existed. The one other +# method is `UnpackedBView`'s (src/execution/unpackedb.jl), which reads B in +# place from its own storage: the kernel only ever consumes B one scalar at a +# time, so unlike A it needs no particular layout to be read unpacked. +# Dispatch is on `packed_b`'s TYPE, so the choice is made at compile time. +@inline _b_step_load(packed_b::PB, kernel, j::Int, p::Int) where {PB} = + panel_load(packed_b, packed_b_offset(kernel, j, p)) + # Fully unrolled, closure-free K-step body: one vector load per A row-vector, # one scalar load per B column, NVECA*NR FMAs, generated as straight-line code. @generated function _accumulate_step( @@ -101,7 +111,7 @@ end for v in 0:(NVECA - 1) ] load_b = [ - :($(bvars[j + 1]) = panel_load(packed_b, packed_b_offset(kernel, $j, p))) + :($(bvars[j + 1]) = _b_step_load(packed_b, kernel, $j, p)) for j in 0:(NR - 1) ] diff --git a/test/execution/test_unpackedb.jl b/test/execution/test_unpackedb.jl new file mode 100644 index 0000000..605d909 --- /dev/null +++ b/test/execution/test_unpackedb.jl @@ -0,0 +1,287 @@ +# The unpacked-B path (src/execution/unpackedb.jl): `execute!` runs the +# microkernel against B read in place through an `UnpackedBView` instead of a +# packed panel. Three properties are checked: +# +# 1. `_use_unpacked_b` selects exactly the plans it should: an eligible +# kernel (SIMD/planar/fmaddsub, never 1m or scalar), `Qm <= +# _UNPACKED_B_MMAX`, and a K composite that is an affine ramp with B step +# +-1; and `_UNPACKED_B_MODE` overrides it both ways. +# 2. The result is BITWISE identical (`==`) to the packed path: the K-step +# arithmetic is the same `_accumulate_step`, in the same order, on the +# same B values -- only the address they are read from differs. Padding +# columns alias a valid column instead of holding zeros, and are never +# stored. Also `≈` the oracle and a brute-force reference. +# 3. Under `:always` the packed-B buffer is never written (NaN poison), and +# the call stays allocation-free. +# +# `_hp_ref` (test/execution/test_halfpack.jl, included before this file) is the +# brute-force reference. + +const _ub_mode = QuasiStrided._UNPACKED_B_MODE +const _ub_use = QuasiStrided._use_unpacked_b + +# Run `execute!` under `mode` on a fresh fixture from `mk()`; returns the +# output array and whether the packed-B buffer was left untouched. +function _ub_run(mk, mode::Symbol, alpha, beta; plankw...) + Cv, Av, indA, Bv, indB, indC = mk() + plan = plan_contract(Cv, Av, indA, Bv, indB, indC; plankw...) + fill!(plan.workspace.packed_b, NaN) + old = _ub_mode[] + _ub_mode[] = mode + try + ret = execute!(plan, alpha, beta) + @test ret === plan.Cstorage + finally + _ub_mode[] = old + end + return Array(Cv), all(isnan, plan.workspace.packed_b), plan +end + +# Matmul fixture with a choice of B layout; NaN gaps poison C if read. +# :kunit B[k,n] dense (K step +1) +# :kneg B reversed along k (K step -1) +# :nunit B stored [n,k], viewed transposed (K step N) +# :kgap every other k of a 2K-row B (K step 2) +function _ub_mm_maker(::Type{T}, Ma, Ka, Na, seed; blayout = :kunit, Cfill = nothing) where {T} + return function () + rng = MersenneTwister(seed) + Amat = randn(rng, T, Ma, Ka) + Bmat = randn(rng, T, Ka, Na) + Cmat = Cfill === nothing ? randn(rng, T, Ma, Na) : fill(convert(T, Cfill), Ma, Na) + Bv = if blayout === :kunit + StridedView(Bmat) + elseif blayout === :kneg + big = fill(convert(T, NaN), Ka + 4, Na) + big[(Ka + 2):-1:3, :] .= Bmat + StridedView(view(big, (Ka + 2):-1:3, :)) + elseif blayout === :nunit + permutedims(StridedView(permutedims(Bmat, (2, 1))), (2, 1)) + elseif blayout === :kgap + big = fill(convert(T, NaN), 2Ka, Na) + big[1:2:(2Ka), :] .= Bmat + StridedView(view(big, 1:2:(2Ka), :)) + else + error("unknown blayout $blayout") + end + @assert Array(Bv) == Bmat + return (StridedView(Cmat), StridedView(Amat), (1, 2), Bv, (2, 3), (1, 3)) + end +end + +const _UB_TYPES = (Float64, Float32, ComplexF64, ComplexF32) +const _UB_AB = ((1.0, 0.0), (2.5, -0.75), (1.0, 1.0), (-0.5, 1.25)) + +@testset "unpacked B: which plans read B in place ($T)" for T in _UB_TYPES + mk(blayout, M) = _ub_mm_maker(T, M, 12, 9, 1; blayout = blayout) + for blayout in (:kunit, :kneg) + p = plan_contract(mk(blayout, 20)()...) + @test _ub_use(p) + @test QuasiStrided._unpacked_b_kernel_eligible(p.kernel) + # The mode override wins both ways. + _ub_mode[] = :never + @test !_ub_use(p) + _ub_mode[] = :always + @test _ub_use(p) + _ub_mode[] = :auto + end + # A strided or gapped K in B stays packed ... + for blayout in (:nunit, :kgap) + @test !_ub_use(plan_contract(mk(blayout, 20)()...)) + end + # ... as does a K = 1 outer product (rank-0 K group, step 0) ... + p = plan_contract( + StridedView(zeros(T, 20, 9)), StridedView(randn(T, 20)), (1,), StridedView(randn(T, 9)), (2,), (1, 2) + ) + @test !_ub_use(p) + # ... and an M extent above the cutoff. + MMAX = QuasiStrided._UNPACKED_B_MMAX + @test _ub_use(plan_contract(mk(:kunit, MMAX)()...)) + @test !_ub_use(plan_contract(mk(:kunit, MMAX + 1)()...)) + # Kernels that address the panel directly never take the view. + if T <: Real + ps = plan_contract(mk(:kunit, 20)()...; kernel = ScalarKernel(Val(4), Val(3), T)) + @test !_ub_use(ps) + else + po = plan_contract(mk(:kunit, 20)()...; kernel = QuasiStrided.OneMKernel(Val(4), Val(6), T, Val(4))) + @test !_ub_use(po) + end + # A scattered (non-ramp) K composite is ineligible: B[k1,k2,n] against + # A[m,k2,k1] -- both K maps ramps on their own, not together. + d = 3 + Aarr = randn(T, 7, d, d) + Barr = randn(T, d, d, 5) + Cm = zeros(T, 7, 5) + pk = plan_contract(StridedView(Cm), StridedView(Aarr), (1, 3, 2), StridedView(Barr), (2, 3, 4), (1, 4)) + @test !first(QuasiStrided.affine_ramp(pk.kgroup)) + @test !_ub_use(pk) +end + +@testset "unpacked B: bitwise the packed path on matmul shapes ($T)" for T in _UB_TYPES + shapes = ((1, 1, 1), (5, 3, 7), (16, 16, 16), (17, 9, 13), (40, 7, 30), (33, 64, 5), (2, 100, 1)) + for (idx, (M, K, N)) in enumerate(shapes), blayout in (:kunit, :kneg), (alpha, beta) in _UB_AB + mk = _ub_mm_maker(T, M, K, N, 10 + idx; blayout = blayout) + C_packed, _, _ = _ub_run(mk, :never, alpha, beta) + C_unp, untouched, plan = _ub_run(mk, :always, alpha, beta) + @test untouched + @test C_unp == C_packed + Cv0, Av0, _, Bv0, _, _ = mk() + C_tw = let (Cv, Av, iA, Bv, iB, iC) = mk() + execute_tilewise!(plan_contract(Cv, Av, iA, Bv, iB, iC), alpha, beta) + Array(Cv) + end + @test C_unp ≈ C_tw + @test C_unp ≈ _hp_ref(Array(Av0), (1, 2), Array(Bv0), (2, 3), Array(Cv0), (1, 3), alpha, beta) + # The automatic rule picks the view for every one of these but the + # K = 1 shape (a rank-0 K group has step 0, not +-1). + @test _ub_use(plan) == (K > 1) + end +end + +@testset "unpacked B: several jc/pc/ic blocks, tails and explicit kernels ($T)" for T in _UB_TYPES + p0 = plan_contract(_ub_mm_maker(T, 64, 8, 8, 2)()...) + kernels = if T <: Real + (p0.kernel, SIMDKernel(Val(8), Val(6), T), SIMDKernel(Val(2 * lanewidth(p0.kernel)), Val(3), T)) + else + W = QuasiStrided.lanewidth(p0.kernel) + (p0.kernel, QuasiStrided.PlanarKernel(Val(W), Val(5), T, Val(W)), QuasiStrided.FMAddSubKernel(Val(W), Val(5), T, Val(W))) + end + for k in kernels, (mc, kc, nc) in ((8, 3, 6), (mr(k), 5, nr(k)), (3 * mr(k), 64, 1), (40, 1, 100)) + for (M, K, N) in ((11, 17, 19), (4, 25, 3), (1, 3, 1), (2 * mr(k) + 1, 13, 2 * nr(k) + 1)) + mk = _ub_mm_maker(T, M, K, N, 7 * K + N + M) + C_packed, _, _ = _ub_run(mk, :never, 1.5, 0.5; kernel = k, mc = mc, kc = kc, nc = nc) + C_unp, untouched, _ = _ub_run(mk, :always, 1.5, 0.5; kernel = k, mc = mc, kc = kc, nc = nc) + @test untouched + @test C_unp == C_packed + Cv0, Av0, _, Bv0, _, _ = mk() + @test C_unp ≈ _hp_ref(Array(Av0), (1, 2), Array(Bv0), (2, 3), Array(Cv0), (1, 3), 1.5, 0.5) + end + end +end + +@testset "unpacked B: strided/gapped K, scattered N, permuted C, conj ($T)" for T in _UB_TYPES + alpha, beta = T <: Complex ? (T(0.75, -1.0), T(0.5, 0.5)) : (T(0.75), T(0.5)) + # (1) K strided in B: ineligible by the rule, but must be CORRECT when + # forced -- the view works for any K axis. + for blayout in (:nunit, :kgap) + mk = _ub_mm_maker(T, 21, 14, 11, 5; blayout = blayout) + C_packed, _, _ = _ub_run(mk, :never, alpha, beta) + C_unp, untouched, _ = _ub_run(mk, :always, alpha, beta) + @test untouched && C_unp == C_packed + end + # (2) Non-ramp N composite: C[m,n1,n2] = A[m,k] B[k,n1,n2] with C reversed + # along n2, so N slivers straddle irregular offsets (the buffer path). + M, K, N1, N2 = 20, 6, 13, 3 + mk2 = function () + rng = MersenneTwister(77) + Amat = randn(rng, T, M, K) + Barr = randn(rng, T, K, N1, N2) + Cfull = randn(rng, T, M, N1, N2) + Cr = view(Cfull, :, :, N2:-1:1) + return (StridedView(Cr), StridedView(Amat), (1, 2), StridedView(Barr), (2, 3, 4), (1, 3, 4)) + end + C_packed, _, _ = _ub_run(mk2, :never, alpha, beta) + C_unp, untouched, plan = _ub_run(mk2, :always, alpha, beta) + @test !first(QuasiStrided.affine_ramp(plan.ngroup)) + @test _ub_use(plan) + @test untouched && C_unp == C_packed + Cv0, Av0, _, Bv0, _, _ = mk2() + @test C_unp ≈ _hp_ref(Array(Av0), (1, 2), Array(Bv0), (2, 3, 4), Array(Cv0), (1, 3, 4), alpha, beta) + # (3) Scattered K composite forced on: correct through the PtrScatterAxis + # K axis. + d = 3 + mk3 = function () + rng = MersenneTwister(78) + Aarr = randn(rng, T, 7, d, d) + Barr = randn(rng, T, d, d, 5) + Cm = randn(rng, T, 7, 5) + return (StridedView(Cm), StridedView(Aarr), (1, 3, 2), StridedView(Barr), (2, 3, 4), (1, 4)) + end + C_packed, _, _ = _ub_run(mk3, :never, alpha, beta) + C_unp, untouched, _ = _ub_run(mk3, :always, alpha, beta) + @test untouched && C_unp == C_packed + Cv0, Av0, _, Bv0, _, _ = mk3() + @test C_unp ≈ _hp_ref(Array(Av0), (1, 3, 2), Array(Bv0), (2, 3, 4), Array(Cv0), (1, 4), alpha, beta) + # (4) Permuted A / negative-stride B / offset C (the scattered fixture). + C_packed = let (Cv, Av, iA, Bv, iB, iC) = scattered_fixture(T) + _ub_mode[] = :never + execute!(plan_contract(Cv, Av, iA, Bv, iB, iC), alpha, beta) + _ub_mode[] = :auto + Array(Cv) + end + C_unp = let (Cv, Av, iA, Bv, iB, iC) = scattered_fixture(T) + _ub_mode[] = :always + execute!(plan_contract(Cv, Av, iA, Bv, iB, iC), alpha, beta) + _ub_mode[] = :auto + Array(Cv) + end + @test C_unp == C_packed + # (5) Conjugation: the flag, a conj-wrapped view, and both (which cancel). + if T <: Complex + for conjA in (false, true), conjB in (false, true), wrap in (false, true) + mk = _ub_mm_maker(T, 9, 7, 8, 91) + run(mode) = let (Cv, Av, iA, Bv, iB, iC) = mk() + _ub_mode[] = mode + p = plan_contract(Cv, Av, iA, wrap ? conj(Bv) : Bv, iB, iC; conjA = conjA, conjB = conjB) + execute!(p, alpha, beta) + _ub_mode[] = :auto + Array(Cv) + end + Cn = run(:never) + Ca = run(:always) + @test Ca == Cn + Cv0, Av0, _, Bv0, _, _ = mk() + @test Ca ≈ _hp_ref( + Array(Av0), (1, 2), Array(Bv0), (2, 3), Array(Cv0), (1, 3), alpha, beta; + conjA = conjA, conjB = conjB ⊻ wrap + ) + end + end +end + +@testset "unpacked B: beta = 0 never reads C, K = 0 and alpha = 0 untouched ($T)" for T in _UB_TYPES + mk = _ub_mm_maker(T, 10, 6, 7, 3; Cfill = NaN) + C_unp, untouched, _ = _ub_run(mk, :always, 2.0, 0.0) + @test untouched && all(isfinite, C_unp) + Cv0, Av0, _, Bv0, _, _ = mk() + @test C_unp ≈ 2.0 .* (Array(Av0) * Array(Bv0)) + # alpha = 0 and K = 0 take the beta-only short circuit before the choice. + Cv, Av, iA, Bv, iB, iC = _ub_mm_maker(T, 10, 6, 7, 3)() + C0 = Array(Cv) + _ub_mode[] = :always + execute!(plan_contract(Cv, Av, iA, Bv, iB, iC), 0.0, 0.5) + _ub_mode[] = :auto + @test Array(Cv) ≈ 0.5 .* C0 +end + +@testset "unpacked B: out-of-bounds B is still rejected before any write ($T)" for T in (Float64, ComplexF64) + M, K, N = 10, 6, 5 + Cm = zeros(T, M, N) + Bshort = StridedView(randn(T, K * N - 1), (K, N), (1, K), 0) # one element short + plan = plan_contract(StridedView(Cm), StridedView(randn(T, M, K)), (1, 2), Bshort, (2, 3), (1, 3)) + @test _ub_use(plan) + _ub_mode[] = :always + @test_throws BoundsError execute!(plan, 1.0, 0.0) + _ub_mode[] = :auto + @test all(iszero, Cm) +end + +@testset "unpacked B: allocation-free in steady state ($T)" for T in _UB_TYPES + Cmat = zeros(T, 40, 37) + plan = _mm_plan(Cmat, randn(T, 40, 50), randn(T, 50, 37); kc = 16, nc = 12) + @test _ub_use(plan) + _ub_mode[] = :always + @test _steady_allocs!(execute!, plan, Cmat) == 0 skip = (VERSION < v"1.11") + _ub_mode[] = :auto + @test Cmat ≈ reshape(plan.Astorage, 40, 50) * reshape(plan.Bstorage, 50, 37) +end + +@testset "unpacked B: the TensorOperations backend result is unchanged ($T)" for T in _UB_TYPES + A = randn(T, 12, 9); B = randn(T, 9, 14) + C1 = zeros(T, 12, 14); C2 = zeros(T, 12, 14) + _ub_mode[] = :never + TO.tensorcontract!(C1, A, ((1,), (2,)), false, B, ((1,), (2,)), false, ((1, 2), ()), 1.0, 0.0, QuasiStridedBackend()) + _ub_mode[] = :auto + TO.tensorcontract!(C2, A, ((1,), (2,)), false, B, ((1,), (2,)), false, ((1, 2), ()), 1.0, 0.0, QuasiStridedBackend()) + @test C2 == C1 + @test C2 ≈ A * B +end diff --git a/test/runtests.jl b/test/runtests.jl index cafc495..4afb63c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -51,6 +51,8 @@ using QuasiStrided: AxisGroup, axis_length, offsets, fill_offsets!, BlockDescrip include("execution/test_macro_blocking.jl") include("execution/test_direct.jl") include("execution/test_halfpack.jl") + # Uses test_halfpack.jl's `_hp_ref` reference (and `_HPDenseMat`). + include("execution/test_unpackedb.jl") # Last among the functional tests: it does a bare `using TensorOperations`, # which exports its own `scalartype` and would otherwise conflict with the From 803496ca1fd0f2458fa0cc3b0c39b62344a843d4 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 10:45:29 -0400 Subject: [PATCH 12/53] Add a K-vectorized dot (gemv) path for M == 1 / N == 1 With a degenerate free extent the register microkernel is the wrong tool: it vectorizes along M, so at M == 1 every A vector is 1/mr full, and with the roles swapped (N == 1, or the planner putting the matrix operand's free composite in M) the matrix operand goes through a pack that gathers its K-major rows one element at a time -- that gather dominated the suite's `C[cde] = A[ab] B[abcde]` cases. `execute!` now tries `_try_execute_dot!` first when Qm == 1 or Qn == 1. It applies when the MATRIX operand's K map is the unit ramp (the vector operand may have any K layout: it is gathered once per K block, with its conjugation folded in), its storage is a DenseVector, and Qk >= one vector. It computes NB outputs at a time (8 real, 4 complex), W-wide FMAs along K, the K tail as one masked step, one horizontal reduction per output; complex uses two accumulator planes against the gathered vector and its pair-swapped copy (no shuffles in the loop). A conjugated matrix operand becomes a flipped gather plus an output conj. The gather buffer is the plan's packed-A workspace; every address is bounds-checked per block before use. `_DOT_MODE` overrides the choice for tests and benchmarks. Measured on ccqlin038 (Cascade Lake, Julia 1.12.7), `execute!` only, interleaved medians, nest -> dot (OpenBLAS gemv): Float64 1x36x216 4.9 -> 1.9 us (1.0) 1x64x512 18.9 -> 4.2 us (3.4) 1x144x1728 176 -> 81 us (76) 1x256x4096 728 -> 331 us (323) ComplexF64 1x36x216 12.1 -> 2.6 us (2.4) 1x64x512 41.7 -> 7.6 us (8.6) 1x144x1728 304 -> 157 us (152) 1x256x4096 1359 -> 680 us (679) Co-Authored-By: Claude Opus 5.5 --- src/QuasiStrided.jl | 1 + src/execution/dot.jl | 422 +++++++++++++++++++++++++++++++++++++ src/execution/execute.jl | 7 + test/execution/test_dot.jl | 255 ++++++++++++++++++++++ test/runtests.jl | 3 +- 5 files changed, 687 insertions(+), 1 deletion(-) create mode 100644 src/execution/dot.jl create mode 100644 test/execution/test_dot.jl diff --git a/src/QuasiStrided.jl b/src/QuasiStrided.jl index 5ac5f8e..0a9d6e5 100644 --- a/src/QuasiStrided.jl +++ b/src/QuasiStrided.jl @@ -58,6 +58,7 @@ include("execution/oracle.jl") include("execution/direct.jl") include("execution/halfpack.jl") include("execution/unpackedb.jl") +include("execution/dot.jl") # --- Integrations --- include("integrations/tensoroperations.jl") diff --git a/src/execution/dot.jl b/src/execution/dot.jl new file mode 100644 index 0000000..7c076e6 --- /dev/null +++ b/src/execution/dot.jl @@ -0,0 +1,422 @@ +# The dot-product ("gemv") path for a degenerate free extent: `M == 1` or +# `N == 1`, when the operand that still has a free extent -- the MATRIX +# operand, `Qfree x K` -- is stored with K unit-stride. +# +# Why the register microkernel is the wrong tool here: it vectorizes along M. +# With `M == 1` every A vector is 1/mr full, and with the roles swapped +# (`N == 1`) the matrix operand feeds M through a pack that gathers its K-major +# rows one element at a time -- on the suite's `C[cde] = A[ab] B[abcde]` +# cases that gather is 70% (Float64) of the call. A gemv wants to vectorize +# along K instead: `C[q] = sum_k v[k] * Mat[q, k]` with `Mat[q, :]` a +# contiguous run, i.e. one W-wide FMA per W elements of the matrix, which is +# read exactly once, plus one horizontal reduction per output. Nothing is +# packed except the K-vector `v` (gathered once per K block, with its +# conjugation folded in). This is what `?gemv` does for the transposed case. +# +# Selected by `execute!` through `_try_execute_dot!` (see there for the +# eligibility rule and the measurement), before the five-loop nest; on an +# ineligible plan it declines and the nest runs. `_DOT_MODE` overrides the +# automatic choice for tests and benchmarks. + +const _DOT_MODE = Ref{Symbol}(:auto) + +# One full hardware vector register of `real(T)`: 8 Float64 on AVX-512, 4 on +# AVX2; the fallback lane width where the register width is unknown. +@inline function _dot_lanewidth(::Type{T}) where {T} + R = real(T) + vb = target_profile().vector_bytes + return (vb > 0 && vb % sizeof(R) == 0) ? vb ÷ sizeof(R) : _default_lanewidth(R) +end + +# Outputs computed together, each with its own accumulator(s): 8 for a real +# `T` (8 FMA chains in flight cover the FMA latency on every x86 we ship for), +# 4 for a complex one (two accumulator planes each, so 8 chains as well). +_dot_group_width(::Type{T}) where {T} = T <: Complex ? 4 : 8 + +# The affine step of map `p` of `g` alone, or `nothing` when that map is not +# a single ramp in the logical coordinate. `affine_ramp` (src/layout/axis_group.jl) +# asks the same of EVERY map at once; here the two operands' K maps are judged +# separately, because only the matrix operand's needs to be contiguous (the +# vector is gathered through its own offsets, whatever they are). Same +# singleton-skipping and Int128 comparison as `affine_ramp`. +function _map_ramp_step(g::AxisGroup{D, P}, p::Int) where {D, P} + step = 0 + run = 1 + started = false + for d in 1:D + L = g.lengths[d] + L == 0 && return 0 + L == 1 && continue + S = g.strides[p][d] + if !started + step = S + run = L + started = true + else + Int128(run) * Int128(step) == Int128(S) || return nothing + run *= L + end + end + return step +end + +""" + _try_execute_dot!(plan::ContractPlan, alphaT, betaT, Qm, Qn, Qk) -> Bool + +Run the dot-product path if `plan` is eligible and return `true`; return +`false` without touching anything otherwise. Eligible when `_DOT_MODE[]` is +not `:never` and + + * `Qm == 1` (the matrix operand is B, its free composite the N group) or + `Qn == 1` (the matrix operand is A, the M group); + * the matrix operand's K map is the unit ramp `offset(k) = k` (checked per + map by `_map_ramp_step`, so a permuted or strided K on the VECTOR operand + does not disqualify), and its storage is a `DenseVector{T}` (the K runs + are read with raw-pointer vector loads); + * `Qk >= W` for the lane width `W = _dot_lanewidth(T)`: below one vector + the path would be all scalar tail; + * the plan's packed-A buffer can hold one K block of the gathered vector + (always true for the shipped kernel shapes; checked, not assumed). + +Measured 2026-09-27, ccqlin038 (Cascade Lake, AVX-512, Julia 1.12.7), the +suite's `dim{6,8,12,16}_0_2_3_gemm_ready` cases `C[cde] = A[ab] B[abcde]` (K += dim^2, N = dim^3), `execute!` time, this path vs the nest on the same tree +(`benchmark/bench_degenerate.jl`, interleaved medians), and OpenBLAS `gemv` +on the same data: + + 1x36x216 1x64x512 1x144x1728 1x256x4096 + Float64 4.9 -> 1.9 18.9 -> 4.2 176 -> 81 728 -> 331 us (gemv 1.0 / 3.4 / 76 / 323) + ComplexF64 12.1 -> 2.6 41.7 -> 7.6 304 -> 157 1359 -> 680 us (gemv 2.4 / 8.6 / 152 / 679) + +The nest's time there is the gather pack of the matrix operand, which this +path does not perform at all. The K tail is one masked step (not a scalar +loop): 1x36x216 Float64 went 3.0 -> 1.9 us with it. +""" +function _try_execute_dot!( + plan::ContractPlan{T}, alphaT::T, betaT::T, Qm::Int, Qn::Int, Qk::Int + ) where {T} + _dot_applicable(plan, Qm, Qn, Qk) || return false + W = _dot_lanewidth(T) + matB = Qm == 1 + # `W` is a runtime value; the ladder hands the kernel a literal. + if W == 16 + _execute_dot!(plan, alphaT, betaT, matB, Val(16)) + elseif W == 8 + _execute_dot!(plan, alphaT, betaT, matB, Val(8)) + elseif W == 4 + _execute_dot!(plan, alphaT, betaT, matB, Val(4)) + else + _execute_dot!(plan, alphaT, betaT, matB, Val(2)) + end + return true +end + +# The eligibility rule of `_try_execute_dot!`, as a side-effect-free predicate. +function _dot_applicable(plan::ContractPlan{T}, Qm::Int, Qn::Int, Qk::Int) where {T} + _DOT_MODE[] === :never && return false + (Qm == 1 || Qn == 1) || return false + Qk >= _dot_lanewidth(T) || return false + if Qm == 1 + plan.Bstorage isa DenseVector{T} || return false + _map_ramp_step(plan.kgroup, 2) == 1 || return false + else + plan.Astorage isa DenseVector{T} || return false + _map_ramp_step(plan.kgroup, 1) == 1 || return false + end + # The gathered vector (and, complex, its pair-swapped copy) lives in the + # packed-A buffer, reinterpreted as `T`. + need = (T <: Complex ? 2 : 1) * plan.blocking.kc + return (length(plan.workspace.packed_a) * sizeof(real(T))) ÷ sizeof(T) >= need +end + +# Resolve the operand roles ONCE into concretely typed arguments: the matrix +# operand (`m*`), the vector operand (`v*`), the matrix's free composite `g` +# with its two offset buffers (map 1 = matrix, map 2 = C), the block length +# those buffers are sized for, and the K buffer that receives the VECTOR's K +# offsets. Branching on `matB` inside `_dot_nest!` would make every storage +# type a Union. +function _execute_dot!(plan::ContractPlan{T}, alphaT::T, betaT::T, matB::Bool, ::Val{W}) where {T, W} + ws = plan.workspace + if matB + _dot_nest!( + plan, ws, plan.Bstorage, plan.Bbase, plan.btransform, + plan.Astorage, plan.Abase, plan.atransform, + plan.ngroup, ws.n_buf_B, ws.n_buf_C, plan.blocking.nc, ws.k_buf_A, + alphaT, betaT, Val(W) + ) + else + _dot_nest!( + plan, ws, plan.Astorage, plan.Abase, plan.atransform, + plan.Bstorage, plan.Bbase, plan.btransform, + plan.mgroup, ws.m_buf_A, ws.m_buf_C, plan.blocking.mc, ws.k_buf_B, + alphaT, betaT, Val(W) + ) + end + return plan.Cstorage +end + +# Conjugation is folded rather than applied per element in the kernel: +# `sum conj(m)*v == conj(sum m*conj(v))`, so a conjugated matrix operand +# becomes a flip of the gathered vector plus one `conj` of each output, and +# the vector's own transform composes with that flip by parity (XOR), the +# same rule `plan_contract` uses for a flag against a view's `op`. +@inline _is_conj(::typeof(conj)) = true +@inline _is_conj(::typeof(identity)) = false + +# `(lo, hi)` of `buf[1:count]`, `count >= 1`. +@inline function _buffer_range(buf::Vector{Int}, count::Int) + @inbounds lo = hi = buf[1] + @inbounds for t in 2:count + v = buf[t] + lo = min(lo, v) + hi = max(hi, v) + end + return (lo, hi) +end + +function _dot_nest!( + plan::ContractPlan{T}, ws, mstorage::SM, mbase::Int, mtrans::FM, + vstorage::SV, vbase::Int, vtrans::FV, g::G, bufM::Vector{Int}, bufC::Vector{Int}, + blocklen::Int, vkbuf::Vector{Int}, alphaT::T, betaT::T, ::Val{W} + ) where {T, SM, FM, SV, FV, G <: AxisGroup, W} + Qf = axis_length(g) + Qk = axis_length(plan.kgroup) + kc_eff = plan.blocking.kc + Cstorage = plan.Cstorage + cbase = plan.Cbase + lenm = length(mstorage) + lenv = length(vstorage) + lenc = length(Cstorage) + mconj = _is_conj(mtrans) + vflip = _is_conj(vtrans) ⊻ mconj + NB = _dot_group_width(T) + + # `ws`: the gather buffer and every offset buffer. `mstorage`: the K runs + # are raw-pointer vector loads. + GC.@preserve ws mstorage begin + gptr = reinterpret(Ptr{T}, pointer(ws.packed_a)) + mptr = pointer(mstorage) + pc = 0 + firstblock = true + while pc < Qk + kblock = min(kc_eff, Qk - pc) + beta_eff = firstblock ? betaT : one(T) + + # The vector's K offsets for this block (both K maps are filled; + # only the vector's is read -- the matrix's is the unit ramp). + fill_offsets!((ws.k_buf_A, ws.k_buf_B), plan.kgroup, pc, kblock) + checked_span_bounds(vbase, _buffer_range(vkbuf, kblock), (0, 0), lenv) + if vflip + _dot_gather!(gptr, vstorage, vbase, vkbuf, kblock, conj) + else + _dot_gather!(gptr, vstorage, vbase, vkbuf, kblock, identity) + end + + q0 = 0 + while q0 < Qf + qcount = min(blocklen, Qf - q0) + fill_offsets!((bufM, bufC), g, q0, qcount) + # Every address this block reads/writes, validated before any + # of it: the matrix rows against this K block (the matrix's + # K offsets are `pc .. pc+kblock-1` by the unit-ramp + # eligibility), and the C elements. + checked_span_bounds(mbase, _buffer_range(bufM, qcount), (pc, pc + kblock - 1), lenm) + checked_span_bounds(cbase, _buffer_range(bufC, qcount), (0, 0), lenc) + if mconj + _dot_block!( + Cstorage, cbase, bufC, mptr, mbase + pc, bufM, qcount, gptr, kblock, + alphaT, beta_eff, Val(W), Val(NB), Val(true) + ) + else + _dot_block!( + Cstorage, cbase, bufC, mptr, mbase + pc, bufM, qcount, gptr, kblock, + alphaT, beta_eff, Val(W), Val(NB), Val(false) + ) + end + q0 += qcount + end + + firstblock = false + pc += kblock + end + end + return nothing +end + +# Gather `v[t] = transform(vstorage[vbase + vkbuf[t+1] + 1])`, `0 <= t < +# kblock`, into `gptr`; for a complex `T` also its pair-swapped copy +# `Complex(imag, real)` at `gptr + kblock`, which the complex kernel +# multiplies by to get the imaginary part without a per-step shuffle. The +# caller has bounds-checked the vector region. +@inline function _dot_gather!( + gptr::Ptr{T}, vstorage::SV, vbase::Int, vkbuf::Vector{Int}, kblock::Int, transform::F + ) where {T, SV, F} + @inbounds for t in 0:(kblock - 1) + z = transform(vstorage[vbase + vkbuf[t + 1] + 1])::T + unsafe_store!(gptr + sizeof(T) * t, z) + if T <: Complex + unsafe_store!(gptr + sizeof(T) * (kblock + t), Complex(imag(z), real(z))) + end + end + return nothing +end + +# One block of outputs, `NB` at a time. The last group's padding outputs alias +# the block's last valid output (computed, never stored) -- the same trick as +# `UnpackedBView`'s padding columns. +@inline function _dot_block!( + Cstorage::SC, cbase::Int, bufC::Vector{Int}, mptr::Ptr{T}, mbase_k::Int, + bufM::Vector{Int}, qcount::Int, gptr::Ptr{T}, kblock::Int, + alpha::T, beta::T, ::Val{W}, ::Val{NB}, ::Val{MCONJ} + ) where {SC, T, W, NB, MCONJ} + q = 0 + while q < qcount + nvalid = min(NB, qcount - q) + mbases = _dot_bases(bufM, q, nvalid, mbase_k, Val(NB)) + sums = _dot_group(mptr, gptr, mbases, kblock, Val(W), Val(NB)) + @inbounds for j in 1:nvalid + s = MCONJ ? conj(sums[j]) : sums[j] + _axpby_at!(Cstorage, cbase + bufC[q + j] + 1, alpha, s, beta) + end + q += nvalid + end + return nothing +end + +# `(mbase_k + bufM[q + min(j, nvalid)])` for `j in 1:NB`, as a literal tuple +# expression: an `ntuple` closure over the loop-carried `q` would box it. +@generated function _dot_bases(bufM::Vector{Int}, q::Int, nvalid::Int, mbase_k::Int, ::Val{NB}) where {NB} + ex = [:(mbase_k + @inbounds(bufM[q + min($j, nvalid)])) for j in 1:NB] + return quote + Base.@_inline_meta + return $(Expr(:tuple, ex...)) + end +end + +# ---------------------------------------------------------------------------- +# The group kernels: `NB` dot products of length `kblock` between the gathered +# vector at `gptr` and the matrix rows at `mptr + mbases[j]`, all unit-stride. +# A W-wide main loop, one masked step for the `kblock % W` tail, and a +# horizontal reduction; every tuple index is a literal (Cliff B, as in the +# microkernels), which is why the step bodies are `@generated`. +# ---------------------------------------------------------------------------- + +# --- real ------------------------------------------------------------------ + +@inline function _dot_group( + mptr::Ptr{T}, gptr::Ptr{T}, mbases::NTuple{NB, Int}, kblock::Int, ::Val{W}, ::Val{NB} + ) where {T <: Real, NB, W} + accs = ntuple(_ -> zero(Vec{W, T}), Val(NB)) + kmain = (kblock ÷ W) * W + t = 0 + while t < kmain + accs = _dot_step_real(accs, mptr, gptr, mbases, t, nothing) + t += W + end + # The K tail (`kblock % W` elements) as ONE masked step: masked-off lanes + # load as zero and are never read from memory, so no address past a row's + # end is touched. + if t < kblock + accs = _dot_step_real(accs, mptr, gptr, mbases, t, _dot_tailmask(Val(W), kblock - t)) + end + return _dot_reduce_real(accs) +end + +# Lanes `0 .. rem-1` of a `W`-lane mask, `0 < rem < W`. +@inline _dot_tailmask(::Val{W}, rem::Int) where {W} = + Vec{W, Int}(ntuple(i -> i - 1, Val(W))) < rem + +# One W-wide K step for every output of the group; with a `mask`, the masked +# tail step (`mask === nothing` for the main loop, resolved at compile time; +# passed explicitly -- a default argument adds a non-inlined wrapper method). +@generated function _dot_step_real( + accs::NTuple{NB, Vec{W, T}}, mptr::Ptr{T}, gptr::Ptr{T}, mbases::NTuple{NB, Int}, t::Int, + mask::M + ) where {NB, W, T, M} + sz = sizeof(T) + mk = M === Nothing ? () : (:mask,) + loads = [:($(Symbol(:m, j)) = vload(Vec{$W, $T}, mptr + $sz * (mbases[$j] + t), $(mk...))) for j in 1:NB] + fmas = [:(muladd($(Symbol(:m, j)), vv, accs[$j])) for j in 1:NB] + return quote + Base.@_inline_meta + vv = vload(Vec{$W, $T}, gptr + $sz * t, $(mk...)) + $(loads...) + return $(Expr(:tuple, fmas...)) + end +end + +@generated function _dot_reduce_real(accs::NTuple{NB, Vec{W, T}}) where {NB, W, T} + ex = [:(sum(accs[$j])) for j in 1:NB] + return quote + Base.@_inline_meta + return $(Expr(:tuple, ex...)) + end +end + +# --- complex --------------------------------------------------------------- +# +# Interleaved (native) layout throughout, `W` REAL lanes = `W ÷ 2` complex +# elements per vector. For output `j`, two accumulator planes over the K loop: +# +# P1 += m .* v lanes (mr*vr, mi*vi, ...) -> re = sum(P1 .* (+1,-1,+1,-1,...)) +# P2 += m .* swap(v) lanes (mr*vi, mi*vr, ...) -> im = sum(P2) +# +# with `swap(v)` the gathered pair-swapped copy at `gptr + kblock`. Two FMAs +# per matrix vector, no shuffle in the loop. `accs[j]` is `P1_j`, `accs[NB+j]` +# is `P2_j`. + +@inline function _dot_group( + mptr::Ptr{T}, gptr::Ptr{T}, mbases::NTuple{NB, Int}, kblock::Int, ::Val{W}, ::Val{NB} + ) where {T <: Complex, NB, W} + R = real(T) + WC = W ÷ 2 + accs = ntuple(_ -> zero(Vec{W, R}), Val(2 * NB)) + kmain = (kblock ÷ WC) * WC + t = 0 + while t < kmain + accs = _dot_step_cplx(accs, mptr, gptr, gptr + sizeof(T) * kblock, mbases, t, nothing) + t += WC + end + # The K tail as one masked step over its `2 * (kblock - t)` real lanes. + if t < kblock + accs = _dot_step_cplx( + accs, mptr, gptr, gptr + sizeof(T) * kblock, mbases, t, + _dot_tailmask(Val(W), 2 * (kblock - t)) + ) + end + return _dot_reduce_cplx(accs, T) +end + +@generated function _dot_step_cplx( + accs::NTuple{NA, Vec{W, R}}, mptr::Ptr{T}, gptr::Ptr{T}, gsptr::Ptr{T}, + mbases::NTuple{NB, Int}, t::Int, mask::M + ) where {NA, W, R, T, NB, M} + NA == 2 * NB || return :(throw(ArgumentError("_dot_step_cplx: expected 2NB accumulators"))) + T === Complex{R} || return :(throw(ArgumentError("_dot_step_cplx: lane type must be real(T)"))) + sz = sizeof(T) # bytes per complex element; offsets below are in elements + mk = M === Nothing ? () : (:mask,) + loads = [ + :($(Symbol(:m, j)) = vload(Vec{$W, $R}, reinterpret(Ptr{$R}, mptr + $sz * (mbases[$j] + t)), $(mk...))) + for j in 1:NB + ] + p1 = [:(muladd($(Symbol(:m, j)), vv, accs[$j])) for j in 1:NB] + p2 = [:(muladd($(Symbol(:m, j)), vs, accs[$(NB + j)])) for j in 1:NB] + return quote + Base.@_inline_meta + vv = vload(Vec{$W, $R}, reinterpret(Ptr{$R}, gptr + $sz * t), $(mk...)) + vs = vload(Vec{$W, $R}, reinterpret(Ptr{$R}, gsptr + $sz * t), $(mk...)) + $(loads...) + return $(Expr(:tuple, p1..., p2...)) + end +end + +@generated function _dot_reduce_cplx(accs::NTuple{NA, Vec{W, R}}, ::Type{T}) where {NA, W, R, T} + NB = NA ÷ 2 + sgn = ntuple(i -> isodd(i) ? one(R) : -one(R), W) + ex = [:(Complex(sum(accs[$j] * SGN), sum(accs[$(NB + j)]))) for j in 1:NB] + return quote + Base.@_inline_meta + SGN = Vec{$W, $R}($sgn) + return $(Expr(:tuple, ex...)) + end +end diff --git a/src/execution/execute.jl b/src/execution/execute.jl index 6b9aaac..9f8b682 100644 --- a/src/execution/execute.jl +++ b/src/execution/execute.jl @@ -77,6 +77,13 @@ function execute!(plan::ContractPlan{T}, alpha::Number, beta::Number) where {T} return plan.Cstorage end + # Degenerate extents take a dedicated path instead of the nest when their + # layout allows: a K-contiguous matrix operand with `M == 1` or `N == 1` + # goes to the K-vectorized dot path (src/execution/dot.jl). It declines + # (returns `false`) on an ineligible plan, and the nest below runs. + (Qm == 1 || Qn == 1) && _try_execute_dot!(plan, alphaT, betaT, Qm, Qn, Qk) && + return plan.Cstorage + mc_eff = plan.blocking.mc kc_eff = plan.blocking.kc nc_eff = plan.blocking.nc diff --git a/test/execution/test_dot.jl b/test/execution/test_dot.jl new file mode 100644 index 0000000..a4dd0bd --- /dev/null +++ b/test/execution/test_dot.jl @@ -0,0 +1,255 @@ +# The dot-product path (src/execution/dot.jl): `execute!` on a plan with +# `M == 1` or `N == 1` whose matrix operand is K-contiguous runs a K-vectorized +# gemv instead of the five-loop nest. Checked here: +# +# 1. `_try_execute_dot!` takes exactly the eligible plans (degenerate free +# extent, unit-ramp K on the matrix operand, `DenseVector` storage, +# `Qk >= W`), declines the rest without touching C, and obeys `_DOT_MODE`. +# 2. On eligible plans the result matches the nest (`_DOT_MODE[] = :never`), +# the oracle and a brute-force reference to a tolerance (the K summation +# order differs, so never `==`), over all four eltypes, conj flags and +# views, alpha/beta regimes, permuted vector and output operands, K tails +# and several K blocks, and the scalar-output case `M == N == 1`. +# 3. beta = 0 never reads C; out-of-bounds operands are rejected before any +# write; the call is allocation-free. +# +# `_hp_ref` (test/execution/test_halfpack.jl) is the brute-force reference. + +const _dot_mode = QuasiStrided._DOT_MODE + +const _DOT_TYPES = (Float64, Float32, ComplexF64, ComplexF32) + +# Run the plan under `mode`; returns the output array. +function _dot_run(mk, mode::Symbol, alpha, beta; plankw...) + Cv, Av, indA, Bv, indB, indC = mk() + plan = plan_contract(Cv, Av, indA, Bv, indB, indC; plankw...) + old = _dot_mode[] + _dot_mode[] = mode + try + @test execute!(plan, alpha, beta) === plan.Cstorage + finally + _dot_mode[] = old + end + return Array(Cv), plan +end + +# Whether the automatic path would take `plan` (without running it). +function _dot_eligible(plan) + Qm = axis_length(plan.mgroup); Qn = axis_length(plan.ngroup); Qk = axis_length(plan.kgroup) + return QuasiStrided._dot_applicable(plan, Qm, Qn, Qk) +end + +# The suite's `C[cde] = A[ab] * B[abcde]` shape (M = 1, matrix B K-fastest) +# at leg dimension `d`, plus variants: +# :plain as is +# :permvec A stored [b,a]: the VECTOR's K map is not a ramp (matrix is) +# :permC C stored [e,d,c] and viewed back: scattered output offsets +# :offsetB B a view with a nonzero offset into a larger parent +function _dot_gemv_maker(::Type{T}, d, seed; variant = :plain, Cfill = nothing) where {T} + return function () + rng = MersenneTwister(seed) + A = randn(rng, T, d, d) + B = randn(rng, T, d, d, d, d, d) + C = Cfill === nothing ? randn(rng, T, d, d, d) : fill(convert(T, Cfill), d, d, d) + Av = variant === :permvec ? permutedims(StridedView(permutedims(A, (2, 1))), (2, 1)) : StridedView(A) + Bv = if variant === :offsetB + big = fill(convert(T, NaN), d, d, d, d, d + 2) + big[:, :, :, :, 2:(d + 1)] .= B + StridedView(view(big, :, :, :, :, 2:(d + 1))) + else + StridedView(B) + end + Cv = variant === :permC ? permutedims(StridedView(permutedims(C, (3, 2, 1))), (3, 2, 1)) : StridedView(C) + @assert isequal(Array(Av), A) && isequal(Array(Bv), B) && isequal(Array(Cv), C) + return (Cv, Av, (1, 2), Bv, (1, 2, 3, 4, 5), (3, 4, 5)) + end +end + +# N = 1 with the matrix operand A K-fastest: C[a,b] = A[k,a,b] * v[k]. +function _dot_n1_maker(::Type{T}, d, K, seed) where {T} + return function () + rng = MersenneTwister(seed) + A = randn(rng, T, K, d, d) + v = randn(rng, T, K) + C = randn(rng, T, d, d) + return (StridedView(C), StridedView(A), (3, 1, 2), StridedView(v), (3,), (1, 2)) + end +end + +const _DOT_AB = ((1.0, 0.0), (2.5, -0.75), (1.0, 1.0), (-0.5, 1.25)) + +@testset "dot path: which plans it takes ($T)" for T in _DOT_TYPES + W = QuasiStrided._dot_lanewidth(T) + @test W >= 2 + # The suite shape, both variants of the vector operand: eligible. + for variant in (:plain, :permvec, :permC, :offsetB) + p = plan_contract(_dot_gemv_maker(T, 4, 1; variant = variant)()...) + @test 1 in (axis_length(p.mgroup), axis_length(p.ngroup)) + @test _dot_eligible(p) + end + # N = 1 with A K-fastest: eligible; with A K-slowest: not. + @test _dot_eligible(plan_contract(_dot_n1_maker(T, 4, 16, 2)()...)) + pk = plan_contract( + StridedView(zeros(T, 4, 4)), StridedView(randn(T, 4, 4, 16)), (1, 2, 3), + StridedView(randn(T, 16)), (3,), (1, 2) + ) + @test !_dot_eligible(pk) + # Below one vector of K: declined. + ps = plan_contract( + StridedView(zeros(T, 1, 9)), StridedView(randn(T, 1, W - 1)), (1, 2), + StridedView(randn(T, W - 1, 9)), (2, 3), (1, 3) + ) + @test !_dot_eligible(ps) + # Neither extent degenerate: declined. + @test !_dot_eligible(_mm_plan(zeros(T, 3, 5), randn(T, 3, 20), randn(T, 20, 5))) + # The mode override. + p = plan_contract(_dot_gemv_maker(T, 4, 1)()...) + _dot_mode[] = :never + @test !_dot_eligible(p) + _dot_mode[] = :auto + @test _dot_eligible(p) + # Non-DenseVector matrix storage is declined (raw-pointer loads need it). + K = 4W + wrapped = StridedView(_HPDenseMat(randn(T, K, 5)), (K, 5), (1, K), 0) + pw = plan_contract(StridedView(zeros(T, 1, 5)), StridedView(randn(T, 1, K)), (1, 2), wrapped, (2, 3), (1, 3)) + @test !_dot_eligible(pw) +end + +@testset "dot path: the suite's M = 1 gemv, all variants and regimes ($T)" for T in _DOT_TYPES + for d in (2, 4, 6), variant in (:plain, :permvec, :permC, :offsetB), (alpha, beta) in _DOT_AB + mk = _dot_gemv_maker(T, d, 10 * d; variant = variant) + C_nest, _ = _dot_run(mk, :never, alpha, beta) + C_dot, plan = _dot_run(mk, :auto, alpha, beta) + @test _dot_eligible(plan) == (d * d >= QuasiStrided._dot_lanewidth(T)) + @test C_dot ≈ C_nest + Cv0, Av0, iA, Bv0, iB, iC = mk() + @test C_dot ≈ _hp_ref(Array(Av0), iA, Array(Bv0), iB, Array(Cv0), iC, alpha, beta) + C_tw = let (Cv, Av, iA, Bv, iB, iC) = mk() + execute_tilewise!(plan_contract(Cv, Av, iA, Bv, iB, iC), alpha, beta) + Array(Cv) + end + @test C_dot ≈ C_tw + end +end + +@testset "dot path: N = 1, scalar output, K tails and several K blocks ($T)" for T in _DOT_TYPES + W = QuasiStrided._dot_lanewidth(T) + for (d, K) in ((3, W), (4, W + 1), (5, 3W - 1), (2, 1000)), (alpha, beta) in _DOT_AB + mk = _dot_n1_maker(T, d, K, 3 * K + d) + # `kc = 64` forces several K blocks at K = 1000; `mc = 7` several + # output blocks against a 25-long free composite. + C_nest, _ = _dot_run(mk, :never, alpha, beta; kc = 64, mc = 7) + C_dot, plan = _dot_run(mk, :auto, alpha, beta; kc = 64, mc = 7) + @test _dot_eligible(plan) + @test C_dot ≈ C_nest + Cv0, Av0, iA, Bv0, iB, iC = mk() + @test C_dot ≈ _hp_ref(Array(Av0), iA, Array(Bv0), iB, Array(Cv0), iC, alpha, beta) + end + # Scalar output C[] = sum_k a[k] b[k], K not a multiple of W. + K = 5W + 3 + mk0 = () -> begin + rng = MersenneTwister(99) + (StridedView(fill(randn(rng, T))), StridedView(randn(rng, T, K)), (1,), StridedView(randn(rng, T, K)), (1,), ()) + end + C_nest, _ = _dot_run(mk0, :never, 1.5, -0.5) + C_dot, plan = _dot_run(mk0, :auto, 1.5, -0.5) + @test _dot_eligible(plan) + @test C_dot ≈ C_nest + Cv0, Av0, _, Bv0, _, _ = mk0() + @test C_dot[] ≈ 1.5 * sum(Array(Av0) .* Array(Bv0)) - 0.5 * Cv0[] +end + +@testset "dot path: conjugation flags and views ($T)" for T in (ComplexF64, ComplexF32) + alpha, beta = T(0.75, -1.0), T(0.5, 0.5) + for conjA in (false, true), conjB in (false, true), wrapA in (false, true), wrapB in (false, true) + mk = _dot_gemv_maker(T, 4, 5) + run(mode) = let (Cv, Av, iA, Bv, iB, iC) = mk() + _dot_mode[] = mode + p = plan_contract(Cv, wrapA ? conj(Av) : Av, iA, wrapB ? conj(Bv) : Bv, iC === () ? iB : iB, iC; conjA = conjA, conjB = conjB) + @test mode === :never || _dot_eligible(p) + execute!(p, alpha, beta) + _dot_mode[] = :auto + Array(Cv) + end + Cn = run(:never) + Cd = run(:auto) + @test Cd ≈ Cn + Cv0, Av0, iA, Bv0, iB, iC = mk() + @test Cd ≈ _hp_ref( + Array(Av0), iA, Array(Bv0), iB, Array(Cv0), iC, alpha, beta; + conjA = conjA ⊻ wrapA, conjB = conjB ⊻ wrapB + ) + end + # N = 1 with a conjugated MATRIX operand (A): the conj is folded into the + # vector and an output conj. + for conjA in (false, true), conjB in (false, true) + mk = _dot_n1_maker(T, 3, 20, 8) + Cn, _ = _dot_run(mk, :never, alpha, beta; conjA = conjA, conjB = conjB) + Cd, plan = _dot_run(mk, :auto, alpha, beta; conjA = conjA, conjB = conjB) + @test _dot_eligible(plan) + @test Cd ≈ Cn + Cv0, Av0, iA, Bv0, iB, iC = mk() + @test Cd ≈ _hp_ref(Array(Av0), iA, Array(Bv0), iB, Array(Cv0), iC, alpha, beta; conjA = conjA, conjB = conjB) + end +end + +@testset "dot path: beta = 0 never reads C ($T)" for T in _DOT_TYPES + mk = _dot_gemv_maker(T, 4, 6; Cfill = NaN) + C_dot, plan = _dot_run(mk, :auto, 2.0, 0.0) + @test _dot_eligible(plan) + @test all(isfinite, C_dot) + Cv0, Av0, iA, Bv0, iB, iC = mk() + @test C_dot ≈ _hp_ref(Array(Av0), iA, Array(Bv0), iB, zeros(T, size(Cv0)), iC, 2.0, 0.0) +end + +@testset "dot path: out-of-bounds operands are rejected before any write ($T)" for T in (Float64, ComplexF64) + K, N = 24, 5 + # Matrix (B) one element short, vector one short, C one short. + mkB = () -> ( + StridedView(zeros(T, 1, N)), StridedView(randn(T, 1, K)), (1, 2), + StridedView(randn(T, K * N - 1), (K, N), (1, K), 0), (2, 3), (1, 3), + ) + mkA = () -> ( + StridedView(zeros(T, 1, N)), StridedView(randn(T, K - 1), (1, K), (1, 1), 0), (1, 2), + StridedView(randn(T, K, N)), (2, 3), (1, 3), + ) + mkC = () -> ( + StridedView(zeros(T, N - 1), (1, N), (1, 1), 0), StridedView(randn(T, 1, K)), (1, 2), + StridedView(randn(T, K, N)), (2, 3), (1, 3), + ) + for mk in (mkB, mkA, mkC) + Cv, Av, iA, Bv, iB, iC = mk() + plan = plan_contract(Cv, Av, iA, Bv, iB, iC) + @test _dot_eligible(plan) + @test_throws BoundsError execute!(plan, 1.0, 0.0) + @test all(iszero, plan.Cstorage) + end +end + +@testset "dot path: allocation-free in steady state ($T)" for T in _DOT_TYPES + Cv, Av, iA, Bv, iB, iC = _dot_gemv_maker(T, 6, 4)() + plan = plan_contract(Cv, Av, iA, Bv, iB, iC; oracle = false) + @test _dot_eligible(plan) + execute!(plan, 1.0, 0.0) + execute!(plan, 1.0, 0.0) + @test (@allocated execute!(plan, 1.0, 0.0)) == 0 skip = (VERSION < v"1.11") + # Several K blocks and output blocks too. + Cv, Av, iA, Bv, iB, iC = _dot_n1_maker(T, 5, 300, 4)() + plan = plan_contract(Cv, Av, iA, Bv, iB, iC; oracle = false, kc = 64, mc = 8) + @test _dot_eligible(plan) + execute!(plan, 1.0, 0.0) + @test (@allocated execute!(plan, 1.0, 0.0)) == 0 skip = (VERSION < v"1.11") +end + +@testset "dot path: through the TensorOperations backend ($T)" for T in _DOT_TYPES + d = 6 + A = randn(T, d, d); B = randn(T, d, d, d, d, d) + C1 = zeros(T, d, d, d); C2 = zeros(T, d, d, d) + pA, pB, pAB = TO.contract_indices((:a, :b), (:a, :b, :c, :d, :e), (:c, :d, :e)) + _dot_mode[] = :never + TO.tensorcontract!(C1, A, pA, false, B, pB, false, pAB, 1.0, 0.0, QuasiStridedBackend()) + _dot_mode[] = :auto + TO.tensorcontract!(C2, A, pA, false, B, pB, false, pAB, 1.0, 0.0, QuasiStridedBackend()) + @test C2 ≈ C1 + @test vec(C2) ≈ transpose(reshape(B, d * d, d^3)) * vec(A) +end diff --git a/test/runtests.jl b/test/runtests.jl index 4afb63c..cdc335e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -51,8 +51,9 @@ using QuasiStrided: AxisGroup, axis_length, offsets, fill_offsets!, BlockDescrip include("execution/test_macro_blocking.jl") include("execution/test_direct.jl") include("execution/test_halfpack.jl") - # Uses test_halfpack.jl's `_hp_ref` reference (and `_HPDenseMat`). + # Both use test_halfpack.jl's `_hp_ref` reference (and `_HPDenseMat`). include("execution/test_unpackedb.jl") + include("execution/test_dot.jl") # Last among the functional tests: it does a bare `using TensorOperations`, # which exports its own `scalartype` and would otherwise conflict with the From 468b453021b7092fce3416302e897958049ee926 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 10:45:38 -0400 Subject: [PATCH 13/53] Add a streaming outer-product path for real K == 1 contractions Through the nest a K = 1 contraction is all fixed cost: every micro-tile runs one K step and a full store, cld(M,mr)*cld(N,nr) of them (176 tiles for a 128x128 Float64 outer product at 16x6), for an output a plain streaming write covers in a fraction of the time. `execute!` now tries `_try_execute_outer!` when Qk == 1: for a real T whose M composite is the unit ramp in both A and C, DenseVector A and C storage and Qm >= one vector, each C column is `alpha * (A .* B[n]) + beta * C`, W rows at a time with a scalar row tail -- no packing, no tiles. The arithmetic is the nest's term for term (bitwise equal except on a signed zero). N and B may have any layout. Complex is left to the nest: it is already within ~10% of OpenBLAS from 63x63 up. `_OUTER_MODE` overrides the choice for tests and benchmarks. Measured on ccqlin038 (Cascade Lake, Julia 1.12.7), Float64, `execute!` only, interleaved medians, nest -> outer (OpenBLAS): 16x16 319 -> 227 ns (84) 63x63 2.40 -> 1.16 us (0.81) 128x128 6.15 -> 2.70 us (3.07) Co-Authored-By: Claude Opus 5.5 --- src/QuasiStrided.jl | 1 + src/execution/execute.jl | 7 +- src/execution/outer.jl | 162 ++++++++++++++++++++++++++++++ test/execution/test_outer.jl | 186 +++++++++++++++++++++++++++++++++++ test/runtests.jl | 3 +- 5 files changed, 356 insertions(+), 3 deletions(-) create mode 100644 src/execution/outer.jl create mode 100644 test/execution/test_outer.jl diff --git a/src/QuasiStrided.jl b/src/QuasiStrided.jl index 0a9d6e5..ce5c117 100644 --- a/src/QuasiStrided.jl +++ b/src/QuasiStrided.jl @@ -59,6 +59,7 @@ include("execution/direct.jl") include("execution/halfpack.jl") include("execution/unpackedb.jl") include("execution/dot.jl") +include("execution/outer.jl") # --- Integrations --- include("integrations/tensoroperations.jl") diff --git a/src/execution/execute.jl b/src/execution/execute.jl index 9f8b682..4bcd4d3 100644 --- a/src/execution/execute.jl +++ b/src/execution/execute.jl @@ -79,10 +79,13 @@ function execute!(plan::ContractPlan{T}, alpha::Number, beta::Number) where {T} # Degenerate extents take a dedicated path instead of the nest when their # layout allows: a K-contiguous matrix operand with `M == 1` or `N == 1` - # goes to the K-vectorized dot path (src/execution/dot.jl). It declines - # (returns `false`) on an ineligible plan, and the nest below runs. + # goes to the K-vectorized dot path (src/execution/dot.jl); a real + # `K == 1` outer product with unit-stride M goes to the streaming + # outer-product path (src/execution/outer.jl). Each declines (returns + # `false`) on an ineligible plan, and the nest below runs. (Qm == 1 || Qn == 1) && _try_execute_dot!(plan, alphaT, betaT, Qm, Qn, Qk) && return plan.Cstorage + Qk == 1 && _try_execute_outer!(plan, alphaT, betaT, Qm, Qn) && return plan.Cstorage mc_eff = plan.blocking.mc kc_eff = plan.blocking.kc diff --git a/src/execution/outer.jl b/src/execution/outer.jl new file mode 100644 index 0000000..68f15b7 --- /dev/null +++ b/src/execution/outer.jl @@ -0,0 +1,162 @@ +# The outer-product path for `K == 1`: `C[m, n] = alpha * A[m] * B[n] + beta * +# C[m, n]`, for a real `T` whose M composite is unit-stride in both A and C. +# +# Through the nest a K = 1 contraction is all fixed cost: every micro-tile +# runs one K step of FMAs and then a full store, and there are `cld(M, mr) * +# cld(N, nr)` of them -- 176 tiles for a 128 x 128 Float64 outer product, +# at ~26 ns each once the store is inlined (src/microkernels/simd.jl), i.e. +# 4.6 us for 131 KB of output that a plain streaming write covers in ~1 us. +# This path is that streaming write: for each n, one W-wide `A * B[n]` per W +# rows of C, no packing, no tiles, no accumulator. Its arithmetic is the +# nest's, term for term -- `r = a * b` (a K = 1 accumulation from zero) and +# then `_store_tile_vector!`'s `alpha * r` / `muladd(alpha, r, C)` / +# `muladd(alpha, r, beta * C)` -- so the two paths agree bitwise except on a +# signed zero (an FMA from `+0.0` returns `+0.0` where `a * b` returns +# `-0.0`). +# +# Real `T` only: the complex kernels' K = 1 tiles are not the bottleneck +# there (the per-call floor is), and a complex outer product would need its +# own interleaved arithmetic. Selected by `execute!` through +# `_try_execute_outer!`; `_OUTER_MODE` overrides the choice for tests and +# benchmarks. + +const _OUTER_MODE = Ref{Symbol}(:auto) + +""" + _try_execute_outer!(plan::ContractPlan, alphaT, betaT, Qm, Qn) -> Bool + +Run the outer-product path if `plan` is eligible and return `true`; return +`false` without touching anything otherwise. Eligible when `_OUTER_MODE[]` is +not `:never`, `T` is real, `Qk == 1` (the caller checks), the M composite is +the unit ramp `offset(m) = m` in BOTH its A and C maps (`_map_ramp_step`), +`Qm >= W` for the lane width `W = _dot_lanewidth(T)`, and A and C are +`DenseVector{T}` (raw-pointer vector loads/stores). N and B may have any +layout: B is read one scalar per column through the N composite's offsets. + +Measured 2026-09-27, ccqlin038 (Cascade Lake, AVX-512, Julia 1.12.7), the +suite's `dim*_1_0_1_gemm_ready` cases `C[a,b] = A[a] B[b]`, Float64, +`execute!` time, this path vs the (store-inlined) nest on the same tree +(`benchmark/bench_degenerate.jl`, interleaved medians), and OpenBLAS on the +same data: 16x16 319 -> 227 ns (84), 63x63 2.40 -> 1.16 us (0.81), 128x128 +6.15 -> 2.70 us (3.07). ComplexF64 is not taken: the nest is already within +~10% of OpenBLAS there from 63x63 up (4.6 vs 4.2 us, 15.1 vs 15.6 us). +""" +function _try_execute_outer!( + plan::ContractPlan{T}, alphaT::T, betaT::T, Qm::Int, Qn::Int + ) where {T} + _outer_applicable(plan, Qm) || return false + W = _dot_lanewidth(T) + if W == 16 + _execute_outer!(plan, alphaT, betaT, Qm, Qn, Val(16)) + elseif W == 8 + _execute_outer!(plan, alphaT, betaT, Qm, Qn, Val(8)) + elseif W == 4 + _execute_outer!(plan, alphaT, betaT, Qm, Qn, Val(4)) + else + _execute_outer!(plan, alphaT, betaT, Qm, Qn, Val(2)) + end + return true +end + +# The eligibility rule of `_try_execute_outer!` (`Qk == 1` is the caller's), +# as a side-effect-free predicate. +function _outer_applicable(plan::ContractPlan{T}, Qm::Int) where {T} + _OUTER_MODE[] === :never && return false + T <: Real || return false + (plan.Astorage isa DenseVector{T} && plan.Cstorage isa DenseVector{T}) || return false + Qm >= _dot_lanewidth(T) || return false + _map_ramp_step(plan.mgroup, 1) == 1 || return false + return _map_ramp_step(plan.mgroup, 2) == 1 +end + +function _execute_outer!( + plan::ContractPlan{T}, alphaT::T, betaT::T, Qm::Int, Qn::Int, ::Val{W} + ) where {T, W} + ws = plan.workspace + Astorage = plan.Astorage + Bstorage = plan.Bstorage + Cstorage = plan.Cstorage + Bbase = plan.Bbase + bufB = ws.n_buf_B + bufC = ws.n_buf_C + nc_eff = plan.blocking.nc + lenB = length(Bstorage) + lenC = length(Cstorage) + + # A's M rows, once: `Abase + m`, `0 <= m < Qm` (the K offset is 0). + checked_span_bounds(plan.Abase, (0, Qm - 1), (0, 0), length(Astorage)) + + # `ws`: the offset buffers; A and C: raw-pointer loads/stores. + GC.@preserve ws Astorage Cstorage begin + ap = pointer(Astorage) + sizeof(T) * plan.Abase + cp = pointer(Cstorage) + sizeof(T) * plan.Cbase + jc = 0 + while jc < Qn + nblock = min(nc_eff, Qn - jc) + fill_offsets!((bufB, bufC), plan.ngroup, jc, nblock) + # Every address this block touches, before any of it: B's + # `nblock` scalars and the `Qm x nblock` rectangle of C. + checked_span_bounds(Bbase, _buffer_range(bufB, nblock), (0, 0), lenB) + checked_span_bounds(plan.Cbase, (0, Qm - 1), _buffer_range(bufC, nblock), lenC) + _outer_block!( + cp, bufC, ap, Bstorage, Bbase, bufB, nblock, Qm, + alphaT, betaT, plan.btransform, Val(W) + ) + jc += nblock + end + end + return plan.Cstorage +end + +# One N block: column `n` of C is `alpha * (A .* B[n]) + beta * C[:, n]`, W +# rows at a time with a scalar row tail. The `beta` regime is chosen once per +# block; inside, the expressions are `_store_tile_vector!`'s with `acc = a * b`. +@inline function _outer_block!( + cp::Ptr{T}, bufC::Vector{Int}, ap::Ptr{T}, Bstorage::SB, Bbase::Int, + bufB::Vector{Int}, nblock::Int, Qm::Int, alpha::T, beta::T, btransform::F, ::Val{W} + ) where {T, SB, F, W} + sz = sizeof(T) + mmain = (Qm ÷ W) * W + @inbounds for n in 1:nblock + b = btransform(Bstorage[Bbase + bufB[n] + 1])::T + bv = Vec{W, T}(b) + cpn = cp + sz * bufC[n] + m = 0 + if iszero(beta) + while m < mmain + r = vload(Vec{W, T}, ap + sz * m) * bv + vstore(alpha * r, cpn + sz * m) + m += W + end + while m < Qm + r = unsafe_load(ap + sz * m) * b + unsafe_store!(cpn + sz * m, alpha * r) + m += 1 + end + elseif isone(beta) + while m < mmain + r = vload(Vec{W, T}, ap + sz * m) * bv + vstore(muladd(alpha, r, vload(Vec{W, T}, cpn + sz * m)), cpn + sz * m) + m += W + end + while m < Qm + r = unsafe_load(ap + sz * m) * b + unsafe_store!(cpn + sz * m, muladd(alpha, r, unsafe_load(cpn + sz * m))) + m += 1 + end + else + betav = Vec{W, T}(beta) + while m < mmain + r = vload(Vec{W, T}, ap + sz * m) * bv + vstore(muladd(alpha, r, betav * vload(Vec{W, T}, cpn + sz * m)), cpn + sz * m) + m += W + end + while m < Qm + r = unsafe_load(ap + sz * m) * b + unsafe_store!(cpn + sz * m, muladd(alpha, r, beta * unsafe_load(cpn + sz * m))) + m += 1 + end + end + end + return nothing +end diff --git a/test/execution/test_outer.jl b/test/execution/test_outer.jl new file mode 100644 index 0000000..b5ad046 --- /dev/null +++ b/test/execution/test_outer.jl @@ -0,0 +1,186 @@ +# The outer-product path (src/execution/outer.jl): `execute!` on a real plan +# with `K == 1` and unit-stride M in A and C streams `alpha * A * B[n]` column +# by column instead of running the nest. Checked here: +# +# 1. `_try_execute_outer!` takes exactly the eligible plans (real `T`, +# `K == 1`, unit-ramp M in both A and C, `Qm >= W`, dense A/C storage), +# declines the rest untouched, and obeys `_OUTER_MODE`. +# 2. On eligible plans the result matches the nest (`_OUTER_MODE[] = +# :never`), the oracle and a brute-force reference: `≈`, since the nest's +# FMA from `+0.0` and this path's plain product differ on a signed zero. +# 3. beta = 0 never reads C; out-of-bounds operands are rejected before any +# write; the call is allocation-free. + +const _outer_mode = QuasiStrided._OUTER_MODE + +function _outer_run(mk, mode::Symbol, alpha, beta; plankw...) + Cv, Av, indA, Bv, indB, indC = mk() + plan = plan_contract(Cv, Av, indA, Bv, indB, indC; plankw...) + old = _outer_mode[] + _outer_mode[] = mode + try + @test execute!(plan, alpha, beta) === plan.Cstorage + finally + _outer_mode[] = old + end + return Array(Cv), plan +end + +# Whether the automatic path would take `plan` (without running it). +_outer_eligible(plan) = + axis_length(plan.kgroup) == 1 && QuasiStrided._outer_applicable(plan, axis_length(plan.mgroup)) + +# `C[a, b] = A[a] * B[b]` (the suite's `*_1_0_1_gemm_ready`) at extents +# `(M, N)`, with variants: +# :plain dense everything +# :bstrided B every other element of a longer vector +# :cpermC C stored [b, a] and viewed back (M not unit-stride in C) +# :agap A every other element (M not unit-stride in A) +# :coffset C a view with an offset into a larger parent, M unit-stride +# :multiM M a composite of two labels, C[a1,a2,b] with (a1,a2) folding +function _outer_maker(::Type{T}, M, N, seed; variant = :plain, Cfill = nothing) where {T} + return function () + rng = MersenneTwister(seed) + a = randn(rng, T, M) + b = randn(rng, T, N) + C = Cfill === nothing ? randn(rng, T, M, N) : fill(convert(T, Cfill), M, N) + Av = if variant === :agap + big = fill(convert(T, NaN), 2M) + big[1:2:(2M)] .= a + StridedView(view(big, 1:2:(2M))) + else + StridedView(a) + end + Bv = if variant === :bstrided + big = fill(convert(T, NaN), 2N) + big[2:2:(2N)] .= b + StridedView(view(big, 2:2:(2N))) + else + StridedView(b) + end + Cv = if variant === :cpermC + permutedims(StridedView(permutedims(C, (2, 1))), (2, 1)) + elseif variant === :coffset + big = fill(convert(T, NaN), M + 3, N + 2) + big[2:(M + 1), 2:(N + 1)] .= C + StridedView(view(big, 2:(M + 1), 2:(N + 1))) + else + StridedView(C) + end + @assert isequal(Array(Av), a) && isequal(Array(Bv), b) && isequal(Array(Cv), C) + if variant === :multiM + M1 = 4 + @assert M % M1 == 0 + return ( + StridedView(reshape(C, M1, M ÷ M1, N)), StridedView(reshape(a, M1, M ÷ M1)), (1, 2), + Bv, (3,), (1, 2, 3), + ) + end + return (Cv, Av, (1,), Bv, (2,), (1, 2)) + end +end + +const _OUTER_AB = ((1.0, 0.0), (2.5, -0.75), (1.0, 1.0), (-0.5, 1.25)) + +@testset "outer path: which plans it takes ($T)" for T in (Float64, Float32, ComplexF64, ComplexF32) + W = QuasiStrided._dot_lanewidth(T) + for variant in (:plain, :bstrided, :coffset, :multiM) + p = plan_contract(_outer_maker(T, 16, 9, 1; variant = variant)()...) + @test axis_length(p.kgroup) == 1 + @test _outer_eligible(p) == (T <: Real) + end + if T <: Real + # M not unit-stride in A: declined. + @test !_outer_eligible(plan_contract(_outer_maker(T, 16, 9, 1; variant = :agap)()...)) + # C stored [b, a]: the planner swaps the roles, so the label that is + # unit-stride in C becomes M (here the 9-long `b`, whose operand is + # dense too) and the path applies iff that M still fills a vector. + pc = plan_contract(_outer_maker(T, 16, 9, 1; variant = :cpermC)()...) + @test axis_length(pc.mgroup) == 9 + @test _outer_eligible(pc) == (9 >= W) + # Below one vector of M: declined. K > 1: not an outer product. + @test !_outer_eligible(plan_contract(_outer_maker(T, W - 1, 9, 1)()...)) + @test !_outer_eligible(_mm_plan(zeros(T, 16, 9), randn(T, 16, 2), randn(T, 2, 9))) + # The mode override. + p = plan_contract(_outer_maker(T, 16, 9, 1)()...) + _outer_mode[] = :never + @test !_outer_eligible(p) + _outer_mode[] = :auto + @test _outer_eligible(p) + end +end + +@testset "outer path: matches the nest, the oracle and a reference ($T)" for T in (Float64, Float32) + W = QuasiStrided._dot_lanewidth(T) + shapes = ((W, 1), (W, 7), (W + 3, 5), (2W + 1, 13), (63, 63), (128, 128), (40, 3)) + for (idx, (M, N)) in enumerate(shapes), variant in (:plain, :bstrided, :coffset, :cpermC, :agap), (alpha, beta) in _OUTER_AB + mk = _outer_maker(T, M, N, 10 + idx; variant = variant) + C_nest, _ = _outer_run(mk, :never, alpha, beta) + C_out, plan = _outer_run(mk, :auto, alpha, beta) + # :agap is never eligible (A has stride 2); :cpermC only when the + # planner swapped the roles so that M is unit-stride in C as well. + @test _outer_eligible(plan) == ( + variant !== :agap && all(==((1,)), plan.mgroup.strides) && axis_length(plan.mgroup) >= W + ) + @test C_out ≈ C_nest + Cv0, Av0, iA, Bv0, iB, iC = mk() + @test C_out ≈ _hp_ref(Array(Av0), iA, Array(Bv0), iB, Array(Cv0), iC, alpha, beta) + C_tw = let (Cv, Av, iA, Bv, iB, iC) = mk() + execute_tilewise!(plan_contract(Cv, Av, iA, Bv, iB, iC), alpha, beta) + Array(Cv) + end + @test C_out ≈ C_tw + end + # Composite M (two labels folding to one unit ramp) and several N blocks. + for (M, N) in ((32, 40), (128, 7)) + mk = _outer_maker(T, M, N, 50 + M; variant = :multiM) + C_nest, _ = _outer_run(mk, :never, 1.5, 0.5; nc = 6) + C_out, plan = _outer_run(mk, :auto, 1.5, 0.5; nc = 6) + @test _outer_eligible(plan) + @test C_out ≈ C_nest + Cv0, Av0, iA, Bv0, iB, iC = mk() + @test C_out ≈ _hp_ref(Array(Av0), iA, Array(Bv0), iB, Array(Cv0), iC, 1.5, 0.5) + end +end + +@testset "outer path: beta = 0 never reads C, bounds, allocations ($T)" for T in (Float64, Float32) + W = QuasiStrided._dot_lanewidth(T) + mk = _outer_maker(T, 2W + 3, 9, 4; Cfill = NaN) + C_out, plan = _outer_run(mk, :auto, 2.0, 0.0) + @test _outer_eligible(plan) + @test all(isfinite, C_out) + Cv0, Av0, _, Bv0, _, _ = mk() + @test C_out ≈ 2.0 .* (Array(Av0) * transpose(Array(Bv0))) + + # One element short on each operand: rejected before any write. + M, N = 2W, 5 + mkA = () -> (StridedView(zeros(T, M, N)), StridedView(randn(T, M - 1), (M,), (1,), 0), (1,), StridedView(randn(T, N)), (2,), (1, 2)) + mkB = () -> (StridedView(zeros(T, M, N)), StridedView(randn(T, M)), (1,), StridedView(randn(T, N - 1), (N,), (1,), 0), (2,), (1, 2)) + mkC = () -> (StridedView(zeros(T, M * N - 1), (M, N), (1, M), 0), StridedView(randn(T, M)), (1,), StridedView(randn(T, N)), (2,), (1, 2)) + for mk in (mkA, mkB, mkC) + Cv, Av, iA, Bv, iB, iC = mk() + plan = plan_contract(Cv, Av, iA, Bv, iB, iC) + @test _outer_eligible(plan) + @test_throws BoundsError execute!(plan, 1.0, 0.0) + @test all(iszero, plan.Cstorage) + end + + Cv, Av, iA, Bv, iB, iC = _outer_maker(T, 64, 50, 6)() + plan = plan_contract(Cv, Av, iA, Bv, iB, iC; oracle = false, nc = 12) + @test _outer_eligible(plan) + execute!(plan, 1.0, 0.0) + execute!(plan, 1.0, 0.0) + @test (@allocated execute!(plan, 1.0, 0.0)) == 0 skip = (VERSION < v"1.11") +end + +@testset "outer path: through the TensorOperations backend ($T)" for T in (Float64, Float32, ComplexF64) + a = randn(T, 63); b = randn(T, 63) + C1 = zeros(T, 63, 63); C2 = zeros(T, 63, 63) + pA, pB, pAB = TO.contract_indices((:a,), (:b,), (:a, :b)) + _outer_mode[] = :never + TO.tensorcontract!(C1, a, pA, false, b, pB, false, pAB, 1.0, 0.0, QuasiStridedBackend()) + _outer_mode[] = :auto + TO.tensorcontract!(C2, a, pA, false, b, pB, false, pAB, 1.0, 0.0, QuasiStridedBackend()) + @test C2 ≈ C1 + @test C2 ≈ a * transpose(b) +end diff --git a/test/runtests.jl b/test/runtests.jl index cdc335e..55564a8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -51,9 +51,10 @@ using QuasiStrided: AxisGroup, axis_length, offsets, fill_offsets!, BlockDescrip include("execution/test_macro_blocking.jl") include("execution/test_direct.jl") include("execution/test_halfpack.jl") - # Both use test_halfpack.jl's `_hp_ref` reference (and `_HPDenseMat`). + # All three use test_halfpack.jl's `_hp_ref` reference (and `_HPDenseMat`). include("execution/test_unpackedb.jl") include("execution/test_dot.jl") + include("execution/test_outer.jl") # Last among the functional tests: it does a bare `using TensorOperations`, # which exports its own `scalartype` and would otherwise conflict with the From ad158f2ecc2a766535a508dd258f283652fff11f Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 10:45:44 -0400 Subject: [PATCH 14/53] Add benchmark/bench_degenerate.jl: A/B of the outer, dot and unpacked-B paths Times `execute!` with each path's mode override set to :never (the nest) and :auto, interleaved rep by rep, plus OpenBLAS on the equivalent dense matrices; the numbers in the three path commits come from it. Co-Authored-By: Claude Opus 5.5 --- benchmark/bench_degenerate.jl | 86 +++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 benchmark/bench_degenerate.jl diff --git a/benchmark/bench_degenerate.jl b/benchmark/bench_degenerate.jl new file mode 100644 index 0000000..94e0331 --- /dev/null +++ b/benchmark/bench_degenerate.jl @@ -0,0 +1,86 @@ +# A/B of the three automatically selected small/degenerate-shape paths against +# the five-loop nest on the same tree, via their mode overrides: +# +# outer K = 1 outer product C[a,b] = A[a] B[b] _OUTER_MODE +# dot M = 1 gemv C[cde] = A[ab] B[abcde] _DOT_MODE +# unpackedb small square matmul C[a,c] = A[a,b] B[b,c] _UNPACKED_B_MODE +# +# `execute!` only (planning excluded), plus OpenBLAS `mul!` on the equivalent +# dense matrices for scale. Modes are interleaved rep by rep; each rep times a +# batch of calls sized to ~200 us. Prints median ns per call and the +# [25%, 75%] spread. +# +# JULIA_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 taskset -c 8 \ +# julia --project=benchmark benchmark/bench_degenerate.jl [reps] + +using QuasiStrided +using QuasiStrided: plan_contract, execute! +using StridedViews: StridedView +using LinearAlgebra +using Statistics: median, quantile +using Random +using Printf + +LinearAlgebra.BLAS.set_num_threads(1) +const REPS = length(ARGS) >= 1 ? parse(Int, ARGS[1]) : 31 + +function batchsize(f!) + f!() + t0 = time_ns(); f!(); t = max(time_ns() - t0, 1) + return clamp(round(Int, 200_000 / t), 1, 100_000) +end + +# Interleaved timing of several closures; returns per-closure (median, q25, q75) ns. +function interleaved(fs::Vector) + ns = [batchsize(f) for f in fs] + ts = [Float64[] for _ in fs] + for _ in 1:REPS, (i, f) in enumerate(fs) + n = ns[i] + t0 = time_ns() + for _ in 1:n + f() + end + push!(ts[i], (time_ns() - t0) / n) + end + return [(median(t), quantile(t, 0.25), quantile(t, 0.75)) for t in ts] +end + +fmt((m, lo, hi)) = @sprintf("%9.0f [%6.0f,%6.0f]", m, lo, hi) + +function withmode(f, ref::Ref{Symbol}, mode::Symbol) + return () -> begin + old = ref[]; ref[] = mode + f() + ref[] = old + end +end + +function row(name, ref, plan, blasf) + ex = () -> execute!(plan, 1.0, 0.0) + r = interleaved(Any[withmode(ex, ref, :never), withmode(ex, ref, :auto), blasf]) + @printf("%-28s nest %s new %s blas %s new/nest %.2f new/blas %.2f\n", + name, fmt(r[1]), fmt(r[2]), fmt(r[3]), r[2][1] / r[1][1], r[2][1] / r[3][1]) + return flush(stdout) +end + +println("# ns per call: median [q25, q75], $REPS interleaved reps; host $(gethostname())") +for T in (Float64, ComplexF64) + println("## $T") + for d in (16, 63, 128) + a = randn(T, d); b = randn(T, d); C = zeros(T, d, d) + plan = plan_contract(StridedView(C), StridedView(a), (1,), StridedView(b), (2,), (1, 2)) + am = reshape(a, d, 1); bm = reshape(b, 1, d) + row("outer $(d)x$(d)", QuasiStrided._OUTER_MODE, plan, () -> mul!(C, am, bm)) + end + for d in (6, 8, 12, 16) + A = randn(T, d, d); B = randn(T, d, d, d, d, d); C = zeros(T, d, d, d) + plan = plan_contract(StridedView(C), StridedView(A), (1, 2), StridedView(B), (1, 2, 3, 4, 5), (3, 4, 5)) + Bm = reshape(B, d^2, d^3); av = vec(A); cv = vec(C) + row("dot 1x$(d^2)x$(d^3)", QuasiStrided._DOT_MODE, plan, () -> mul!(cv, transpose(Bm), av)) + end + for d in (16, 32, 64, 128) + A = randn(T, d, d); B = randn(T, d, d); C = zeros(T, d, d) + plan = plan_contract(StridedView(C), StridedView(A), (1, 2), StridedView(B), (2, 3), (1, 3)) + row("unpackedB $(d)^3", QuasiStrided._UNPACKED_B_MODE, plan, () -> mul!(C, A, B)) + end +end From 0835cd784389ce13ebafe3b1f3ad9acd8fcde2cd Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 11:05:49 -0400 Subject: [PATCH 15/53] Keep the unpacked-B tile loop out of line Inlined into `_execute_nest!`, `_micro_tiles_unpacked_b!` grew the nest for every plan, including packed-B ones: ~+4% on TCCG ccsd_2 dim16 Float64 (K = 256, B packed) on ccqlin038, 3-tree interleaved comparison. Out of line (one call per (jc, pc, ic) block) the packed plans are back to the pre-unpacked-B time and the unpacked 16^3/32^3 matmuls are unchanged (~305 ns / ~1.2-1.3 us). Co-Authored-By: Claude Opus 5.5 --- src/execution/unpackedb.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/execution/unpackedb.jl b/src/execution/unpackedb.jl index 0251d40..84f4fd4 100644 --- a/src/execution/unpackedb.jl +++ b/src/execution/unpackedb.jl @@ -148,12 +148,18 @@ end # per the guardrail at the top of src/execution/macrokernel.jl), so each view # is concretely typed and the micro-tile calls specialize on it. # +# `@noinline`: one call per (jc, pc, ic) block, and inlined it grows +# `_execute_nest!` for EVERY plan, including the packed ones -- measured +# 2026-09-27 (ccqlin038) as ~+4% on the packed-B TCCG case ccsd_2 dim16 +# Float64 (7.2 -> 7.5 us), gone out of line, with no measurable cost to the +# unpacked 16^3/32^3 matmuls. +# # Bounds: the addresses a view can read are `Bbase + colbase[j] + koffset(p)` # for the sliver's own columns (padding aliases a valid one) and this panel's # K offsets -- a subset of the `(rng_kB, rng_nB)` rectangle the caller has # already passed through `checked_span_bounds` (check 1 of 3), which is why # the element loads are `@inbounds`. C is covered by the caller's check 3. -@inline function _micro_tiles_unpacked_b!( +@noinline function _micro_tiles_unpacked_b!( kernel::K, plan::ContractPlan, ws, rowsB_k::KA, m_slivers::Int, n_slivers::Int, MRk::Int, NRk::Int, MRp::Int, kblock::Int, alphaT, beta_eff ) where {K, KA <: Axis} From c741fb566e68bc3bd1181acb9a4d1812f9974fbf Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 11:06:22 -0400 Subject: [PATCH 16/53] Add benchmark/submit_degenerate.sh for per-ISA node verification Runs bench_degenerate.jl and the bench_to_suite.jl :contract subset the unpacked-B/dot/outer paths target, with submit_bench_to_suite.sh's julia resolution (rocky8 reservation fallback). Not submitted from here. Co-Authored-By: Claude Opus 5.5 --- benchmark/submit_degenerate.sh | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 benchmark/submit_degenerate.sh diff --git a/benchmark/submit_degenerate.sh b/benchmark/submit_degenerate.sh new file mode 100755 index 0000000..73382a3 --- /dev/null +++ b/benchmark/submit_degenerate.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Slurm job verifying the degenerate/small-shape paths (unpacked B, dot, outer) +# on one node: benchmark/bench_degenerate.jl (each path vs the nest, same +# tree) and the bench_to_suite.jl :contract subset those paths target. Single +# core, single node. One job per microarchitecture, from the repo root -- the +# paths pick different lane widths per ISA (AVX2 on rome: W = 4 Float64): +# +# sbatch --reservation=rocky8 --constraint=icelake benchmark/submit_degenerate.sh +# sbatch --reservation=rocky8 --constraint=genoa benchmark/submit_degenerate.sh +# sbatch --reservation=rocky8 --constraint=rome benchmark/submit_degenerate.sh +# +# See https://wiki.flatironinstitute.org/SCC/Software/Slurm. +#SBATCH --job-name=qs-degenerate +#SBATCH --partition=ccq +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=16G +#SBATCH --time=02:00:00 +#SBATCH --output=benchmark/results/slurm-%j.out + +set -euo pipefail +cd "${SLURM_SUBMIT_DIR:?SLURM_SUBMIT_DIR not set -- run this script via sbatch, not directly}" + +# Same julia resolution as submit_bench_to_suite.sh (module julia is broken +# on the rocky8 reservation; fall back to $QS_JULIA or juliaup 1.12). +module load modules/2.5-beta1 2>/dev/null || true +module load julia/1.12.6 2>/dev/null || true +MODJULIA=$(command -v julia 2>/dev/null || true) +case "$MODJULIA" in /mnt/sw/*) ;; *) MODJULIA="" ;; esac +JULIA=${QS_JULIA:-${MODJULIA:-$(ls -d "$HOME"/.julia/juliaup/julia-1.12*/bin/julia 2>/dev/null | tail -1)}} +[ -x "${JULIA:-}" ] || { echo "no julia found; set QS_JULIA" >&2; exit 127; } +echo "julia = $JULIA" + +export JULIA_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 +"$JULIA" --project=benchmark -e 'using Pkg; Pkg.instantiate()' + +OUTDIR="benchmark/results/degenerate-${SLURM_JOB_ID}-$(hostname -s)" +mkdir -p "$OUTDIR" +"$JULIA" --project=benchmark benchmark/bench_degenerate.jl 31 | tee "$OUTDIR/bench_degenerate.txt" +"$JULIA" --project=benchmark benchmark/bench_to_suite.jl \ + --categories contract --sources synthetic,batched,tccg \ + --synthetic-sizes 6,8,12,16,32,63,128 --batched-sizes 16,32,64 --tccg-sizes 8,16 \ + --dtypes Float64,ComplexF64 --outdir "$OUTDIR" + +echo "=== node check ===" +echo "SLURM_JOB_NODELIST = ${SLURM_JOB_NODELIST:-?}" +lscpu | grep -i "model name\|cache\|flags" | cut -c1-200 || true From 62ad7d90d1be0ff766ca7d4c43ff497df8d6e322 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 18:42:26 -0400 Subject: [PATCH 17/53] Compile only the chosen kernel and execution path: allocation-free dispatch barriers The first `@tensor backend = QuasiStridedBackend()` call in a fresh process had grown from ~3 s (main) to 28 s (Float64) / 87 s (ComplexF64) on integration/perf, and the test suite from ~14 to ~121 min (benchmark/probes/probe_ttfx.jl, ccqlin038, Julia 1.12.7). SnoopCompile on the first ComplexF64 call: 4146 method instances inferred, 85 s of inference, 40 instances of `execute!`/`_execute_nest!` (the unrolled kernel ladder's 10 menu kernels x 4 conj/identity transform pairs, every arm compiled although one runs), 320 of `_dot_nest!` (40 plans x 4 lane widths x 2 matrix sides) and 40 of `_micro_tiles_unpacked_b!`. Float64: 6 plans (3 kernels x 2 M/N orientations), 26 s of inference, of which the nest alone was 9 s (1.5 s per instance, ~15x main's: every `Union` sliver axis split into up to four inlined copies of the tile at each pack and tile call). Fix: every runtime choice is made BEFORE a dynamic call and passed through it as a singleton, so inference stops at the call and only the callee actually reached is compiled. The call boxes nothing (src/execution/barrier.jl): static choices cross as singletons (`Val` of the shape, the method, the transforms, a path marker), the operand storages as themselves (already heap objects), and everything else in a typed slot (`Base.RefValue{P}`) owned by the workspace and reused across calls. No slot holds an operand array, so a pooled workspace retains no user data. * Kernel barrier (`_plan_with_kernel`, src/planning/plan.jl): the automatic `(shape, method)` -- now including the run-length demotion, applied on shapes (`_demote_shape_for_run`) -- is turned into a `Val(shape)` by a menu ladder (`_menu_val`) and the plan is built in `_plan_resolved`, specialised on that one kernel and transform pair. Replaces `_with_menu_kernel`/`_plan_demoted`. * Path barrier (`execute!`, src/execution/execute.jl): `_select_path` picks the dot path (`_DotPath{matB, W}`), the outer path (`_OuterPath{W}`) or the nest (`_NestPath{unpacked_b, aff}`) -- the same rules, in the same order, as before -- and `_execute_path!` runs only that one. The TO adapter's continuation (`_Execute`) predicts the path before the kernel barrier (`_path_hint`) so a TensorOperations call crosses ONE barrier; the prediction is checked against the plan (`_hint_holds`) before use. `_try_execute_dot!`/`_try_execute_outer!` are gone; `_dot_applicable`, `_outer_applicable`, `_use_unpacked_b` keep their meaning. * Static sliver axis types (`_NestPath`'s `aff`): a map that is an affine ramp makes every block descriptor regular, so its sliver axis is always an `AffineAxis` (`_axis_of(..., Val(true))`) and the `PtrScatterAxis` arm is not compiled. Also for the half-packed nest (`_HalfPackPath`). * Tile loops out of line (`_micro_tiles_packed_b!`, `_half_packed_micro_tiles!` now `@noinline`, like `_micro_tiles_unpacked_b!`): inference/LLVM cost is superlinear in function size; test_halfpack.jl inference 207 -> 152 s. * `ContractWorkspace` is a `mutable struct` with `const` fields: one heap object, so plans, slots and the task pool hold a pointer instead of copying 21 GC-tracked fields. This alone took the 8x8x8 call through `tensorcontract!` from 0.47 to 0.38 us (Float64) and 0.54 to 0.45 us (ComplexF64). * `_default_shape` returns a concrete `RealMethod` for real `T` (the FMAddSub small-M demotion is complex-only), and `_path_hint` takes the method's unpacked-B eligibility as a `Bool`: a hint call union-split on the method was emitted out of line and boxed the request (256 B/call). Measured, ccqlin038 (Cascade Lake), Julia 1.12.7, cores 4-7: probe_ttfx.jl integration/perf -> this (main) using 0.6 s -> 0.5 s first Float64 call 28.0 s -> 2.7-2.9 s (3.0 s) first ComplexF64 call 87.0 s -> 2.0 s (3.0 s) first K=1 outer 10.1 s -> 0.5 s (2.0 s) SnoopCompile, first call alone (inferred MIs total / in QuasiStrided): Float64 1062 / 686 -> 521 / 254 (inference 26.8 -> 2.4 s) ComplexF64 4146 / 2895 -> 806 / 265 (inference 85.3 -> 2.7 s) (the first call of a process, whichever eltype, also infers ~1300 more instances of TensorOperations/Base code, ~1.3 s, as on main.) Package precompile 3.2 -> 3.4 s, pkgimage 1.88 -> 2.18 MiB. Pkg.test(): 121 min -> 15.1 min, all 189763 pass probe_call_floor.jl (median ns/call, 3 interleaved runs each, 0 B/call both): Float64 8x8x8 465 -> 370 16^3 643 -> 510 d4 outer 465 -> 351 ComplexF64 8x8x8 522 -> 462 16^3 1355 -> 1171 d4 outer 816 -> 714 Warm throughput (same host, interleaved): dim63_2_2_2_gemm_ready Float64 80-84 -> 84-85 GF/s; dim96_1_3_1_contract_scrambled 0.422 -> 0.424 s; benchmark/bench_degenerate.jl outer/dot/unpacked-B paths unchanged within run-to-run noise. Julia 1.13.0: full suite passes (10.5 min). Julia 1.10.12: the same 62 failures and 1 error as integration/perf itself (all in tests: allocation assertions not skipped below 1.11, a test depending on a skipped call's side effect, `sort(::Tuple)`), fixed separately. Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_ttfx.jl | 9 + benchmark/profile_buckets.jl | 2 +- src/QuasiStrided.jl | 2 + src/execution/barrier.jl | 157 ++++++++++++++++ src/execution/dot.jl | 64 +++---- src/execution/execute.jl | 268 +++++++++++++++++++++------ src/execution/halfpack.jl | 50 +++-- src/execution/macrokernel.jl | 15 ++ src/execution/outer.jl | 47 ++--- src/execution/unpackedb.jl | 29 ++- src/execution/workspace.jl | 72 ++++--- src/integrations/tensoroperations.jl | 33 ++-- src/planning/defaults.jl | 12 +- src/planning/kernel_selection.jl | 60 +++--- src/planning/plan.jl | 124 +++++++++---- test/execution/test_dot.jl | 2 +- test/execution/test_outer.jl | 2 +- test/execution/test_workspace.jl | 6 +- 18 files changed, 696 insertions(+), 258 deletions(-) create mode 100644 benchmark/probes/probe_ttfx.jl create mode 100644 src/execution/barrier.jl diff --git a/benchmark/probes/probe_ttfx.jl b/benchmark/probes/probe_ttfx.jl new file mode 100644 index 0000000..8bd4099 --- /dev/null +++ b/benchmark/probes/probe_ttfx.jl @@ -0,0 +1,9 @@ +t0 = time(); using TensorOperations, QuasiStrided; tload = time() - t0 +function first(T) + A = randn(T, 20, 30, 10); B = randn(T, 30, 15); C = zeros(T, 20, 10, 15) + t = @elapsed @tensor backend = QuasiStridedBackend() C[a, b, n] = A[a, k, b] * B[k, n] + return t +end +ts = [(T, first(T)) for T in (Float64, ComplexF64)] +t2 = @elapsed @tensor backend = QuasiStridedBackend() D[a, b] := randn(8, 1)[a, k] * randn(1, 9)[k, b] +println("load ", round(tload, digits = 1), " s; first call ", join(["$T $(round(t, digits = 1)) s" for (T, t) in ts], ", "), "; first K=1 outer ", round(t2, digits = 1), " s") diff --git a/benchmark/profile_buckets.jl b/benchmark/profile_buckets.jl index 7229083..ccf1276 100644 --- a/benchmark/profile_buckets.jl +++ b/benchmark/profile_buckets.jl @@ -40,7 +40,7 @@ const SPECIFIC_QS_BUCKETS = [ "planning", [ "plan_contract", "_planned", "_plan_contract", "_plan_with_kernel", - "_plan_demoted", "_with_menu_kernel", "_classify_labels", "_order_free_labels", + "_plan_resolved", "_menu_val", "_classify_labels", "_order_free_labels", "_prefer_swap", "_default_kernel", "_default_shape", "_resolved_defaults", "_kernel_from_shape", "default_blocking", # These would land in "planning" through the ancestor walk anyway diff --git a/src/QuasiStrided.jl b/src/QuasiStrided.jl index 571d834..f8a01ff 100644 --- a/src/QuasiStrided.jl +++ b/src/QuasiStrided.jl @@ -48,6 +48,8 @@ include("planning/blocking.jl") include("planning/defaults.jl") # `ContractPlan` holds a `ContractWorkspace`, so the workspace comes first. include("execution/workspace.jl") +# The dispatch barriers' slots and path markers, used by the plan and `execute!`. +include("execution/barrier.jl") include("planning/plan.jl") # --- Execution: the five-loop nest, the tile-by-tile oracle, the opt-in diff --git a/src/execution/barrier.jl b/src/execution/barrier.jl new file mode 100644 index 0000000..c44cf51 --- /dev/null +++ b/src/execution/barrier.jl @@ -0,0 +1,157 @@ +# Dispatch barriers: how a contraction reaches the ONE kernel and the ONE +# execution path it runs, compiling nothing else, while allocating nothing. +# +# The problem. The kernel (a menu shape, chosen from the host and the extents) +# and the execution path (the five-loop nest with B packed or read in place, +# the dot path at one of four lane widths, the outer-product path at one of +# four) are runtime choices, and everything below them is specialised on +# them. A static branch over every choice -- an unrolled ladder with a +# concrete call in each arm -- is allocation-free and cheap to run, but Julia +# infers and compiles every arm the first time the ladder itself is compiled: +# the whole menu (ten kernels for ComplexF64) times both B paths times the +# degenerate paths, and for a complex eltype times the four `conj`/`identity` +# transform pairs. Measured on ccqlin038 (Julia 1.12.7, SnoopCompile, the +# first `@tensor` call of benchmark/probes/probe_ttfx.jl): 40 `execute!` +# instances and 320 `_dot_nest!` instances, 85 s of inference, for ONE +# ComplexF64 contraction; 6 plans and 26 s for Float64. +# +# The fix is a dynamic call -- inference stops at it, and only the callee the +# call actually reaches is ever compiled -- arranged so that it boxes nothing. +# `jl_apply_generic` passes every argument as a boxed pointer, so a call +# allocates exactly for the arguments that are not already heap objects or +# singletons. Every barrier below therefore passes: +# +# * the static choices (kernel shape, method, transforms, path) as +# SINGLETON values -- a `Val` or an empty struct each; +# * the operand storages (`parent` of each `StridedView`, a `Memory` or +# `Vector`) as themselves, which are already heap objects; +# * everything else -- groups, bases, blocking, scalars, the workspace -- +# through a typed slot, a `Base.RefValue{P}` owned by the workspace's +# `_SlotCache` and reused across calls: written just before the call, +# read back as the callee's first action. +# +# The payload never holds an operand array (they travel as arguments), so a +# pooled, task-lifetime workspace retains no user data between calls; it +# may hold the workspace itself, which is a cycle within the workspace, not a +# retention. Measured in isolation (Julia 1.12.7): the barrier -- slot write +# plus the dynamic call, method-cache hit -- costs ~15 ns and 0 bytes over +# the equivalent static branch. +# +# Reentrancy/threading: a slot belongs to one workspace, and a workspace to +# one task at a time (the pool is task-local; an explicitly shared workspace +# was never safe to use concurrently). The callee copies the payload out +# before doing anything that could run another contraction. + +# The typed slot for payload type `P` in `ws`, created on first use. The +# common case, the same payload type as the previous barrier crossing with +# this workspace, is a single type-tag test. +@inline function _barrier_slot!(ws::ContractWorkspace, ::Type{P}) where {P} + cache = ws.slots + slot = cache.last + slot isa Base.RefValue{P} && return slot + return _barrier_slot_slow!(cache, P) +end + +# No workspace yet (`plan_contract(...; workspace = nothing)`, which builds a +# fresh one after the kernel is known): a one-off slot, which allocates, on a +# path that allocates a whole workspace anyway. +@inline _barrier_slot!(::Nothing, ::Type{P}) where {P} = Base.RefValue{P}() + +@noinline function _barrier_slot_slow!(cache::_SlotCache, ::Type{P}) where {P} + slot = get!(() -> Base.RefValue{P}(), cache.slots, P)::Base.RefValue{P} + cache.last = slot + return slot +end + +# ---------------------------------------------------------------------------- +# Execution paths +# ---------------------------------------------------------------------------- +# +# What `execute!` runs once the empty-output and beta-only short-circuits are +# past, as a singleton value, so that it can cross a barrier for free and +# select exactly one `_execute_path!` method. Chosen by `_select_path` +# (src/execution/execute.jl). + +# The five-loop nest (src/execution/execute.jl). `UNPACKED_B`: B read in +# place by the microkernel (`_use_unpacked_b`, src/execution/unpackedb.jl) +# rather than packed. `AFF`: which of the six sliver axes the nest builds are +# statically `AffineAxis` rather than `Union{AffineAxis,PtrScatterAxis}`, one +# `Bool` per map in the order M-in-A, M-in-C, N-in-B, N-in-C, K-in-A, K-in-B +# (each group's maps in their own order) -- true where that map of its +# composite is an affine ramp (`_map_ramp_step`), so that every sliver +# descriptor of it is regular. Each `Union` axis is split at every pack and +# tile call it reaches, so a plan whose maps are all ramps (the common case) +# compiles one variant of each where the `Union` compiled two to four. +struct _NestPath{UNPACKED_B, AFF} end + +@inline _is_ramp_map(g::AxisGroup, p::Int) = _map_ramp_step(g, p) !== nothing + +# The nest path for a plan with these groups. N's B map is left `false` when B +# is read in place: that path builds no B sliver axis (`_unpacked_b_view` +# reads the descriptor itself), so the flag would only split specialisations. +@inline _nest_path(unpacked_b::Bool, mgroup::AxisGroup, ngroup::AxisGroup, kgroup::AxisGroup) = + _nest_path_from_flags( + unpacked_b, + _is_ramp_map(mgroup, 1), _is_ramp_map(mgroup, 2), + !unpacked_b && _is_ramp_map(ngroup, 1), _is_ramp_map(ngroup, 2), + _is_ramp_map(kgroup, 1), _is_ramp_map(kgroup, 2) +) + +# Seven runtime `Bool`s to the `_NestPath` singleton they name, looked up in a +# table of all 128: one index computation and one load. The value is typed +# `Any` -- it is only ever passed through a barrier, which dispatches on it, +# or tested with `isa` -- so neither a runtime `_NestPath{u, aff}()` (which +# applies the type dynamically, ~100 ns) nor a 128-leaf literal decision +# tree (0.27 s of inference on the first call) is needed. +@inline function _nest_path_from_flags(flags::Vararg{Bool, 7}) + index = 1 + for i in 1:7 + index += flags[i] << (i - 1) + end + return @inbounds _NEST_PATHS[index] +end + +const _NEST_PATHS = let + table = Vector{Any}(undef, 128) + for index in 1:128 + bits = ntuple(i -> ((index - 1) >> (i - 1)) & 1 == 1, 7) + table[index] = _NestPath{bits[1], bits[2:end]}() + end + table +end + +# The opt-in half-packed nest (`execute_half_packed!`, src/execution/halfpack.jl), +# with `_NestPath`'s six static affine-axis flags. Its M-in-A flag is always +# `false`: that path builds no A sliver axis (A is read in place). +struct _HalfPackPath{AFF} end + +@inline _half_pack_path(mgroup::AxisGroup, ngroup::AxisGroup, kgroup::AxisGroup) = + @inbounds _HALF_PACK_PATHS[ + 1 + (_is_ramp_map(mgroup, 2) << 1) + + (_is_ramp_map(ngroup, 1) << 2) + (_is_ramp_map(ngroup, 2) << 3) + + (_is_ramp_map(kgroup, 1) << 4) + (_is_ramp_map(kgroup, 2) << 5), +] + +const _HALF_PACK_PATHS = let + table = Vector{Any}(undef, 64) + for index in 1:64 + table[index] = _HalfPackPath{ntuple(i -> ((index - 1) >> (i - 1)) & 1 == 1, 6)}() + end + table +end + +# The K-vectorized dot path (src/execution/dot.jl) at lane width `W`; `MATB`: +# the matrix operand is B (`Qm == 1`), else A (`Qn == 1`). +struct _DotPath{MATB, W} end + +# The streaming outer-product path (src/execution/outer.jl) at lane width `W`. +struct _OuterPath{W} end + +# `Val(W)`'s singleton for the lane widths the degenerate paths compile, +# from a runtime `W` (`_dot_lanewidth`), as a literal per arm. +@inline function _lane_path(::Type{P}, W::Int) where {P} + W == 16 && return P{16}() + W == 8 && return P{8}() + W == 4 && return P{4}() + return P{2}() +end diff --git a/src/execution/dot.jl b/src/execution/dot.jl index 7c076e6..ee2ad30 100644 --- a/src/execution/dot.jl +++ b/src/execution/dot.jl @@ -13,9 +13,9 @@ # packed except the K-vector `v` (gathered once per K block, with its # conjugation folded in). This is what `?gemv` does for the transposed case. # -# Selected by `execute!` through `_try_execute_dot!` (see there for the -# eligibility rule and the measurement), before the five-loop nest; on an -# ineligible plan it declines and the nest runs. `_DOT_MODE` overrides the +# Selected by `execute!` through `_select_path` when `_dot_applicable` holds +# (see there for the eligibility rule and the measurement), before the +# five-loop nest; on an ineligible plan the nest runs. `_DOT_MODE` overrides the # automatic choice for tests and benchmarks. const _DOT_MODE = Ref{Symbol}(:auto) @@ -61,11 +61,11 @@ function _map_ramp_step(g::AxisGroup{D, P}, p::Int) where {D, P} end """ - _try_execute_dot!(plan::ContractPlan, alphaT, betaT, Qm, Qn, Qk) -> Bool + _dot_applicable(plan::ContractPlan, Qm, Qn, Qk) -> Bool -Run the dot-product path if `plan` is eligible and return `true`; return -`false` without touching anything otherwise. Eligible when `_DOT_MODE[]` is -not `:never` and +Whether `execute!` takes the dot-product path on `plan` (`_select_path`, +src/execution/execute.jl, which then runs it as `_DotPath{Qm == 1, W}` with +`W = _dot_lanewidth(T)`). Eligible when `_DOT_MODE[]` is not `:never` and * `Qm == 1` (the matrix operand is B, its free composite the N group) or `Qn == 1` (the matrix operand is A, the M group); @@ -92,49 +92,37 @@ The nest's time there is the gather pack of the matrix operand, which this path does not perform at all. The K tail is one masked step (not a scalar loop): 1x36x216 Float64 went 3.0 -> 1.9 us with it. """ -function _try_execute_dot!( - plan::ContractPlan{T}, alphaT::T, betaT::T, Qm::Int, Qn::Int, Qk::Int - ) where {T} - _dot_applicable(plan, Qm, Qn, Qk) || return false - W = _dot_lanewidth(T) - matB = Qm == 1 - # `W` is a runtime value; the ladder hands the kernel a literal. - if W == 16 - _execute_dot!(plan, alphaT, betaT, matB, Val(16)) - elseif W == 8 - _execute_dot!(plan, alphaT, betaT, matB, Val(8)) - elseif W == 4 - _execute_dot!(plan, alphaT, betaT, matB, Val(4)) - else - _execute_dot!(plan, alphaT, betaT, matB, Val(2)) - end - return true +function _dot_applicable(plan::ContractPlan{T}, Qm::Int, Qn::Int, Qk::Int) where {T} + return _dot_applicable(T, plan.Astorage, plan.Bstorage, plan.kgroup, Qm, Qn, Qk) && + _dot_capacity_ok(plan) end -# The eligibility rule of `_try_execute_dot!`, as a side-effect-free predicate. -function _dot_applicable(plan::ContractPlan{T}, Qm::Int, Qn::Int, Qk::Int) where {T} +# Every clause but the workspace capacity, on the plan's parts, so that +# `plan_contract` can predict the path before the plan exists (`_path_hint`). +function _dot_applicable(::Type{T}, Astorage, Bstorage, kgroup::AxisGroup, Qm::Int, Qn::Int, Qk::Int) where {T} _DOT_MODE[] === :never && return false (Qm == 1 || Qn == 1) || return false Qk >= _dot_lanewidth(T) || return false if Qm == 1 - plan.Bstorage isa DenseVector{T} || return false - _map_ramp_step(plan.kgroup, 2) == 1 || return false + Bstorage isa DenseVector{T} || return false + _map_ramp_step(kgroup, 2) == 1 || return false else - plan.Astorage isa DenseVector{T} || return false - _map_ramp_step(plan.kgroup, 1) == 1 || return false + Astorage isa DenseVector{T} || return false + _map_ramp_step(kgroup, 1) == 1 || return false end - # The gathered vector (and, complex, its pair-swapped copy) lives in the - # packed-A buffer, reinterpreted as `T`. + return true +end + +# The gathered vector (and, complex, its pair-swapped copy) lives in the +# packed-A buffer, reinterpreted as `T`. Always true for an automatically +# chosen kernel (the buffer holds `mr >= 2` K columns of reals per K step); +# `nothing` is the prediction's "assume so". +function _dot_capacity_ok(plan::ContractPlan{T}) where {T} need = (T <: Complex ? 2 : 1) * plan.blocking.kc return (length(plan.workspace.packed_a) * sizeof(real(T))) ÷ sizeof(T) >= need end +_dot_capacity_ok(::Nothing) = true -# Resolve the operand roles ONCE into concretely typed arguments: the matrix -# operand (`m*`), the vector operand (`v*`), the matrix's free composite `g` -# with its two offset buffers (map 1 = matrix, map 2 = C), the block length -# those buffers are sized for, and the K buffer that receives the VECTOR's K -# offsets. Branching on `matB` inside `_dot_nest!` would make every storage -# type a Union. function _execute_dot!(plan::ContractPlan{T}, alphaT::T, betaT::T, matB::Bool, ::Val{W}) where {T, W} ws = plan.workspace if matB diff --git a/src/execution/execute.jl b/src/execution/execute.jl index 4bcd4d3..783fa3a 100644 --- a/src/execution/execute.jl +++ b/src/execution/execute.jl @@ -50,7 +50,13 @@ is rejected exactly when a per-tile check would reject it; `execute_tilewise!` runs the per-tile checked path as an independent oracle for both the values and the rejections. -Scoped to that path deliberately: the `Qk == 0 || alpha == 0` beta-only +Degenerate extents run a dedicated path instead of the nest where their layout +allows (the dot and outer-product paths; `_select_path` has the order), and +the nest reads B in place where that pays (`_use_unpacked_b`). Whichever runs +is reached through a dispatch barrier (src/execution/barrier.jl), so a plan +compiles only the path it takes. + +Scoped to the nest deliberately: the `Qk == 0 || alpha == 0` beta-only short-circuit above it goes to `_scale_all_of_C!`, which performs no storage-bounds check at all (it writes through `scale_tile!`'s `@inbounds` path, relying on the `AxisGroup`s' own construction-time validation). It is @@ -59,58 +65,186 @@ named here only so this paragraph is not read as a claim about it. function execute!(plan::ContractPlan{T}, alpha::Number, beta::Number) where {T} alphaT = convert(T, alpha) betaT = convert(T, beta) + _execute_short_circuit!(plan, alphaT, betaT) && return plan.Cstorage + _execute_across_barrier!(plan, alphaT, betaT, _select_path(plan)) + return plan.Cstorage +end - kernel = plan.kernel - MRk = mr(kernel) - NRk = nr(kernel) - +# The two short-circuits every execution path shares, in order; `true` when +# the call is finished. Empty output: a no-op, nothing read or written at +# all. Nothing to contract (`Qk == 0` or `alpha == 0`): a beta-only pass, A +# and B never read. +@inline function _execute_short_circuit!(plan::ContractPlan{T}, alphaT::T, betaT::T) where {T} Qm = axis_length(plan.mgroup) Qn = axis_length(plan.ngroup) - Qk = axis_length(plan.kgroup) - - # Empty output: no-op, nothing read or written at all. - (Qm == 0 || Qn == 0) && return plan.Cstorage + (Qm == 0 || Qn == 0) && return true + if axis_length(plan.kgroup) == 0 || iszero(alphaT) + _scale_all_of_C!(plan, betaT, mr(plan.kernel), nr(plan.kernel), Qm, Qn) + return true + end + return false +end - # Nothing to contract: beta-only pass, A and B never read. - if Qk == 0 || iszero(alphaT) - _scale_all_of_C!(plan, betaT, MRk, NRk, Qm, Qn) - return plan.Cstorage +""" + _select_path(plan::ContractPlan) -> path + +The execution path `execute!` runs on `plan` past its short-circuits, as a +singleton (src/execution/barrier.jl). Degenerate extents take a dedicated +path instead of the nest when their layout allows: a K-contiguous matrix +operand with `M == 1` or `N == 1` goes to the K-vectorized dot path +(`_dot_applicable`, src/execution/dot.jl); a real `K == 1` outer product with +unit-stride M goes to the streaming outer-product path (`_outer_applicable`, +src/execution/outer.jl). Otherwise the nest, reading B in place where +`_use_unpacked_b` (src/execution/unpackedb.jl) says so. + +Each path's own check is the whole rule, so this is only their order: dot +first, then outer, then the nest. +""" +@inline _select_path(plan::ContractPlan{T}) where {T} = _select_path( + T, _unpacked_b_kernel_eligible(plan.kernel), + plan.Astorage, plan.Bstorage, plan.Cstorage, plan.mgroup, plan.ngroup, plan.kgroup, plan +) + +# The rule on the plan's parts, so that `plan_contract` can PREDICT it before +# the plan exists (`_path_hint`): `unpack_ok` is the kernel's eligibility for +# the unpacked-B path, and `capacity` the plan whose workspace must hold the +# dot path's gathered vector, or `nothing` to assume it does (true for every +# automatically chosen kernel; see `_dot_capacity_ok`). The prediction is +# re-checked against the plan's own `_select_path` before it is trusted +# (`_execute_hinted!`). +@inline function _select_path( + ::Type{T}, unpack_ok::Bool, Astorage, Bstorage, Cstorage, + mgroup::AxisGroup, ngroup::AxisGroup, kgroup::AxisGroup, capacity + ) where {T} + Qm = axis_length(mgroup) + Qn = axis_length(ngroup) + Qk = axis_length(kgroup) + if (Qm == 1 || Qn == 1) && _dot_applicable(T, Astorage, Bstorage, kgroup, Qm, Qn, Qk) && + _dot_capacity_ok(capacity) + W = _dot_lanewidth(T) + return Qm == 1 ? _lane_path(_DotPath{true}, W) : _lane_path(_DotPath{false}, W) end + Qk == 1 && _outer_applicable(T, Astorage, Cstorage, mgroup, Qm) && + return _lane_path(_OuterPath, _dot_lanewidth(T)) + return _nest_path(unpack_ok && _unpacked_b_rule(mgroup, kgroup), mgroup, ngroup, kgroup) +end - # Degenerate extents take a dedicated path instead of the nest when their - # layout allows: a K-contiguous matrix operand with `M == 1` or `N == 1` - # goes to the K-vectorized dot path (src/execution/dot.jl); a real - # `K == 1` outer product with unit-stride M goes to the streaming - # outer-product path (src/execution/outer.jl). Each declines (returns - # `false`) on an ineligible plan, and the nest below runs. - (Qm == 1 || Qn == 1) && _try_execute_dot!(plan, alphaT, betaT, Qm, Qn, Qk) && - return plan.Cstorage - Qk == 1 && _try_execute_outer!(plan, alphaT, betaT, Qm, Qn) && return plan.Cstorage +# `execute!`'s path barrier (src/execution/barrier.jl): run `path` on `plan` +# through a dynamic call specialised on it, so the paths a plan does not take +# are never compiled for it. The plan crosses in the workspace's slot with its +# storages stripped (they cross as arguments), together with the scalars. +@inline function _execute_across_barrier!(plan::ContractPlan{T}, alphaT::T, betaT::T, path) where {T} + core = _strip_storage(plan) + slot = _barrier_slot!(plan.workspace, Tuple{typeof(core), T, T}) + slot[] = (core, alphaT, betaT) + Base.inferencebarrier(_execute_resolved!)( + path, slot, plan.Astorage, plan.Bstorage, plan.Cstorage + ) + return nothing +end + +function _execute_resolved!( + path::P, slot::Base.RefValue{Tuple{R, T, T}}, Astorage::SA, Bstorage::SB, Cstorage::SC + ) where {P, R <: ContractPlan, T, SA, SB, SC} + core, alphaT, betaT = slot[] + _execute_path!(_with_storage(core, Astorage, Bstorage, Cstorage), alphaT, betaT, path) + return nothing +end - mc_eff = plan.blocking.mc - kc_eff = plan.blocking.kc - nc_eff = plan.blocking.nc +@inline _strip_storage(p::ContractPlan) = ContractPlan( + p.kernel, p.mgroup, p.ngroup, p.kgroup, p.blocking, + nothing, p.Abase, nothing, p.Bbase, nothing, p.Cbase, + p.atransform, p.btransform, p.workspace +) +@inline _with_storage(p::ContractPlan, Astorage, Bstorage, Cstorage) = ContractPlan( + p.kernel, p.mgroup, p.ngroup, p.kgroup, p.blocking, + Astorage, p.Abase, Bstorage, p.Bbase, Cstorage, p.Cbase, + p.atransform, p.btransform, p.workspace +) + +# The continuation the TensorOperations adapter hands `_planned` +# (src/planning/plan.jl): `execute!` on the plan inside the planning barrier, +# with `alpha`/`beta` already converted to `eltype(C)`. A callable struct +# rather than a closure so the captured scalars are concretely typed fields. +struct _Execute{T} + alpha::T + beta::T +end +(e::_Execute)(plan::ContractPlan) = (execute!(plan, e.alpha, e.beta); nothing) + +# The execution path predicted from the request, before the kernel barrier, +# so the barrier's callee is specialised on the path as well as the kernel +# (`_select_path` without the capacity check; `unpack_ok` is the unpacked-B +# eligibility of the kernel the barrier will build, from its method). +@inline _path_hint(::_Execute, req::_PlanRequest{T}, unpack_ok::Bool) where {T} = _select_path( + T, unpack_ok, + req.Astorage, req.Bstorage, req.Cstorage, req.mgroup, req.ngroup, req.kgroup, nothing +) + +@inline _continue(e::_Execute{T}, plan::ContractPlan{T}, hint) where {T} = + (_execute_hinted!(plan, e.alpha, e.beta, hint); nothing) +@inline _continue(e::_Execute{T}, plan::ContractPlan{T}, ::Nothing) where {T} = + (execute!(plan, e.alpha, e.beta); nothing) + +# `execute!` with the path predicted before the plan was built: when the +# prediction provably equals the plan's own `_select_path` (`_hint_holds`) -- +# always, for an automatically chosen kernel -- the path runs statically, with +# no second barrier; otherwise the plan's own choice runs through +# `execute!`'s barrier. +@inline function _execute_hinted!(plan::ContractPlan{T}, alphaT::T, betaT::T, hint) where {T} + _execute_short_circuit!(plan, alphaT, betaT) && return nothing + if _hint_holds(plan, hint) + _execute_path!(plan, alphaT, betaT, hint) + else + _execute_across_barrier!(plan, alphaT, betaT, _select_path(plan)) + end + return nothing +end +# Whether `_path_hint`'s prediction equals `_select_path(plan)`. The two +# evaluate the same rule on the same groups, storages and modes and differ in +# exactly two inputs: the unpacked-B eligibility of the kernel (predicted from +# its method; a compile-time property of the kernel type, so this folds) and +# the dot path's workspace capacity (assumed by the prediction). +@inline _hint_holds(plan::ContractPlan, ::_NestPath) = + _unpacked_b_method_eligible(complex_method(plan.kernel)) === _unpacked_b_kernel_eligible(plan.kernel) +@inline _hint_holds(plan::ContractPlan, ::_DotPath) = _dot_capacity_ok(plan) +@inline _hint_holds(plan::ContractPlan, ::_OuterPath) = true + +_execute_path!(plan::ContractPlan, alphaT, betaT, ::_DotPath{MATB, W}) where {MATB, W} = + (_execute_dot!(plan, alphaT, betaT, MATB, Val(W)); nothing) + +_execute_path!(plan::ContractPlan, alphaT, betaT, ::_OuterPath{W}) where {W} = ( + _execute_outer!( + plan, alphaT, betaT, axis_length(plan.mgroup), axis_length(plan.ngroup), Val(W) + ); nothing +) + +function _execute_path!( + plan::ContractPlan{T}, alphaT::T, betaT::T, path::_NestPath + ) where {T} + kernel = plan.kernel # A reused workspace may be oversized: every extent below comes from the # *current* block, never from a buffer's length. ws = plan.workspace - # Panels below borrow pointers into ws.packed_a/_b (src/packing/panel.jl), as do # the PtrScatterAxes from `_axis_of`; this is their lifetime. - return GC.@preserve ws begin + GC.@preserve ws begin _execute_nest!( - plan, ws, kernel, MRk, NRk, Qm, Qn, Qk, - mc_eff, kc_eff, nc_eff, alphaT, betaT + plan, ws, kernel, mr(kernel), nr(kernel), + axis_length(plan.mgroup), axis_length(plan.ngroup), axis_length(plan.kgroup), + plan.blocking.mc, plan.blocking.kc, plan.blocking.nc, alphaT, betaT, path ) end + return nothing end -# Split out so the `GC.@preserve` above has one obvious scope. +# Split out so the `GC.@preserve` in `_execute_path!` has one obvious scope. function _execute_nest!( plan::ContractPlan{T}, ws, kernel::K, MRk::Int, NRk::Int, Qm::Int, Qn::Int, Qk::Int, mc_eff::Int, kc_eff::Int, nc_eff::Int, - alphaT::T, betaT::T - ) where {T, K} + alphaT::T, betaT::T, ::_NestPath{UNPACKED_B, AFF} + ) where {T, K, UNPACKED_B, AFF} # GUARDRAIL: reals per sliver per LOGICAL K step, which is what addresses # the packed panels. NOT interchangeable with `MRk`/`NRk`, which keep their # meaning everywhere else here (sliver counts, block extents, @@ -137,9 +271,13 @@ function _execute_nest!( lenC = length(plan.Cstorage) # Whether B is read in place by the microkernel instead of packed - # (src/execution/unpackedb.jl). Decided once per call; the B pack below - # is skipped and loops 2/1 run through `_micro_tiles_unpacked_b!`. - unpacked_b = _use_unpacked_b(plan) + # (src/execution/unpackedb.jl): decided by `_select_path` and static + # here, so a plan compiles only the B path it takes. When set, the B pack + # below is skipped and loops 2/1 run through `_micro_tiles_unpacked_b!`. + unpacked_b = UNPACKED_B + # Which of the six sliver axes are statically affine (`_NestPath`): M in + # A, M in C, N in B, N in C, K in A, K in B. + aff_mA, aff_mC, aff_nB, aff_nC, aff_kA, aff_kB = map(Val, AFF) # --- loop 5: jc over N in steps of nc_eff --- jc = 0 @@ -181,8 +319,8 @@ function _execute_nest!( descriptor_offset_range(dB, ws.k_buf_B, 0), ) end - colsA_k = _axis_of(dK_A, ws.k_buf_A, 0) - rowsB_k = _axis_of(dK_B, ws.k_buf_B, 0) + colsA_k = _axis_of(dK_A, ws.k_buf_A, 0, aff_kA) + rowsB_k = _axis_of(dK_B, ws.k_buf_B, 0, aff_kB) # HOISTED CHECK 1 of 3 -- the whole B panel of this (jc, pc). # `rowsB_k` is shared by every N-sliver and `rng_nB` is the union @@ -199,7 +337,7 @@ function _execute_nest!( if !unpacked_b for s in 0:(n_slivers - 1) sfirst = s * NRk - colsB = _axis_of(ws.n_desc_B[s + 1], ws.n_buf_B, sfirst) + colsB = _axis_of(ws.n_desc_B[s + 1], ws.n_buf_B, sfirst, aff_nB) bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) _pack_sliver!( unsafe_pack_b!, bpanel, plan.Bstorage, plan.Bbase, rowsB_k, colsB, @@ -242,7 +380,7 @@ function _execute_nest!( # Pack the whole A panel for this (jc, pc, ic): every M-sliver. for r in 0:(m_slivers - 1) rfirst = r * MRk - rowsA = _axis_of(ws.m_desc_A[r + 1], ws.m_buf_A, rfirst) + rowsA = _axis_of(ws.m_desc_A[r + 1], ws.m_buf_A, rfirst, aff_mA) apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) _pack_sliver!( unsafe_pack_a!, apanel, plan.Astorage, plan.Abase, rowsA, colsA_k, @@ -254,23 +392,13 @@ function _execute_nest!( if unpacked_b _micro_tiles_unpacked_b!( kernel, plan, ws, rowsB_k, m_slivers, n_slivers, - MRk, NRk, MRp, kblock, alphaT, beta_eff + MRk, NRk, MRp, kblock, alphaT, beta_eff, aff_mC, aff_nC ) else - for s in 0:(n_slivers - 1) - sfirst = s * NRk - colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst) - bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) - for r in 0:(m_slivers - 1) - rfirst = r * MRk - rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst) - apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) - unsafe_execute_micro_tile!( - kernel, plan.Cstorage, plan.Cbase, rowsC, colsC, - apanel, bpanel, kblock, alphaT, beta_eff - ) - end - end + _micro_tiles_packed_b!( + kernel, plan, ws, m_slivers, n_slivers, + MRk, NRk, MRp, NRp, kblock, alphaT, beta_eff, aff_mC, aff_nC + ) end ic += mblock @@ -286,6 +414,38 @@ function _execute_nest!( return plan.Cstorage end +# Loops 2/1 of `_execute_nest!` over the packed A and B panels of one +# (jc, pc, ic) block. `@noinline`, like its unpacked-B twin +# `_micro_tiles_unpacked_b!` (src/execution/unpackedb.jl): one call per block, +# and the register-tile code it inlines -- the whole microkernel and its +# store, once per C axis type -- is most of a nest's size. Inference and LLVM +# cost grow faster than linearly in function size, so keeping it in its own +# function measurably shortens compilation: the same split of the half-packed +# nest cut test/execution/test_halfpack.jl's inference from 207 to 163 s +# (SnoopCompile, --check-bounds=yes, ccqlin038, Julia 1.12.7). The block's C +# region was validated by the caller (check 3 of 3). +@noinline function _micro_tiles_packed_b!( + kernel::K, plan::ContractPlan, ws, m_slivers::Int, n_slivers::Int, + MRk::Int, NRk::Int, MRp::Int, NRp::Int, kblock::Int, alphaT, beta_eff, + aff_mC::Val{MC}, aff_nC::Val{NC} + ) where {K, MC, NC} + for s in 0:(n_slivers - 1) + sfirst = s * NRk + colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst, aff_nC) + bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) + for r in 0:(m_slivers - 1) + rfirst = r * MRk + rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst, aff_mC) + apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) + unsafe_execute_micro_tile!( + kernel, plan.Cstorage, plan.Cbase, rowsC, colsC, + apanel, bpanel, kblock, alphaT, beta_eff + ) + end + end + return nothing +end + # ---------------------------------------------------------------------------- # contract! # ---------------------------------------------------------------------------- diff --git a/src/execution/halfpack.jl b/src/execution/halfpack.jl index c399674..8201cfc 100644 --- a/src/execution/halfpack.jl +++ b/src/execution/halfpack.jl @@ -205,19 +205,32 @@ function execute_half_packed!(plan::ContractPlan{T}, alpha::Number, beta::Number (Qm == 0 || Qn == 0 || Qk == 0 || iszero(alphaT)) && return execute!(plan, alpha, beta) (_half_pack_static_eligible(plan) && Qm == mr(kernel)) || return execute!(plan, alpha, beta) - (ok, dM_A, dM_C) = _half_pack_m_descriptors!(plan) + (ok, _, _) = _half_pack_m_descriptors!(plan) ok || return execute!(plan, alpha, beta) + # Through `execute!`'s dispatch barrier (src/execution/barrier.jl), so the + # nest is compiled only for the axis types this plan's maps produce. + _execute_across_barrier!( + plan, alphaT, betaT, _half_pack_path(plan.mgroup, plan.ngroup, plan.kgroup) + ) + return plan.Cstorage +end + +function _execute_path!(plan::ContractPlan{T}, alphaT::T, betaT::T, path::_HalfPackPath) where {T} + # Recomputed rather than carried across the barrier: two descriptors, + # already known to be eligible. + (_, dM_A, dM_C) = _half_pack_m_descriptors!(plan) ws = plan.workspace Astorage = plan.Astorage # `ws`: the packed B panel and every `PtrScatterAxis` borrow its buffers. # `Astorage`: the in-place A loads are raw-pointer loads. - return GC.@preserve ws Astorage begin + GC.@preserve ws Astorage begin _execute_half_packed_nest!( - plan, ws, kernel, dM_A, dM_C, Qn, Qk, - plan.blocking.kc, plan.blocking.nc, alphaT, betaT + plan, ws, plan.kernel, dM_A, dM_C, axis_length(plan.ngroup), axis_length(plan.kgroup), + plan.blocking.kc, plan.blocking.nc, alphaT, betaT, path ) end + return nothing end # `_execute_nest!` (src/execution/execute.jl) with the M side fixed at one full @@ -225,8 +238,11 @@ end # pack -- is `_execute_nest!`'s verbatim. function _execute_half_packed_nest!( plan::ContractPlan{T}, ws, kernel::K, dM_A::BlockDescriptor, dM_C::BlockDescriptor, - Qn::Int, Qk::Int, kc_eff::Int, nc_eff::Int, alphaT::T, betaT::T - ) where {T, K} + Qn::Int, Qk::Int, kc_eff::Int, nc_eff::Int, alphaT::T, betaT::T, + ::_HalfPackPath{AFF} + ) where {T, K, AFF} + # The static affine-axis flags (`_NestPath`'s order; M in A is unused). + _, aff_mC, aff_nB, aff_nC, aff_kA, aff_kB = map(Val, AFF) MRk = mr(kernel) NRk = nr(kernel) NRp = packed_b_per_k(kernel) @@ -292,8 +308,8 @@ function _execute_half_packed_nest!( descriptor_offset_range(dB, ws.k_buf_B, 0), ) end - colsA_k = _axis_of(dK_A, ws.k_buf_A, 0) - rowsB_k = _axis_of(dK_B, ws.k_buf_B, 0) + colsA_k = _axis_of(dK_A, ws.k_buf_A, 0, aff_kA) + rowsB_k = _axis_of(dK_B, ws.k_buf_B, 0, aff_kB) # HOISTED CHECK (B) -- the whole B panel of this (jc, pc), exactly # as `_execute_nest!`'s check 1. @@ -310,7 +326,7 @@ function _execute_half_packed_nest!( # Pack the whole B panel for this (jc, pc): every N-sliver. for s in 0:(n_slivers - 1) sfirst = s * NRk - colsB = _axis_of(ws.n_desc_B[s + 1], ws.n_buf_B, sfirst) + colsB = _axis_of(ws.n_desc_B[s + 1], ws.n_buf_B, sfirst, aff_nB) bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) _pack_sliver!( unsafe_pack_b!, bpanel, plan.Bstorage, plan.Bbase, rowsB_k, colsB, @@ -322,7 +338,7 @@ function _execute_half_packed_nest!( _half_packed_micro_tiles!( kernel, plan.Cstorage, plan.Cbase, dM_C, ws, plan.Astorage, arowbase, colsA_k, MRk, NRk, NRp, - n_slivers, kblock, alphaT, beta_eff + n_slivers, kblock, alphaT, beta_eff, aff_mC, aff_nC ) firstpanel = false @@ -339,17 +355,19 @@ end # the guardrail at the top of src/execution/macrokernel.jl), so the # `UnpackedAView` built below is concretely typed and the micro-tile calls # specialize on it. `unsafe_execute_micro_tile!`'s precondition is the caller's -# hoisted C check. -@inline function _half_packed_micro_tiles!( +# hoisted C check. `@noinline` for compile time, as `_micro_tiles_packed_b!` +# (src/execution/execute.jl): inlined, this nest's inference was 60 of the +# 207 s test/execution/test_halfpack.jl spends inferring; out of line, 14. +@noinline function _half_packed_micro_tiles!( kernel::K, Cstorage::SC, Cbase::Int, dM_C::BlockDescriptor, ws, Astorage::SA, arowbase::Int, colsA_k::CA, MRk::Int, NRk::Int, NRp::Int, - n_slivers::Int, kblock::Int, alphaT, beta_eff - ) where {K, SC, SA, CA <: Axis} + n_slivers::Int, kblock::Int, alphaT, beta_eff, aff_mC::Val{MC}, aff_nC::Val{NC} + ) where {K, SC, SA, CA <: Axis, MC, NC} apanel = UnpackedAView(Astorage, arowbase, colsA_k, MRk) for s in 0:(n_slivers - 1) sfirst = s * NRk - rowsC = _axis_of(dM_C, ws.m_buf_C, 0) - colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst) + rowsC = _axis_of(dM_C, ws.m_buf_C, 0, aff_mC) + colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst, aff_nC) bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) unsafe_execute_micro_tile!( kernel, Cstorage, Cbase, rowsC, colsC, diff --git a/src/execution/macrokernel.jl b/src/execution/macrokernel.jl index 0d439ac..0a3f30c 100644 --- a/src/execution/macrokernel.jl +++ b/src/execution/macrokernel.jl @@ -13,6 +13,21 @@ PtrScatterAxis(pointer(buffer, first + 1), d.count) end +# The same, with the sliver's axis type fixed statically by the execution +# path (`_NestPath`, src/execution/barrier.jl): `Val(true)` where the +# composite's map is an affine ramp, which makes every block descriptor +# `describe_block` or `_ramp_descriptor` produces from it regular, so the axis +# is always an `AffineAxis` and no `PtrScatterAxis` arm is compiled +# downstream. The `regular` test stays as a cheap guard on that invariant. +@inline _axis_of(d::BlockDescriptor, buffer::Vector{Int}, first::Int, ::Val{false}) = + _axis_of(d, buffer, first) +@inline function _axis_of(d::BlockDescriptor, ::Vector{Int}, ::Int, ::Val{true}) + d.regular || _throw_irregular_ramp_descriptor() + return AffineAxis(d.base, d.stride, d.count) +end +@noinline _throw_irregular_ramp_descriptor() = + throw(AssertionError("an affine-ramp map produced an irregular block descriptor")) + # `pack!` is pack_a!/pack_b! -- or their `unsafe_pack_a!`/`unsafe_pack_b!` # siblings (src/packing/pack.jl) -- as a plain function, specialized on, never a # closure; A and B differ only in which of rows/cols is the k axis, which the diff --git a/src/execution/outer.jl b/src/execution/outer.jl index 68f15b7..07fead5 100644 --- a/src/execution/outer.jl +++ b/src/execution/outer.jl @@ -16,20 +16,21 @@ # # Real `T` only: the complex kernels' K = 1 tiles are not the bottleneck # there (the per-call floor is), and a complex outer product would need its -# own interleaved arithmetic. Selected by `execute!` through -# `_try_execute_outer!`; `_OUTER_MODE` overrides the choice for tests and -# benchmarks. +# own interleaved arithmetic. Selected by `execute!` through `_select_path` +# when `_outer_applicable` holds; `_OUTER_MODE` overrides the choice for tests +# and benchmarks. const _OUTER_MODE = Ref{Symbol}(:auto) """ - _try_execute_outer!(plan::ContractPlan, alphaT, betaT, Qm, Qn) -> Bool + _outer_applicable(plan::ContractPlan, Qm) -> Bool -Run the outer-product path if `plan` is eligible and return `true`; return -`false` without touching anything otherwise. Eligible when `_OUTER_MODE[]` is -not `:never`, `T` is real, `Qk == 1` (the caller checks), the M composite is +Whether `execute!` takes the outer-product path on a `Qk == 1` plan +(`_select_path`, src/execution/execute.jl, which checks `Qk` and then runs it +as `_OuterPath{W}` with `W = _dot_lanewidth(T)`). Eligible when +`_OUTER_MODE[]` is not `:never`, `T` is real, the M composite is the unit ramp `offset(m) = m` in BOTH its A and C maps (`_map_ramp_step`), -`Qm >= W` for the lane width `W = _dot_lanewidth(T)`, and A and C are +`Qm >= W`, and A and C are `DenseVector{T}` (raw-pointer vector loads/stores). N and B may have any layout: B is read one scalar per column through the N composite's offsets. @@ -41,32 +42,18 @@ same data: 16x16 319 -> 227 ns (84), 63x63 2.40 -> 1.16 us (0.81), 128x128 6.15 -> 2.70 us (3.07). ComplexF64 is not taken: the nest is already within ~10% of OpenBLAS there from 63x63 up (4.6 vs 4.2 us, 15.1 vs 15.6 us). """ -function _try_execute_outer!( - plan::ContractPlan{T}, alphaT::T, betaT::T, Qm::Int, Qn::Int - ) where {T} - _outer_applicable(plan, Qm) || return false - W = _dot_lanewidth(T) - if W == 16 - _execute_outer!(plan, alphaT, betaT, Qm, Qn, Val(16)) - elseif W == 8 - _execute_outer!(plan, alphaT, betaT, Qm, Qn, Val(8)) - elseif W == 4 - _execute_outer!(plan, alphaT, betaT, Qm, Qn, Val(4)) - else - _execute_outer!(plan, alphaT, betaT, Qm, Qn, Val(2)) - end - return true -end +_outer_applicable(plan::ContractPlan{T}, Qm::Int) where {T} = + _outer_applicable(T, plan.Astorage, plan.Cstorage, plan.mgroup, Qm) -# The eligibility rule of `_try_execute_outer!` (`Qk == 1` is the caller's), -# as a side-effect-free predicate. -function _outer_applicable(plan::ContractPlan{T}, Qm::Int) where {T} +# The rule on the plan's parts, so that `plan_contract` can predict the path +# before the plan exists (`_path_hint`). +function _outer_applicable(::Type{T}, Astorage, Cstorage, mgroup::AxisGroup, Qm::Int) where {T} _OUTER_MODE[] === :never && return false T <: Real || return false - (plan.Astorage isa DenseVector{T} && plan.Cstorage isa DenseVector{T}) || return false + (Astorage isa DenseVector{T} && Cstorage isa DenseVector{T}) || return false Qm >= _dot_lanewidth(T) || return false - _map_ramp_step(plan.mgroup, 1) == 1 || return false - return _map_ramp_step(plan.mgroup, 2) == 1 + _map_ramp_step(mgroup, 1) == 1 || return false + return _map_ramp_step(mgroup, 2) == 1 end function _execute_outer!( diff --git a/src/execution/unpackedb.jl b/src/execution/unpackedb.jl index 84f4fd4..97c79f9 100644 --- a/src/execution/unpackedb.jl +++ b/src/execution/unpackedb.jl @@ -70,8 +70,14 @@ end @inline _unpacked_b_kernel_eligible(::FMAddSubKernel) = true @inline _unpacked_b_kernel_eligible(::Any) = false +# The same, for the kernel an automatic `(shape, method)` choice builds +# (`_kernel_type`: `SIMDKernel`, `PlanarKernel`, `FMAddSubKernel`, `OneMKernel`), +# as `plan_contract` needs it to predict the path before the kernel exists. +@inline _unpacked_b_method_eligible(::Union{RealMethod, PlanarMethod, FMAddSubMethod}) = true +@inline _unpacked_b_method_eligible(::Any) = false + # Test/benchmark override of the automatic rule: `:auto` (the rule below), -# `:always`, `:never`. Read once per `execute!`. +# `:always`, `:never`. Read once per `execute!` (by `_select_path`). const _UNPACKED_B_MODE = Ref{Symbol}(:auto) """ @@ -110,13 +116,17 @@ with a measured win on both dtypes; 512 is a wash. The strided-K wins at small K are left on the table deliberately: the same rule would have to model L1 set conflicts to avoid the 1.2-2.1x losses above. """ -@inline function _use_unpacked_b(plan::ContractPlan) - _unpacked_b_kernel_eligible(plan.kernel) || return false +@inline _use_unpacked_b(plan::ContractPlan) = + _unpacked_b_kernel_eligible(plan.kernel) && _unpacked_b_rule(plan.mgroup, plan.kgroup) + +# Everything but the kernel's eligibility, on the plan's groups (`_select_path` +# also applies it before the plan exists, src/execution/execute.jl). +@inline function _unpacked_b_rule(mgroup::AxisGroup, kgroup::AxisGroup) mode = _UNPACKED_B_MODE[] mode === :always && return true mode === :never && return false - axis_length(plan.mgroup) <= _UNPACKED_B_MMAX || return false - (k_ramp, k_step) = affine_ramp(plan.kgroup) + axis_length(mgroup) <= _UNPACKED_B_MMAX || return false + (k_ramp, k_step) = affine_ramp(kgroup) return k_ramp && abs(k_step[2]) == 1 end @@ -161,20 +171,21 @@ end # the element loads are `@inbounds`. C is covered by the caller's check 3. @noinline function _micro_tiles_unpacked_b!( kernel::K, plan::ContractPlan, ws, rowsB_k::KA, m_slivers::Int, n_slivers::Int, - MRk::Int, NRk::Int, MRp::Int, kblock::Int, alphaT, beta_eff - ) where {K, KA <: Axis} + MRk::Int, NRk::Int, MRp::Int, kblock::Int, alphaT, beta_eff, + aff_mC::Val{MC}, aff_nC::Val{NC} + ) where {K, KA <: Axis, MC, NC} Bstorage = plan.Bstorage Bbase = plan.Bbase btransform = plan.btransform for s in 0:(n_slivers - 1) sfirst = s * NRk - colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst) + colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst, aff_nC) bview = _unpacked_b_view( kernel, Bstorage, Bbase, ws.n_desc_B[s + 1], ws.n_buf_B, sfirst, rowsB_k, btransform ) for r in 0:(m_slivers - 1) rfirst = r * MRk - rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst) + rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst, aff_mC) apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) unsafe_execute_micro_tile!( kernel, plan.Cstorage, plan.Cbase, rowsC, colsC, diff --git a/src/execution/workspace.jl b/src/execution/workspace.jl index 7ff3c66..d5aec30 100644 --- a/src/execution/workspace.jl +++ b/src/execution/workspace.jl @@ -5,6 +5,20 @@ # `Vector{Int}` -- acquired as non-temporaries, so only the packed panels # route through the allocator. +# The workspace's typed slots for `_barrier_slot!` (src/execution/barrier.jl): +# one `Base.RefValue{P}` per payload type `P` ever sent across a dispatch +# barrier with this workspace, plus the most recently used one, so that the +# common case -- the same contraction type as last time -- is one type-tag +# compare. Mutable (and heap-allocated once, with the workspace) because the +# slots must outlive every call: a payload is written into its slot and read +# back on the far side of a dynamic call, which is what lets that call pass +# only already-boxed arguments and allocate nothing. +mutable struct _SlotCache + last::Any + const slots::IdDict{Any, Any} +end +_SlotCache() = _SlotCache(nothing, IdDict{Any, Any}()) + """ ContractWorkspace{T,VT<:AbstractVector} @@ -33,44 +47,58 @@ are allocated only under `oracle = true`. The four register-tile-sized `tile_*` offset buffers are always allocated: the beta-only pass of both drivers uses them, and so does the oracle's tile loop. +A `mutable struct` with every field `const`: nothing is ever reassigned +(buffers grow in place, `reserve!`), but a workspace must be ONE heap object, +so that a plan holds a single pointer to it and a dispatch barrier +(src/execution/barrier.jl) passes or stores it without copying its twenty-one +fields: every copy of an immutable workspace -- into a plan, through a +barrier slot, out of the task pool's `Dict` -- was twenty-one GC-tracked +pointer copies. Measured through `tensorcontract!` (ccqlin038, Julia 1.12.7, +pooled workspace, 8x8x8): Float64 0.47 -> 0.38 us, ComplexF64 0.54 -> 0.45 +us per call. + Field layout is an implementation detail, not part of the public interface. """ -struct ContractWorkspace{T, VT <: AbstractVector} +mutable struct ContractWorkspace{T, VT <: AbstractVector} # Macro-block-sized offset buffers: one fill_offsets! per jc/pc/ic block, # reused by every sliver inside it. - m_buf_A::Vector{Int} - m_buf_C::Vector{Int} - n_buf_B::Vector{Int} - n_buf_C::Vector{Int} - k_buf_A::Vector{Int} - k_buf_B::Vector{Int} + const m_buf_A::Vector{Int} + const m_buf_C::Vector{Int} + const n_buf_B::Vector{Int} + const n_buf_C::Vector{Int} + const k_buf_A::Vector{Int} + const k_buf_B::Vector{Int} # Per-sliver descriptors classified from the buffers above (3-arg # describe_block), reused across the pc/ic loops of a given jc/ic. - m_desc_A::Vector{BlockDescriptor} - m_desc_C::Vector{BlockDescriptor} - n_desc_B::Vector{BlockDescriptor} - n_desc_C::Vector{BlockDescriptor} + const m_desc_A::Vector{BlockDescriptor} + const m_desc_C::Vector{BlockDescriptor} + const n_desc_B::Vector{BlockDescriptor} + const n_desc_C::Vector{BlockDescriptor} # Packed macro-panel buffers: cld(mc,MRk)/cld(nc,NRk) slivers at kc-eff # depth. The ONLY allocator-routed temporaries in this struct. - packed_a::VT - packed_b::VT + const packed_a::VT + const packed_b::VT # One register tile's offsets (MR/NR-sized): the beta-only pass and the # oracle's tile loop. - tile_m_buf_A::Vector{Int} - tile_m_buf_C::Vector{Int} - tile_n_buf_B::Vector{Int} - tile_n_buf_C::Vector{Int} + const tile_m_buf_A::Vector{Int} + const tile_m_buf_C::Vector{Int} + const tile_n_buf_B::Vector{Int} + const tile_n_buf_C::Vector{Int} # execute_tilewise!'s own K-panel buffers (blocking.kc-sized). Deliberately # NOT shared with the macro buffers above: the oracle must have no packed # or K-offset state in common with the code it checks. - tw_k_buf_A::Vector{Int} - tw_k_buf_B::Vector{Int} - tw_packed_a::VT - tw_packed_b::VT + const tw_k_buf_A::Vector{Int} + const tw_k_buf_B::Vector{Int} + const tw_packed_a::VT + const tw_packed_b::VT + + # Typed payload slots for the dispatch barriers (`_barrier_slot!`, + # src/execution/barrier.jl). Not a buffer: never sized, grown or released. + const slots::_SlotCache # GUARDRAIL, the one invariant the loose `VT` bound needs: the packed # panels hold `real(T)`, NEVER `T`. Enforced here so a mis-paired (T, VT) @@ -98,7 +126,7 @@ struct ContractWorkspace{T, VT <: AbstractVector} m_desc_A, m_desc_C, n_desc_B, n_desc_C, packed_a, packed_b, tile_m_buf_A, tile_m_buf_C, tile_n_buf_B, tile_n_buf_C, - tw_k_buf_A, tw_k_buf_B, tw_packed_a, tw_packed_b, + tw_k_buf_A, tw_k_buf_B, tw_packed_a, tw_packed_b, _SlotCache(), ) end end diff --git a/src/integrations/tensoroperations.jl b/src/integrations/tensoroperations.jl index 8d4c5de..5a467a1 100644 --- a/src/integrations/tensoroperations.jl +++ b/src/integrations/tensoroperations.jl @@ -315,36 +315,27 @@ function TO.tensorcontract!( ) Cv, Av, Bv, indA, indB, indC, α′, β′ = _qs_prepare(C, A, pA, B, pB, pAB, α, β) # `plan_contract`'s body with `execute!` as the continuation (`_planned`, - # src/planning/plan.jl): the plan is built and run inside the - # `_plan_contract` barrier, where its type is concrete, so - # `execute!(plan, ...)` is a static call and the plan never escapes this - # frame -- `plan = plan_contract(...); execute!(plan, ...)` would instead - # dispatch `execute!` at runtime on a Union-typed plan and box it (the - # kernel choice is a runtime value; for a complex `T` its Union spans the - # whole menu). Measured on ccqlin038 (Julia 1.12.7, 8x8x8, pooled - # workspace): 1232 B and 1.1 us per Float64 call, 2032 B and 5.5 us per - # ComplexF64 call before this and the planning-side changes it is paired - # with (statically sized label tuples, the per-(profile, eltype) defaults - # cache and the static kernel ladder); 0 B and ~0.5 / ~0.7 us after. + # src/planning/plan.jl, and `_Execute`, src/execution/execute.jl): the + # plan is built and run on the far side of the kernel barrier, where its + # type is concrete and the execution path is already fixed, so the plan + # never escapes that frame and nothing is boxed -- `plan = + # plan_contract(...); execute!(plan, ...)` would instead return a plan of + # runtime type and box it. Measured on ccqlin038 (Julia 1.12.7, 8x8x8, + # pooled workspace): 1232 B and 1.1 us per Float64 call, 2032 B and 5.5 + # us per ComplexF64 call before this and the planning-side changes it is + # paired with (statically sized label tuples, the per-(profile, eltype) + # defaults cache); 0 B and ~0.4 / ~0.45 us with the dispatch barriers + # (src/execution/barrier.jl) and the heap-allocated workspace. # `_planned` `reserve!`s the pooled workspace itself. oracle=false: the # backend path never needs `execute_tilewise!`'s buffers. _planned( - _QSExecute(α′, β′), Cv, Av, indA, Bv, indB, indC, + _Execute(α′, β′), Cv, Av, indA, Bv, indB, indC, nothing, conjA, conjB, nothing, nothing, nothing, _qs_task_workspace(eltype(C)), allocator, false ) return C end -# The continuation the method above hands `_planned`: `execute!` on the plan, -# with `alpha`/`beta` already converted to `eltype(C)`. A callable struct -# rather than a closure so the captured scalars are concretely typed fields. -struct _QSExecute{T} - alpha::T - beta::T -end -(e::_QSExecute)(plan::ContractPlan) = (execute!(plan, e.alpha, e.beta); nothing) - # NOT merged with the method above, although the two differ only in allocator # handling: this one needs the plan back after `execute!` (to `release!` its # allocator-owned workspace in `finally`), so it goes through `plan_contract` diff --git a/src/planning/defaults.jl b/src/planning/defaults.jl index 283247b..81f74e7 100644 --- a/src/planning/defaults.jl +++ b/src/planning/defaults.jl @@ -123,15 +123,19 @@ _default_kernel(::Type{T}) where {T} = # `_kernel_from_shape` builds at `shape`. # # Returns the `(shape, method)` pair as plain values rather than the kernel: -# `plan_contract` consumes it through `_with_menu_kernel`, so the choice is -# made without ever holding a menu-wide kernel Union (ten members for -# ComplexF64), which its function barrier would otherwise have to dispatch on -# at runtime. `_default_kernel` below builds the kernel from the same pair. +# `plan_contract` carries the shape across its kernel barrier as a singleton +# `Val` (`_plan_with_kernel`, src/planning/plan.jl), so the choice is made +# without ever holding a menu-wide kernel Union (ten members for ComplexF64). +# `_default_kernel` below builds the kernel from the same pair. @inline function _default_shape(::Type{T}, Qm::Int, Qn::Int) where {T} d = _resolved_defaults(T) method = _default_method(T) shape = _extent_shape(d.shape, T, method, Qm) (Qm > 0 && Qm < shape[1]) || return (shape, method) + # The FMAddSub small-M demotion is complex-only (`_small_m_candidates` is + # empty for a real `T`); saying so statically keeps a real `T`'s method a + # concrete `RealMethod` rather than a Union with `FMAddSubMethod`. + T <: Complex || return (d.fitted, method) small = _small_m_shape(d.small_m, Qm) small === nothing || return (small, FMAddSubMethod()) return (d.fitted, method) diff --git a/src/planning/kernel_selection.jl b/src/planning/kernel_selection.jl index 0a4afc8..26f677f 100644 --- a/src/planning/kernel_selection.jl +++ b/src/planning/kernel_selection.jl @@ -174,27 +174,22 @@ _kernel_from_shape(shape::Tuple{Int, Int, Int}, ::Type{T}) where {T} = end """ - _with_menu_kernel(g, shape, T, method, args...) -> g(kernel, args...) - -`_kernel_from_shape`'s continuation-passing twin: build the `method` kernel -for `T` at `shape` and hand it to `g` together with `args...`, so that `g` -runs on a CONCRETE kernel type in every arm of the unrolled menu ladder. -`_kernel_from_shape`'s return type is the Union of the whole menu, which a -consumer has to dispatch on at runtime once the Union exceeds four members -(ComplexF64's does: six planar plus four FMAddSub shapes); here the dispatch -stays a chain of literal tuple compares and every `g` call is static. Same -menu, same constructor and the same two throws as `_kernel_from_shape`. This -is how `plan_contract` reaches its function barrier when the caller named no -kernel; see `_plan_with_kernel` in src/planning/plan.jl for the measurement. + _menu_val(shape, T, method) -> Val{shape}() + +`Val(shape)` for a `shape` in `kernel_shapes(T, method)`, built as a literal +in an unrolled ladder over the menu (a plain `Val(shape)` would construct the +type at runtime), with the same two throws as `_kernel_from_shape`. The +result is a singleton, so it crosses `plan_contract`'s kernel barrier +(`_plan_with_kernel`, src/planning/plan.jl) without boxing, and the callee is +specialised on this one shape; building only menu shapes keeps that set of +specialisations closed. """ -@generated function _with_menu_kernel( - g::G, shape::Tuple{Int, Int, Int}, ::Type{T}, method::M, args... - ) where {G, T, M} +@generated function _menu_val(shape::Tuple{Int, Int, Int}, ::Type{T}, method::M) where {T, M} K = _kernel_type(M.instance, T) K === nothing && return :(_throw_no_kernel(shape, T, method)) ex = :(_throw_shape_not_in_menu(shape, T, method)) for (MR, NR, W) in reverse(kernel_shapes(T, M.instance)) - ex = :(shape === ($MR, $NR, $W) ? g($K(Val($MR), Val($NR), T, Val($W)), args...) : $ex) + ex = :(shape === ($MR, $NR, $W) ? Val(($MR, $NR, $W)) : $ex) end return ex end @@ -533,14 +528,31 @@ _small_m_shape(key::Val, profile::TargetProfile, ::Type{T}, Qm::Int) where {T} = # short-circuit holds the fraction is `1.0` anyway, so the order changes no # decision. function _demote_for_run(::Type{T}, kernel, run::Int, Qm::Int, Qk::Int) where {T} - T <: Real || return kernel - kmax = T === Float64 ? _RUN_DEMOTE_KMAX_F64 : _RUN_DEMOTE_KMAX_F32 - Qk > kmax && return kernel - Qm == run && return kernel - run % mr(kernel) == 0 && return kernel - _RUN_DEMOTE_BROKEN_ENOUGH < 1.0 && _unbroken_fraction(Qm, run, mr(kernel)) > _RUN_DEMOTE_BROKEN_ENOUGH && - return kernel method = complex_method(kernel) + target = _run_demotion_target(T, mr(kernel), method, run, Qm, Qk) + return target === nothing ? kernel : _kernel_from_shape(target, T, method) +end + +# The same rule on a `(shape, method)` pair, as `plan_contract` applies it +# before its kernel barrier (`_plan_with_kernel`, src/planning/plan.jl): the +# kernel built from `shape` has `mr == shape[1]` for every method. +@inline function _demote_shape_for_run( + ::Type{T}, shape::NTuple{3, Int}, method, run::Int, Qm::Int, Qk::Int + ) where {T} + target = _run_demotion_target(T, shape[1], method, run, Qm, Qk) + return target === nothing ? shape : target +end + +# The menu shape `_demote_for_run` demotes a kernel of register height `mrk` +# (under `method`) to, or `nothing` to keep it. +function _run_demotion_target(::Type{T}, mrk::Int, method, run::Int, Qm::Int, Qk::Int) where {T} + T <: Real || return nothing + kmax = T === Float64 ? _RUN_DEMOTE_KMAX_F64 : _RUN_DEMOTE_KMAX_F32 + Qk > kmax && return nothing + Qm == run && return nothing + run % mrk == 0 && return nothing + _RUN_DEMOTE_BROKEN_ENOUGH < 1.0 && _unbroken_fraction(Qm, run, mrk) > _RUN_DEMOTE_BROKEN_ENOUGH && + return nothing best = nothing for shape in kernel_shapes(T, method) m = shape[1] @@ -548,7 +560,7 @@ function _demote_for_run(::Type{T}, kernel, run::Int, Qm::Int, Qk::Int) where {T best = shape end end - return best === nothing ? kernel : _kernel_from_shape(best, T, method) + return best end # Deepest `Qk` at which run-length demotion still wins: it crosses over from a diff --git a/src/planning/plan.jl b/src/planning/plan.jl index 7a7d6ec..407a994 100644 --- a/src/planning/plan.jl +++ b/src/planning/plan.jl @@ -144,9 +144,10 @@ end # fixed by the label tuples' lengths, so `GM`/`GN`/`GK` are concrete), the # operand storage/base pairs, the leading unit-stride run of the chosen M # composite (for `_demote_for_run`), the blocking overrides and the -# workspace/allocator choice. One value, so the kernel-resolution ladder -# (`_plan_with_kernel`) forwards a single argument, and so the swapped and -# as-is orientations differ only in how it is filled. +# workspace/allocator choice. One value, so kernel resolution +# (`_plan_with_kernel`) forwards a single argument -- and, with its storages +# stripped, sends it through the kernel barrier's slot -- and so the swapped +# and as-is orientations differ only in how it is filled. # # `T` is a phantom parameter (the storage element type, `eltype(C)`), carried # so the barrier can check the kernel against it without re-deriving it. @@ -301,46 +302,93 @@ end # Kernel resolution, the last step before the barrier. A caller-named kernel # goes straight through, never demoted. An automatic one is chosen as a -# `(shape, method)` value (`_default_shape`) and built INSIDE -# `_with_menu_kernel`'s unrolled ladder, so that `_plan_demoted` -- and -# through it `_demote_for_run` and `_plan_contract` -- always runs on a -# concrete kernel type and every call below this line is static. +# `(shape, method)` value -- `_default_shape`, then the run-length demotion +# (`_demote_shape_for_run`) -- and the plan is built on the far side of a +# dispatch barrier (src/execution/barrier.jl) specialised on that ONE shape, +# so that only the chosen kernel's planning and execution code is ever +# compiled. # -# Why not simply `_plan_contract(_default_kernel(T, Qm, Qn), ...)`: that -# value's type is the Union of `T`'s whole menu, ten members for ComplexF64, -# past the four Julia union-splits, so the barrier call became a -# `jl_apply_generic` that boxed the request and -- for reasons this file does -# not claim to understand -- missed the method cache more often than not. -# Measured on ccqlin038 (Julia 1.12.7, 8x8x8 ComplexF64, pooled workspace): -# `plan_contract` 0.78 us and 704 B/call with the Union-typed call; the same -# tail with a concrete kernel 0.09 us and 0 B. Float64 was already static -# (a two-member Union) and is unchanged by this. +# Two earlier forms, and why neither: +# +# * `_plan_contract(_default_kernel(T, Qm, Qn), ...)`: the value's type is +# the Union of `T`'s whole menu, ten members for ComplexF64, so the call +# was a `jl_apply_generic` that boxed the request (0.78 us and 704 B per +# call, ccqlin038, Julia 1.12.7, 8x8x8 ComplexF64). +# * An unrolled ladder over the menu with a static, concrete call in every +# arm (0 B, ~0.1 us), which compiles EVERY arm the first time: 40 plans +# (10 kernels x 4 transform pairs) and 85 s of inference for the first +# ComplexF64 `@tensor` call, 6 plans and 26 s for Float64 +# (benchmark/probes/probe_ttfx.jl, SnoopCompile). +# +# The barrier is a dynamic call whose arguments are all singletons (`Val` of +# the shape, the method, the transforms, the execution-path hint) or heap +# objects (the slot, the storages): 0 B, one method-cache hit per call. @inline _plan_with_kernel(kernel, atransform, btransform, req::_PlanRequest) = - _plan_contract(kernel, atransform, btransform, req) + _plan_contract(kernel, atransform, btransform, req, nothing) @inline function _plan_with_kernel(::Nothing, atransform, btransform, req::_PlanRequest{T}) where {T} - shape, method = _default_shape(T, axis_length(req.mgroup), axis_length(req.ngroup)) - return _with_menu_kernel(_plan_demoted, shape, T, method, atransform, btransform, req) + Qm = axis_length(req.mgroup) + shape, method = _default_shape(T, Qm, axis_length(req.ngroup)) + shape = _demote_shape_for_run(T, shape, method, req.run, Qm, axis_length(req.kgroup)) + # Built from the menu (throwing for a shape outside it), so the callee is + # only ever specialised on a menu shape. + vshape = _menu_val(shape, T, method) + # Which execution path the continuation will take, predicted here so the + # callee is specialised on it too (`_path_hint`; `nothing` when the + # continuation does not execute). Only the method's unpacked-B + # eligibility is passed, as a `Bool`: `method` may be a Union + # (`PlanarMethod`/`FMAddSubMethod`, or with no `T <: Complex` guard + # upstream even for a real `T`), and a call union-split on it was + # measured to be emitted out of line, boxing `req` (256 B per call). + hint = _path_hint(req.f, req, _unpacked_b_method_eligible(method)) + core = _strip_storage(req) + slot = _barrier_slot!(req.workspace, typeof(core)) + slot[] = core + return Base.inferencebarrier(_plan_resolved)( + vshape, method, atransform, btransform, hint, slot, + req.Astorage, req.Bstorage, req.Cstorage + ) end -# Run-length demotion (`_demote_for_run`) of an automatically chosen kernel, -# keyed on the chosen M orientation's own run (`req.run`), then the barrier. -@inline function _plan_demoted(kernel, atransform, btransform, req::_PlanRequest{T}) where {T} - kernel_final = _demote_for_run( - T, kernel, req.run, axis_length(req.mgroup), axis_length(req.kgroup) - ) - return _plan_contract(kernel_final, atransform, btransform, req) +# The far side of the kernel barrier: everything from here down is static, on +# one concrete kernel, transform pair and path hint. +function _plan_resolved( + ::Val{S}, method::M, atransform::TA, btransform::TB, hint::H, + slot::Base.RefValue{R}, Astorage::SA, Bstorage::SB, Cstorage::SC + ) where {S, M, TA, TB, H, T, R <: _PlanRequest{T}, SA, SB, SC} + req = _with_storage(slot[], Astorage, Bstorage, Cstorage) + return _plan_contract(_kernel_from_shape(S, T, method), atransform, btransform, req, hint) end -# Function barrier: the `conj`/`identity` transform Unions die here (and, on -# the real path, `_demote_for_run`'s small kernel Union), so `ContractPlan`'s -# `Kern`, `TA` and `TB` are concrete and `execute!` sees no abstract type. -# `TA`/`TB` each get their own bound parameter for the same reason `K` does. -# The continuation `req.f` runs in here rather than on the returned plan: -# `f(plan)` sees a concrete plan type, so an executor passed as `f` needs no -# further dispatch and no boxed plan. +# The request without its three operand storages (`nothing` in their place), +# which is what crosses the barrier in the slot; the storages cross as +# arguments, so no slot ever retains a user array. +@inline _strip_storage(req::_PlanRequest{T}) where {T} = _plan_request( + T, req.f, req.mgroup, req.ngroup, req.kgroup, + nothing, req.Abase, nothing, req.Bbase, nothing, req.Cbase, + req.run, req.mc, req.kc, req.nc, req.workspace, req.allocator, req.oracle +) +@inline _with_storage(req::_PlanRequest{T}, Astorage, Bstorage, Cstorage) where {T} = _plan_request( + T, req.f, req.mgroup, req.ngroup, req.kgroup, + Astorage, req.Abase, Bstorage, req.Bbase, Cstorage, req.Cbase, + req.run, req.mc, req.kc, req.nc, req.workspace, req.allocator, req.oracle +) + +# The execution path a continuation will take on the plan (src/execution/ +# barrier.jl), or `nothing` for one that does not execute it. Only +# `_Execute` (src/execution/execute.jl) executes. +@inline _path_hint(f, req::_PlanRequest, unpack_ok::Bool) = nothing + +# Plan construction, on a concrete kernel and transform pair: reached through +# the kernel barrier above for an automatic kernel, and directly for a +# caller-named one (whose transform Unions then die here, in a function +# barrier), so `ContractPlan`'s `Kern`, `TA` and `TB` are concrete and +# `execute!` sees no abstract type. `TA`/`TB` each get their own bound +# parameter for the same reason `K` does. The continuation `req.f` runs in +# here rather than on the returned plan: `f(plan)` sees a concrete plan type, +# so an executor passed as `f` needs no further dispatch and no boxed plan. function _plan_contract( - kernel::K, atransform::TA, btransform::TB, req::_PlanRequest{T} - ) where {K, TA, TB, T} + kernel::K, atransform::TA, btransform::TB, req::_PlanRequest{T}, hint::H + ) where {K, TA, TB, T, H} scalartype(kernel) === T || throw(ArgumentError("kernel scalar type $(scalartype(kernel)) does not match eltype(C) = $T")) @@ -380,5 +428,9 @@ function _plan_contract( req.Astorage, req.Abase, req.Bstorage, req.Bbase, req.Cstorage, req.Cbase, atransform, btransform, ws ) - return req.f(plan) + return _continue(req.f, plan, hint) end + +# Apply the continuation. `hint` is `_path_hint`'s prediction, used only by +# an executing continuation (`_Execute`, src/execution/execute.jl). +@inline _continue(f::F, plan::ContractPlan, hint) where {F} = f(plan) diff --git a/test/execution/test_dot.jl b/test/execution/test_dot.jl index a4dd0bd..9704862 100644 --- a/test/execution/test_dot.jl +++ b/test/execution/test_dot.jl @@ -2,7 +2,7 @@ # `M == 1` or `N == 1` whose matrix operand is K-contiguous runs a K-vectorized # gemv instead of the five-loop nest. Checked here: # -# 1. `_try_execute_dot!` takes exactly the eligible plans (degenerate free +# 1. the dot path takes exactly the eligible plans (degenerate free # extent, unit-ramp K on the matrix operand, `DenseVector` storage, # `Qk >= W`), declines the rest without touching C, and obeys `_DOT_MODE`. # 2. On eligible plans the result matches the nest (`_DOT_MODE[] = :never`), diff --git a/test/execution/test_outer.jl b/test/execution/test_outer.jl index b5ad046..3230854 100644 --- a/test/execution/test_outer.jl +++ b/test/execution/test_outer.jl @@ -2,7 +2,7 @@ # with `K == 1` and unit-stride M in A and C streams `alpha * A * B[n]` column # by column instead of running the nest. Checked here: # -# 1. `_try_execute_outer!` takes exactly the eligible plans (real `T`, +# 1. the outer-product path takes exactly the eligible plans (real `T`, # `K == 1`, unit-ramp M in both A and C, `Qm >= W`, dense A/C storage), # declines the rest untouched, and obeys `_OUTER_MODE`. # 2. On eligible plans the result matches the nest (`_OUTER_MODE[] = diff --git a/test/execution/test_workspace.jl b/test/execution/test_workspace.jl index 1947f36..6ce9551 100644 --- a/test/execution/test_workspace.jl +++ b/test/execution/test_workspace.jl @@ -3,7 +3,11 @@ # SIMDKernel default. # ===================================================================== -_ws_lengths(ws) = map(f -> length(getfield(ws, f)), fieldnames(typeof(ws))) +# Buffer fields only: the workspace also holds its dispatch-barrier slot cache, +# which is not a buffer and has no length. +_ws_lengths(ws) = [ + length(getfield(ws, f)) for f in fieldnames(typeof(ws)) if getfield(ws, f) isa AbstractVector +] @testset "plan_contract: SIMDKernel is the engine-wide default kernel" begin for T in (Float64, Float32) From e7e341ed5e4ef36eceb4be2a5c9fb82d01d92906 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 18:59:04 -0400 Subject: [PATCH 18/53] Fix the three test-suite failures on Julia 1.10 integration/perf (and the previous commit, unchanged) failed 62 tests and errored one on Julia 1.10.12, all in the tests themselves: * test_plan_contract.jl: `sort(::Tuple)` is Julia 1.11+; sort a collected vector (the default sort is stable) and rebuild the tuple. * test_unpackedb.jl: the steady-state allocation check is skipped below 1.11, and a skipped `@test` does not evaluate its expression, so the product check after it saw an unwritten C. Measure outside the `@test`. * test_plan_contract.jl (K order, 32 B/call) and test_fmaddsub_store_fastpath.jl (FMAddSub store, 80 B/call): allocation assertions that allocate on 1.10; skipped below 1.11 like the suite's other allocation assertions. Runic also re-indented one `for` header in test_plan_contract.jl. Pkg.test(), ccqlin038: Julia 1.10.12 184058 pass / 183 broken (skipped) in 11.8 min; Julia 1.12.7 189763 pass in 15.1 min. Co-Authored-By: Claude Opus 5.5 --- test/execution/test_unpackedb.jl | 5 ++++- .../test_fmaddsub_store_fastpath.jl | 4 +++- test/planning/test_plan_contract.jl | 21 +++++++++++-------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/test/execution/test_unpackedb.jl b/test/execution/test_unpackedb.jl index 605d909..ef1e57d 100644 --- a/test/execution/test_unpackedb.jl +++ b/test/execution/test_unpackedb.jl @@ -270,7 +270,10 @@ end plan = _mm_plan(Cmat, randn(T, 40, 50), randn(T, 50, 37); kc = 16, nc = 12) @test _ub_use(plan) _ub_mode[] = :always - @test _steady_allocs!(execute!, plan, Cmat) == 0 skip = (VERSION < v"1.11") + # Measured outside the `@test`: a skipped `@test` does not evaluate its + # expression, and the product check below needs the last run's result. + allocs = _steady_allocs!(execute!, plan, Cmat) + @test allocs == 0 skip = (VERSION < v"1.11") _ub_mode[] = :auto @test Cmat ≈ reshape(plan.Astorage, 40, 50) * reshape(plan.Bstorage, 50, 37) end diff --git a/test/microkernels/test_fmaddsub_store_fastpath.jl b/test/microkernels/test_fmaddsub_store_fastpath.jl index eabfa51..126fe0e 100644 --- a/test/microkernels/test_fmaddsub_store_fastpath.jl +++ b/test/microkernels/test_fmaddsub_store_fastpath.jl @@ -208,7 +208,9 @@ end d = DestinationTile(storage, 0, AffineAxis(0, 1, MR), AffineAxis(0, MR, NR)) for (alpha, beta) in fsfp_ab(T) store_tile!(d, acc, alpha, beta, k) - @test (@allocated store_tile!(d, acc, alpha, beta, k)) == 0 + # Julia 1.10 allocates here (80 B per call, measured); skipped + # below 1.11 like the suite's other allocation assertions. + @test (@allocated store_tile!(d, acc, alpha, beta, k)) == 0 skip = (VERSION < v"1.11") end end end diff --git a/test/planning/test_plan_contract.jl b/test/planning/test_plan_contract.jl index 4c849f1..057e206 100644 --- a/test/planning/test_plan_contract.jl +++ b/test/planning/test_plan_contract.jl @@ -412,8 +412,9 @@ end # Expected: each composite ascending in |C-stride|; the same set of # labels as `_classify_labels` produced. - msorted = sort(mlab; by = cstride) - nsorted = sort(nlab; by = cstride) + # (`sort` of a Tuple needs Julia 1.11; the default sort is stable.) + msorted = Tuple(sort(collect(mlab); by = cstride)) + nsorted = Tuple(sort(collect(nlab); by = cstride)) @test _lo_order(mlab, indC, Cv) == msorted @test _lo_order(nlab, indC, Cv) == nsorted mrun = _lo_run(msorted, indC, Cv) @@ -893,8 +894,10 @@ end @test @inferred(f(0)) === (4, 3, 2) @test @inferred(g()) isa NTuple{3, Int} f(0); g() - @test (@allocated f(0)) == 0 - @test (@allocated g()) == 0 + # Julia 1.10 allocates here (32 B per call, measured); skipped below 1.11 + # like the suite's other allocation assertions. + @test (@allocated f(0)) == 0 skip = (VERSION < v"1.11") + @test (@allocated g()) == 0 skip = (VERSION < v"1.11") # Zero and one K label: returned as given, without the model. h1() = QuasiStrided._order_contract_labels((2,), (1, 2), Av, (1,), (2, 5), Bs, (5,), 9, 40) h0() = QuasiStrided._order_contract_labels((), (1,), Av, (1,), (5,), Bs, (5,), 9, 40) @@ -1019,11 +1022,11 @@ end alpha = T <: Complex ? T(1.3, -0.4) : T(1.3) beta = T <: Complex ? T(0.7, 0.2) : T(0.7) for (indA, indB, indC) in ( - ((1, 2, 3, 4), (4, 3, 2, 5), (1, 5)), # scrambled, C[a,e] (reordered K) - ((1, 2, 3, 4), (4, 3, 2, 5), (5, 1)), # scrambled, C[e,a] (swap for real T) - ((4, 3, 2, 1), (2, 3, 4, 5), (1, 5)), # A transposed, K inner in both - ((4, 3, 2, 1), (3, 2, 4, 5), (5, 1)), # both scrambled, C transposed - ), (conjA, conjB) in ((false, false), (true, false), (true, true)) + ((1, 2, 3, 4), (4, 3, 2, 5), (1, 5)), # scrambled, C[a,e] (reordered K) + ((1, 2, 3, 4), (4, 3, 2, 5), (5, 1)), # scrambled, C[e,a] (swap for real T) + ((4, 3, 2, 1), (2, 3, 4, 5), (1, 5)), # A transposed, K inner in both + ((4, 3, 2, 1), (3, 2, 4, 5), (5, 1)), # both scrambled, C transposed + ), (conjA, conjB) in ((false, false), (true, false), (true, true)) (T <: Real) && conjA && continue A = _ko_array(T, indA, ext) B = _ko_array(T, indB, ext) From e44aba5fd1bab1456994d95a37d3111a9d4fd1a5 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Sun, 27 Sep 2026 19:00:57 -0400 Subject: [PATCH 19/53] Runic-format the benchmark scripts and probes added by the perf branches Layout only; the FormatCheck workflow runs Runic over the whole tree. Co-Authored-By: Claude Opus 5.5 --- benchmark/bench_degenerate.jl | 6 ++-- benchmark/bench_large_gemm_peak.jl | 40 ++++++++++++++-------- benchmark/probes/probe_call_floor.jl | 6 ++-- benchmark/probes/probe_case_ab.jl | 10 +++--- benchmark/probes/probe_msplit_prototype.jl | 2 +- benchmark/probes/probe_network_korder.jl | 14 +++++--- benchmark/probes/probe_pack_order.jl | 6 ++-- 7 files changed, 53 insertions(+), 31 deletions(-) diff --git a/benchmark/bench_degenerate.jl b/benchmark/bench_degenerate.jl index 94e0331..9f83760 100644 --- a/benchmark/bench_degenerate.jl +++ b/benchmark/bench_degenerate.jl @@ -58,8 +58,10 @@ end function row(name, ref, plan, blasf) ex = () -> execute!(plan, 1.0, 0.0) r = interleaved(Any[withmode(ex, ref, :never), withmode(ex, ref, :auto), blasf]) - @printf("%-28s nest %s new %s blas %s new/nest %.2f new/blas %.2f\n", - name, fmt(r[1]), fmt(r[2]), fmt(r[3]), r[2][1] / r[1][1], r[2][1] / r[3][1]) + @printf( + "%-28s nest %s new %s blas %s new/nest %.2f new/blas %.2f\n", + name, fmt(r[1]), fmt(r[2]), fmt(r[3]), r[2][1] / r[1][1], r[2][1] / r[3][1] + ) return flush(stdout) end diff --git a/benchmark/bench_large_gemm_peak.jl b/benchmark/bench_large_gemm_peak.jl index 46de611..21de942 100644 --- a/benchmark/bench_large_gemm_peak.jl +++ b/benchmark/bench_large_gemm_peak.jl @@ -43,9 +43,11 @@ csv_io = open(CSV_PATH, "w") println(csv_io, "arm,dtype,case,shape,M,K,N,mc,kc,nc,round,seconds,gflops") function log_row(arm, T, case, shape, M, K, N, blk, round, t) gf = 2.0 * M * K * N / t / 1.0e9 - println(csv_io, "$arm,$T,$case,$(shape),$M,$K,$N,$(blk === nothing ? "" : blk.mc),", + println( + csv_io, "$arm,$T,$case,$(shape),$M,$K,$N,$(blk === nothing ? "" : blk.mc),", "$(blk === nothing ? "" : blk.kc),$(blk === nothing ? "" : blk.nc),$round,", - @sprintf("%.9f", t), ",", @sprintf("%.3f", gf)) + @sprintf("%.9f", t), ",", @sprintf("%.3f", gf) + ) flush(csv_io) return gf end @@ -62,8 +64,10 @@ end println("bench_large_gemm_peak.jl cpu = ", Sys.CPU_NAME, " profile = ", target_profile()) for T in REAL_DTYPES - println(" ", T, ": derived shape ", _derived_shape(target_profile(), T), " default kernel ", - shapestr(shape_of(_default_kernel(T))), " blocking ", default_blocking(_default_kernel(T))) + println( + " ", T, ": derived shape ", _derived_shape(target_profile(), T), " default kernel ", + shapestr(shape_of(_default_kernel(T))), " blocking ", default_blocking(_default_kernel(T)) + ) end # --------------------------------------------------------------------------- @@ -157,7 +161,9 @@ for T in REAL_DTYPES push!(plans, ("default:" * shapestr(shape_of(pdef.kernel)), pdef)) reps = max(3, round(Int, 0.2 / (2.0 * M * K * N / 30.0e9))) for r in 1:ROUNDS, (name, plan) in plans - t = @elapsed for _ in 1:reps; execute!(plan, one(T), zero(T)); end + t = @elapsed for _ in 1:reps + execute!(plan, one(T), zero(T)) + end record!("stepdown", T, "M$M", name, M, K * reps, N, plan.blocking, r, t) end end @@ -176,11 +182,13 @@ for T in REAL_DTYPES MR = mr(kernel) model = default_blocking(kernel) kc34 = max(1, (target_profile().l1d.bytes * 3 ÷ 4) ÷ (nr(kernel) * sizeof(T))) - grid = unique([ - (model.mc, model.kc), (model.mc ÷ 2 ÷ MR * MR, model.kc), (model.mc * 2, model.kc), - (max(MR, model.mc * model.kc ÷ kc34 ÷ MR * MR), kc34), (model.mc, kc34), - (max(MR, model.mc ÷ 2 ÷ MR * MR), 2 * model.kc), - ]) + grid = unique( + [ + (model.mc, model.kc), (model.mc ÷ 2 ÷ MR * MR, model.kc), (model.mc * 2, model.kc), + (max(MR, model.mc * model.kc ÷ kc34 ÷ MR * MR), kc34), (model.mc, kc34), + (max(MR, model.mc ÷ 2 ÷ MR * MR), 2 * model.kc), + ] + ) plans = [] for (mc, kc) in grid (mc >= MR && kc >= 1) || continue @@ -204,9 +212,11 @@ open(SUMMARY_PATH, "w") do io print_env_header(io, "bench_large_gemm_peak.jl") println(io, "profile = ", target_profile()) for T in REAL_DTYPES - println(io, T, ": derived shape ", _derived_shape(target_profile(), T), + println( + io, T, ": derived shape ", _derived_shape(target_profile(), T), " default kernel ", shapestr(shape_of(_default_kernel(T))), - " blocking ", default_blocking(_default_kernel(T))) + " blocking ", default_blocking(_default_kernel(T)) + ) end keys_ = unique([(r.arm, r.dtype, r.case, r.shape) for r in rows]) lastarm = "" @@ -217,8 +227,10 @@ open(SUMMARY_PATH, "w") do io lastarm = arm end v = [r.gf for r in rows if (r.arm, r.dtype, r.case, r.shape) == key] - println(io, " ", rpad(string(T), 8), rpad(case, 12), rpad(shape, 22), - @sprintf("%7.1f", median(v)), " (", @sprintf("%.1f", minimum(v)), "..", @sprintf("%.1f", maximum(v)), ")") + println( + io, " ", rpad(string(T), 8), rpad(case, 12), rpad(shape, 22), + @sprintf("%7.1f", median(v)), " (", @sprintf("%.1f", minimum(v)), "..", @sprintf("%.1f", maximum(v)), ")" + ) end end println(read(SUMMARY_PATH, String)) diff --git a/benchmark/probes/probe_call_floor.jl b/benchmark/probes/probe_call_floor.jl index f6c32b8..b67628c 100644 --- a/benchmark/probes/probe_call_floor.jl +++ b/benchmark/probes/probe_call_floor.jl @@ -50,7 +50,9 @@ for T in (Float64, ComplexF64), (name, sa, sb, sc, pA, pB, pAB) in CASES A, B, C = rand(T, sa), rand(T, sb), zeros(T, sc) for (bname, backend) in (("QS", QuasiStridedBackend()), ("BLAS", TO.StridedBLAS())) med, lo, hi, b = measure(C, A, pA, B, pB, pAB, backend) - @printf("%s\t%-10s\t%-10s\t%-4s\t%8.1f ns\t[%7.1f, %7.1f]\t%6.0f B\n", - label, T, name, bname, med, lo, hi, b) + @printf( + "%s\t%-10s\t%-10s\t%-4s\t%8.1f ns\t[%7.1f, %7.1f]\t%6.0f B\n", + label, T, name, bname, med, lo, hi, b + ) end end diff --git a/benchmark/probes/probe_case_ab.jl b/benchmark/probes/probe_case_ab.jl index 0f6fc8f..0a6f3cd 100644 --- a/benchmark/probes/probe_case_ab.jl +++ b/benchmark/probes/probe_case_ab.jl @@ -51,13 +51,13 @@ run!(QuasiStridedBackend()); println("max abs diff QS vs BLAS = ", maximum(abs, tb = Float64[]; tq = Float64[] for round in 1:ROUNDS for _ in 1:REPS - t0 = time_ns(); run!(StridedBLAS()); push!(tb, (time_ns() - t0) / 1e9) + t0 = time_ns(); run!(StridedBLAS()); push!(tb, (time_ns() - t0) / 1.0e9) end for _ in 1:REPS - t0 = time_ns(); run!(QuasiStridedBackend()); push!(tq, (time_ns() - t0) / 1e9) + t0 = time_ns(); run!(QuasiStridedBackend()); push!(tq, (time_ns() - t0) / 1.0e9) end - @printf("round %d: BLAS %s QS %s\n", round, join((@sprintf("%.3f", x) for x in tb[end-REPS+1:end]), ","), join((@sprintf("%.3f", x) for x in tq[end-REPS+1:end]), ",")) + @printf("round %d: BLAS %s QS %s\n", round, join((@sprintf("%.3f", x) for x in tb[(end - REPS + 1):end]), ","), join((@sprintf("%.3f", x) for x in tq[(end - REPS + 1):end]), ",")) end -@printf("StridedBLAS median %.4f s (min %.4f, max %.4f) %.2f GF/s\n", median(tb), minimum(tb), maximum(tb), flops / median(tb) / 1e9) -@printf("QuasiStrided median %.4f s (min %.4f, max %.4f) %.2f GF/s\n", median(tq), minimum(tq), maximum(tq), flops / median(tq) / 1e9) +@printf("StridedBLAS median %.4f s (min %.4f, max %.4f) %.2f GF/s\n", median(tb), minimum(tb), maximum(tb), flops / median(tb) / 1.0e9) +@printf("QuasiStrided median %.4f s (min %.4f, max %.4f) %.2f GF/s\n", median(tq), minimum(tq), maximum(tq), flops / median(tq) / 1.0e9) @printf("QS/BLAS = %.3f\n", median(tq) / median(tb)) diff --git a/benchmark/probes/probe_msplit_prototype.jl b/benchmark/probes/probe_msplit_prototype.jl index 438ed34..cfef2fe 100644 --- a/benchmark/probes/probe_msplit_prototype.jl +++ b/benchmark/probes/probe_msplit_prototype.jl @@ -118,6 +118,6 @@ for round in 1:2 execute!(p, one(T), zero(T)) @assert C ≈ Cref "$name wrong result" t = median_time_s(() -> execute!(p, one(T), zero(T)); reps = REPS) - @printf("round %d %-22s %9.4f s %7.2f GF/s\n", round, name, t, flops / t / 1e9) + @printf("round %d %-22s %9.4f s %7.2f GF/s\n", round, name, t, flops / t / 1.0e9) end end diff --git a/benchmark/probes/probe_network_korder.jl b/benchmark/probes/probe_network_korder.jl index e838f04..22a0562 100644 --- a/benchmark/probes/probe_network_korder.jl +++ b/benchmark/probes/probe_network_korder.jl @@ -79,8 +79,10 @@ Cnew = run(QuasiStridedBackend()) decisions = unique(e -> (e.klabels, e.kA, e.kB, e.indA, e.indB, e.sizeA, e.sizeB), LOG) println("pairwise contractions with >1 K label: $(length(decisions)) distinct (of $(length(LOG)) planned)") for e in decisions - println(" K=", e.klabels, " lengths=", e.kL, " A: size=", e.sizeA, " ind=", e.indA, " Kstrides=", e.kA, " B: size=", e.sizeB, " ind=", e.indB, " Kstrides=", e.kB, - " Qm=", e.Qm, " Qn=", e.Qn, " cost(A,B) old=", e.costs[1], " new=", e.costs[2], e.changed ? " CHANGED: new=$(e.new) simple=$(e.simple)" : " (unchanged)") + println( + " K=", e.klabels, " lengths=", e.kL, " A: size=", e.sizeA, " ind=", e.indA, " Kstrides=", e.kA, " B: size=", e.sizeB, " ind=", e.indB, " Kstrides=", e.kB, + " Qm=", e.Qm, " Qn=", e.Qn, " cost(A,B) old=", e.costs[1], " new=", e.costs[2], e.changed ? " CHANGED: new=$(e.new) simple=$(e.simple)" : " (unchanged)" + ) end MODE[] = :old Cold = run(QuasiStridedBackend()) @@ -90,7 +92,7 @@ println("max|new-old| = ", maximum(abs, Cnew .- Cold), " max|new-blas| = ", max # Whole network, interleaved. function tmed(f) f() - ts = [(t0 = time_ns(); f(); (time_ns() - t0) / 1e9) for _ in 1:REPS] + ts = [(t0 = time_ns(); f(); (time_ns() - t0) / 1.0e9) for _ in 1:REPS] return median(ts) end for round in 1:2 @@ -120,6 +122,8 @@ for e in decisions MODE[] = :simple; tsim = tmed(f); MODE[] = :simple; tsim2 = tmed(f) @printf(" pairwise K=%s: simple %.3e,%.3e s simple/old %.3f\n", e.klabels, tsim, tsim2, min(tsim, tsim2) / min(told, told2)) flops = 2.0 * prod(size(C)) * prod(e.kL) - @printf(" pairwise K=%s A%s B%s: old %.3e,%.3e s new %.3e,%.3e s new/old %.3f (%.1f GF/s new)\n", - e.klabels, e.sizeA, e.sizeB, told, told2, tnew, tnew2, min(tnew, tnew2) / min(told, told2), flops / min(tnew, tnew2) / 1e9) + @printf( + " pairwise K=%s A%s B%s: old %.3e,%.3e s new %.3e,%.3e s new/old %.3f (%.1f GF/s new)\n", + e.klabels, e.sizeA, e.sizeB, told, told2, tnew, tnew2, min(tnew, tnew2) / min(told, told2), flops / min(tnew, tnew2) / 1.0e9 + ) end diff --git a/benchmark/probes/probe_pack_order.jl b/benchmark/probes/probe_pack_order.jl index dc3ca9a..7632f6a 100644 --- a/benchmark/probes/probe_pack_order.jl +++ b/benchmark/probes/probe_pack_order.jl @@ -47,8 +47,10 @@ Apar = randn(T, ntuple(_ -> DIM + PAD, length(la))...) A = view(Apar, ntuple(_ -> 1:DIM, length(la))...) B = randn(T, ntuple(_ -> DIM, length(lb))...) C = zeros(T, ntuple(_ -> DIM, length(rhs))...) -plan = plan_contract(StridedView(C), StridedView(A), indA, StridedView(B), indB, indC; - oracle = false, kernel = _kernel_from_shape(SHAPE, T)) +plan = plan_contract( + StridedView(C), StridedView(A), indA, StridedView(B), indB, indC; + oracle = false, kernel = _kernel_from_shape(SHAPE, T) +) pointer(plan.Astorage) == pointer(Apar) || error("swapped plan: A does not feed M") const Ast = plan.Astorage # the flat storage the engine indexes (0-based offsets) From f2e818b0d3890056be7e7a12260ffa5232c787db Mon Sep 17 00:00:00 2001 From: lkdvos Date: Mon, 28 Sep 2026 08:19:40 -0400 Subject: [PATCH 20/53] test_target: allow Julia 1.10's stale "znver3" name for AMD Zen 4/5 CI (lts, ubuntu-latest) failed "CPUID probe is an independent fallback": `_isa_rank(probe) <= _isa_rank(table)` evaluated `2 <= 1`, i.e. the live CPUID probe read AVX-512 while `_UARCH_ISA[Sys.CPU_NAME]` said AVX2. Julia 1.10's CPU detector (LLVM 15) has no `znver4` -- the name is absent from its libjulia-internal, present from 1.11 on -- so it names AMD Zen 4 and Zen 5 hosts "znver3". GitHub's ubuntu runners include Zen 4 EPYCs; the same test passed on Julia 1 (1.13.1), which names the CPU correctly. The TEST was wrong, not the code: the table entry is right for genuine Zen 3 (AVX2 only), and `_detect_isa` already takes the narrower of name and probe, so such a host safely gets AVX2 shapes. The assertion exists to catch a table entry too narrow for its microarchitecture, which a stale name is not. It is now skipped only for "znver3" below Julia 1.11 (still enforced there on 1.11+ and for every other name), and a new assertion checks for every listed name that `_detect_isa` returns the narrower of the two. The "runs on this host" testset also logs the detected profile, CPU name and probe, so a host-dependent failure can be read off the CI log. Co-Authored-By: Claude Opus 5.5 --- test/hardware/test_target.jl | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test/hardware/test_target.jl b/test/hardware/test_target.jl index 17423e1..929db46 100644 --- a/test/hardware/test_target.jl +++ b/test/hardware/test_target.jl @@ -14,6 +14,8 @@ using QuasiStrided: TargetProfile, CacheLevel, target_profile, cache_topology, @testset "target detection" begin @testset "runs on this host without throwing" begin p = target_profile() + # Logged, so a host-dependent CI failure can be read off the log. + @info "target profile" p Sys.CPU_NAME QuasiStrided._isa_from_cpuid() VERSION @test p isa TargetProfile @test p.isa in VALID_ISAS @test p.arch === Sys.ARCH @@ -42,8 +44,23 @@ using QuasiStrided: TargetProfile, CacheLevel, target_profile, cache_topology, # AVX2-only live probe) -- `_detect_isa` itself now defers to the # narrower of the two (src/hardware/target.jl), so only a probe # reporting something WIDER than the table is a real table bug. - (table === :miss || probe === :unknown) || + # + # Except where the NAME is stale: a Julia whose CPU detector + # predates a microarchitecture reports it under an older name. + # Julia 1.10 (LLVM 15) has no `znver4`, and names AMD Zen 4 and + # Zen 5 (AVX-512) "znver3" (AVX2) -- a GitHub ubuntu runner on + # Zen 4 then probes AVX-512 against an AVX2 entry. That entry is + # right for real Zen 3, so it is not a table bug; the name just + # cannot tell the two apart. There only the reconciliation itself + # is checked (below, for every name). + stale = VERSION < v"1.11" ? ("znver3",) : () + (table === :miss || probe === :unknown || Sys.CPU_NAME in stale) || @test QuasiStrided._isa_rank(probe) <= QuasiStrided._isa_rank(table) + # `_detect_isa` is the narrower of a listed name and a live probe. + if table !== :miss && probe !== :unknown + @test _detect_isa() === + (QuasiStrided._isa_rank(probe) < QuasiStrided._isa_rank(table) ? probe : table) + end # Base's CPUID submodule is undocumented: if these names move, # `_isa_from_cpuid` silently returns :unknown and every unlisted # x86 CPU quietly loses the derived shape. Fail loudly instead. From c9332ce5f75a5c3a154194706317c59d854a8831 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Mon, 28 Sep 2026 08:19:40 -0400 Subject: [PATCH 21/53] K-order plan test: size the case from the detected L2 share macOS CI (both Julia versions) failed "K order: plan_contract flips contract_scrambled at an L2-exceeding size": the plan kept indA order (40, 40, 5) instead of reordering to (5, 40, 40). The case's footprint was fixed at b*c*e*64 B = 40*40*40*64 = 4.096 MB, commented "beyond every measured core's L2 share". GitHub's macos-26-arm64 runner is a 3-vCPU virtual M1 whose sysctl reports hw.perflevel0.l2cachesize = 12582912 and hw.perflevel0.cpusperl2 = 3, so `_l2_core_bytes` = 4 MiB = 4.194 MB: the lines fit, and the model correctly keeps indA order. The TEST encoded an x86 L2 size; the detection and the model are right. `_cache_topology_darwin` reads the cluster L2 and its sharing count, and `_l2_core_bytes` divides it (a real M1/M2 with 4 cores per cluster gives 3/4 MB); undetected L2 falls back to 1 MB. `_KO_BIG`'s e extent is now derived from the detected share (never below the old 40, so x86 hosts run the unchanged case), and the flip test asserts that premise. The correctness testset sharing `_KO_BIG` thereby also exercises the reordered K on every host, as its "(reordered K)" comment intends. test/forced_isa_runner.jl gains optional QS_FAKE_L1D / QS_FAKE_L2 / QS_FAKE_L2_SHARING knobs (unset: the all-zero levels it used before) to model such a host. Verified on x86 by installing the runner's profile (:neon, 16 B, 32 regs, L1d 128 KB, L2 12 MB / 3): the old test reproduces exactly the six CI failures, the new one passes there, with sharing 4 (real M1) and with sharing 0 (undetected: 12 MB share, e = 123). Co-Authored-By: Claude Opus 5.5 --- test/forced_isa_runner.jl | 20 ++++++++++++++++++-- test/planning/test_plan_contract.jl | 16 +++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/test/forced_isa_runner.jl b/test/forced_isa_runner.jl index c51b0f9..c1dbea3 100644 --- a/test/forced_isa_runner.jl +++ b/test/forced_isa_runner.jl @@ -11,6 +11,13 @@ # # Useful triples: avx512/64/32, avx2/32/16, neon/16/32, unknown/0/0. # +# The cache levels default to "not detected" (all zero); set the L1d/L2 size +# (bytes) and the number of CPUs sharing L2 to model a specific host, e.g. +# GitHub's macOS arm64 runner (a 3-vCPU virtual M1, per its sysctl): +# +# QS_FAKE_ISA=neon QS_FAKE_VB=16 QS_FAKE_NREG=32 QS_FAKE_L1D=131072 \ +# QS_FAKE_L2=12582912 QS_FAKE_L2_SHARING=3 julia --project=. test/forced_isa_runner.jl +# # Expected residue: `hardware/test_target.jl`'s "runs on this host without throwing" # compares a fresh `_detect_target()` against the stored profile and therefore # fails by construction here -- that one failure is this harness, not the @@ -23,13 +30,22 @@ _QS._TARGET[] = _QS.TargetProfile( Symbol(get(ENV, "QS_FAKE_ISA", "avx512")), Sys.ARCH, "forced-isa-runner", parse(Int, get(ENV, "QS_FAKE_VB", "64")), parse(Int, get(ENV, "QS_FAKE_NREG", "32")), - _QS.CacheLevel(), _QS.CacheLevel(), _QS.CacheLevel() + # Unset: exactly `CacheLevel()` ("not detected"), as before these knobs. + let l1d = parse(Int, get(ENV, "QS_FAKE_L1D", "0")) + _QS.CacheLevel(l1d, 0, 0, l1d > 0 ? 1 : 0) + end, + _QS.CacheLevel( + parse(Int, get(ENV, "QS_FAKE_L2", "0")), 0, 0, + parse(Int, get(ENV, "QS_FAKE_L2_SHARING", "0")) + ), + _QS.CacheLevel() ) let p = _QS.target_profile() println( "forced target: isa=", p.isa, " vector_bytes=", p.vector_bytes, - " nregisters=", p.nregisters + " nregisters=", p.nregisters, " l1d=", p.l1d.bytes, " l2=", p.l2.bytes, + "/", p.l2.sharing ) end diff --git a/test/planning/test_plan_contract.jl b/test/planning/test_plan_contract.jl index 057e206..896b9ac 100644 --- a/test/planning/test_plan_contract.jl +++ b/test/planning/test_plan_contract.jl @@ -823,9 +823,17 @@ end # Labels: a=1 b=2 c=3 d=4 e=5, distinct extents so a wrong order shows up as a # wrong stride, not a coincidentally equal one. `_KO_EXT` is small (every # operand fits any L2); `_KO_BIG` makes B's lines-touched-before-`d`-advances -# footprint b*c*e*64 B = 4 MB, beyond every measured core's L2 share. +# footprint b*c*e*64 B exceed THIS host's L2 share (`_l2_core_bytes`), so the +# model reorders K wherever the suite runs. A fixed e = 40 (4.1 MB) sits just +# below the 4 MB share of GitHub's macOS arm64 runner (a 3-vCPU virtual M1: +# 12 MB L2 over `hw.perflevel0.cpusperl2` = 3), where the model rightly keeps +# indA order; so e is derived from the detected share, never below 40. +function _ko_big(l2 = QuasiStrided._l2_core_bytes(Float64)) + e = max(40, fld(l2, 40 * 40 * 64) + 1) + return Dict(1 => 9, 2 => 40, 3 => 40, 4 => 5, 5 => e) +end const _KO_EXT = Dict(1 => 9, 2 => 3, 3 => 4, 4 => 5, 5 => 6) -const _KO_BIG = Dict(1 => 9, 2 => 40, 3 => 40, 4 => 5, 5 => 40) +const _KO_BIG = _ko_big() _ko_array(::Type{T}, ind, ext = _KO_EXT) where {T} = randn(T, Tuple(ext[l] for l in ind)...) _ko_stride(v, ind, l) = Base.strides(v)[findfirst(==(l), ind)] @@ -989,6 +997,8 @@ end @testset "K order: plan_contract flips contract_scrambled at an L2-exceeding size" begin T = Float64 ext = _KO_BIG + # The premise, on this host's detected L2 share (what `plan_contract` reads). + @test ext[2] * ext[3] * ext[5] * 64 > QuasiStrided._l2_core_bytes(T) Av = StridedView(_ko_array(T, (1, 2, 3, 4), ext)) Bs = StridedView(_ko_array(T, (4, 3, 2, 5), ext)) # B[d,c,b,e] Cv = StridedView(zeros(T, ext[1], ext[5])) @@ -1004,7 +1014,7 @@ end @test p.kgroup.strides[1] == (9, 360, 14400) # The swapped orientation (B feeds M: C stored as (e, a), an N run of - # 40 >= mr) uses the same K order with the maps exchanged. + # e >= 40 >= mr) uses the same K order with the maps exchanged. Ct = StridedView(zeros(T, ext[5], ext[1])) kernel = SIMDKernel(Val(8), Val(6), T) pt = plan_contract(Ct, Av, (1, 2, 3, 4), Bs, (4, 3, 2, 5), (5, 1); kernel = kernel) From 06b94489020f6623c828add2934ce3423aaa78b6 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Mon, 28 Sep 2026 08:19:40 -0400 Subject: [PATCH 22/53] test_outer: keep the "below one vector" case from swapping at W = 2 macOS CI failed `!_outer_eligible(plan_contract(_outer_maker(T, W - 1, 9, 1)()...))` for Float64. W = `_dot_lanewidth(Float64)` = 2 on NEON (16 B vectors), the same W `_outer_applicable` uses, so both agree on "fills one vector". But M = W - 1 = 1 makes C a 1 x 9 matrix whose N axis is unit-stride too, so the planner swaps the roles (as for :cpermC) and the printed plan has M = 9 (mgroup lengths (9,)), unit-stride in both B and C: a genuine outer product, correctly eligible. On AVX-512/AVX2 M = 7/3 is not a singleton, so no swap -- the test encoded W > 2. The TEST was wrong. The below-a-vector check now uses a (W-1) x (W-1) C (no long N to swap to), and the (W-1) x 9 case asserts the bound against the plan's actual M. Correctness of the swapped path on a length that W does not divide (vector body plus scalar tail) is added to the "matches the nest, the oracle and a reference" testset: C = 1 x 9, all four alpha/beta regimes. Verified on x86 under a forced NEON profile (vector_bytes = 16, so the path runs at W = 2 with Vec{2}): passes there and on the native AVX-512 profile. Co-Authored-By: Claude Opus 5.5 --- test/execution/test_outer.jl | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test/execution/test_outer.jl b/test/execution/test_outer.jl index 3230854..9649c0b 100644 --- a/test/execution/test_outer.jl +++ b/test/execution/test_outer.jl @@ -98,8 +98,16 @@ const _OUTER_AB = ((1.0, 0.0), (2.5, -0.75), (1.0, 1.0), (-0.5, 1.25)) pc = plan_contract(_outer_maker(T, 16, 9, 1; variant = :cpermC)()...) @test axis_length(pc.mgroup) == 9 @test _outer_eligible(pc) == (9 >= W) - # Below one vector of M: declined. K > 1: not an outer product. - @test !_outer_eligible(plan_contract(_outer_maker(T, W - 1, 9, 1)()...)) + # Below one vector of M: declined. N is kept below W too: where W = 2 + # (NEON Float64) `W - 1` is a singleton axis, which makes C's N axis + # unit-stride as well, so with a long N the planner legitimately + # swaps the roles and takes the 9-long N as M (as :cpermC above). + @test !_outer_eligible(plan_contract(_outer_maker(T, W - 1, W - 1, 1)()...)) + # With a long N the bound applies to the M the plan ends up with. + p1 = plan_contract(_outer_maker(T, W - 1, 9, 1)()...) + @test axis_length(p1.mgroup) in (W - 1, 9) + @test _outer_eligible(p1) == (axis_length(p1.mgroup) >= W) + # K > 1: not an outer product. @test !_outer_eligible(_mm_plan(zeros(T, 16, 9), randn(T, 16, 2), randn(T, 2, 9))) # The mode override. p = plan_contract(_outer_maker(T, 16, 9, 1)()...) @@ -141,6 +149,20 @@ end Cv0, Av0, iA, Bv0, iB, iC = mk() @test C_out ≈ _hp_ref(Array(Av0), iA, Array(Bv0), iB, Array(Cv0), iC, 1.5, 0.5) end + # A singleton M beside a long N: C's N axis is unit-stride too, so the + # planner swaps the roles and the path runs on the 9-long N, with a scalar + # tail wherever W does not divide 9 -- the plan NEON Float64 (W = 2) + # takes for `M = W - 1` above. + for (alpha, beta) in _OUTER_AB + mk = _outer_maker(T, 1, 9, 77) + C_nest, _ = _outer_run(mk, :never, alpha, beta) + C_out, plan = _outer_run(mk, :auto, alpha, beta) + @test axis_length(plan.mgroup) == 9 + @test _outer_eligible(plan) == (9 >= W) + @test C_out ≈ C_nest + Cv0, Av0, iA, Bv0, iB, iC = mk() + @test C_out ≈ _hp_ref(Array(Av0), iA, Array(Bv0), iB, Array(Cv0), iC, alpha, beta) + end end @testset "outer path: beta = 0 never reads C, bounds, allocations ($T)" for T in (Float64, Float32) From 18ab1f428739d4b2b933281c5f7f815e8ba016bb Mon Sep 17 00:00:00 2001 From: lkdvos Date: Mon, 28 Sep 2026 09:08:02 -0400 Subject: [PATCH 23/53] Step the MV = 4 real tile down to MV = 2 where C's unit run breaks its slivers PR #10's MV = 4 AVX-512 rule regressed ccsd_t_2/4 and ao2mo_2 (dim 16/24, Float64) by 1.5-2.5x. Mechanism (plan dumps, benchmark/probes/ probe_mv4_plans.jl; stage breakdown, probe_stage_breakdown.jl): C's only unit-stride run in these cases is 16 or 24 long and lies on the N side as given. At MV = 2 the swap decision (_prefer_swap: run_n >= mr_swapped) cleared it with mr = 16 and fed that run to M; with mr = 32 it declined, so the plan kept M on a stride-16 C axis and every tile went through the scattered store (ccsd_t_2_dim16: kernel+store 97% of the time, 9.5 vs 23.5 GF/s). ao2mo_2 is the same, the as-is orientation having stepped down to MR = 16 on Qm = 16 while the swapped one was compared at 32. _store_shape: the MV = 4 shape steps down to its MV = 2 sibling when C's leading unit run along M is at least one half tile but does not make every tall sliver unit-stride (Qm != run, run % MR != 0). It is applied per orientation inside _default_shape (which gains a run argument, default Qm = no layout known), so _candidate_mrs feeds the swap the tile each side will really run, and again in _plan_with_kernel with the chosen side's run. Static per plan, no allocation, the shape still crosses the kernel barrier as one Val. Measured (probe_mv4_store_sweep.jl, named kernels, Cascade Lake): a broken tall sliver loses to a whole half-height one by up to 5x at shallow K and is still behind at Qk = 256-1024 (Float64 run 16: 0.95-0.98; Float32 run 32: 0.80 at 256, 0.98 at 512); where run is MR-aligned MV = 4 wins at every Qk measured. probe_mv4_regress.jl, ccqlin038 (us, fix / main): ccsd_t_2 dim16 24167 / 27760, ccsd_t_4 dim16 22603 / 26830, ao2mo_2 dim16 81 / 99, ccsd_t_2 dim24 350602 / 483285, ao2mo_2 dim24 771 / 802 -- the plans are now identical to MV = 2's for all five. Co-Authored-By: Claude Opus 5.5 --- benchmark/probes/probe_mv4_plans.jl | 71 +++++++++++++++++++++++ benchmark/probes/probe_mv4_regress.jl | 19 ++++++ benchmark/probes/probe_mv4_store_sweep.jl | 67 +++++++++++++++++++++ src/planning/defaults.jl | 10 +++- src/planning/kernel_selection.jl | 54 +++++++++++++++++ src/planning/plan.jl | 19 +++--- test/planning/test_kernel_selection.jl | 69 ++++++++++++++++++++++ 7 files changed, 299 insertions(+), 10 deletions(-) create mode 100644 benchmark/probes/probe_mv4_plans.jl create mode 100644 benchmark/probes/probe_mv4_regress.jl create mode 100644 benchmark/probes/probe_mv4_store_sweep.jl diff --git a/benchmark/probes/probe_mv4_plans.jl b/benchmark/probes/probe_mv4_plans.jl new file mode 100644 index 0000000..89e419b --- /dev/null +++ b/benchmark/probes/probe_mv4_plans.jl @@ -0,0 +1,71 @@ +# Plan dump for the MV = 4 regression cases (probe_mv4_regress.jl's list, or +# `--cases id1,id2`): every pairwise plan a case builds (register shape, +# whether the operand roles were swapped, blocking, extents, C's M/N strides, +# the M composite's unit-stride run), then the case's median time. +# +# julia -t 1 --project=benchmark benchmark/probes/probe_mv4_plans.jl --mode fix +# +# `--mode`: `fix` (this tree as is), `nostore` (the `_store_shape` step-down +# disabled: the PR #10 head's selection), `mv2` (the real AVX-512 rule at +# MV = 2: main's selection). The modes redefine package internals, so run +# one mode per process. + +using TensorOperations, QuasiStrided, TensorOperationsBenchmarks, Statistics, Printf +using QuasiStrided: ContractPlan, mr, nr, axis_length, RealMethod +const TOB = TensorOperationsBenchmarks +TensorOperations.LinearAlgebra.BLAS.set_num_threads(1) +include(joinpath(@__DIR__, "..", "harness.jl")) + +const MODE = argopt("mode", "fix") +if MODE == "nostore" + @eval QuasiStrided @inline _store_shape( + shape::NTuple{3, Int}, ::Type{T}, method::RealMethod, Qm::Int, run::Int + ) where {T} = shape +elseif MODE == "mv2" + @eval QuasiStrided _rule_mv(::Val{:avx512}, ::RealMethod) = 2 + QuasiStrided._DEFAULTS_F64[] = nothing + QuasiStrided._DEFAULTS_F32[] = nothing +elseif MODE != "fix" + error("unknown --mode $MODE") +end + +const LOG = Ref(false) +const SEEN = Set{String}() +function log_plan(plan) + g = plan.mgroup; h = plan.ngroup + s = @sprintf( + " MR=%-3d Qm=%-6d Qn=%-6d Qk=%-5d blk=(%d,%d,%d) M(C)=%s N(C)=%s K(A,B)=%s", + mr(plan.kernel), axis_length(g), axis_length(h), axis_length(plan.kgroup), + plan.blocking.mc, plan.blocking.kc, plan.blocking.nc, + string(g.strides[2]), string(h.strides[2]), string(plan.kgroup.strides) + ) + s in SEEN || (push!(SEEN, s); println(s)) + return nothing +end + +@eval QuasiStrided @inline function _continue(e::_Execute{T}, plan::ContractPlan{T}, hint) where {T} + $(LOG)[] && $(log_plan)(plan) + return (_execute_hinted!(plan, e.alpha, e.beta, hint); nothing) +end + +ids = split(argopt("cases", "ao2mo_2_dim16,ao2mo_2_dim24,ccsd_t_2_dim16,ccsd_t_4_dim16,ccsd_t_2_dim24,mps_1site_D64,mps_2site_D64"), ',') +dtype = parse_dtypes(argopt("dtype", "Float64"))[1] +cases = filter(c -> c.id in ids, vcat(TOB._tccg_cases((8, 16, 24)), TOB._mps_cases((64, 128, 256)))) +println("mode = $MODE T = $dtype default shape = ", QuasiStrided._resolved_defaults(dtype).shape) +for c in cases + s = c.spec + if s isa TOB.ContractSpec + dims(I) = ntuple(i -> s.dims[I[i]], length(I)) + A = randn(dtype, dims(s.IA)); B = randn(dtype, dims(s.IB)); C = zeros(dtype, dims(s.IC)) + pA, pB, pAB = TensorOperations.contract_indices(s.IA, s.IB, s.IC) + f = () -> TensorOperations.tensorcontract!(C, A, pA, false, B, pB, false, pAB, one(dtype), zero(dtype), QuasiStridedBackend()) + else + ts = [randn(dtype, ntuple(i -> s.dims[abs(il[i])], length(il))...) for il in s.indexlists] + f = () -> ncon(ts, s.indexlists, s.conjlist; order = s.order, output = s.output, backend = QuasiStridedBackend()) + end + println(c.id) + f() + LOG[] = true; empty!(SEEN); f(); LOG[] = false + ts_ = [(@elapsed f()) for _ in 1:21] + @printf(" median %.1f us (min %.1f, max %.1f)\n", median(ts_) * 1.0e6, minimum(ts_) * 1.0e6, maximum(ts_) * 1.0e6) +end diff --git a/benchmark/probes/probe_mv4_regress.jl b/benchmark/probes/probe_mv4_regress.jl new file mode 100644 index 0000000..7abb613 --- /dev/null +++ b/benchmark/probes/probe_mv4_regress.jl @@ -0,0 +1,19 @@ +using TensorOperations, QuasiStrided, TensorOperationsBenchmarks, Statistics +const TOB = TensorOperationsBenchmarks +LinearAlgebra = TensorOperations.LinearAlgebra; LinearAlgebra.BLAS.set_num_threads(1) +cases = filter(c -> c.id in ("ao2mo_2_dim16", "ao2mo_2_dim24", "ccsd_t_2_dim16", "ccsd_t_4_dim16", "ccsd_t_2_dim24", "ccsd_1_dim16"), TOB._tccg_cases((16, 24))) +append!(cases, filter(c -> c.id in ("mps_1site_D64", "mps_2site_D64"), TOB._mps_cases((64,)))) +for c in cases + s = c.spec + if s isa TOB.ContractSpec + dims(I) = ntuple(i -> s.dims[I[i]], length(I)) + A = randn(dims(s.IA)); B = randn(dims(s.IB)); C = zeros(dims(s.IC)) + pA, pB, pAB = TensorOperations.contract_indices(s.IA, s.IB, s.IC) + f = () -> TensorOperations.tensorcontract!(C, A, pA, false, B, pB, false, pAB, 1.0, 0.0, QuasiStridedBackend()) + else + ts = [randn(ntuple(i -> s.dims[abs(il[i])], length(il))...) for il in s.indexlists] + f = () -> ncon(ts, s.indexlists, s.conjlist; order = s.order, output = s.output, backend = QuasiStridedBackend()) + end + f(); ts_ = [(@elapsed f()) for _ in 1:15] + println(rpad(c.id, 18), " ", round(median(ts_) * 1.0e6, digits = 1), " us") +end diff --git a/benchmark/probes/probe_mv4_store_sweep.jl b/benchmark/probes/probe_mv4_store_sweep.jl new file mode 100644 index 0000000..8b62c96 --- /dev/null +++ b/benchmark/probes/probe_mv4_store_sweep.jl @@ -0,0 +1,67 @@ +# The MV = 4 real AVX-512 tile against its MV = 2 sibling as a function of the +# contracted depth `Qk` and of the C layout of the M composite: a leading +# unit-stride run of `r` rows in C, then a jump (C = [a, n, b], M = (a, b), +# `a` of extent `r`). Each kernel is named explicitly, so no swap and no +# run-length demotion applies: this is the raw store-layout x K-depth trade. +# +# julia -t 1 --project=benchmark benchmark/probes/probe_mv4_store_sweep.jl \ +# --dtype Float64 --runs 16,24,32,4096 --ks 8,16,24,32,48,64,128,256 \ +# --qm 2048 --qn 512 --rounds 5 +# +# `--runs` value >= `--qm` means a fully unit-stride M (C = [m, n]). +# Prints, per (r, Qk), the median GF/s of each shape over `--rounds` +# interleaved rounds (each a median of `--reps` calls) and the MV4/MV2 ratio. + +using QuasiStrided +using QuasiStrided: plan_contract, execute!, _kernel_from_shape, mr +using StridedViews +using Statistics: median +using Printf +using Random + +include(joinpath(@__DIR__, "..", "harness.jl")) + +const T = parse_dtypes(argopt("dtype", "Float64"))[1] +const RUNS = parse.(Int, split(argopt("runs", "16,24,32,4096"), ',')) +const KS = parse.(Int, split(argopt("ks", "8,16,24,32,48,64,128,256"), ',')) +const QM = argopt("qm", 2048) +const QN = argopt("qn", 512) +const ROUNDS = argopt("rounds", 5) +const REPS = argopt("reps", 0) +const W = 64 ÷ sizeof(T) +const SHAPES = ((2W, 6, W), (4W, 6, W)) + +function setup(r, qk) + rng = MersenneTwister(1) + if r >= QM + A = randn(rng, T, QM, qk); B = randn(rng, T, qk, QN); C = zeros(T, QM, QN) + return (C, A, (1, -1), B, (-1, 2), (1, 2)) + end + mb = QM ÷ r + # labels: a = 1, b = 2, n = 3, k = -1 + A = randn(rng, T, r, mb, qk); B = randn(rng, T, qk, QN); C = zeros(T, r, QN, mb) + return (C, A, (1, 2, -1), B, (-1, 3), (1, 3, 2)) +end + +println("T = $T Qm = $QM Qn = $QN rounds = $ROUNDS shapes = $SHAPES") +@printf("%6s %5s %10s %10s %8s %8s\n", "run", "Qk", "MV2 GF/s", "MV4 GF/s", "MV4/MV2", "spread") +for r in RUNS, qk in KS + C, A, iA, B, iB, iC = setup(r, qk) + plans = map(SHAPES) do s + plan_contract( + StridedView(C), StridedView(A), iA, StridedView(B), iB, iC; + kernel = _kernel_from_shape(s, T), oracle = false + ) + end + flops = 2.0 * QM * QN * qk + reps = REPS > 0 ? REPS : clamp(round(Int, 2.0e8 / flops), 5, 400) + g = [Float64[] for _ in SHAPES] + for _ in 1:ROUNDS, (i, p) in enumerate(plans) + push!(g[i], flops / median_time_s(() -> execute!(p, one(T), zero(T)); reps) / 1.0e9) + end + ratios = g[2] ./ g[1] + @printf( + "%6d %5d %10.2f %10.2f %8.3f %8.3f\n", r, qk, median(g[1]), median(g[2]), + median(ratios), maximum(ratios) - minimum(ratios) + ) +end diff --git a/src/planning/defaults.jl b/src/planning/defaults.jl index 81f74e7..7aa15cd 100644 --- a/src/planning/defaults.jl +++ b/src/planning/defaults.jl @@ -127,10 +127,16 @@ _default_kernel(::Type{T}) where {T} = # `Val` (`_plan_with_kernel`, src/planning/plan.jl), so the choice is made # without ever holding a menu-wide kernel Union (ten members for ComplexF64). # `_default_kernel` below builds the kernel from the same pair. -@inline function _default_shape(::Type{T}, Qm::Int, Qn::Int) where {T} +# +# `run` is C's leading unit-stride run along this orientation's M composite +# (`_leading_unit_run`); the real `MV = 4` shape also steps down where it +# would break the tall slivers' unit-stride store (`_store_shape`). The +# default `run = Qm` (a fully unit-stride M) is "no layout known", which never +# steps down. +@inline function _default_shape(::Type{T}, Qm::Int, Qn::Int, run::Int = Qm) where {T} d = _resolved_defaults(T) method = _default_method(T) - shape = _extent_shape(d.shape, T, method, Qm) + shape = _store_shape(_extent_shape(d.shape, T, method, Qm), T, method, Qm, run) (Qm > 0 && Qm < shape[1]) || return (shape, method) # The FMAddSub small-M demotion is complex-only (`_small_m_candidates` is # empty for a real `T`); saying so statically keeps a real `T`'s method a diff --git a/src/planning/kernel_selection.jl b/src/planning/kernel_selection.jl index 26f677f..23a256c 100644 --- a/src/planning/kernel_selection.jl +++ b/src/planning/kernel_selection.jl @@ -442,6 +442,60 @@ _extent_shape(profile::TargetProfile, ::Type{T}, method, Qm::Int) where {T} = return cld(Qm, half[1]) * half[1] < cld(Qm, MR) * MR ? half : shape end +""" + _store_shape(shape, T, method, Qm, run) -> (MR, NR, W) + +`shape`, or -- real method only, and only when `shape` is the `MV = 4` rule +shape -- its `MV = 2` sibling `(MR ÷ 2, NR, W)` when C's leading unit-stride +run along M (`run`, `_leading_unit_run`) does not make every tall register +sliver unit-stride (`Qm != run` and `run % MR != 0`) but is at least one half +tile long. Applied per orientation, BEFORE the swap decision +(`_candidate_mrs`, src/planning/plan.jl), and again to the chosen one, so the +swap and the run-length demotion (`_demote_for_run`) see the tile that will +actually run. Always a member of `kernel_shapes(T, method)`. + +Why: a sliver that is not unit-stride in C takes the scattered store, whose +cost per tile is fixed while the kernel's work per tile grows with `Qk`, and +the `MV = 4` tile's 1.2x kernel rate does not pay for it at any `Qk` the +measurement below reached. Measured 2026-09-28, ccqlin038 (Cascade Lake), +Julia 1.12.7, `benchmark/probes/probe_mv4_store_sweep.jl` (named kernels, no +swap, no run demotion; Qm = 1920, Qn = 480; C = [a, n, b] so `run` = extent of +`a`), GF/s ratio `MV = 4` / `MV = 2`: + + Float64 (16 vs 32 rows) Qk: 16 32 64 128 256 512 1024 + run = 16 (MV2 whole, MV4 broken) 0.31 0.46 0.59 0.68-0.76 0.95-0.98 0.89 0.95 + run = 24 (both broken) 0.55 0.64 0.77 0.88 1.03 + run = 48 (MV2 whole, MV4 half) 0.55 0.69 0.84 0.94 1.07 1.07 1.00 + run = 32, 64, 96 (both whole) 1.10-1.13 at Qk = 16..128 + Float32 (32 vs 64 rows) + run = 32 0.19 0.29 0.43 0.61 0.80 0.98 + run = 48 (both broken) 0.49 0.56 0.65 0.78 0.92 1.04 + run = 96 0.39 0.51 0.69 0.85 1.00 1.05 + run = 64, 128 (both whole) 1.04-1.24 at every Qk + +Past the crossover (Qk ~ 200-500, and only where part of the tall slivers +is whole) the tall tile gains at most ~7%, and since `kc` caps a panel's +depth near there the ratio stops growing; below it the half tile is up to 5x +faster. So the rule has no `Qk` term. This is the regression the `MV = 4` +rule first shipped with: the swap decision (`_prefer_swap`) compared C's +16-long runs of `ccsd_t_2_dim16` / `ao2mo_2_dim16` against `mr = 32`, +declined the swap `MV = 2` had taken, and ran every tile through the +scattered store (2.2-2.6x slower end to end). + +Below one half tile (`run < 2W`) the tall shape is kept: both tiles store +scattered there, and the tall one measured 0.88-1.00x at Qk <= 32 (where the +run demotion replaces either by the tile that divides `run`) and 1.05-1.16x +from Qk = 64 (Float64, run = 8). +""" +@inline _store_shape(shape::NTuple{3, Int}, ::Type{T}, method, Qm::Int, run::Int) where {T} = shape + +@inline function _store_shape(shape::NTuple{3, Int}, ::Type{T}, method::RealMethod, Qm::Int, run::Int) where {T} + MR, NR, W = shape + (MR == 4 * W && Qm != run && run % MR != 0 && run >= 2 * W) || return shape + half = (2 * W, NR, W) + return half in kernel_shapes(T, method) ? half : shape +end + # Small-M demotion target for complex `T` on AVX-512: the native-width # (`W == lanes`) FMAddSub menu shape that pads `Qm` least (`cld(Qm, MR) * MR`), # ties by the larger `MR * NR` tile; `nothing` (keep the planar fitted shape) diff --git a/src/planning/plan.jl b/src/planning/plan.jl index 407a994..bf3c029 100644 --- a/src/planning/plan.jl +++ b/src/planning/plan.jl @@ -257,7 +257,7 @@ function _planned( # decision. The kernel itself is resolved by `_plan_with_kernel`, after # the orientation is fixed -- and resolved as a `(shape, method)` value, # not a kernel object, so no menu-wide Union is ever held in this frame. - mr_asis, mr_swapped = _candidate_mrs(T, kernel, Qm, Qn) + mr_asis, mr_swapped = _candidate_mrs(T, kernel, Qm, Qn, run_m, run_n) # The swap is for real element types only. Extending it to complex # kernels, which now also have a vectorized store, is a deliberately @@ -291,12 +291,15 @@ end # `mr` of the kernel each orientation would run: the as-is one at `(Qm, Qn)` # and the swapped one at `(Qn, Qm)`. A caller-named kernel runs either way; # an automatic one is `_default_shape`'s pick, whose two candidates differ -# only when the small-M demotion applies to one orientation. The swapped -# candidate is resolved for a real `T` only, the only `T` that can swap. -@inline _candidate_mrs(::Type{T}, kernel, Qm::Int, Qn::Int) where {T} = (mr(kernel), mr(kernel)) -@inline function _candidate_mrs(::Type{T}, ::Nothing, Qm::Int, Qn::Int) where {T} - mr_asis = _default_shape(T, Qm, Qn)[1][1] - mr_swapped = T <: Real ? _default_shape(T, Qn, Qm)[1][1] : mr_asis +# when an extent demotion applies to one orientation or when the real `MV = 4` +# tile steps down for one orientation's C run (`_store_shape`, keyed on that +# orientation's own `run`). The swapped candidate is resolved for a real `T` +# only, the only `T` that can swap. +@inline _candidate_mrs(::Type{T}, kernel, Qm::Int, Qn::Int, run_m::Int, run_n::Int) where {T} = + (mr(kernel), mr(kernel)) +@inline function _candidate_mrs(::Type{T}, ::Nothing, Qm::Int, Qn::Int, run_m::Int, run_n::Int) where {T} + mr_asis = _default_shape(T, Qm, Qn, run_m)[1][1] + mr_swapped = T <: Real ? _default_shape(T, Qn, Qm, run_n)[1][1] : mr_asis return mr_asis, mr_swapped end @@ -327,7 +330,7 @@ end _plan_contract(kernel, atransform, btransform, req, nothing) @inline function _plan_with_kernel(::Nothing, atransform, btransform, req::_PlanRequest{T}) where {T} Qm = axis_length(req.mgroup) - shape, method = _default_shape(T, Qm, axis_length(req.ngroup)) + shape, method = _default_shape(T, Qm, axis_length(req.ngroup), req.run) shape = _demote_shape_for_run(T, shape, method, req.run, Qm, axis_length(req.kgroup)) # Built from the menu (throwing for a shape outside it), so the callee is # only ever specialised on a menu shape. diff --git a/test/planning/test_kernel_selection.jl b/test/planning/test_kernel_selection.jl index dc6ec18..1a27ee5 100644 --- a/test/planning/test_kernel_selection.jl +++ b/test/planning/test_kernel_selection.jl @@ -169,6 +169,75 @@ using QuasiStrided: TargetProfile, CacheLevel, target_profile, cache_topology, @test ext(p, ComplexF64, PlanarMethod(), 3) === _derived_shape(p, ComplexF64, PlanarMethod()) end end + + @testset "_store_shape: the MV = 4 real shape steps down where C's run breaks its slivers" begin + st = QuasiStrided._store_shape + for T in (Float64, Float32) + p = synthetic(:avx512, 64) + tall = _derived_shape(p, T) + MR, NR, W = tall + half = (2 * W, NR, W) + Qm = 64 * MR + # Every tall sliver unit-stride: whole M, or a run that MR divides. + for run in (Qm, MR, 2 * MR, 5 * MR) + @test st(tall, T, RealMethod(), Qm, run) === tall + end + @test st(tall, T, RealMethod(), 3 * W, 3 * W) === tall # Qm == run + # A run of at least one half tile that MR does not divide. + for run in (2 * W, 3 * W, 6 * W, MR + 2 * W, Qm - 1) + @test st(tall, T, RealMethod(), Qm, run) === half + end + # Below one half tile (or no run at all, 1): both would scatter. + for run in (1, W, 2 * W - 1) + @test st(tall, T, RealMethod(), Qm, run) === tall + end + # Only the tall shape steps down; nothing else is touched. + @test st(half, T, RealMethod(), Qm, 3 * W) === half + fb = _fallback_shape(T) + @test st(fb, T, RealMethod(), Qm, 3 * W) === fb + cs = _derived_shape(p, ComplexF64, PlanarMethod()) + @test st(cs, ComplexF64, PlanarMethod(), Qm, 3 * W) === cs + # Through the per-plan shape on the live host. + if _derived_shape(target_profile(), T) === tall + @test QuasiStrided._default_shape(T, Qm, 256, 2 * W)[1] === half + @test QuasiStrided._default_shape(T, Qm, 256, MR)[1] === tall + @test QuasiStrided._default_shape(T, Qm, 256)[1] === tall # no run given + end + end + end + + @testset "_store_shape feeds the swap: ccsd_t_2 / ao2mo_2 at dim 16 swap onto the 16-run" begin + # The regression the MV = 4 rule first shipped with: C's only unit + # run is 16 long (on the N side as given), and the swap decision + # compared it against mr = 32 and declined the swap, leaving every + # tile on the scattered store. With the step-down the swapped side + # runs the 16-row tile, which the run makes unit-stride, and swaps. + T = Float64 + if _derived_shape(target_profile(), T)[1] == 32 + d = 16 + # ccsd_t_2: C = abcijk, A = ijmb, B = mkac + a, b, c, i, j, k, m = 1, 2, 3, 4, 5, 6, -1 + Av = StridedView(randn(T, d, d, d, d)) + Bv = StridedView(randn(T, d, d, d, d)) + Cv = StridedView(zeros(T, d, d, d, d, d, d)) + plan = QuasiStrided.plan_contract( + Cv, Av, (i, j, m, b), Bv, (m, k, a, c), (a, b, c, i, j, k); oracle = false + ) + @test mr(plan.kernel) == 16 + @test plan.Astorage === parent(Bv) # swapped + # ao2mo_2: C = abrs, A = qb, B = aqrs + q, r, s = -1, 3, 4 + A2 = StridedView(randn(T, d, d)) + B2 = StridedView(randn(T, d, d, d, d)) + C2 = StridedView(zeros(T, d, d, d, d)) + plan2 = QuasiStrided.plan_contract( + C2, A2, (q, b), B2, (a, q, r, s), (a, b, r, s); oracle = false + ) + @test mr(plan2.kernel) == 16 + @test plan2.Astorage === parent(B2) + @test axis_length(plan2.mgroup) == d^3 + end + end end From 651d350344466952b74c01fc705556712000033f Mon Sep 17 00:00:00 2001 From: lkdvos Date: Mon, 28 Sep 2026 10:00:07 -0400 Subject: [PATCH 24/53] benchmark: delete one-off experiment scripts and probes Keep the upstream-suite driver, the general engine benchmarks, the two acceptance probes (probe_ttfx.jl, probe_call_floor.jl) and one generic Slurm submit script (submit.sh) in place of the per-experiment ones. Co-Authored-By: Claude Opus 5.5 --- benchmark/bench_axis_group.jl | 165 ------ benchmark/bench_ccsd_t_store.jl | 517 ------------------ benchmark/bench_default_vs_legacy.jl | 109 ---- benchmark/bench_driver.jl | 279 ---------- benchmark/bench_half_packing.jl | 242 --------- benchmark/bench_kernel_shape.jl | 355 ------------ benchmark/bench_pack_loops.jl | 297 ----------- benchmark/bench_real_path_guard.jl | 142 ----- benchmark/bench_skip_packing.jl | 133 ----- benchmark/bench_store_path.jl | 563 -------------------- benchmark/bench_tensoroperations.jl | 230 -------- benchmark/probes/fmaddsub_codegen.jl | 117 ---- benchmark/probes/probe_call_breakdown.jl | 65 --- benchmark/probes/probe_case_ab.jl | 63 --- benchmark/probes/probe_ccsd_t_layout.jl | 128 ----- benchmark/probes/probe_julia_versions.sh | 36 -- benchmark/probes/probe_korder_changes.jl | 56 -- benchmark/probes/probe_msplit_prototype.jl | 123 ----- benchmark/probes/probe_network_korder.jl | 129 ----- benchmark/probes/probe_pack_order.jl | 116 ---- benchmark/probes/probe_run_demote_kdepth.jl | 304 ----------- benchmark/probes/probe_stage_breakdown.jl | 302 ----------- benchmark/probes/probe_storage_type.jl | 75 --- benchmark/profile_buckets.jl | 250 --------- benchmark/profile_to_suite.jl | 517 ------------------ benchmark/submit.sh | 46 ++ benchmark/submit_blocking_model.sh | 47 -- benchmark/submit_complex_efficiency.sh | 50 -- benchmark/submit_degenerate.sh | 48 -- benchmark/submit_fmaddsub.sh | 71 --- benchmark/submit_half_packing.sh | 52 -- benchmark/submit_large_gemm_peak.sh | 80 --- benchmark/submit_profile_to_suite.sh | 53 -- benchmark/submit_scrambled_verify.sh | 104 ---- benchmark/submit_skip_packing.sh | 54 -- benchmark/submit_vs_openblas.sh | 53 -- 36 files changed, 46 insertions(+), 5925 deletions(-) delete mode 100644 benchmark/bench_axis_group.jl delete mode 100644 benchmark/bench_ccsd_t_store.jl delete mode 100644 benchmark/bench_default_vs_legacy.jl delete mode 100644 benchmark/bench_driver.jl delete mode 100644 benchmark/bench_half_packing.jl delete mode 100644 benchmark/bench_kernel_shape.jl delete mode 100644 benchmark/bench_pack_loops.jl delete mode 100644 benchmark/bench_real_path_guard.jl delete mode 100644 benchmark/bench_skip_packing.jl delete mode 100644 benchmark/bench_store_path.jl delete mode 100644 benchmark/bench_tensoroperations.jl delete mode 100644 benchmark/probes/fmaddsub_codegen.jl delete mode 100644 benchmark/probes/probe_call_breakdown.jl delete mode 100644 benchmark/probes/probe_case_ab.jl delete mode 100644 benchmark/probes/probe_ccsd_t_layout.jl delete mode 100644 benchmark/probes/probe_julia_versions.sh delete mode 100644 benchmark/probes/probe_korder_changes.jl delete mode 100644 benchmark/probes/probe_msplit_prototype.jl delete mode 100644 benchmark/probes/probe_network_korder.jl delete mode 100644 benchmark/probes/probe_pack_order.jl delete mode 100644 benchmark/probes/probe_run_demote_kdepth.jl delete mode 100644 benchmark/probes/probe_stage_breakdown.jl delete mode 100644 benchmark/probes/probe_storage_type.jl delete mode 100644 benchmark/profile_buckets.jl delete mode 100644 benchmark/profile_to_suite.jl create mode 100644 benchmark/submit.sh delete mode 100755 benchmark/submit_blocking_model.sh delete mode 100755 benchmark/submit_complex_efficiency.sh delete mode 100755 benchmark/submit_degenerate.sh delete mode 100755 benchmark/submit_fmaddsub.sh delete mode 100755 benchmark/submit_half_packing.sh delete mode 100644 benchmark/submit_large_gemm_peak.sh delete mode 100755 benchmark/submit_profile_to_suite.sh delete mode 100755 benchmark/submit_scrambled_verify.sh delete mode 100755 benchmark/submit_skip_packing.sh delete mode 100755 benchmark/submit_vs_openblas.sh diff --git a/benchmark/bench_axis_group.jl b/benchmark/bench_axis_group.jl deleted file mode 100644 index 706a540..0000000 --- a/benchmark/bench_axis_group.jl +++ /dev/null @@ -1,165 +0,0 @@ -# Benchmark harness for the grouped-axis indexing layer -# (`src/layout/axis_group.jl`). See Julia-Tensor-Indexing-Agent-Spec.md section 12 -# for what this is meant to measure and why. -# -# Run with: -# julia --project=. benchmark/bench_axis_group.jl -# -# This measures, separately from setup/compilation: -# 1. Repeated reference `offsets` calls (per-element divrem decode). -# 2. Interval generation with preallocated buffers (`fill_offsets!`). -# 3. Generation plus classification (`block_descriptors!`). -# 4. Classification alone on already-filled buffers (`describe_block`). -# -# across affine and irregular layouts, one and several maps, small -# microtile-like counts and larger panel-like counts, and aligned and -# nonaligned interval starts — both for an unnormalized group and its -# `normalize_group` result, where that is meaningful (an irregular layout -# does not normalize away). -# -# This is a correctness-first indexing layer, not a tuned microkernel: no -# absolute speedup threshold is asserted here. The numbers are reported for -# the record (Julia version, CPU, parameters), per the spec's requirement -# to measure rather than assume. - -using QuasiStrided -using QuasiStrided: AxisGroup, axis_length, offsets, fill_offsets!, describe_block, - block_descriptors!, normalize_group -using Printf - -# ------------------------------------------------------------------------ -# Layouts under test -# ------------------------------------------------------------------------ - -# Affine, single map, unit stride: the maximally regular case. -affine1_unnorm = AxisGroup((8, 8), ((1, 8),)) # foldable: normalize_group -> rank 1 -affine1_norm = normalize_group(affine1_unnorm) # (64,), stride (1,) - -# Affine, two maps (as in a paired A/C group), unit + nonunit stride. -affine2_unnorm = AxisGroup((8, 8), ((1, 8), (1, 24))) -affine2_norm = normalize_group(affine2_unnorm) - -# Irregular for at least one map: mismatched strides across dims prevent a -# single affine run (this is the section 8 `G` fixture's shape, extended). -irregular2 = AxisGroup((8, 8), ((1, 3), (1, 40))) # map 2 irregular over the join -# irregular2 does not jointly fold (per-map strides disagree), so -# normalize_group(irregular2) == irregular2 up to singleton removal (none -# here); we do not separately benchmark a "normalized" arm for it. - -layouts = [ - ("affine1_unnorm", affine1_unnorm), - ("affine1_norm", affine1_norm), - ("affine2_unnorm", affine2_unnorm), - ("affine2_norm", affine2_norm), - ("irregular2", irregular2), -] - -# Interval shapes: (label, count, aligned_start_fn) -# "microtile-like" (e.g. MR=8) and "panel-like" (bigger) counts, at both an -# aligned (block-boundary) and a nonaligned start. -interval_shapes = [ - ("micro_aligned", 8, g -> 0), - ("micro_unaligned", 8, g -> 3), - ("panel_aligned", 48, g -> 0), - ("panel_unaligned", 48, g -> 5), -] - -# ------------------------------------------------------------------------ -# Timing helpers: separate warmup/compilation from steady-state execution. -# ------------------------------------------------------------------------ - -function time_offsets_reference(g, first, count; reps = 2000) - # Warmup (forces compilation before timing). - s = 0 - for t in 0:(count - 1) - s += sum(offsets(g, first + t)) - end - t0 = time_ns() - for _ in 1:reps - for t in 0:(count - 1) - s += sum(offsets(g, first + t)) - end - end - elapsed = (time_ns() - t0) / 1.0e9 - return elapsed / reps, s # seconds per rep; return s to prevent DCE -end - -function time_fill_offsets!(bufs, g, first, count; reps = 2000) - fill_offsets!(bufs, g, first, count) # warmup - allocs = @allocated fill_offsets!(bufs, g, first, count) - t0 = time_ns() - for _ in 1:reps - fill_offsets!(bufs, g, first, count) - end - elapsed = (time_ns() - t0) / 1.0e9 - return elapsed / reps, allocs -end - -function time_block_descriptors!(bufs, g, first, count; reps = 2000) - block_descriptors!(bufs, g, first, count) # warmup - allocs = @allocated block_descriptors!(bufs, g, first, count) - t0 = time_ns() - for _ in 1:reps - block_descriptors!(bufs, g, first, count) - end - elapsed = (time_ns() - t0) / 1.0e9 - return elapsed / reps, allocs -end - -function time_describe_block(bufs, count; reps = 2000) - P = length(bufs) - descs = ntuple(p -> describe_block(bufs[p], count), P) # warmup - allocs = @allocated ntuple(p -> describe_block(bufs[p], count), P) - t0 = time_ns() - for _ in 1:reps - ntuple(p -> describe_block(bufs[p], count), P) - end - elapsed = (time_ns() - t0) / 1.0e9 - return elapsed / reps, allocs -end - -# ------------------------------------------------------------------------ -# Run -# ------------------------------------------------------------------------ - -println("Julia version: ", VERSION) -println("CPU: ", Sys.cpu_info()[1].model, " (", length(Sys.cpu_info()), " logical cores reported)") -println() - -@printf( - "%-16s %-16s %-10s %10s %12s %12s %10s\n", - "layout", "interval", "count", "offsets/el", "fill/call", "descr/call", "classify" -) -@printf( - "%-16s %-16s %-10s %10s %12s %12s %10s\n", - "", "", "", "(ns)", "(ns, alloc)", "(ns, alloc)", "(ns)" -) - -for (glabel, g) in layouts - P = length(g.strides) - Q = axis_length(g) - for (ilabel, count, startfn) in interval_shapes - count > Q && continue - first = startfn(g) - first + count > Q && (first = Q - count) # clamp into domain - - t_offsets, _ = time_offsets_reference(g, first, count) - t_offsets_per_el = t_offsets / count - - bufs = ntuple(_ -> Vector{Int}(undef, count), P) - t_fill, a_fill = time_fill_offsets!(bufs, g, first, count) - t_bd, a_bd = time_block_descriptors!(bufs, g, first, count) - - fill_offsets!(bufs, g, first, count) - t_cls, _ = time_describe_block(bufs, count) - - @printf( - "%-16s %-16s %-10d %10.2f %8.1f/%-3d %8.1f/%-3d %10.2f\n", - glabel, ilabel, count, - t_offsets_per_el * 1.0e9, - t_fill * 1.0e9, a_fill, - t_bd * 1.0e9, a_bd, - t_cls * 1.0e9 - ) - end -end diff --git a/benchmark/bench_ccsd_t_store.jl b/benchmark/bench_ccsd_t_store.jl deleted file mode 100644 index d17e508..0000000 --- a/benchmark/bench_ccsd_t_store.jl +++ /dev/null @@ -1,517 +0,0 @@ -# Times four TCCG quantum-chemistry contractions (6-index output, 1 -# contracted index) across several "arms": -# -# ccsd_t_1: C[a,b,c,i,j,k] = A[i,j,m,a] * B[m,k,b,c] -# ccsd_t_2: C[a,b,c,i,j,k] = A[i,j,m,b] * B[m,k,a,c] -# ccsd_t_3: C[a,b,c,i,j,k] = A[i,j,m,c] * B[m,k,a,b] -# ccsd_t_4: C[a,b,c,i,j,k] = A[i,k,m,b] * B[m,j,a,c] -# -# Arm 1: TO.tensorcontract! under StridedNative()/StridedBLAS()/QuasiStridedBackend(). -# Arm 2: QuasiStrided.plan_contract/execute! directly, with the same label -# order TO.tensorcontract! derives internally (via _qs_labels). -# Arm 3: same as Arm 2, but with whichever operand carries `a` (C's -# stride-1 axis) permuted so `a` is that operand's own first -# physical axis. Only `a` moves; the other labels keep their -# relative order. -# Arm 4-M: same as Arm 2, but operand A's free (M) axes are reordered by -# increasing |C-stride| of their label ("M sorted by |C-stride|"). -# Contracted axes keep their positions. -# Arm 4-N: same for operand B's free (N) axes. -# Arm 4both: 4-M and 4-N together. -# Arm 5: 4both, plus an M/N *orientation swap* whenever C's stride-1 -# label is not on A: B is fed into plan_contract's A (M-role) -# slot and vice versa, so C's stride-1 label lands in the M -# composite for every case. For ccsd_t_1 no swap is needed and -# Arm 5 is a repeat of Arm 4both. -# -# NOTE: `plan_contract` performs the label sort and orientation -# swap that Arms 3/4-*/5 emulate by hand INTERNALLY, unconditionally, on -# every call -- so on current `src/planning/labels.jl`, Arm 2's own label order is -# already re-sorted by `_order_free_labels` before these arms' extra operand -# permutations are even applied. These arms describe PRE-FIX semantics; they -# are kept as a working prototype/regression record of what motivated the -# fix, not as arms that still change engine behaviour today. -# -# All operand permutations (Arms 3, 4-*, 5) are `permutedims` on a -# `StridedView`, i.e. lazy views with permuted strides -- no data is copied. -# They emulate a planning-time label-order change with zero extra data -# movement. -# -# julia --project=. benchmark/bench_ccsd_t_store.jl [options] -# -# Options: -# --smoke dims=(8,) only, quick check -# --dims 8,16 -# --dtypes Float64,Float32 -# -# Writes benchmark/results/-/{bench_ccsd_t_store.csv, -# summary_ccsd_t_store.txt,PROVENANCE_ccsd_t_store.txt}. - -using TensorOperations -import TensorOperations as TO -using TensorOperations: StridedNative, StridedBLAS -using QuasiStrided -using QuasiStrided: QuasiStridedBackend -using StridedViews -using StridedViews: StridedView -using Random - -include(joinpath(@__DIR__, "harness.jl")) - -const SMOKE = hasflag("smoke") -const DIMS = SMOKE ? (8,) : parse_ints(argopt("dims", "8,16")) -const CORRECTNESS_DTYPES = parse_dtypes(argopt("dtypes", "Float64,Float32")) - -# `arm3_target` names which operand carries `a` (C's stride-1 axis) and -# therefore gets permuted for Arm 3; `arm3_perm` is that operand's own -# 4-axis permutation moving `a` to the front. -struct CaseSpec - name::String - IA::NTuple{4, Symbol} - IB::NTuple{4, Symbol} - IC::NTuple{6, Symbol} - arm3_target::Symbol - arm3_perm::NTuple{4, Int} -end - -const CASES = [ - CaseSpec( - "ccsd_t_1", (:i, :j, :m, :a), (:m, :k, :b, :c), (:a, :b, :c, :i, :j, :k), - :A, (4, 1, 2, 3) # A is (i,j,m,a); move a (axis 4) to front -> (a,i,j,m) - ), - CaseSpec( - "ccsd_t_2", (:i, :j, :m, :b), (:m, :k, :a, :c), (:a, :b, :c, :i, :j, :k), - :B, (3, 1, 2, 4) # B is (m,k,a,c); move a (axis 3) to front -> (a,m,k,c) - ), - CaseSpec( - "ccsd_t_3", (:i, :j, :m, :c), (:m, :k, :a, :b), (:a, :b, :c, :i, :j, :k), - :B, (3, 1, 2, 4) # B is (m,k,a,b); move a (axis 3) to front -> (a,m,k,b) - ), - CaseSpec( - "ccsd_t_4", (:i, :k, :m, :b), (:m, :j, :a, :c), (:a, :b, :c, :i, :j, :k), - :B, (3, 1, 2, 4) # B is (m,j,a,c); move a (axis 3) to front -> (a,m,j,c) - ), -] - -# Arm identifiers, in run order. Arm 1 is the adapter path (three backends); -# everything else is the direct plan_contract/execute! API. -const DIRECT_ARMS = ("2", "3", "4M", "4N", "4both", "5") - -# dim=16 QuasiStrided calls run ~1-2s each; keep those at 15 reps, everything -# else (dim=8, and dim=16 StridedNative/StridedBLAS) at 21. -function reps_for(dim::Int, arm::AbstractString, backend_name::AbstractString) - dim == 8 && return 21 - (arm == "1" && backend_name != "QuasiStrided") && return 21 - return 15 -end - -# Tallies M/N register-slivers of a plan's mgroup/ngroup by whether their -# C-side BlockDescriptor is `regular && stride == 1`. -function tally_group_slivers(group, reg_tile::Int) - Q = QuasiStrided.axis_length(group) - buf1 = Vector{Int}(undef, reg_tile) - buf2 = Vector{Int}(undef, reg_tile) - n_unit = 0 - n_other = 0 - first_desc = nothing - first = 0 - while first < Q - count = min(reg_tile, Q - first) - (_, dC) = QuasiStrided.block_descriptors!((buf1, buf2), group, first, count) - if dC.regular && dC.stride == 1 - n_unit += 1 - else - n_other += 1 - end - first_desc === nothing && (first_desc = dC) - first += count - end - return (n_unit = n_unit, n_other = n_other, first_desc = first_desc) -end - -function print_layout_diagnostic(io::IO, label::String, plan) - println(io, " [$label] typeof(plan.Cstorage) = ", typeof(plan.Cstorage)) - MRk = mr(plan.kernel) - NRk = nr(plan.kernel) - mt = tally_group_slivers(plan.mgroup, MRk) - nt = tally_group_slivers(plan.ngroup, NRk) - println( - io, " [$label] M-slivers: regular&&unit-stride = ", mt.n_unit, - " other = ", mt.n_other, " (first C-descriptor: ", mt.first_desc, ")" - ) - println( - io, " [$label] N-slivers: regular&&unit-stride = ", nt.n_unit, - " other = ", nt.n_other, " (first C-descriptor: ", nt.first_desc, ")" - ) - return (mt = mt, nt = nt) -end - -# ---------------------------------------------------------------------------- -# Label-order helpers for Arms 4/5 -# ---------------------------------------------------------------------------- - -# Symbol names for a QuasiStrided label tuple, for readable logs. -function label_syms(ind, indA, IA, indB, IB) - sym = Dict{Int, Symbol}() - for (l, s) in zip(indA, IA) - sym[l] = s - end - for (l, s) in zip(indB, IB) - sym[l] = s - end - return Tuple(sym[l] for l in ind) -end - -# Permutation of one operand's axes that reorders its *free* labels (positive, -# i.e. present in C) by increasing |C-stride|, leaving contracted (negative) -# labels at their original positions. `cstride` maps label -> C stride. -function csorted_perm(ind::NTuple{N, Int}, cstride::Dict{Int, Int}) where {N} - free_pos = [p for p in 1:N if ind[p] > 0] - free_sorted = sort(free_pos; by = p -> abs(cstride[ind[p]])) - perm = collect(1:N) - for (slot, src) in zip(free_pos, free_sorted) - perm[slot] = src - end - return Tuple(perm) -end - -# `permutedims` convention: output axis c takes input axis perm[c]. -permute_labels(ind::NTuple{N, Int}, perm::NTuple{N, Int}) where {N} = ntuple(i -> ind[perm[i]], N) - -# ---------------------------------------------------------------------------- -# Correctness gate -# ---------------------------------------------------------------------------- - -rtol_for(::Type{Float64}) = 1.0e-10 -rtol_for(::Type{Float32}) = 1.0e-4 - -# ---------------------------------------------------------------------------- -# Output -# ---------------------------------------------------------------------------- - -const OUTDIR = results_dir() -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "bench_ccsd_t_store.csv") -const SUMMARY_PATH = joinpath(OUTDIR, "summary_ccsd_t_store.txt") -const PROVENANCE_PATH = joinpath(OUTDIR, "PROVENANCE_ccsd_t_store.txt") - -csv_io = open(CSV_PATH, "w") -println( - csv_io, - "arm,backend,dtype,case,dim,reps,median_seconds,gflops,rows_unit_stride_frac,cols_stride,indA,indB,swapped" -) -function log_row( - arm::AbstractString, backend::AbstractString, T::Type, case::AbstractString, dim::Int, - reps::Int, t::Float64, gf::Float64, - rows_unit_stride_frac, cols_stride, indA, indB, swapped - ) - println( - csv_io, - "$arm,$backend,$T,$case,$dim,$reps,", - @sprintf("%.9f", t), ",", - @sprintf("%.6f", gf), ",", - rows_unit_stride_frac === nothing ? "" : @sprintf("%.6f", rows_unit_stride_frac), ",", - cols_stride === nothing ? "" : string(cols_stride), ",", - indA === nothing ? "" : "\"$(indA)\"", ",", - indB === nothing ? "" : "\"$(indB)\"", ",", - swapped === nothing ? "" : string(swapped) - ) - return flush(csv_io) -end - -summary_io = open(SUMMARY_PATH, "w") -print_env_header(summary_io, "bench_ccsd_t_store.jl") -println(summary_io, "smoke = ", SMOKE) -println(summary_io, "dims = ", collect(DIMS)) - -print_env_header(stdout, "bench_ccsd_t_store.jl") -println("smoke = ", SMOKE) -println("dims = ", collect(DIMS)) - -function both(msg) - println(msg) - return println(summary_io, msg) -end - -const rng = MersenneTwister(0xC7_5D_00_03) - -canary_results = Float64[] -push!(canary_results, run_canary(rng, "A (start)")) - -n_mismatches = 0 -n_bitwise_diffs = 0 - -# (case, dim, dtype, arm-key) -> median seconds, for the end-of-run ratio table. -const RESULTS = Dict{Tuple{String, Int, DataType, String}, Float64}() - -# Plans, checks, prints layout diagnostics, times and logs one direct-API arm. -# Returns the output array on success, `nothing` on a correctness mismatch. -function run_direct_arm!( - arm::AbstractString, case::CaseSpec, dim::Int, ::Type{T}, - Cbase, Cref, Aop::StridedView, indA, Bop::StridedView, indB, indC, - swapped::Bool, note::AbstractString - ) where {T} - both(" [arm$arm] indA=$indA indB=$indB indC=$indC $note") - C = copy(Cbase) - plan = plan_contract(StridedView(C), Aop, indA, Bop, indB, indC) - execute!(plan, one(T), zero(T)) - ok = isapprox(C, Cref; rtol = rtol_for(T)) - diag = print_layout_diagnostic(stdout, "arm$arm", plan) - print_layout_diagnostic(summary_io, "arm$arm", plan) - if !ok - global n_mismatches += 1 - both("MISMATCH: arm=$arm case=$(case.name) dim=$dim dtype=$T -- ABORTED (not timed)") - return nothing - end - reps = reps_for(dim, arm, "QuasiStrided") - t = median_time_s(() -> execute!(plan, one(T), zero(T)); reps = reps) - gf = gflops(T, dim^3, dim, dim^3, t) - mfrac = diag.mt.n_unit / (diag.mt.n_unit + diag.mt.n_other) - ncol_stride = diag.nt.first_desc.regular ? diag.nt.first_desc.stride : "irregular" - log_row(arm, "QuasiStrided-direct", T, case.name, dim, reps, t, gf, mfrac, ncol_stride, indA, indB, swapped) - RESULTS[(case.name, dim, T, arm)] = t - both(" arm=$(rpad(arm, 5)) QuasiStrided-direct reps=$reps median=$(@sprintf("%.6e", t)) s gflops=$(@sprintf("%.3f", gf))") - return C -end - -for T in CORRECTNESS_DTYPES - for dim in DIMS - for case in CASES - pA, pB, pAB = TO.contract_indices(case.IA, case.IB, case.IC) - - Aarr = randn(rng, T, dim, dim, dim, dim) - Barr = randn(rng, T, dim, dim, dim, dim) - Cbase = zeros(T, dim, dim, dim, dim, dim, dim) # physical order a,b,c,i,j,k - - # Reference, computed once per (case,dim,dtype) with StridedBLAS. - Cref = copy(Cbase) - TO.tensorcontract!(Cref, Aarr, pA, false, Barr, pB, false, pAB, one(T), zero(T), StridedBLAS()) - - rtol = rtol_for(T) - - both("\n== case=$(case.name) dim=$dim dtype=$T ==") - - # ----------------------------------------------------------- - # Arm 1: adapter path, one backend at a time. - # ----------------------------------------------------------- - for (bname, backend) in ( - ("StridedNative", StridedNative()), - ("StridedBLAS", StridedBLAS()), - ("QuasiStrided", QuasiStridedBackend()), - ) - C = copy(Cbase) - TO.tensorcontract!(C, Aarr, pA, false, Barr, pB, false, pAB, one(T), zero(T), backend) - ok = isapprox(C, Cref; rtol = rtol) - if !ok - global n_mismatches += 1 - both("MISMATCH: arm=1 backend=$bname case=$(case.name) dim=$dim dtype=$T -- ABORTED (not timed)") - continue - end - reps = reps_for(dim, "1", bname) - t = median_time_s(() -> TO.tensorcontract!(C, Aarr, pA, false, Barr, pB, false, pAB, one(T), zero(T), backend); reps = reps) - gf = gflops(T, dim^3, dim, dim^3, t) - log_row("1", bname, T, case.name, dim, reps, t, gf, nothing, nothing, nothing, nothing, nothing) - RESULTS[(case.name, dim, T, "1-" * bname)] = t - both(" arm=1 backend=$(rpad(bname, 14)) reps=$reps median=$(@sprintf("%.6e", t)) s gflops=$(@sprintf("%.3f", gf))") - end - - # ----------------------------------------------------------- - # Arm 2: direct API, adapter's own label order (via _qs_labels, - # the exact function TO.tensorcontract! for QuasiStridedBackend - # calls internally -- see src/integrations/tensoroperations.jl). - # ----------------------------------------------------------- - indA2, indB2, indC2 = QuasiStrided._qs_labels(pA, pB, pAB) - Av = StridedView(Aarr) - Bv = StridedView(Barr) - C2 = run_direct_arm!( - "2", case, dim, T, Cbase, Cref, Av, indA2, Bv, indB2, indC2, false, - "(adapter's own label order)" - ) - - # ----------------------------------------------------------- - # Arm 3: "C-local" label order control -- permute whichever - # operand carries `a` (C's stride-1 axis) so that `a` is that - # operand's own first physical axis. Lazy view, no copy. - # ----------------------------------------------------------- - if case.arm3_target === :A - Aperm3 = permutedims(Av, case.arm3_perm) - indA3 = permute_labels(indA2, case.arm3_perm) - Bperm3 = Bv - indB3 = indB2 - else - Aperm3 = Av - indA3 = indA2 - Bperm3 = permutedims(Bv, case.arm3_perm) - indB3 = permute_labels(indB2, case.arm3_perm) - end - C3 = run_direct_arm!( - "3", case, dim, T, Cbase, Cref, Aperm3, indA3, Bperm3, indB3, indC2, false, - "(permuted operand: $(case.arm3_target), `a` moved to front)" - ) - - # ----------------------------------------------------------- - # Arms 4-M / 4-N / 4both: reorder each operand's free axes by - # increasing |C-stride| of their label (lazy views). - # ----------------------------------------------------------- - Cstr = strides(Cbase) - cstride = Dict{Int, Int}(indC2[i] => Cstr[i] for i in 1:6) - permA4 = csorted_perm(indA2, cstride) - permB4 = csorted_perm(indB2, cstride) - Aperm4 = permutedims(Av, permA4) - indA4 = permute_labels(indA2, permA4) - Bperm4 = permutedims(Bv, permB4) - indB4 = permute_labels(indB2, permB4) - symsA4 = label_syms(indA4, indA2, case.IA, indB2, case.IB) - symsB4 = label_syms(indB4, indA2, case.IA, indB2, case.IB) - both(" [arm4] C strides by label: " * join(("$(label_syms((l,), indA2, case.IA, indB2, case.IB)[1])=$(cstride[l])" for l in indC2), " ")) - both(" [arm4] permA=$permA4 -> A axes $symsA4 ; permB=$permB4 -> B axes $symsB4") - - C4M = run_direct_arm!( - "4M", case, dim, T, Cbase, Cref, Aperm4, indA4, Bv, indB2, indC2, false, - "(A's M-axes sorted by |C-stride|)" - ) - C4N = run_direct_arm!( - "4N", case, dim, T, Cbase, Cref, Av, indA2, Bperm4, indB4, indC2, false, - "(B's N-axes sorted by |C-stride|)" - ) - C4both = run_direct_arm!( - "4both", case, dim, T, Cbase, Cref, Aperm4, indA4, Bperm4, indB4, indC2, false, - "(both sorted)" - ) - - # ----------------------------------------------------------- - # Arm 5: 4both + orientation swap when C's stride-1 label is - # not on A. Swapping the operand slots leaves the contraction - # itself unchanged (A*B summed over the shared label; real - # dtypes, so no conjugation to track). - # ----------------------------------------------------------- - lbl_unit = indC2[argmin(abs.(Cstr))] - sym_unit = label_syms((lbl_unit,), indA2, case.IA, indB2, case.IB)[1] - swap5 = !(lbl_unit in indA2) - if swap5 - C5 = run_direct_arm!( - "5", case, dim, T, Cbase, Cref, Bperm4, indB4, Aperm4, indA4, indC2, true, - "(both sorted + SWAPPED: original B in M-role slot; C's stride-1 label `$sym_unit` was on B)" - ) - else - C5 = run_direct_arm!( - "5", case, dim, T, Cbase, Cref, Aperm4, indA4, Bperm4, indB4, indC2, false, - "(both sorted, no swap needed: C's stride-1 label `$sym_unit` already on A; repeat of 4both)" - ) - end - - # Bonus: bitwise equality between the direct-API outputs. - outs = (("2", C2), ("3", C3), ("4M", C4M), ("4N", C4N), ("4both", C4both), ("5", C5)) - if C2 !== nothing - parts = String[] - for (nm, Cx) in outs[2:end] - Cx === nothing && continue - eq = (Cx == C2) - eq || (global n_bitwise_diffs += 1) - push!(parts, "arm$nm==arm2:$(eq)") - end - both(" [bitwise] " * join(parts, " ")) - end - end - end -end - -push!(canary_results, run_canary(rng, "B (middle)")) -push!(canary_results, run_canary(rng, "A' (end)")) -close(csv_io) - -canary_spread = relative_spread(canary_results) -println("\ncanary spread (max-min)/min = ", @sprintf("%.4f", canary_spread)) -println(summary_io, "\ncanary median times (s): ", canary_results) -println(summary_io, "canary relative spread (max-min)/min: ", @sprintf("%.4f", canary_spread)) -both("\ncorrectness mismatches (aborted, not timed): $n_mismatches") -both("bitwise inequalities vs arm 2 (informational): $n_bitwise_diffs") - -# ---------------------------------------------------------------------------- -# Ratio table: median seconds per arm and ratio vs Arm 1 (QuasiStrided -# backend, the adapter path) and vs Arm 1 StridedBLAS. -# ---------------------------------------------------------------------------- - -const TABLE_ARMS = ("1-StridedBLAS", "1-QuasiStrided", DIRECT_ARMS...) - -function print_ratio_table(io::IO) - println(io, "\n# median seconds per arm; in brackets: time / Arm-1-QuasiStrided time (lower is better)") - hdr = rpad("case", 9) * rpad("dim", 4) * rpad("dtype", 8) - for a in TABLE_ARMS - hdr *= rpad(a, 22) - end - println(io, hdr) - for T in CORRECTNESS_DTYPES, dim in DIMS, case in CASES - base = get(RESULTS, (case.name, dim, T, "1-QuasiStrided"), NaN) - row = rpad(case.name, 9) * rpad(string(dim), 4) * rpad(string(T), 8) - for a in TABLE_ARMS - t = get(RESULTS, (case.name, dim, T, a), NaN) - row *= isnan(t) ? rpad("n/a", 22) : rpad(@sprintf("%.3e [%.3f]", t, t / base), 22) - end - println(io, row) - end - println(io, "\n# speedup of each arm over Arm-1-QuasiStrided (Arm1-QS time / arm time; >1 is faster), and Arm-1-StridedBLAS time / arm time") - hdr = rpad("case", 9) * rpad("dim", 4) * rpad("dtype", 8) - for a in TABLE_ARMS - hdr *= rpad(a, 22) - end - println(io, hdr) - for T in CORRECTNESS_DTYPES, dim in DIMS, case in CASES - base = get(RESULTS, (case.name, dim, T, "1-QuasiStrided"), NaN) - blas = get(RESULTS, (case.name, dim, T, "1-StridedBLAS"), NaN) - row = rpad(case.name, 9) * rpad(string(dim), 4) * rpad(string(T), 8) - for a in TABLE_ARMS - t = get(RESULTS, (case.name, dim, T, a), NaN) - row *= isnan(t) ? rpad("n/a", 22) : rpad(@sprintf("%.2fx / blas %.2fx", base / t, blas / t), 22) - end - println(io, row) - end - return nothing -end - -print_ratio_table(stdout) -print_ratio_table(summary_io) -close(summary_io) - -# ---------------------------------------------------------------------------- -# Provenance -# ---------------------------------------------------------------------------- - -function safe_run(cmd) - return try - read(cmd, String) - catch e - "unavailable ($(sprint(showerror, e)))" - end -end - -open(PROVENANCE_PATH, "w") do io - println(io, "git_commit = ", git_commit()) - println(io, "command = julia --project=. benchmark/bench_ccsd_t_store.jl ", join(ARGS, " ")) - println(io, "hostname = ", gethostname()) - println(io, "cpu = ", Sys.CPU_NAME) - println(io, "julia = ", VERSION) - println(io, "TensorOperations = ", pkgversion(TensorOperations)) - println(io, "nthreads = ", NTHREADS, " blas_threads = ", BLAS_THREADS) - println(io, "dtypes = ", collect(CORRECTNESS_DTYPES)) - println(io, "dims = ", collect(DIMS)) - println(io, "arms = 1 (StridedNative/StridedBLAS/QuasiStrided adapter), ", join(DIRECT_ARMS, ", "), " (direct API)") - println(io, "reps = 21 (dim=8, and dim=16 StridedNative/StridedBLAS); 15 (dim=16 QuasiStrided-engine rows, noted per-row)") - println(io, "smoke = ", SMOKE) - println(io, "date = ", now()) - println(io, "correctness mismatches (aborted, not timed) = ", n_mismatches) - println(io, "bitwise inequalities vs arm 2 (informational) = ", n_bitwise_diffs) - println(io, "canary median times (s) = ", canary_results) - println(io, "canary relative spread (max-min)/min = ", @sprintf("%.4f", canary_spread)) - println( - io, - "caveat = single machine ($(gethostname())), single measurement session; ", - "not averaged across machines or repeated sessions." - ) - println(io, "\n# Machine-load check at provenance-write time (uptime):") - println(io, safe_run(`uptime`)) - println(io, "# Machine-load check at provenance-write time (top -bn1 | head -15):") - println(io, safe_run(pipeline(`top -bn1`, `head -15`))) -end - -println("\nDone. Results in ", OUTDIR) diff --git a/benchmark/bench_default_vs_legacy.jl b/benchmark/bench_default_vs_legacy.jl deleted file mode 100644 index 87a4c9a..0000000 --- a/benchmark/bench_default_vs_legacy.jl +++ /dev/null @@ -1,109 +0,0 @@ -# The configuration this package ships now against the one it shipped before -# hardware detection existed. Both measured as a user would get them -- no -# pinned mc/kc/nc, no grid. It doubles as a regression guard. -# -# julia --project=. benchmark/bench_default_vs_legacy.jl - -include(joinpath(@__DIR__, "harness.jl")) - -using QuasiStrided: _default_kernel, default_blocking, _fallback_blocking, _fallback_shape - -# The pre-detection configuration, reconstructed. -function legacy_config(::Type{T}) where {T} - MR, NR, W = _fallback_shape(T) - return (SIMDKernel(Val(MR), Val(NR), T, Val(W)), _fallback_blocking(T)) -end - -# What `contract!` / `plan_contract` / QuasiStridedBackend use by default now. -function derived_config(::Type{T}) where {T} - kernel = _default_kernel(T) - return (kernel, default_blocking(kernel)) -end - -# The derived side passes NOTHING, so it takes the real default path including -# the extent-aware demotion; naming the kernel would bypass that. -function time_legacy(::Type{T}, fx, kernel, b; reps = 21) where {T} - plan = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = b.mc, kc = b.kc, nc = b.nc - ) - execute!(plan, one(T), zero(T)) - return median_time_s(() -> execute!(plan, one(T), zero(T)); reps = reps) -end - -function time_default(::Type{T}, fx; reps = 21) where {T} - plan = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC) - execute!(plan, one(T), zero(T)) - return (median_time_s(() -> execute!(plan, one(T), zero(T)); reps = reps), plan.kernel) -end - -print_env_header(stdout, "bench_default_vs_legacy.jl") -println("target = ", QuasiStrided.target_profile()) - -const OUTDIR = results_dir() -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "default_vs_legacy.csv") - -rng = MersenneTwister(0xB3_C4_0003) -csv = open(CSV_PATH, "w") -println(csv, "dtype,shape,legacy_MR,legacy_NR,legacy_W,new_MR,new_NR,new_W,legacy_s,new_s,speedup") - -canaries = Float64[] -push!(canaries, run_canary(rng, "A (start)")) - -for T in DTYPES - kold, bold = legacy_config(T) - knew, bnew = derived_config(T) - println("\n== ", T, " ==") - println(" legacy : MR=", mr(kold), " NR=", nr(kold), " W=", lanewidth(kold), " ", bold) - println(" derived: MR=", mr(knew), " NR=", nr(knew), " W=", lanewidth(knew), " ", bnew) - println(" (per-contraction demotion may pick a smaller shape; see last column)") - println( - rpad(" shape", 30), lpad("legacy s", 14), lpad("new s", 14), - lpad("speedup", 10), lpad("shape used", 12) - ) - - speedups = Float64[] - for spec in vcat(MAIN_SHAPES, SMALL_SHAPES) - fx = build_plain(T, spec, rng) - told = time_legacy(T, fx, kold, bold) - tnew, kused = time_default(T, fx) - push!(speedups, told / tnew) - println( - rpad(" " * spec.name, 30), lpad(@sprintf("%.6e", told), 14), - lpad(@sprintf("%.6e", tnew), 14), lpad(@sprintf("%.3f", told / tnew), 10), - lpad(string(mr(kused), "x", nr(kused), "/W", lanewidth(kused)), 12) - ) - println( - csv, "$T,$(spec.name),$(mr(kold)),$(nr(kold)),$(lanewidth(kold)),", - "$(mr(kused)),$(nr(kused)),$(lanewidth(kused)),", - @sprintf("%.9f,%.9f,%.4f", told, tnew, told / tnew) - ) - end - - fxs = build_scattered(T, rng) - told = time_legacy(T, fxs, kold, bold) - tnew, kused = time_default(T, fxs) - push!(speedups, told / tnew) - println( - rpad(" scattered_a64k64b16n64", 30), lpad(@sprintf("%.6e", told), 14), - lpad(@sprintf("%.6e", tnew), 14), lpad(@sprintf("%.3f", told / tnew), 10), - lpad(string(mr(kused), "x", nr(kused), "/W", lanewidth(kused)), 12) - ) - println( - csv, "$T,scattered_a64k64b16n64,$(mr(kold)),$(nr(kold)),$(lanewidth(kold)),", - "$(mr(kused)),$(nr(kused)),$(lanewidth(kused)),", - @sprintf("%.9f,%.9f,%.4f", told, tnew, told / tnew) - ) - - println( - " geomean speedup = ", @sprintf("%.3f", geomean(speedups)), - " worst = ", @sprintf("%.3f", minimum(speedups)), - " best = ", @sprintf("%.3f", maximum(speedups)) - ) - push!(canaries, run_canary(rng, "after $T")) -end - -close(csv) -println("\ncanary spread (max-min)/min = ", @sprintf("%.4f", relative_spread(canaries))) -println("Results in ", CSV_PATH) diff --git a/benchmark/bench_driver.jl b/benchmark/bench_driver.jl deleted file mode 100644 index 22e4bec..0000000 --- a/benchmark/bench_driver.jl +++ /dev/null @@ -1,279 +0,0 @@ -# Block-size sweep. -# Times `execute!` over a grid of shapes x (mc,kc,nc) for -# ScalarKernel/SIMDKernel x Float64/Float32, against `execute_tilewise!` and -# a `LinearAlgebra.mul!` line, and reports a geomean ranking. No cache model -# or probing by design -- just measured grid points. -# -# `mul!` is BLAS on a plain dense matmul of the same M/K/N: a reference -# point, NOT a target -- a "slower than mul!" ratio is not a problem. -# -# julia --project=. benchmark/bench_driver.jl -# -# writes results + PROVENANCE.txt to benchmark/results/-/. - -include(joinpath(@__DIR__, "harness.jl")) - -using QuasiStrided: execute_tilewise! - -const GRID_F64 = full_grid((64, 128, 256, 512), (128, 256, 512), (768, 1536, 3072)) -const GRID_F32 = full_grid((96, 192, 384, 768), (192, 384, 768), (1152, 2304, 4608)) - -grid_for(::Type{Float64}) = GRID_F64 -grid_for(::Type{Float32}) = GRID_F32 - -# ScalarKernel stays at the historical (8,6) reference shape; SIMDKernel uses -# whatever `_default_kernel` derives for this machine, so this sweep validates -# mc/nc at the shape the engine actually ships. -kernels_for(::Type{T}) where {T} = ( - ScalarKernel = ScalarKernel(Val(8), Val(6), T), - SIMDKernel = QuasiStrided._default_kernel(T), -) - -function print_header(io::IO) - print_env_header(io, "bench_driver.jl") - println(io, "target = ", QuasiStrided.target_profile()) - for T in DTYPES - for (kname, k) in pairs(kernels_for(T)) - println( - io, "kernel(", T, ", ", kname, "): MR=", mr(k), " NR=", nr(k), - k isa SIMDKernel ? " W=" * string(lanewidth(k)) : "" - ) - end - println(io, "default_blocking(", T, ") = ", QuasiStrided.default_blocking(kernels_for(T).SIMDKernel)) - end - return nothing -end -print_header(stdout) - -# Output location. -const OUTDIR = results_dir() -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "bench.csv") -const CANARY_PATH = joinpath(OUTDIR, "canary.csv") -const SUMMARY_PATH = joinpath(OUTDIR, "summary.txt") -const PROVENANCE_PATH = joinpath(OUTDIR, "PROVENANCE.txt") - -csv_io = open(CSV_PATH, "w") -println( - csv_io, - "kernel,dtype,shape,Ma,Ka,Na,method,mc,kc,nc,reps,median_seconds" -) - -function log_row(kernel_name, T, shapename, Ma, Ka, Na, method, mc, kc, nc, reps, t) - println( - csv_io, - "$kernel_name,$T,$shapename,$Ma,$Ka,$Na,$method,$mc,$kc,$nc,$reps,", - @sprintf("%.9f", t) - ) - return flush(csv_io) -end - -# Canary: a small, fixed case run at the start / middle / end of the sweep -# (A, B, A' pattern) to catch drift (thermal throttling, background load). -const CANARY_SHAPE = ShapeSpec("canary_64^3", 64, 64, 64) -const CANARY_COMBO = (128, 256, 1536) - -function run_canary(rng, label::String) - fx = build_plain(Float64, CANARY_SHAPE, rng) - kernel = SIMDKernel(Val(8), Val(6), Float64) - mc, kc, nc = CANARY_COMBO - plan = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = mc, kc = kc, nc = nc - ) - t = median_time_s(() -> execute!(plan, 1.0, 0.0); reps = 15) - println("canary[$label] median = $(t) s") - return t -end - -# Main grid sweep over the small/medium shapes. -rng = MersenneTwister(0xB3_C4_0001) - -raw_execute = Vector{NamedTuple}() # for ranking - -canary_results = Float64[] -push!(canary_results, run_canary(rng, "A (start)")) - -for T in DTYPES - for (kname, kernel) in pairs(kernels_for(T)) - combos = grid_for(T) - for (mc, kc, nc) in combos - for spec in MAIN_SHAPES - fx = build_plain(T, spec, rng) - plan = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = mc, kc = kc, nc = nc - ) - t = median_time_s(() -> execute!(plan, 1.0, 0.0); reps = 9) - log_row( - kname, T, spec.name, spec.Ma, spec.Ka, spec.Na, - "execute!", mc, kc, nc, 9, t - ) - push!( - raw_execute, - ( - kernel = String(kname), dtype = T, shape = spec.name, - mc = mc, kc = kc, nc = nc, t = t, - ) - ) - end - @info "grid progress" kernel = kname dtype = T combo = (mc, kc, nc) - end - - # execute_tilewise!, once per (kernel,dtype,shape) at the center - # combo -- the old driver has no macro blocking to sweep, so - # repeating it at every grid point would be pure waste. - center = combos[cld(length(combos), 2)] - mc, kc, nc = center - for spec in MAIN_SHAPES - fx = build_plain(T, spec, rng) - plan = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = mc, kc = kc, nc = nc - ) - t = median_time_s(() -> execute_tilewise!(plan, 1.0, 0.0); reps = 9) - log_row( - kname, T, spec.name, spec.Ma, spec.Ka, spec.Na, - "execute_tilewise!", mc, kc, nc, 9, t - ) - end - end -end - -push!(canary_results, run_canary(rng, "B (middle)")) - -# mul! reference line (once per dtype/shape -- it doesn't depend on -# kernel/blocking at all). -for T in DTYPES - for spec in MAIN_SHAPES - fx = build_plain(T, spec, rng) - t = median_time_s(() -> mul!(fx.Cmat, fx.Amat, fx.Bmat); reps = 9) - log_row("mul!", T, spec.name, spec.Ma, spec.Ka, spec.Na, "mul!", 0, 0, 0, 9, t) - end -end - -# Extra shapes (large cuboid + scattered-C fixture): only at each dtype's -# center combo, both kernels, to check the grid-derived winner generalizes -# without paying for the full grid at these more expensive shapes. -for T in DTYPES - for (kname, kernel) in pairs(kernels_for(T)) - mc, kc, nc = grid_for(T)[cld(length(grid_for(T)), 2)] - - for spec in EXTRA_SHAPES - fx = build_plain(T, spec, rng) - plan = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = mc, kc = kc, nc = nc - ) - t_exec = median_time_s(() -> execute!(plan, 1.0, 0.0); reps = 9) - log_row( - kname, T, spec.name, spec.Ma, spec.Ka, spec.Na, - "execute!", mc, kc, nc, 9, t_exec - ) - plan_tw = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = mc, kc = kc, nc = nc - ) - t_tw = median_time_s(() -> execute_tilewise!(plan_tw, 1.0, 0.0); reps = 9) - log_row( - kname, T, spec.name, spec.Ma, spec.Ka, spec.Na, - "execute_tilewise!", mc, kc, nc, 9, t_tw - ) - if kname == :ScalarKernel # mul! doesn't depend on kernel; log once - t_mul = median_time_s(() -> mul!(fx.Cmat, fx.Amat, fx.Bmat); reps = 9) - log_row("mul!", T, spec.name, spec.Ma, spec.Ka, spec.Na, "mul!", 0, 0, 0, 9, t_mul) - end - end - - # Scattered-C fixture: no plain-matmul mul! reference (it's a - # genuinely 3-index case; a 2-D mul! isn't a fair comparison point). - fx = build_scattered(T, rng) - plan = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = mc, kc = kc, nc = nc - ) - t_exec = median_time_s(() -> execute!(plan, 1.0, 0.0); reps = 9) - log_row(kname, T, "scattered_a64k64b16n64", 64, 64, 64, "execute!", mc, kc, nc, 9, t_exec) - plan_tw = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = mc, kc = kc, nc = nc - ) - t_tw = median_time_s(() -> execute_tilewise!(plan_tw, 1.0, 0.0); reps = 9) - log_row( - kname, T, "scattered_a64k64b16n64", 64, 64, 64, - "execute_tilewise!", mc, kc, nc, 9, t_tw - ) - end -end - -push!(canary_results, run_canary(rng, "A' (end)")) -close(csv_io) - -# Canary spread report. -canary_spread = (maximum(canary_results) - minimum(canary_results)) / minimum(canary_results) -open(CANARY_PATH, "w") do io - println(io, "label,median_seconds") - for (lbl, t) in zip(("A_start", "B_middle", "Aprime_end"), canary_results) - println(io, "$lbl,", @sprintf("%.9f", t)) - end - println(io, "# relative spread (max-min)/min = ", @sprintf("%.4f", canary_spread)) -end -println("canary spread (max-min)/min = ", @sprintf("%.4f", canary_spread)) - -# Ranking: geomean ratio of execute! time across all (kernel,shape) pairs -# in the main grid, per dtype, at each (mc,kc,nc) combo. Each shape/kernel -# time is first normalized by that (kernel,dtype,shape)'s own minimum -# across the combo grid, so shapes of very different absolute cost weigh -# equally in the geomean. -rank_combos(raw, T::DataType) = rank_by(raw, T, r -> (r.mc, r.kc, r.nc)) - -open(SUMMARY_PATH, "w") do io - println(io, "# Ranking summary") - println(io, "canary median times (s): ", canary_results) - println(io, "canary relative spread (max-min)/min: ", @sprintf("%.4f", canary_spread)) - chosen = Dict{DataType, Tuple{Int, Int, Int}}() - for T in DTYPES - ranked = rank_combos(raw_execute, T) - println(io, "\n== $T ranking (combo => geomean ratio, best first) ==") - for (combo, g) in ranked - println(io, " $combo => ", @sprintf("%.4f", g)) - end - best_combo, best_g = ranked[1] - best_size = prod(best_combo) - # Prefer the smallest-footprint combo within 6% of the best geomean - # (this project's own noise-floor convention: sub-6% differences - # are treated as noise, not a real ranking signal). - within_noise = filter(x -> x[2] <= best_g * 1.06, ranked) - winner = first(sort(within_noise; by = x -> prod(x[1]))) - chosen[T] = winner[1] - println( - io, - "chosen $T default: mc,kc,nc = ", winner[1], - " (geomean ratio ", @sprintf("%.4f", winner[2]), "; unconstrained best was ", - best_combo, " at ", @sprintf("%.4f", best_g), ")" - ) - runner_up = length(ranked) > 1 ? ranked[2] : nothing - println(io, "runner-up: ", runner_up) - end - println(io, "\nchosen defaults: ", chosen) - global CHOSEN = chosen -end -println(read(SUMMARY_PATH, String)) - -# Provenance. -commit = git_commit() -open(PROVENANCE_PATH, "w") do io - println(io, "git_commit = ", commit) - println(io, "command = julia --project=. benchmark/bench_driver.jl") - println(io, "cpu = ", Sys.CPU_NAME) - println(io, "julia = ", VERSION) - println(io, "nthreads = ", NTHREADS, " blas_threads = ", BLAS_THREADS) - println(io, "date = ", now()) - println(io, "grid_f64 = ", GRID_F64) - println(io, "grid_f32 = ", GRID_F32) - println(io, "main_shapes = ", [s.name for s in MAIN_SHAPES]) - println(io, "extra_shapes = ", [s.name for s in EXTRA_SHAPES], " + scattered_a64k64b16n64") - println(io, "chosen_defaults = ", CHOSEN) -end - -println("\nDone. Results in ", OUTDIR) diff --git a/benchmark/bench_half_packing.jl b/benchmark/bench_half_packing.jl deleted file mode 100644 index 7e004f8..0000000 --- a/benchmark/bench_half_packing.jl +++ /dev/null @@ -1,242 +0,0 @@ -# Empirical measurement for `execute_half_packed!` (src/execution/halfpack.jl): -# B packed as usual, A read in place when the whole M extent is exactly one -# full SIMDKernel register tile (`Qm == mr(kernel)`) -- versus the fully -# packed `execute!`. -# -# julia --project=. benchmark/bench_half_packing.jl -# -# Point: profiling (see benchmark/profile_buckets.jl and the investigation -# behind this file) found packing to be a genuine 30-38% of `execute!`'s own -# time in the 8-32-element range, most of it attributable to a redundant -# copy-then-immediately-reread round trip through a freshly packed A panel -# when A already has the layout the microkernel could read directly. This -# script sweeps M fixed at exactly `mr(kernel)` (the only shape -# `execute_half_packed!` takes the in-place path for) across a range of N/K -# from small to large, to see where -- if anywhere -- skipping A's packing -# pays off and by how much, and whether that shrinks or grows with N/K. - -# This script's first sweep (below) only ever measured the simplest eligible -# case: a plain, contiguous, dense M-by-K/K-by-N matrix pair. Two follow-up -# sweeps extend that: -# -# - "non-ramp K": A's own rows are still unit-stride (eligible), but the K -# composite as a WHOLE (both operands' maps) is not a single affine ramp, -# so `_axis_of` hands the in-place A read a `PtrScatterAxis` -- a per-K-step -# indexed lookup instead of a compile-time stride multiply. This is the -# realistic case for a genuine multi-label tensor contraction (as opposed -# to a plain two-index GEMM), and it was previously only correctness- -# tested (test/execution/test_halfpack.jl), never benchmarked. -# - "alternate kernel shape": the first sweep only ever measured M pinned to -# the DEFAULT kernel's `mr` on this machine. This sweeps a second, -# non-default menu shape too. - -include(joinpath(@__DIR__, "harness.jl")) - -using QuasiStrided: execute_half_packed!, _default_kernel, kernel_shapes, _kernel_from_shape, - affine_ramp - -const REPS_TINY = 51 -const REPS_MAIN = 21 - -# (K, N) pairs at M = mr(kernel) exactly. Doubling ladder from tiny K/N (where -# the packing-avoidance premise should matter most, relative to a near-zero -# baseline cost) up through shapes large enough that B's own packing/compute -# dominates and any A-side saving should vanish into noise. -const HALF_PACKING_KN = [ - (1, 1), (2, 2), (4, 4), (8, 8), (16, 16), (32, 32), (63, 63), - (64, 64), (128, 128), (256, 256), (512, 512), (1024, 1024), - (2048, 2048), (4096, 512), (512, 4096), -] - -reps_for(K::Int, N::Int) = (K * N <= 64 * 64) ? REPS_TINY : REPS_MAIN - -function time_packed(::Type{T}, fx; reps::Int) where {T} - plan = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC) - execute!(plan, one(T), zero(T)) - t = median_time_s(() -> execute!(plan, one(T), zero(T)); reps = reps) - return t -end - -function time_half_packed(::Type{T}, fx; reps::Int) where {T} - plan = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC) - execute_half_packed!(plan, one(T), zero(T)) - t = median_time_s(() -> execute_half_packed!(plan, one(T), zero(T)); reps = reps) - return (t, plan) -end - -# --------------------------------------------------------------------------- -# "Non-ramp K" fixture: A[m,k1,k2] dense (m fastest, then k1, then k2) so A's -# OWN map is a clean ramp; B[k2,k1,n] dense (k2 fastest) so B's map, over the -# SAME (k1,k2) label order, is not -- stride(k2 in B) = 1 is far smaller than -# stride(k1 in B) = k2n, the opposite of what folding k1,k2 into one ramp -# needs. `affine_ramp` requires EVERY map to fold, so the whole K composite -# (shared by both operands) classifies as non-ramp, even though A's own -# addresses alone would trivially ramp -- exactly the generic multi-label -# case, and the in-place A read pays for it (indexed `PtrScatterAxis` lookups, -# not a compile-time stride multiply) regardless of which operand "caused" it. -function build_nonramp_k(::Type{T}, M::Int, k1n::Int, k2n::Int, N::Int, rng) where {T} - Amat = randn(rng, T, M, k1n, k2n) - Bmat = randn(rng, T, k2n, k1n, N) - Cmat = zeros(T, M, N) - Av = StridedView(Amat) - Bv = StridedView(Bmat) - Cv = StridedView(Cmat) - # indA labels: m=1, k1=2, k2=3. indB labels B's own physical axis order - # (k2, k1, n) = (3, 2, 4). indC: (m, n) = (1, 4). - return (Av = Av, indA = (1, 2, 3), Bv = Bv, indB = (3, 2, 4), Cv = Cv, indC = (1, 4)) -end - -const NONRAMP_K1 = 4 # fixed; k2 scales to reach each target K = k1n*k2n below -const NONRAMP_TOTAL_K_N = [ - (4, 1), (8, 2), (16, 8), (32, 32), (64, 64), (128, 128), - (256, 256), (512, 512), (1024, 1024), -] - -function run_nonramp_sweep(csv, ::Type{T}, rng) where {T} - default_kernel = _default_kernel(T) - M = mr(default_kernel) - for (K, N) in NONRAMP_TOTAL_K_N - k1n = NONRAMP_K1 - k2n = cld(K, k1n) - fx = build_nonramp_k(T, M, k1n, k2n, N, rng) - plan_check = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC) - if affine_ramp(plan_check.kgroup)[1] - @warn "non-ramp K fixture unexpectedly ramps; skipping" T K N - continue - end - reps = reps_for(k1n * k2n, N) - - t_packed = time_packed(T, fx; reps = reps) - (t_half, plan) = time_half_packed(T, fx; reps = reps) - gf_packed = gflops(T, M, k1n * k2n, N, t_packed) - gf_half = gflops(T, M, k1n * k2n, N, t_half) - speedup = t_packed / t_half - - println( - csv, - "$T,$M,$(k1n * k2n),$N,$reps,$t_packed,$t_half,$gf_packed,$gf_half,$speedup,nonramp_k" - ) - @printf( - "[nonramp-K] %-10s M=%-4d K=%-5d(=%dx%d) N=%-5d packed %10.3e s (%6.2f GF/s) half %10.3e s (%6.2f GF/s) speedup %6.3fx\n", - string(T), M, k1n * k2n, k1n, k2n, N, t_packed, gf_packed, t_half, gf_half, speedup - ) - end - return nothing -end - -# --------------------------------------------------------------------------- -# Alternate (non-default) kernel shapes: the first and non-ramp sweeps above -# only ever pin M to mr() of the DEFAULT kernel this machine's ISA resolves -# to. Sweep every OTHER menu shape too (kernel_shapes(T), real method), at a -# fixed mid-range (K, N), to check the win isn't an artifact of one specific -# register shape. -const ALT_SHAPE_K, ALT_SHAPE_N = 128, 128 - -function run_alt_shape_sweep(csv, ::Type{T}, rng) where {T} - default_kernel = _default_kernel(T) - for shape in kernel_shapes(T) - kernel = _kernel_from_shape(shape, T) - M = mr(kernel) - M == mr(default_kernel) && shape == (mr(default_kernel), nr(default_kernel), lanewidth(default_kernel)) && continue - spec = ShapeSpec("alt_$(M)x$(ALT_SHAPE_K)x$(ALT_SHAPE_N)", M, ALT_SHAPE_K, ALT_SHAPE_N) - fx = build_plain(T, spec, rng) - reps = REPS_MAIN - - plan_p = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; kernel = kernel) - execute!(plan_p, one(T), zero(T)) - t_packed = median_time_s(() -> execute!(plan_p, one(T), zero(T)); reps = reps) - plan_h = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; kernel = kernel) - execute_half_packed!(plan_h, one(T), zero(T)) - t_half = median_time_s(() -> execute_half_packed!(plan_h, one(T), zero(T)); reps = reps) - - gf_packed = gflops(T, M, ALT_SHAPE_K, ALT_SHAPE_N, t_packed) - gf_half = gflops(T, M, ALT_SHAPE_K, ALT_SHAPE_N, t_half) - speedup = t_packed / t_half - - println( - csv, - "$T,$M,$ALT_SHAPE_K,$ALT_SHAPE_N,$reps,$t_packed,$t_half,$gf_packed,$gf_half,$speedup,alt_shape_$(shape[1])x$(shape[2])W$(shape[3])" - ) - @printf( - "[alt-shape] %-10s kernel=%dx%d/W%d packed %10.3e s (%6.2f GF/s) half %10.3e s (%6.2f GF/s) speedup %6.3fx\n", - string(T), shape[1], shape[2], shape[3], t_packed, gf_packed, t_half, gf_half, speedup - ) - end - return nothing -end - -const OUTDIR = results_dir() -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "half_packing.csv") -const PROV_PATH = joinpath(OUTDIR, "half_packing_PROVENANCE.txt") - -function main() - print_env_header(stdout, "bench_half_packing.jl") - csv = open(CSV_PATH, "w") - println(csv, "dtype,M,K,N,reps,t_packed,t_half_packed,gf_packed,gf_half_packed,speedup_half_over_packed,fixture") - - canaries = Float64[] - rng = Random.MersenneTwister(0x4A1FBACC) - push!(canaries, run_canary(rng, "start")) - - for T in DTYPES # Float64, Float32 -- execute_half_packed! is real-only by design - # M is fixed at exactly mr(kernel) for the DEFAULT kernel this dtype - # resolves to on this machine -- the only shape the in-place path takes. - default_kernel = _default_kernel(T) - M = mr(default_kernel) - for (K, N) in HALF_PACKING_KN - spec = ShapeSpec("$(M)x$(K)x$(N)", M, K, N) - fx = build_plain(T, spec, rng) - reps = reps_for(K, N) - - t_packed = time_packed(T, fx; reps = reps) - (t_half, plan) = time_half_packed(T, fx; reps = reps) - gf_packed = gflops(T, M, K, N, t_packed) - gf_half = gflops(T, M, K, N, t_half) - speedup = t_packed / t_half - - println( - csv, - "$T,$M,$K,$N,$reps,$t_packed,$t_half,$gf_packed,$gf_half,$speedup,plain" - ) - @printf( - "%-10s M=%-4d (mr) K=%-5d N=%-5d packed %10.3e s (%6.2f GF/s) half %10.3e s (%6.2f GF/s) half/packed speedup %6.3fx kernel=%dx%d/W%d\n", - string(T), M, K, N, t_packed, gf_packed, t_half, gf_half, speedup, - mr(plan.kernel), nr(plan.kernel), lanewidth(plan.kernel) - ) - end - - run_nonramp_sweep(csv, T, rng) - run_alt_shape_sweep(csv, T, rng) - end - - push!(canaries, run_canary(rng, "end")) - close(csv) - - spread = relative_spread(canaries) - @printf("\ncanary spread: %.1f%% %s\n", 100spread, canaries) - if spread > 0.1 - @warn "canary spread exceeds 10%: the machine was not quiet enough for " * - "a few-percent conclusion. Re-run before believing any of these ratios." - end - - open(PROV_PATH, "w") do io - print_env_header(io, "bench_half_packing.jl") - println(io, "git commit = ", git_commit()) - println(io, "canaries = ", canaries) - @printf(io, "canary spread = %.2f%%\n", 100spread) - println( - io, - "\nM is fixed per dtype at mr(default_kernel(T)) on THIS machine -- see the", - ) - println( - io, - "printed kernel= shape per row. speedup_half_over_packed > 1 means the", - ) - println(io, "in-place-A path won; read half_packing.csv for the full sweep.") - end - println("\nwrote ", CSV_PATH) - return nothing -end - -main() diff --git a/benchmark/bench_kernel_shape.jl b/benchmark/bench_kernel_shape.jl deleted file mode 100644 index f9a2bf9..0000000 --- a/benchmark/bench_kernel_shape.jl +++ /dev/null @@ -1,355 +0,0 @@ -# Register-blocking sweep: (MR, NR, W) x kc for SIMDKernel. -# -# kc is swept jointly with the shape, never held fixed: MR*kc*sizeof(T) is the -# A-micropanel L1 footprint, so doubling MR at fixed kc measures the wrong -# thing. -# -# julia --project=. benchmark/bench_kernel_shape.jl - -include(joinpath(@__DIR__, "harness.jl")) - -using InteractiveUtils: code_native - -# --------------------------------------------------------------------------- -# Candidates -# --------------------------------------------------------------------------- - -# (MR, NR, W). Budget: NV = (MR/W)*NR accumulators + MR/W live A vectors, -# against 32 registers on AVX-512 (the count comes from AVX512VL, not the lane -# width in use). `spill_report` below is the ground truth. BLIS's own choices, -# harvested from the blis_jll 2.0.0+2 artifact: skx MR_d=16 NR_d=14, -# MR_s=32 NR_s=12; haswell MR_d=8 NR_d=6 (= this package's old default). -const CANDIDATES_F64 = [ - (8, 6, 4), # control: today's default (= BLIS haswell dgemm 8x6) - # (16,6,4): NV=24 at 256-bit lanes, kept to separate "more accumulators" - # from "wider accumulators". It does NOT spill here -- 32 ymm registers - # via AVX512VL, see the note above. - (16, 6, 4), - (8, 12, 4), # bigger tile, 256-bit lanes: isolates tile size from width - (8, 6, 8), # 512-bit lanes, small tile: isolates width from tile size - (16, 6, 8), - (16, 8, 8), - (16, 12, 8), - (16, 14, 8), # BLIS skx dgemm family - (24, 8, 8), - (32, 6, 8), - (8, 14, 8), # wide/short: better padding behaviour at small M -] - -const CANDIDATES_F32 = [ - (8, 6, 8), # control: today's default - (16, 12, 8), # 256-bit, bigger tile - (16, 6, 16), - (16, 12, 16), - (16, 14, 16), - (32, 12, 16), # BLIS skx sgemm family - (32, 6, 16), - (48, 8, 16), -] - -candidates_for(::Type{Float64}) = CANDIDATES_F64 -candidates_for(::Type{Float32}) = CANDIDATES_F32 - -# mc/nc pinned at the Stage 0 re-measured defaults; only kc moves with MR. -const PINNED_F64 = (mc = 64, nc = 1536) -const PINNED_F32 = (mc = 96, nc = 1152) -pinned_for(::Type{Float64}) = PINNED_F64 -pinned_for(::Type{Float32}) = PINNED_F32 - -const KC_BASE_F64 = 128 -const KC_BASE_F32 = 384 -kc_base(::Type{Float64}) = KC_BASE_F64 -kc_base(::Type{Float32}) = KC_BASE_F32 - -# The measured default; one holding MR*kc*sizeof(T) at the control's 8 KiB A -# micropanel; and 2x, since top-ranked points sit at the largest kc. -function kc_values(::Type{T}, MR::Int) where {T} - base = kc_base(T) - panel = max(8, round(Int, base * 8 / MR) & ~7) - return unique((base, panel, 2 * base)) -end - -# --------------------------------------------------------------------------- -# Spill detection -# --------------------------------------------------------------------------- - -# A stack store has the [rsp/rbp] operand first, i.e. followed by a comma. -# The match must be pinned to the accumulator's own width: matching any width -# counts the 16-byte GC-frame store every one of these functions emits, and -# reports a phantom spill even at 6 accumulators. -# Matches BOTH `vfmadd*` and `vfnmadd*`. The negated form is not an -# alternative spelling: a planar complex kernel issues exactly `MV*NR` of them -# per K step (the `-Ai*Bi` term of the real output plane) against `3*MV*NR` -# plain ones, so a bare `r"vfmadd"` -- which does NOT match the substring -# `vfnmadd` -- would undercount planar's FMAs by a quarter and make the -# "fmas should equal nv" check below read as a spurious shortfall. -const FMA_RE = r"vfn?madd" - -function stack_store_re(vector_bytes::Int) - word = vector_bytes == 64 ? "zmmword" : - vector_bytes == 32 ? "ymmword" : "xmmword" - return Regex( - "v(?:mov(?:up|ap)[sd]|movdq[ua])\\s+" * word * - "\\s+ptr\\s+\\[r[sb]p[^\\]]*\\]\\s*," - ) -end - -# `spills == 0` passes; `fmas` should equal `nv` (one FMA per accumulator per -# K step) -- a shortfall means LLVM restructured the body. -function spill_report(kernel::SIMDKernel{MR, NR, T, W}) where {MR, NR, T, W} - nv = (MR ÷ W) * NR - acc = QuasiStrided.zero_accumulator(kernel) - io = IOBuffer() - try - code_native( - io, Base.accumulate, - (typeof(kernel), typeof(acc), Vector{T}, Vector{T}, Int); - debuginfo = :none, syntax = :intel - ) - catch err - return (spills = -1, fmas = -1, nv = nv, ok = false, err = string(err)) - end - asm = String(take!(io)) - spills = count(stack_store_re(W * sizeof(T)), asm) - fmas = count(FMA_RE, asm) - return (spills = spills, fmas = fmas, nv = nv, ok = true, err = "") -end - -# --------------------------------------------------------------------------- -# Setup -# --------------------------------------------------------------------------- - -function print_header(io::IO) - print_env_header(io, "bench_kernel_shape.jl") - for T in DTYPES - println(io, "candidates(", T, ") = ", candidates_for(T)) - println(io, "pinned(", T, ") = ", pinned_for(T), " kc_base = ", kc_base(T)) - end - return nothing -end -print_header(stdout) - -const OUTDIR = results_dir() -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "bench_kernel_shape.csv") -const SPILL_PATH = joinpath(OUTDIR, "kernel_shape_spills.csv") -const SUMMARY_PATH = joinpath(OUTDIR, "summary_kernel_shape.txt") -const CANARY_PATH = joinpath(OUTDIR, "canary_kernel_shape.csv") -const PROVENANCE_PATH = joinpath(OUTDIR, "PROVENANCE_kernel_shape.txt") - -const REPS = 9 - -csv_io = open(CSV_PATH, "w") -println(csv_io, "dtype,shape,Ma,Ka,Na,MR,NR,W,NV,mc,kc,nc,reps,median_seconds,allocated_bytes") - -function log_row(T, spec, MR, NR, W, mc, kc, nc, t, alloc) - nv = (MR ÷ W) * NR - println( - csv_io, - "$T,$(spec.name),$(spec.Ma),$(spec.Ka),$(spec.Na),$MR,$NR,$W,$nv,", - "$mc,$kc,$nc,$REPS,", @sprintf("%.9f", t), ",$alloc" - ) - return flush(csv_io) -end - -# --------------------------------------------------------------------------- -# Spill pass (no timing; cheap and decides how to read the timings) -# --------------------------------------------------------------------------- - -# "0 spills" only means something if the detector can see one at all. -function assert_spill_detector_works() - for (MR, NR, T, W) in ((16, 16, Float64, 4), (32, 32, Float32, 16)) - rep = spill_report(SIMDKernel(Val(MR), Val(NR), T, Val(W))) - rep.spills > 0 || error( - "spill detector is broken: ($MR,$NR,$T,W=$W) has NV=$(rep.nv) " * - "accumulators, which cannot fit 32 registers, yet reported " * - "$(rep.spills) spills" - ) - @info "spill detector self-test" shape = (MR, NR, T, W) NV = rep.nv spills = rep.spills - end - return nothing -end -assert_spill_detector_works() - -spill_rows = NamedTuple[] -open(SPILL_PATH, "w") do io - println(io, "dtype,MR,NR,W,NV,spills,fmas,codegen_ok,note") - for T in DTYPES, (MR, NR, W) in candidates_for(T) - kernel = SIMDKernel(Val(MR), Val(NR), T, Val(W)) - rep = spill_report(kernel) - push!(spill_rows, (dtype = T, MR = MR, NR = NR, W = W, rep = rep)) - println( - io, "$T,$MR,$NR,$W,$(rep.nv),$(rep.spills),$(rep.fmas),", - "$(rep.ok),", replace(rep.err, ',' => ';') - ) - @info "spill check" dtype = T shape = (MR, NR, W) NV = rep.nv spills = rep.spills fmas = rep.fmas - end -end - -# --------------------------------------------------------------------------- -# Timing sweep -# --------------------------------------------------------------------------- - -rng = MersenneTwister(0xB3_C4_0002) -raw = NamedTuple[] -canary_results = Float64[] -push!(canary_results, run_canary(rng, "A (start)")) - -const SWEEP_SHAPES = vcat(MAIN_SHAPES, SMALL_SHAPES) - -function measure!(raw, T, spec, fx, kernel, MR, NR, W, mc, kc, nc) - plan = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = mc, kc = kc, nc = nc - ) - execute!(plan, one(T), zero(T)) # warm up before @allocated - alloc = @allocated execute!(plan, one(T), zero(T)) - t = median_time_s(() -> execute!(plan, one(T), zero(T)); reps = REPS) - log_row(T, spec, MR, NR, W, mc, kc, nc, t, alloc) - push!( - raw, ( - kernel = "SIMDKernel", dtype = T, shape = spec.name, - MR = MR, NR = NR, W = W, kc = kc, t = t, alloc = alloc, - ) - ) - return t -end - -for T in DTYPES - pin = pinned_for(T) - for (MR, NR, W) in candidates_for(T) - kernel = SIMDKernel(Val(MR), Val(NR), T, Val(W)) - for kc in kc_values(T, MR) - for spec in SWEEP_SHAPES - fx = build_plain(T, spec, rng) - measure!(raw, T, spec, fx, kernel, MR, NR, W, pin.mc, kc, pin.nc) - end - # scattered / 3-index fixture: the store path that cannot - # vectorize, so a larger register tile may behave differently. - fxs = build_scattered(T, rng) - spec_s = ShapeSpec("scattered_a64k64b16n64", 64, 64, 64) - measure!(raw, T, spec_s, fxs, kernel, MR, NR, W, pin.mc, kc, pin.nc) - @info "sweep progress" dtype = T shape = (MR, NR, W) kc = kc - end - end - push!(canary_results, run_canary(rng, "mid ($T done)")) -end - -push!(canary_results, run_canary(rng, "A' (end)")) -close(csv_io) - -spread = relative_spread(canary_results) -open(CANARY_PATH, "w") do io - println(io, "index,median_seconds") - for (i, t) in enumerate(canary_results) - println(io, "$i,", @sprintf("%.9f", t)) - end - println(io, "# relative spread (max-min)/min = ", @sprintf("%.4f", spread)) -end -println("canary spread (max-min)/min = ", @sprintf("%.4f", spread)) - -# --------------------------------------------------------------------------- -# Ranking and summary -# --------------------------------------------------------------------------- - -# MAIN_SHAPES decide; SMALL_SHAPES are reported apart so a large-shape win -# cannot hide a small-extent padding regression. -const MAIN_NAMES = Set(s.name for s in MAIN_SHAPES) -const SMALL_NAMES = Set(s.name for s in SMALL_SHAPES) - -shape_key(r) = (r.MR, r.NR, r.W, r.kc) -nv_of(key) = (key[1] ÷ key[3]) * key[2] - -control_for(::Type{Float64}) = (8, 6, 4) -control_for(::Type{Float32}) = (8, 6, 8) - -open(SUMMARY_PATH, "w") do io - print_header(io) - println(io, "\ncanary median times (s): ", canary_results) - println(io, "canary relative spread (max-min)/min: ", @sprintf("%.4f", spread)) - - println(io, "\n== spill report (spills must be 0; fmas should equal NV) ==") - for r in spill_rows - println( - io, " ", r.dtype, " (MR,NR,W)=", (r.MR, r.NR, r.W), - " NV=", r.rep.nv, " spills=", r.rep.spills, " fmas=", r.rep.fmas, - r.rep.ok ? "" : " CODEGEN FAILED: " * r.rep.err - ) - end - - for T in DTYPES - main_rows = filter(r -> r.dtype == T && r.shape in MAIN_NAMES, raw) - ranked = rank_by(main_rows, T, shape_key) - - println(io, "\n== $T ranking on MAIN_SHAPES ((MR,NR,W,kc) => geomean ratio) ==") - for (key, g) in ranked - println(io, " ", key, " NV=", nv_of(key), " => ", @sprintf("%.4f", g)) - end - - winner, wg = choose_within_noise(ranked, nv_of) - best_key, best_g = ranked[1] - println( - io, "chosen $T shape: (MR,NR,W,kc) = ", winner, - " (geomean ", @sprintf("%.4f", wg), "; unconstrained best ", - best_key, " at ", @sprintf("%.4f", best_g), ")" - ) - - # Speedup of the best candidate over the control, per shape, so the - # decision is reported in wall-clock terms and not only as a ratio. - ctrl = control_for(T) - println(io, "\n-- $T per-shape time vs control $(ctrl) (>1 means candidate is faster) --") - for name in vcat( - [s.name for s in MAIN_SHAPES], [s.name for s in SMALL_SHAPES], - ["scattered_a64k64b16n64"] - ) - rows = filter(r -> r.dtype == T && r.shape == name, raw) - isempty(rows) && continue - cbest = minimum( - r.t for r in rows - if (r.MR, r.NR, r.W) == ctrl; init = Inf - ) - wbest = minimum( - r.t for r in rows - if (r.MR, r.NR, r.W) == (winner[1], winner[2], winner[3]); init = Inf - ) - tag = name in SMALL_NAMES ? " [small-extent]" : - (name in MAIN_NAMES ? "" : " [scattered]") - println( - io, " ", rpad(name, 26), " control ", @sprintf("%.6e", cbest), - " chosen ", @sprintf("%.6e", wbest), - " speedup ", @sprintf("%.3f", cbest / wbest), tag - ) - end - - println(io, "\n-- $T allocation per execute! (Julia $(VERSION)) --") - for (MR, NR, W) in candidates_for(T) - rows = filter(r -> r.dtype == T && (r.MR, r.NR, r.W) == (MR, NR, W), raw) - isempty(rows) && continue - println( - io, " (MR,NR,W)=", (MR, NR, W), " NV=", (MR ÷ W) * NR, - " max allocated = ", maximum(r.alloc for r in rows), " B" - ) - end - end -end -println(read(SUMMARY_PATH, String)) - -open(PROVENANCE_PATH, "w") do io - println(io, "git_commit = ", git_commit()) - println(io, "command = julia --project=. benchmark/bench_kernel_shape.jl") - println(io, "cpu = ", Sys.CPU_NAME) - println(io, "julia = ", VERSION) - println(io, "nthreads = ", NTHREADS, " blas_threads = ", BLAS_THREADS) - println(io, "date = ", now()) - println(io, "reps = ", REPS) - println(io, "candidates_f64 = ", CANDIDATES_F64) - println(io, "candidates_f32 = ", CANDIDATES_F32) - println(io, "pinned_f64 = ", PINNED_F64, " kc_base_f64 = ", KC_BASE_F64) - println(io, "pinned_f32 = ", PINNED_F32, " kc_base_f32 = ", KC_BASE_F32) - println(io, "main_shapes = ", [s.name for s in MAIN_SHAPES]) - println(io, "small_shapes = ", [s.name for s in SMALL_SHAPES]) - println(io, "canary_spread = ", @sprintf("%.4f", spread)) - println(io, "blis_reference = blis_jll 2.0.0+2 artifact share/blis/config/*/bli_kernel_defs_*.h") -end - -println("\nDone. Results in ", OUTDIR) diff --git a/benchmark/bench_pack_loops.jl b/benchmark/bench_pack_loops.jl deleted file mode 100644 index 11a4e0a..0000000 --- a/benchmark/bench_pack_loops.jl +++ /dev/null @@ -1,297 +0,0 @@ -# Packing-loop microbenchmark: per-call cost of the single `_pack_panel!` loop -# (src/packing/pack.jl), which packs every format (real, planar, 1e, 1m), -# dispatched via `_pack_emit!`/`_pack_emit_zero!`. Times real and complex -# formats on full and tail slivers (ns per element and per real written), and -# its `code_llvm` section dumps `_pack_panel!` for a complex format to check -# its branch structure. -# -# julia -t 1 --project=benchmark benchmark/bench_pack_loops.jl -# -# Every case calls `pack_a!`/`pack_b!` **through `plan.kernel`** (never the -# `_pack_panel!` loop function directly): a -# `plan_contract` call builds a real plan first (default kernel selection for -# the real/complex default cases, an explicitly named `OneMKernel` for the 1m -# case, mirroring `benchmark/profile_to_suite.jl`'s `_onem_default_kernel`), -# and `plan.kernel` is what every timed `pack_a!`/`pack_b!` call is dispatched -# on (src/microkernels/interface.jl forwards `DescriptorKernel` to its `.descriptor`, -# which is where src/packing/pack.jl's real/complex methods live). -# -# Fixtures are built directly against `QSTile`/`PackedPanel` (as -# `test/packing/test_pack*.jl` do), NOT via `execute!`, so each case isolates one -# pack call's cost instead of a whole contraction's. "Full sliver" means the -# tile's valid row/column count equals the kernel's MR/NR (the loop's fast -# branch in the real path, no padding at all); "tail sliver" means valid == -# MR-1/NR-1 (the padding branch), the case a non-multiple extent produces at -# the last block of an M/N sweep. `kc` (the K depth packed per call) is fixed -# at 256 for every case. -# -# Real-dtype A is packed from a deliberately NON-contiguous (stride-2 rows) -# source, so `_pack_a_contiguous_eligible` (src/packing/pack_contiguous.jl) can never -# fire -- `_unit_stride_rows` (src/microkernels/simd.jl) requires stride == 1 -# exactly -- and `pack_a!` falls into `_pack_panel!`'s scalar fallback body, -# the real baseline the complex formats are compared against. Real B -# has no contiguous fast path to dodge either way, so it is packed from an -# ordinary column-major-pitch tile. - -include(joinpath(@__DIR__, "harness.jl")) - -using QuasiStrided: PlanarKernel, OneMKernel, kernel_shapes, OneMMethod, default_blocking, - PackedPanel, packed_panel, AffineAxis, SourceTile, pack_a!, pack_b!, - packed_a_length, packed_b_length, a_format, b_format, reals_per_element, realtype, - ComplexKernelDescriptor, PlanarFormat, packed_a_plane_offset, packed_b_plane_offset, - tile_load, _pack_panel! -using InteractiveUtils - -const REPS = 21 -const KC = 256 # fixed K depth for every packing call -const SHAPE_FOR_PLAN = ShapeSpec("plan_256^3", 256, 256, 256) # only to pick plan.kernel - -const OUTDIR = results_dir() -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "pack_loops.csv") -const PROV_PATH = joinpath(OUTDIR, "pack_loops_PROVENANCE.txt") - -# --------------------------------------------------------------------------- -# Fixture builders: plain QSTile sources, no `execute!` involved. -# --------------------------------------------------------------------------- - -# Non-contiguous A source (stride-2 rows): defeats -# `_pack_a_contiguous_eligible` unconditionally, so `pack_a!` always runs -# `_pack_panel!`'s scalar body for the real dtypes below. -function noncontig_a_tile(::Type{T}, m::Int, kc::Int) where {T} - rowstride = 2 - colstride = 4 * max(m, 1) # generous pitch; no address overlap with rows - len = (m == 0 || kc == 0) ? 1 : (m - 1) * rowstride + (kc - 1) * colstride + 1 - storage = rand(T, len) - return SourceTile(storage, 0, AffineAxis(0, rowstride, m), AffineAxis(0, colstride, kc)) -end - -# Ordinary column-major-pitch B source (unit-stride K rows): B has no -# contiguous fast path either way, so this is simply "packed normally". -function plain_b_tile(::Type{T}, kc::Int, n::Int) where {T} - storage = rand(T, kc * max(n, 1)) - return SourceTile(storage, 0, AffineAxis(0, 1, kc), AffineAxis(0, kc, n)) -end - -# Ordinary contiguous complex A/B tiles (no fast path exists for complex -# packing at all, so contiguity here is just the common case, not a dodge). -function plain_a_tile(::Type{T}, m::Int, kc::Int) where {T} - storage = rand(T, max(m, 1) * kc) - return SourceTile(storage, 0, AffineAxis(0, 1, m), AffineAxis(0, max(m, 1), kc)) -end - -# --------------------------------------------------------------------------- -# Plan builders: get a real `plan.kernel` for each configuration. -# --------------------------------------------------------------------------- - -# Default kernel (real SIMDKernel, or complex PlanarKernel -- the driver's -# unconditional complex default; src/execution/execute.jl) at a plain 256^3 shape. -function default_plan_kernel(::Type{T}) where {T} - rng = Random.MersenneTwister(0x0007A616) - fx = build_plain(T, SHAPE_FOR_PLAN, rng) - plan = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC) - return plan.kernel -end - -# Explicitly-named OneMKernel (1e format on A), mirroring -# `benchmark/profile_to_suite.jl`'s `_onem_default_kernel`. -_onem_default_kernel(::Type{T}) where {T} = - ((MR, NR, W) = kernel_shapes(T, OneMMethod())[end]; OneMKernel(Val(MR), Val(NR), T, Val(W))) - -function onem_plan_kernel(::Type{T}) where {T} - kernel = _onem_default_kernel(T) - b = default_blocking(kernel) - rng = Random.MersenneTwister(0x0007A616) - fx = build_plain(T, SHAPE_FOR_PLAN, rng) - plan = plan_contract( - fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC; - kernel = kernel, mc = b.mc, kc = b.kc, nc = b.nc - ) - return plan.kernel -end - -# --------------------------------------------------------------------------- -# One timed case -# --------------------------------------------------------------------------- - -struct PackCase - label::String - dtype::DataType - format::String # "real" / "planar" / "1e" - operand::Symbol # :A / :B - sliver::String # "full" / "tail" -end - -function run_case!(csv, kernel, case::PackCase) - T = case.dtype - R = realtype(kernel) - MR, NR = mr(kernel), nr(kernel) - pd = case.operand === :A ? MR : NR - valid = case.sliver == "full" ? pd : pd - 1 - valid >= 0 || error("MR/NR too small for a tail sliver: $pd") - - if case.operand === :A - tile = case.format == "real" ? noncontig_a_tile(T, valid, KC) : plain_a_tile(T, valid, KC) - needed = packed_a_length(kernel, KC) - rpe = reals_per_element(a_format(kernel)) - pack! = pack_a! - else - tile = plain_b_tile(T, KC, valid) - needed = packed_b_length(kernel, KC) - rpe = reals_per_element(b_format(kernel)) - pack! = pack_b! - end - buffer = Vector{R}(undef, needed) - packed = packed_panel(buffer, 1, needed) - - f!() = pack!(packed, tile, kernel, identity) - t = median_time_s(f!; reps = REPS) - - n_slots = KC * pd # register-tile slots touched (valid + padding) - ns_per_element = 1.0e9 * t / n_slots - ns_per_real = ns_per_element / rpe - - @printf( - "%-28s %-10s %-7s %-4s %-6s MR=%-3d NR=%-3d valid=%-3d kc=%-4d t=%.9e s ns/elt=%.4f ns/real=%.4f (rpe=%d)\n", - case.label, string(T), case.format, string(case.operand), case.sliver, - MR, NR, valid, KC, t, ns_per_element, ns_per_real, rpe - ) - println( - csv, - "$(case.label),$T,$(case.format),$(case.operand),$(case.sliver),", - "$MR,$NR,$valid,$KC,$t,$ns_per_element,$ns_per_real,$rpe,$REPS" - ) - return (case = case, t = t, ns_per_element = ns_per_element, ns_per_real = ns_per_real, rpe = rpe) -end - -# --------------------------------------------------------------------------- -# Code inspection: real and complex packing share one loop, `_pack_panel!` -# (src/packing/pack.jl), dispatched on `PackFormat` via -# `_pack_emit!`/`_pack_emit_zero!`. This dumps it at the Planar-A call's own -# (complex) argument types, i.e. `format = PlanarFormat()`, -# `Val(PD) = Val(MR)`. `_pack_panel!` branches once per `p` (`if valid == PD` -# outside the K loop), not once per element, so check that this coarse branch -# is present and that no per-element `t < valid ? load : zero` conditional -# load appears in the IR. -# --------------------------------------------------------------------------- - -function dump_pack_panel_llvm_complex(io, ::Type{T}) where {T} - kernel = default_plan_kernel(T) # PlanarKernel{MR,NR,T,W} - d = kernel.descriptor - tile = plain_a_tile(T, mr(kernel), KC) - load = (i, p) -> tile_load(tile, i, p) - plane_offset = (plane, i, p) -> packed_a_plane_offset(d, plane, i, p) - R = realtype(kernel) - needed = packed_a_length(kernel, KC) - buffer = Vector{R}(undef, needed) - packed = packed_panel(buffer, 1, needed) - MR = mr(kernel) - - println(io, "\n--- @code_llvm _pack_panel! at the Planar-A call's argument types ($T) ---") - println(io, "kernel = ", typeof(kernel), " MR=", MR, " format=", PlanarFormat()) - io2 = IOBuffer() - InteractiveUtils.code_llvm( - io2, _pack_panel!, - Tuple{ - typeof(packed), Type{T}, PlanarFormat, Val{MR}, Int, Int, - typeof(identity), typeof(load), typeof(plane_offset), - }; - debuginfo = :none, optimize = true - ) - llvm = String(take!(io2)) - println(io, llvm) - return llvm -end - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -function main() - using_io = stdout - print_env_header(using_io, "bench_pack_loops.jl") - println("REPS = ", REPS, " kc = ", KC) - - csv = open(CSV_PATH, "w") - println(csv, "label,dtype,format,operand,sliver,MR,NR,valid,kc,t_s,ns_per_element,ns_per_real,reals_per_element,reps") - - canaries = Float64[] - crng = Random.MersenneTwister(0x0CA9A121) - push!(canaries, run_canary(crng, "start")) - - results = NamedTuple[] - - # --- complex default (PlanarKernel), A and B separately --- - for T in CDTYPES - kernel = default_plan_kernel(T) - for sliver in ("full", "tail") - push!(results, run_case!(csv, kernel, PackCase("planar_A_$(T)", T, "planar", :A, sliver))) - push!(results, run_case!(csv, kernel, PackCase("planar_B_$(T)", T, "planar", :B, sliver))) - end - end - push!(canaries, run_canary(crng, "after-planar")) - - # --- complex OneMKernel (1e format), A only --- - for T in CDTYPES - kernel = onem_plan_kernel(T) - for sliver in ("full", "tail") - push!(results, run_case!(csv, kernel, PackCase("onem_A_$(T)", T, "1e", :A, sliver))) - end - end - push!(canaries, run_canary(crng, "after-onem")) - - # --- real fallback loop: A non-contiguous, B normal --- - for T in DTYPES - kernel = default_plan_kernel(T) - for sliver in ("full", "tail") - push!(results, run_case!(csv, kernel, PackCase("real_A_$(T)", T, "real", :A, sliver))) - push!(results, run_case!(csv, kernel, PackCase("real_B_$(T)", T, "real", :B, sliver))) - end - end - push!(canaries, run_canary(crng, "end")) - - close(csv) - - spread = relative_spread(canaries) - @printf("\ncanary spread: %.1f%% %s\n", 100spread, canaries) - if spread > 0.1 - @warn "canary spread exceeds 10%: the machine was not quiet enough for " * - "a few-percent conclusion. Re-run before believing any ranking." - end - - machine_load = try - strip(read(`uptime`, String)) - catch - "unknown (uptime failed)" - end - - open(PROV_PATH, "w") do io - print_env_header(io, "bench_pack_loops.jl") - println(io, "git_commit = ", git_commit()) - println(io, "reps = ", REPS, " kc = ", KC) - println(io, "canaries = ", canaries) - @printf(io, "canary spread = %.2f%%\n", 100spread) - println(io, "machine load (uptime, at provenance-write time) = ", machine_load) - println(io) - println(io, "--- microbenchmark table (see pack_loops.csv for the machine-readable form) ---") - for r in results - c = r.case - @printf( - io, "%-28s %-10s %-7s %-4s %-6s t=%.9e s ns/elt=%.4f ns/real=%.4f (rpe=%d)\n", - c.label, string(c.dtype), c.format, string(c.operand), c.sliver, - r.t, r.ns_per_element, r.ns_per_real, r.rpe - ) - end - end - - println("\n--- code_llvm: _pack_panel! branch structure (complex formats) ---") - for T in (ComplexF64, ComplexF32) - dump_pack_panel_llvm_complex(stdout, T) - end - - println("\nwrote ", CSV_PATH) - println("wrote ", PROV_PATH) - return results -end - -main() diff --git a/benchmark/bench_real_path_guard.jl b/benchmark/bench_real_path_guard.jl deleted file mode 100644 index 7c60988..0000000 --- a/benchmark/bench_real_path_guard.jl +++ /dev/null @@ -1,142 +0,0 @@ -# Real-path performance regression guard. The suite proves the real path is -# *correct*, but cannot see a regression caused by (say) a new `ContractPlan` -# type parameter defeating a specialization. Only a measurement can, and this -# is it. -# -# It is NOT `bench_default_vs_legacy.jl`, which compares two configurations -# *within* one tree. This measures the same real default configuration so it -# can be compared *across two trees* -- the working tree against a base -# commit -- which cannot be done in a single process, because -# both trees define a module named `QuasiStrided`. -# -# Usage: run once per tree, back to back, then diff the two CSVs. -# -# julia --project=. benchmark/bench_real_path_guard.jl -# -# HOW TO RUN AND READ IT (the resolution of this guard is ~5%): -# -# * Sequence in ABBA order, with nothing else on the machine. Straight -# base-then-new twice is confounded by wall-clock drift, and an 11-rep -# comparison once invented two regressions that 21 reps erased. -# * A *systematic one-sided* shift across every shape is a regression even -# inside the noise band -- that is what a lost specialization looks like. A -# lone outlier, or a pattern that does not reproduce across rounds, is -# noise. This instrument resolves ~5-6% on geomean, no finer. -# * Judge machine quietness from the middle/end canary pair only. -# `canary[start]` reads ~10% FASTER than the others in every run of this -# script, on both trees -- a warm-up effect in the canary itself, not drift -# -- so the reported "canary spread" trips its own warning at ~11% while -# the machine is in fact quiet. - -include(joinpath(@__DIR__, "harness.jl")) - -const REPS = 21 -const GUARD_SHAPES = vcat(MAIN_SHAPES, SMALL_SHAPES) - -const OUTDIR = results_dir() -mkpath(OUTDIR) - -# The filename carries the commit and a run counter. `results_dir()` is keyed -# only by host and date, so without the counter the second run of the day -# silently overwrites the first -- which happened, and destroyed a round of -# data. The counter matters because the interesting comparison is sometimes the -# *same* tree twice: that is how the run-to-run noise floor gets established. -const RUN_TAG = let - # `QS_GUARD_LABEL`: the tree that is NOT the working copy is usually - # extracted with `git archive`, where `git_commit()` cannot work, so both - # sides would otherwise carry the same tag and a reviewer could not tell a - # two-tree comparison from a same-tree noise run. That happened too. - c = get(ENV, "QS_GUARD_LABEL", git_commit()) - # `git_commit()` returns a human sentence on failure, so keep the filename - # filesystem-safe rather than assuming a hash. - safe = replace(c, r"[^A-Za-z0-9]" => "") - short = length(safe) >= 8 ? safe[1:8] : safe - n = 1 - while isfile(joinpath(OUTDIR, "real_path_guard_$(short)_run$(n).csv")) - n += 1 - end - "$(short)_run$(n)" -end -const CSV_PATH = joinpath(OUTDIR, "real_path_guard_$(RUN_TAG).csv") -const PROV_PATH = joinpath(OUTDIR, "real_path_guard_$(RUN_TAG)_PROVENANCE.txt") - -# Pass no kernel and no blocking: this must take the real default path, -# including the extent-aware demotion, exactly as a user gets it. Naming either -# would bypass the thing under test. -function time_real_default(::Type{T}, fx; reps::Int = REPS) where {T} - plan = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC) - execute!(plan, one(T), zero(T)) - t = median_time_s(() -> execute!(plan, one(T), zero(T)); reps = reps) - return (t, plan.kernel, plan.blocking) -end - -shape_tag(k) = "$(mr(k))x$(nr(k))/W$(lanewidth(k))" - -function main() - print_env_header(stdout, "bench_real_path_guard.jl") - println() - - csv = open(CSV_PATH, "w") - println(csv, "dtype,shape,Ma,Ka,Na,kernel,mc,kc,nc,reps,median_seconds,gflops") - - canaries = Float64[] - crng = Random.MersenneTwister(0x0CA9A121) - push!(canaries, run_canary(crng, "start")) - - for (di, T) in enumerate(DTYPES) - rng = Random.MersenneTwister(0x00C0FFEE + di) - for spec in GUARD_SHAPES - fx = build_plain(T, spec, rng) - t, kernel, b = time_real_default(T, fx) - gf = gflops(T, spec.Ma, spec.Ka, spec.Na, t) - println( - csv, - "$T,$(spec.name),$(spec.Ma),$(spec.Ka),$(spec.Na),", - "$(shape_tag(kernel)),$(b.mc),$(b.kc),$(b.nc),$REPS,$t,$gf" - ) - @printf( - "%-10s %-22s %9.3f ms %7.2f GF/s %s\n", - T, spec.name, 1000t, gf, shape_tag(kernel) - ) - end - di == 1 && push!(canaries, run_canary(crng, "middle")) - end - - # The scattered fixture too: the real default's zero-allocation scattered - # path is where a lost specialization would surface first, and every - # allocation regression this project has had was invisible on plain shapes. - for (di, T) in enumerate(DTYPES) - rng = Random.MersenneTwister(0x5CA7 + di) - fx = build_scattered(T, rng) - t, kernel, b = time_real_default(T, fx) - println( - csv, - "$T,scattered_a64k64b16n64,64,64,1024,", - "$(shape_tag(kernel)),$(b.mc),$(b.kc),$(b.nc),$REPS,$t,NaN" - ) - @printf("%-10s %-22s %9.3f ms (scattered)\n", T, "scattered", 1000t) - end - - push!(canaries, run_canary(crng, "end")) - close(csv) - - spread = (maximum(canaries) - minimum(canaries)) / minimum(canaries) - @printf("\ncanary spread: %.1f%% %s\n", 100spread, canaries) - if spread > 0.1 - @warn "canary spread exceeds 10%: the machine was not quiet enough for " * - "a few-percent conclusion. Re-run before believing any ratio." - end - - open(PROV_PATH, "w") do io - print_env_header(io, "bench_real_path_guard.jl") - println(io, "git commit = ", git_commit()) - println(io, "reps = ", REPS) - println(io, "canaries = ", canaries) - @printf(io, "canary spread = %.2f%%\n", 100spread) - end - - println("\nwrote ", CSV_PATH) - return nothing -end - -main() diff --git a/benchmark/bench_skip_packing.jl b/benchmark/bench_skip_packing.jl deleted file mode 100644 index 828d6cd..0000000 --- a/benchmark/bench_skip_packing.jl +++ /dev/null @@ -1,133 +0,0 @@ -# Empirical crossover sweep for `execute_direct!` (src/execution/direct.jl): -# the unpacked, scalar, BLIS-SUP-style path for small contractions, versus the -# existing packed `execute!`. -# -# julia --project=. benchmark/bench_skip_packing.jl -# -# Point: `plan_contract`/`execute!` always pack A/B into register-format -# panels before running the SIMD microkernel, even for tiny shapes where -# packing's O(MK+NK) cost isn't repaid by the microkernel's cache reuse. -# `execute_direct!` skips packing and the microkernel entirely (a plain scalar -# triple loop over the same AxisGroup-derived offsets `execute!` uses). This -# script sweeps shapes from tiny (M=N=K=1) up past where the packed path -# clearly wins, to find the empirical crossover -- per the project's -# "no claim without a measurement" rule, no threshold is hardcoded anywhere in -# src/ off the back of this file; a human reads the CSV/ranking and decides. -# -# Shapes deliberately include genuinely small M *and* N *and* K together (not -# just one small free extent, as `SMALL_SHAPES` in harness.jl does), plus a -# small-K family around the `dim63_2_1_2` regression shape (K=63) at a few -# M/N sizes, since that is the shape that motivated this work. - -include(joinpath(@__DIR__, "harness.jl")) - -using QuasiStrided: execute_direct! - -const REPS_TINY = 51 # small shapes need more reps: timer-resolution noise -const REPS_MAIN = 21 # matches bench_complex_efficiency.jl's REPS - -# (M, K, N) triples. Doubling ladder from 1 up to past the packed crossover, -# plus asymmetric small-K cases (the dim63_2_1_2 family) and small-M/N cases -# (mirroring SMALL_SHAPES's motivation) at a few different absolute scales. -const SKIP_PACKING_SHAPES = [ - # --- every dim tiny: doubling ladder --- - (1, 1, 1), (2, 2, 2), (4, 4, 4), (8, 8, 8), (16, 16, 16), - (32, 32, 32), (64, 64, 64), (128, 128, 128), (256, 256, 256), - # --- small-K family (dim63_2_1_2's K=63 is the motivating shape) --- - (32, 63, 32), (64, 63, 64), (128, 63, 128), (256, 63, 256), - (32, 1, 32), (64, 1, 64), (256, 1, 256), - (32, 8, 32), (64, 8, 64), (256, 8, 256), - # --- small-M or small-N, K not tiny --- - (4, 64, 256), (256, 64, 4), (4, 256, 256), (256, 256, 4), - (12, 64, 256), (256, 64, 12), - # --- small M and N together, K larger --- - (8, 256, 8), (16, 128, 16), (4, 512, 4), -] - -reps_for(M::Int, K::Int, N::Int) = (M * K * N <= 64^3) ? REPS_TINY : REPS_MAIN - -function time_packed(::Type{T}, fx; reps::Int) where {T} - plan = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC) - execute!(plan, one(T), zero(T)) - t = median_time_s(() -> execute!(plan, one(T), zero(T)); reps = reps) - return t -end - -function time_direct(::Type{T}, fx; reps::Int) where {T} - plan = plan_contract(fx.Cv, fx.Av, fx.indA, fx.Bv, fx.indB, fx.indC) - execute_direct!(plan, one(T), zero(T)) - t = median_time_s(() -> execute_direct!(plan, one(T), zero(T)); reps = reps) - return t -end - -const OUTDIR = results_dir() -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "skip_packing.csv") -const PROV_PATH = joinpath(OUTDIR, "skip_packing_PROVENANCE.txt") - -function main() - print_env_header(stdout, "bench_skip_packing.jl") - csv = open(CSV_PATH, "w") - println(csv, "dtype,M,K,N,reps,t_packed,t_direct,gf_packed,gf_direct,speedup_direct_over_packed") - - canaries = Float64[] - rng = Random.MersenneTwister(0xDEC0DE01) - push!(canaries, run_canary(rng, "start")) - - for T in DTYPES # Float64, Float32 only -- execute_direct! is exercised - # for complex types in test/execution/test_direct.jl; this sweep is - # about the packed-vs-direct crossover, which the real kernels' - # measured menu already anchors, so keep the sweep itself real-only. - for (M, K, N) in SKIP_PACKING_SHAPES - spec = ShapeSpec("$(M)x$(K)x$(N)", M, K, N) - fx = build_plain(T, spec, rng) - reps = reps_for(M, K, N) - - t_packed = time_packed(T, fx; reps = reps) - t_direct = time_direct(T, fx; reps = reps) - gf_packed = gflops(T, M, K, N, t_packed) - gf_direct = gflops(T, M, K, N, t_direct) - speedup = t_packed / t_direct - - println( - csv, - "$T,$M,$K,$N,$reps,$t_packed,$t_direct,$gf_packed,$gf_direct,$speedup" - ) - @printf( - "%-10s %4dx%4dx%4d packed %10.3e s (%6.2f GF/s) direct %10.3e s (%6.2f GF/s) direct/packed speedup %6.3fx\n", - string(T), M, K, N, t_packed, gf_packed, t_direct, gf_direct, speedup - ) - end - end - - push!(canaries, run_canary(rng, "end")) - close(csv) - - spread = relative_spread(canaries) - @printf("\ncanary spread: %.1f%% %s\n", 100spread, canaries) - if spread > 0.1 - @warn "canary spread exceeds 10%: the machine was not quiet enough for " * - "a few-percent conclusion. Re-run before believing any crossover read " * - "off this CSV." - end - - open(PROV_PATH, "w") do io - print_env_header(io, "bench_skip_packing.jl") - println(io, "git commit = ", git_commit()) - println(io, "canaries = ", canaries) - @printf(io, "canary spread = %.2f%%\n", 100spread) - println( - io, - "\nNo threshold is derived automatically here -- read speedup_direct_over_packed", - ) - println( - io, - "in skip_packing.csv against M,K,N and pick the crossover by hand, per shape", - ) - println(io, "family (all-tiny vs small-K vs small-M/N).") - end - println("\nwrote ", CSV_PATH) - return nothing -end - -main() diff --git a/benchmark/bench_store_path.jl b/benchmark/bench_store_path.jl deleted file mode 100644 index 85eccea..0000000 --- a/benchmark/bench_store_path.jl +++ /dev/null @@ -1,563 +0,0 @@ -# Tile-level cost of the scattered-store path (`_store_tile_scattered!`) vs. -# the vectorized fast path in `store_tile!` (src/microkernels/simd.jl), across -# destination-storage variants (`Vector{T}`, `Memory{T}`, strided-hot, -# strided-cold, and MR-tail versions of the first two). Also attempts to -# reproduce the "SIMDKernel reaches 101-103 GFLOP/s" accumulate-only claim -# (Float64, kc=256). Report-only, no fix applied. -# -# julia --project=. benchmark/bench_store_path.jl [options] -# -# Options: -# --reps 21 --batch 50000 --alloc-batch 100 -# --kc 16,256 -# --cold-npos 4096 --cold-step 8191 -# --skip-repro --skip-native - -include(joinpath(@__DIR__, "harness.jl")) - -using QuasiStrided: SIMDKernel, mr, nr, lanewidth, scalartype, - packed_a_length, packed_b_length, packed_a_offset, packed_b_offset, - AffineAxis, DestinationTile, zero_accumulator, accumulate, store_tile!, - execute_tile!, PackedPanel, packed_panel -using InteractiveUtils: code_native - -const REPS = argopt("reps", 21) -const BATCH = argopt("batch", 50_000) # calls per timed rep, for median_time_s -const ALLOC_BATCH = argopt("alloc-batch", 100) # calls per @allocated probe (after warm-up) - -# "Cold" destination: cycles the tile's base across COLD_NPOS positions -# spaced COLD_STEP elements apart (not COLD_NPOS fully disjoint tile-sized -# regions, which would need tens of GB at these shapes) -- an approximation -# of "a large, mostly-cold output", spanning ~270 MB by default. -const COLD_NPOS = argopt("cold-npos", 4096) -const COLD_STEP = argopt("cold-step", 8191) - -# --------------------------------------------------------------------------- -# Kernel shapes -# --------------------------------------------------------------------------- - -# (dtype, (MR,NR,W), label). Float64 (16,6,8) and Float32 (32,6,16) are the -# shipped AVX-512 register shapes; (16,14,8) and (32,6,8) are other swept -# shapes reused here for comparability. -const SHAPES = [ - (Float64, (16, 6, 8), "F64_16x6x8 (shipped default)"), - (Float64, (16, 14, 8), "F64_16x14x8"), - (Float64, (32, 6, 8), "F64_32x6x8"), - (Float32, (32, 6, 16), "F32_32x6x16 (shipped default)"), -] - -const KC_VALUES = parse_ints(argopt("kc", "16,256")) - -shape_label(MR, NR, W) = "$(MR)x$(NR)x$(W)" - -# Packed-panel construction: mirrors test/microkernels/test_simd_kernel.jl's -# `packed_from_matrices` (same offset formulas/order). - -function build_packed(::Type{T}, kernel, kc::Int, rng) where {T} - MR, NR = mr(kernel), nr(kernel) - Amat = rand(rng, T, MR, max(kc, 1)) - Bmat = rand(rng, T, max(kc, 1), NR) - pa = zeros(T, packed_a_length(kernel, kc)) - pb = zeros(T, packed_b_length(kernel, kc)) - for p in 0:(kc - 1), i in 0:(MR - 1) - pa[packed_a_offset(kernel, i, p) + 1] = Amat[i + 1, p + 1] - end - for p in 0:(kc - 1), j in 0:(NR - 1) - pb[packed_b_offset(kernel, j, p) + 1] = Bmat[p + 1, j + 1] - end - return pa, pb -end - -# --------------------------------------------------------------------------- -# Destination variants -# --------------------------------------------------------------------------- - -# `dests` is a concretely-typed Vector (length 1 except D-strided-cold) so -# `bench_execute!`'s inner loop never dynamically dispatches. -struct DestVariant{V} - name::String - dests::V - m::Int - n::Int -end - -function dvec_variant(::Type{T}, MR, NR) where {T} - storage = zeros(T, MR * NR) - rows, cols = AffineAxis(0, 1, MR), AffineAxis(0, MR, NR) - dest = DestinationTile(storage, 0, rows, cols) - return DestVariant("D-vec", [dest], MR, NR) -end - -function dmem_variant(::Type{T}, MR, NR) where {T} - storage = parent(StridedView(zeros(T, MR, NR))) - rows, cols = AffineAxis(0, 1, MR), AffineAxis(0, MR, NR) - dest = DestinationTile(storage, 0, rows, cols) - return DestVariant("D-mem", [dest], MR, NR) -end - -function dvec_tail_variant(::Type{T}, MR, NR) where {T} - m = MR - 3 - storage = zeros(T, m * NR) - rows, cols = AffineAxis(0, 1, m), AffineAxis(0, m, NR) - dest = DestinationTile(storage, 0, rows, cols) - return DestVariant("D-vec-tail", [dest], m, NR) -end - -function dmem_tail_variant(::Type{T}, MR, NR) where {T} - m = MR - 3 - storage = parent(StridedView(zeros(T, m, NR))) - rows, cols = AffineAxis(0, 1, m), AffineAxis(0, m, NR) - dest = DestinationTile(storage, 0, rows, cols) - return DestVariant("D-mem-tail", [dest], m, NR) -end - -max_tile_offset(MR, NR) = (MR - 1) * 4096 + (NR - 1) * 4096 * MR - -function dstrided_hot_variant(::Type{T}, MR, NR) where {T} - storage = zeros(T, max_tile_offset(MR, NR) + 1) - rows, cols = AffineAxis(0, 4096, MR), AffineAxis(0, 4096 * MR, NR) - dest = DestinationTile(storage, 0, rows, cols) - return DestVariant("D-strided-hot", [dest], MR, NR) -end - -function dstrided_cold_variant(::Type{T}, MR, NR) where {T} - tile_span = max_tile_offset(MR, NR) + 1 - storage = zeros(T, COLD_NPOS * COLD_STEP + tile_span) - rows, cols = AffineAxis(0, 4096, MR), AffineAxis(0, 4096 * MR, NR) - dests = [DestinationTile(storage, (p - 1) * COLD_STEP, rows, cols) for p in 1:COLD_NPOS] - return DestVariant("D-strided-cold", dests, MR, NR) -end - -variant_builders(::Type{T}, MR, NR) where {T} = ( - dvec_variant(T, MR, NR), dmem_variant(T, MR, NR), - dstrided_hot_variant(T, MR, NR), dstrided_cold_variant(T, MR, NR), - dvec_tail_variant(T, MR, NR), dmem_tail_variant(T, MR, NR), -) - -# --------------------------------------------------------------------------- -# Timed inner loops -# --------------------------------------------------------------------------- - -# `accumulate` alone (FMA only, no store): destination-independent. Sinks -# into a `Ref` so the compiler cannot hoist/CSE the whole batch away (with no -# real side effect and identical inputs every iteration, that is a genuine -# risk for a pure function called in a tight loop). -function bench_accumulate(kernel, packed_a, packed_b, kc::Int, batch::Int) - T = scalartype(kernel) - sink = Ref(zero(T)) - @inbounds for _ in 1:batch - acc = accumulate(kernel, zero_accumulator(kernel), packed_a, packed_b, kc) - sink[] += first(Tuple(acc[1])) - end - return sink[] -end - -# `execute_tile!`, including the store. Cycles through `dests` (length 1 -# except D-strided-cold) so the array write itself is inside the timed loop. -function bench_execute(kernel, dests::Vector, packed_a, packed_b, kc::Int, alpha, beta, batch::Int) - n = length(dests) - @inbounds for k in 1:batch - d = dests[mod1(k, n)] - execute_tile!(kernel, d, packed_a, packed_b, kc, alpha, beta) - end - return nothing -end - -# Native-code inspection (report-only). A stack store has the [rsp/rbp] -# operand first, i.e. followed by a comma. -const VEC_STORE_RE = r"v(?:mov(?:up|ap)[sd]|movdq[ua])\s+(?:zmmword|ymmword|xmmword)\s+ptr\s+\[([^\]]*)\]\s*," - -function native_store_counts(f, argtypes::Tuple) - io = IOBuffer() - try - code_native(io, f, argtypes; debuginfo = :none, syntax = :intel) - catch err - return (ok = false, err = string(err), total = 0, stack = 0, other = 0) - end - asm = String(take!(io)) - total = 0 - stack = 0 - for m in eachmatch(VEC_STORE_RE, asm) - total += 1 - occursin(r"r[sb]p", m.captures[1]) && (stack += 1) - end - return (ok = true, err = "", total = total, stack = stack, other = total - stack) -end - -# --------------------------------------------------------------------------- -# Output setup -# --------------------------------------------------------------------------- - -const OUTDIR = results_dir() -mkpath(OUTDIR) - -function unique_path(path::String) - isfile(path) || return path - base, ext = splitext(path) - i = 1 - while isfile("$(base)_$(i)$(ext)") - i += 1 - end - chosen = "$(base)_$(i)$(ext)" - @warn "output path already existed; using a suffixed name instead" original = path chosen - return chosen -end - -const CSV_PATH = unique_path(joinpath(OUTDIR, "bench_store_path.csv")) -const SUMMARY_PATH = unique_path(joinpath(OUTDIR, "summary_store_path.txt")) -const PROVENANCE_PATH = unique_path(joinpath(OUTDIR, "PROVENANCE_store_path.txt")) -const CANARY_PATH = unique_path(joinpath(OUTDIR, "canary_store_path.csv")) -native_path(tag) = unique_path(joinpath(OUTDIR, "store_path_native_$(tag).txt")) - -csv_io = open(CSV_PATH, "w") -println( - csv_io, - "kernel_shape,dtype,kc,destination_variant,call,reps,median_seconds,", - "gflops,allocated_bytes,storage_type_string" -) - -const ROWS = NamedTuple[] - -function log_row!(kernel_shape, T, kc, variant, call, t, gf, alloc, storagetype) - println( - csv_io, - kernel_shape, ",", T, ",", kc, ",", variant, ",", call, ",", REPS, ",", - @sprintf("%.9e", t), ",", (gf === nothing ? "" : @sprintf("%.4f", gf)), ",", - alloc, ",", storagetype - ) - flush(csv_io) - push!( - ROWS, ( - shape = kernel_shape, dtype = T, kc = kc, variant = variant, - call = call, t = t, gf = gf, alloc = alloc, storagetype = storagetype, - ) - ) - return nothing -end - -# --------------------------------------------------------------------------- -# Machine-load check (this project's standing measurement discipline: the -# reference machine is not assumed exclusive) -# --------------------------------------------------------------------------- - -function capture_cmd(cmd) - return try - read(cmd, String) - catch err - "unavailable: $err" - end -end - -const UPTIME_OUT = capture_cmd(`uptime`) -const TOP_OUT = capture_cmd(`top -bn1`) -const TOP_HEAD = join(split(TOP_OUT, '\n')[1:min(15, end)], '\n') - -println("machine load at start:\n", UPTIME_OUT) - -# --------------------------------------------------------------------------- -# Canary bracket -# --------------------------------------------------------------------------- - -# Fixed cell timed at start/middle/end: Float64 (16,6,8) shipped default, -# kc=256, D-vec, beta=0. -function run_store_path_canary(rng, label::String) - T = Float64 - MR, NR, W = 16, 6, 8 - kernel = SIMDKernel(Val(MR), Val(NR), T, Val(W)) - kc = 256 - pa, pb = build_packed(T, kernel, kc, rng) - variant = dvec_variant(T, MR, NR) - t = GC.@preserve pa pb begin - panel_a = packed_panel(pa, 1, length(pa)) - panel_b = packed_panel(pb, 1, length(pb)) - bench_execute(kernel, variant.dests, panel_a, panel_b, kc, 1.0, 0.0, 1) # warm-up - median_time_s( - () -> bench_execute(kernel, variant.dests, panel_a, panel_b, kc, 1.0, 0.0, BATCH); - reps = 15 - ) / BATCH - end - println("canary[$label] median per-call = $(t) s") - return t -end - -rng = MersenneTwister(0x5701_0002) -canary_results = Float64[] -push!(canary_results, run_store_path_canary(rng, "A (start)")) - -# --------------------------------------------------------------------------- -# Main sweep -# --------------------------------------------------------------------------- - -for (T, (MR, NR, W), label) in SHAPES - kshape = shape_label(MR, NR, W) - kernel = SIMDKernel(Val(MR), Val(NR), T, Val(W)) - for kc in KC_VALUES - pa, pb = build_packed(T, kernel, kc, rng) - GC.@preserve pa pb begin - panel_a = packed_panel(pa, 1, length(pa)) - panel_b = packed_panel(pb, 1, length(pb)) - - # --- accumulate (destination-independent) --- - bench_accumulate(kernel, panel_a, panel_b, kc, 1) # warm-up - t_acc = median_time_s( - () -> bench_accumulate(kernel, panel_a, panel_b, kc, BATCH); - reps = REPS - ) / BATCH - alloc_acc = @allocated bench_accumulate(kernel, panel_a, panel_b, kc, ALLOC_BATCH) - gf_acc = gflops(T, MR, kc, NR, t_acc) - - for variant in variant_builders(T, MR, NR) - log_row!(kshape, T, kc, variant.name, "accumulate", t_acc, gf_acc, alloc_acc, "n/a (accumulate)") - - storagetype = string(typeof(variant.dests[1].storage)) - - bench_execute(kernel, variant.dests, panel_a, panel_b, kc, 1.0, 0.0, 1) # warm-up - t0 = median_time_s( - () -> bench_execute(kernel, variant.dests, panel_a, panel_b, kc, 1.0, 0.0, BATCH); - reps = REPS - ) / BATCH - alloc0 = @allocated bench_execute(kernel, variant.dests, panel_a, panel_b, kc, 1.0, 0.0, ALLOC_BATCH) - log_row!(kshape, T, kc, variant.name, "execute_tile_beta0", t0, nothing, alloc0, storagetype) - - bench_execute(kernel, variant.dests, panel_a, panel_b, kc, 1.0, 1.0, 1) # warm-up - t1 = median_time_s( - () -> bench_execute(kernel, variant.dests, panel_a, panel_b, kc, 1.0, 1.0, BATCH); - reps = REPS - ) / BATCH - alloc1 = @allocated bench_execute(kernel, variant.dests, panel_a, panel_b, kc, 1.0, 1.0, ALLOC_BATCH) - log_row!(kshape, T, kc, variant.name, "execute_tile_beta1", t1, nothing, alloc1, storagetype) - end - end - @info "sweep progress" dtype = T shape = (MR, NR, W) kc = kc - end - push!(canary_results, run_store_path_canary(rng, "mid ($label done)")) -end - -push!(canary_results, run_store_path_canary(rng, "A' (end)")) -close(csv_io) - -const CANARY_SPREAD = relative_spread(canary_results) -open(CANARY_PATH, "w") do io - println(io, "index,median_seconds_per_call") - for (i, t) in enumerate(canary_results) - println(io, "$i,", @sprintf("%.9e", t)) - end - println(io, "# relative spread (max-min)/min = ", @sprintf("%.4f", CANARY_SPREAD)) -end -println("canary spread (max-min)/min = ", @sprintf("%.4f", CANARY_SPREAD)) - -# Reproduction attempt for "SIMDKernel reaches 101-103 GFLOP/s, ~88% of -# peak", Float64/Float32, kc=256. Times both a plain -# `Vector` panel (matching that table's column) and a `PackedPanel` (the -# real driver's actual panel type). -const REPRO_SHAPES = [ - (Float64, (16, 6, 8)), - (Float64, (16, 14, 8)), - (Float64, (32, 6, 8)), - (Float32, (32, 6, 16)), -] -const REPRO_KC = 256 - -repro_rows = NamedTuple[] -for (T, (MR, NR, W)) in (hasflag("skip-repro") ? [] : REPRO_SHAPES) - kernel = SIMDKernel(Val(MR), Val(NR), T, Val(W)) - pa, pb = build_packed(T, kernel, REPRO_KC, rng) - - bench_accumulate(kernel, pa, pb, REPRO_KC, 1) # warm-up - t_vec = median_time_s(() -> bench_accumulate(kernel, pa, pb, REPRO_KC, BATCH); reps = REPS) / BATCH - gf_vec = gflops(T, MR, REPRO_KC, NR, t_vec) - - t_panel = GC.@preserve pa pb begin - panel_a = packed_panel(pa, 1, length(pa)) - panel_b = packed_panel(pb, 1, length(pb)) - bench_accumulate(kernel, panel_a, panel_b, REPRO_KC, 1) # warm-up - median_time_s(() -> bench_accumulate(kernel, panel_a, panel_b, REPRO_KC, BATCH); reps = REPS) / BATCH - end - gf_panel = gflops(T, MR, REPRO_KC, NR, t_panel) - - push!( - repro_rows, ( - dtype = T, shape = (MR, NR, W), kc = REPRO_KC, - gf_vec = gf_vec, gf_panel = gf_panel, - ) - ) - @info "reproduction attempt" dtype = T shape = (MR, NR, W) gf_as_vector = gf_vec gf_as_packedpanel = gf_panel -end - -# --------------------------------------------------------------------------- -# Native-code inspection: shipped shapes only, D-vec vs D-mem -# --------------------------------------------------------------------------- - -native_rows = NamedTuple[] -for (T, (MR, NR, W), label) in SHAPES - hasflag("skip-native") && continue - occursin("shipped default", label) || continue - kernel = SIMDKernel(Val(MR), Val(NR), T, Val(W)) - kc = 4 - pa, pb = build_packed(T, kernel, kc, rng) - GC.@preserve pa pb begin - panel_a = packed_panel(pa, 1, length(pa)) - panel_b = packed_panel(pb, 1, length(pb)) - for (vname, variant) in ( - ("D-vec", dvec_variant(T, MR, NR)), ("D-mem", dmem_variant(T, MR, NR)), - ) - dest = variant.dests[1] - argtypes = ( - typeof(kernel), typeof(dest), typeof(panel_a), typeof(panel_b), Int, T, T, - ) - rep = native_store_counts(execute_tile!, argtypes) - tag = "$(shape_label(MR, NR, W))_$(vname)" - path = native_path(tag) - io = IOBuffer() - try - code_native(io, execute_tile!, argtypes; debuginfo = :none, syntax = :intel) - catch err - println(io, "code_native FAILED: ", err) - end - open(path, "w") do fio - write(fio, take!(io)) - end - push!(native_rows, (dtype = T, shape = (MR, NR, W), variant = vname, rep = rep, path = path)) - @info "native code dumped" shape = (MR, NR, W) variant = vname total_vec_stores = rep.total stack_stores = rep.stack other_stores = rep.other path = path - end - end -end - -# --------------------------------------------------------------------------- -# Summary -# --------------------------------------------------------------------------- - -function per_element_ns(rows, shape, T, kc, variant, call) - for r in rows - if r.shape == shape && r.dtype == T && r.kc == kc && r.variant == variant && r.call == call - return r.t - end - end - return NaN -end - -open(SUMMARY_PATH, "w") do io - print_env_header(io, "bench_store_path.jl") - println(io, "reps = ", REPS, " batch = ", BATCH, " alloc_batch = ", ALLOC_BATCH) - println(io, "cold_npos = ", COLD_NPOS, " cold_step = ", COLD_STEP) - println(io, "\nmachine load (uptime): ", strip(UPTIME_OUT)) - println(io, "\ncanary median per-call times (s): ", canary_results) - println(io, "canary relative spread (max-min)/min: ", @sprintf("%.4f", CANARY_SPREAD)) - - println(io, "\n== Reproduction attempt: \"101-103 GFLOP/s, ~88% of peak\" (accumulate-only) ==") - println(io, "(Float64 kc=256, plain Vector panel)") - for r in repro_rows - println( - io, " ", r.dtype, " ", r.shape, " kc=", r.kc, - " as-plain-Vector: ", @sprintf("%.2f", r.gf_vec), " GFLOP/s", - " as-PackedPanel: ", @sprintf("%.2f", r.gf_panel), " GFLOP/s" - ) - end - cited_lo, cited_hi = 101.0, 103.0 - any_match = any(r -> r.dtype == Float64 && (cited_lo <= r.gf_vec <= cited_hi || cited_lo <= r.gf_panel <= cited_hi), repro_rows) - println( - io, any_match ? - " MATCH: at least one Float64 cell fell inside the cited 101-103 GFLOP/s band." : - " NO MATCH: no measured cell fell inside the cited 101-103 GFLOP/s band on this run " * - "(reported honestly rather than forced -- see the numbers above for the actual gap)." - ) - - println(io, "\n== Store cost table: per-element store cost (ns) = (median(execute_tile_beta) - median(accumulate)) / (m*n) ==") - for (T, (MR, NR, W), label) in SHAPES - kshape = shape_label(MR, NR, W) - for kc in KC_VALUES - println(io, "\n-- ", label, " kc=", kc, " --") - for variant in ("D-vec", "D-mem", "D-strided-hot", "D-strided-cold", "D-vec-tail", "D-mem-tail") - t_acc = per_element_ns(ROWS, kshape, T, kc, variant, "accumulate") - t0 = per_element_ns(ROWS, kshape, T, kc, variant, "execute_tile_beta0") - t1 = per_element_ns(ROWS, kshape, T, kc, variant, "execute_tile_beta1") - mn = variant in ("D-vec-tail", "D-mem-tail") ? (MR - 3) * NR : MR * NR - cost0 = (t0 - t_acc) / mn * 1.0e9 - cost1 = (t1 - t_acc) / mn * 1.0e9 - println( - io, " ", rpad(variant, 16), - " accumulate=", @sprintf("%.3e", t_acc), " s/call", - " beta0=", @sprintf("%.3e", t0), " s/call (", @sprintf("%7.2f", cost0), " ns/elem store cost)", - " beta1=", @sprintf("%.3e", t1), " s/call (", @sprintf("%7.2f", cost1), " ns/elem store cost)" - ) - end - end - end - - println(io, "\n== D-vec vs D-mem: the core deliverable ==") - println(io, "(today's real driver hands the kernel D-mem's storage type; D-vec is what would take the") - println(io, " fast path IF the guard were fixed to also accept Memory{T} -- this is what a fix could recover)") - for (T, (MR, NR, W), label) in SHAPES - kshape = shape_label(MR, NR, W) - for kc in KC_VALUES - t_acc = per_element_ns(ROWS, kshape, T, kc, "D-vec", "accumulate") - t0_vec = per_element_ns(ROWS, kshape, T, kc, "D-vec", "execute_tile_beta0") - t0_mem = per_element_ns(ROWS, kshape, T, kc, "D-mem", "execute_tile_beta0") - t1_vec = per_element_ns(ROWS, kshape, T, kc, "D-vec", "execute_tile_beta1") - t1_mem = per_element_ns(ROWS, kshape, T, kc, "D-mem", "execute_tile_beta1") - mn = MR * NR - c0v = (t0_vec - t_acc) / mn * 1.0e9 - c0m = (t0_mem - t_acc) / mn * 1.0e9 - c1v = (t1_vec - t_acc) / mn * 1.0e9 - c1m = (t1_mem - t_acc) / mn * 1.0e9 - println( - io, " ", label, " kc=", kc, - " beta0: D-vec=", @sprintf("%7.2f", c0v), " ns/elem D-mem=", @sprintf("%7.2f", c0m), - " ns/elem delta(mem-vec)=", @sprintf("%7.2f", c0m - c0v), " ns/elem" - ) - println( - io, " ", " "^length(label), " ", - " beta1: D-vec=", @sprintf("%7.2f", c1v), " ns/elem D-mem=", @sprintf("%7.2f", c1m), - " ns/elem delta(mem-vec)=", @sprintf("%7.2f", c1m - c1v), " ns/elem" - ) - end - end - - println(io, "\n== Allocation (bytes over ", ALLOC_BATCH, " calls; should be 0 everywhere -- Cliff B's static-index fix applies to BOTH code paths) ==") - for r in ROWS - r.alloc == 0 && continue - println(io, " NONZERO ALLOCATION: ", r.shape, " ", r.dtype, " kc=", r.kc, " ", r.variant, " ", r.call, " => ", r.alloc, " B") - end - if all(r -> r.alloc == 0, ROWS) - println(io, " none found -- every (shape,kc,variant,call) cell measured 0 B.") - end - - println(io, "\n== Native-code stack-store counts (shipped shapes, D-vec vs D-mem; report-only, no fix) ==") - println(io, "(VEC_STORE_RE counts, ", "vmovup(s|d)/vmovap(s|d)/vmovdqu/vmovdqa to a zmmword/ymmword/xmmword ptr; \"stack\" = operand references r[sb]p)") - for r in native_rows - println( - io, " ", r.dtype, " ", r.shape, " ", r.variant, - " total_vector_stores=", r.rep.ok ? r.rep.total : -1, - " stack(rsp/rbp)=", r.rep.ok ? r.rep.stack : -1, - " other(non-stack, e.g. destination/panel pointers)=", r.rep.ok ? r.rep.other : -1, - r.rep.ok ? "" : " CODEGEN FAILED: " * r.rep.err, - " (dump: ", r.path, ")" - ) - end -end -println(read(SUMMARY_PATH, String)) - -open(PROVENANCE_PATH, "w") do io - println(io, "git_commit = ", git_commit()) - println(io, "command = julia --project=. benchmark/bench_store_path.jl") - println(io, "cpu = ", Sys.CPU_NAME) - println(io, "julia = ", VERSION) - println(io, "nthreads = ", NTHREADS, " blas_threads = ", BLAS_THREADS) - println(io, "date = ", now()) - println(io, "reps = ", REPS, " batch = ", BATCH, " alloc_batch = ", ALLOC_BATCH) - println(io, "shapes = ", [(T, s) for (T, s, _) in SHAPES]) - println(io, "kc_values = ", KC_VALUES) - println(io, "canary_spread = ", @sprintf("%.4f", CANARY_SPREAD)) - println(io, "\n--- machine load at script start (uptime) ---") - println(io, UPTIME_OUT) - println(io, "--- machine load at script start (top -bn1, first 15 lines) ---") - println(io, TOP_HEAD) - println( - io, "\nSINGLE-MACHINE CAVEAT: as with every other benchmark in this project, this is one", - "\nCascade Lake / AVX-512 machine, not a portable claim.", - "\nThe reference machine was NOT confirmed exclusive for this run (see uptime/top above);", - "\nthe canary bracket above is the check for drift during the run." - ) -end - -println("\nDone. Results in ", OUTDIR) diff --git a/benchmark/bench_tensoroperations.jl b/benchmark/bench_tensoroperations.jl deleted file mode 100644 index badddcf..0000000 --- a/benchmark/bench_tensoroperations.jl +++ /dev/null @@ -1,230 +0,0 @@ -# Head-to-head timing of `@tensor` under `StridedNative()`, `StridedBLAS()` -# and `QuasiStridedBackend()` on identical inputs, over the same shape grid as -# `benchmark/bench_driver.jl`. An honest comparison, not a search for a win: -# these numbers are why the backend is not hooked into `select_backend`. -# -# julia --project=. benchmark/bench_tensoroperations.jl -# -# writes results + PROVENANCE.txt to benchmark/results/-/. - -using TensorOperations -using TensorOperations: StridedNative, StridedBLAS -using QuasiStrided -using QuasiStrided: QuasiStridedBackend -using LinearAlgebra -using Statistics: median -using Random -using Dates -using Printf - -# Single-core measurement discipline (this project's standing rule; see -# benchmark/bench_driver.jl). -LinearAlgebra.BLAS.set_num_threads(1) -const NTHREADS = Threads.nthreads() -const BLAS_THREADS = LinearAlgebra.BLAS.get_num_threads() -if NTHREADS != 1 - @warn "Threads.nthreads() = $NTHREADS != 1 -- this is NOT the pinned " * - "single-core measurement this project's rules require. Results " * - "below should not be trusted as the reference-machine numbers." -end - -# Warm up once (discarded), then `reps` timed calls; median, not mean. -# Identical convention to benchmark/harness.jl's `median_time_s`. -# -# `reps` was 9, which is below this project's standing discipline: ccqlin038 is -# not reliably exclusive, canary spreads of 4-15% are normal, and an 11-rep -# comparison once invented two regressions that 21 reps erased. 15 is the -# floor; pass more for anything whose conclusion depends on a difference under -# ~10%. -function median_time_s(f!::Function; reps::Int = 15) - f!() # warm-up, discarded - ts = Vector{Float64}(undef, reps) - for r in 1:reps - t0 = time_ns() - f!() - t1 = time_ns() - ts[r] = (t1 - t0) / 1.0e9 - end - return median(ts) -end - -# Same shape grid as benchmark/harness.jl's MAIN_SHAPES + EXTRA_SHAPES -# (matrix-shaped contraction C[m,n] = A[m,k]*B[k,n]), reproduced here verbatim -# rather than included. -# -# The duplication is deliberate: every committed number in `benchmark/results/` -# for this script was taken against these literals. -# Collapsing onto the harness is a safe cleanup only when done together with a -# re-run, not as a drive-by edit. Until then the two must be kept in step by -# hand; they are identical today. -struct ShapeSpec - name::String - Ma::Int - Ka::Int - Na::Int -end - -const MAIN_SHAPES = [ - ShapeSpec("64^3", 64, 64, 64), - ShapeSpec("128^3", 128, 128, 128), - ShapeSpec("256^3", 256, 256, 256), - ShapeSpec("512^3", 512, 512, 512), - ShapeSpec("shallowK_256x24x256", 256, 24, 256), -] -const EXTRA_SHAPES = [ - ShapeSpec("1024x256x1024", 1024, 256, 1024), -] -const ALL_SHAPES = vcat(MAIN_SHAPES, EXTRA_SHAPES) - -const DTYPES = (Float64, Float32) - -const BACKENDS = ( - StridedNative = StridedNative(), - StridedBLAS = StridedBLAS(), - QuasiStrided = QuasiStridedBackend(), -) - -function build(::Type{T}, spec::ShapeSpec, rng) where {T} - A = randn(rng, T, spec.Ma, spec.Ka) - B = randn(rng, T, spec.Ka, spec.Na) - C = zeros(T, spec.Ma, spec.Na) - return A, B, C -end - -# One in-place contraction call under a given backend: `C = A*B` (beta = 0), -# identical expression for all three backends -- only `backend` varies. -function run_contract!(backend, C, A, B) - @tensor backend = backend C[i, j] = A[i, k] * B[k, j] - return C -end - -# Output location. -const OUTDIR = joinpath( - @__DIR__, "results", "$(gethostname())-$(Dates.format(now(), "yyyy-mm-dd"))" -) -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "bench_tensoroperations.csv") -const CANARY_PATH = joinpath(OUTDIR, "canary_tensoroperations.csv") -const PROVENANCE_PATH = joinpath(OUTDIR, "PROVENANCE.txt") - -csv_io = open(CSV_PATH, "w") -println(csv_io, "backend,dtype,shape,Ma,Ka,Na,reps,median_seconds") -function log_row(backend_name, T, spec::ShapeSpec, reps, t) - println( - csv_io, - "$backend_name,$T,$(spec.name),$(spec.Ma),$(spec.Ka),$(spec.Na),$reps,", - @sprintf("%.9f", t) - ) - return flush(csv_io) -end - -println("# QuasiStrided.jl benchmark/bench_tensoroperations.jl") -println("cpu = ", Sys.CPU_NAME) -println("julia = ", VERSION) -println("nthreads = ", NTHREADS, " blas_threads = ", BLAS_THREADS) -println("date = ", now()) - -# Canary: a small, fixed case run at the start / middle / end of the sweep -# (A, B, A' pattern), StridedBLAS on a 64^3 shape -- same spirit as -# bench_driver.jl's canary (drift detection: thermal throttling, background -# load), scoped to a single backend since this script's job is a -# cross-backend comparison, not a per-backend sweep. -const CANARY_SHAPE = ShapeSpec("canary_64^3", 64, 64, 64) -function run_canary(rng, label::String) - A, B, C = build(Float64, CANARY_SHAPE, rng) - t = median_time_s(() -> run_contract!(StridedBLAS(), C, A, B); reps = 15) - println("canary[$label] median = $(t) s") - return t -end - -rng = MersenneTwister(0xB3_C4_0002) - -canary_results = Float64[] -push!(canary_results, run_canary(rng, "A (start)")) - -const REPS = 9 -raw = Vector{NamedTuple}() - -for T in DTYPES - for spec in ALL_SHAPES - # One shared pair of inputs per (dtype, shape): all three backends - # see byte-identical A/B, and each gets its own freshly zeroed C - # (beta = 0 in run_contract! means C's initial content is irrelevant - # to the result, but a fresh buffer per backend avoids any - # in-place-mutation surprises across backends sharing memory). - A, B, _ = build(T, spec, rng) - for (bname, backend) in pairs(BACKENDS) - C = zeros(T, spec.Ma, spec.Na) - t = median_time_s(() -> run_contract!(backend, C, A, B); reps = REPS) - log_row(bname, T, spec, REPS, t) - push!(raw, (backend = String(bname), dtype = T, shape = spec.name, t = t)) - end - @info "grid progress" dtype = T shape = spec.name - end -end - -push!(canary_results, run_canary(rng, "B (middle)")) -push!(canary_results, run_canary(rng, "A' (end)")) -close(csv_io) - -canary_spread = (maximum(canary_results) - minimum(canary_results)) / minimum(canary_results) -open(CANARY_PATH, "w") do io - println(io, "label,median_seconds") - for (lbl, t) in zip(("A_start", "B_middle", "Aprime_end"), canary_results) - println(io, "$lbl,", @sprintf("%.9f", t)) - end - println(io, "# relative spread (max-min)/min = ", @sprintf("%.4f", canary_spread)) -end -println("canary spread (max-min)/min = ", @sprintf("%.4f", canary_spread)) - -# Per-shape/dtype summary: ratio of each backend's time to the fastest backend -# at that point, with a QuasiStrided win reported exactly like a loss. -open(joinpath(OUTDIR, "summary_tensoroperations.txt"), "w") do io - println(io, "# TensorOperations backend benchmark summary") - println(io, "canary median times (s): ", canary_results) - println(io, "canary relative spread (max-min)/min: ", @sprintf("%.4f", canary_spread)) - for T in DTYPES - println(io, "\n== ", T, " ==") - for spec in ALL_SHAPES - rows = filter(r -> r.dtype == T && r.shape == spec.name, raw) - isempty(rows) && continue - best = minimum(r.t for r in rows) - println(io, " ", spec.name, ":") - for r in sort(collect(rows); by = r -> r.t) - println( - io, " ", rpad(r.backend, 14), @sprintf("%.6e s", r.t), - " (", @sprintf("%.3fx", r.t / best), " of fastest)" - ) - end - end - end -end -println(read(joinpath(OUTDIR, "summary_tensoroperations.txt"), String)) - -# Provenance -- same format as the other PROVENANCE.txt files under -# benchmark/results/. -commit = try - strip(read(`git -C $(joinpath(@__DIR__, "..")) rev-parse HEAD`, String)) -catch - "unknown (git rev-parse failed)" -end -open(PROVENANCE_PATH, "w") do io - println(io, "git_commit = ", commit) - println(io, "command = julia --project=. benchmark/bench_tensoroperations.jl") - println(io, "cpu = ", Sys.CPU_NAME) - println(io, "julia = ", VERSION) - println(io, "nthreads = ", NTHREADS, " blas_threads = ", BLAS_THREADS) - println(io, "date = ", now()) - println(io, "backends = ", collect(keys(BACKENDS))) - println(io, "dtypes = ", collect(DTYPES)) - println(io, "shapes = ", [s.name for s in ALL_SHAPES]) - println(io, "reps = ", REPS) - println(io, "caveat = single machine ($(gethostname())), single measurement session; ") - println( - io, - " not averaged across machines or repeated sessions. Numbers are indicative of ", - "this reference machine only." - ) -end - -println("\nDone. Results in ", OUTDIR) diff --git a/benchmark/probes/fmaddsub_codegen.jl b/benchmark/probes/fmaddsub_codegen.jl deleted file mode 100644 index f559d95..0000000 --- a/benchmark/probes/fmaddsub_codegen.jl +++ /dev/null @@ -1,117 +0,0 @@ -# Instruction-selection probe for src/microkernels/fmaddsub.jl, next to the -# planar and 1m kernels at the same shapes: counts, in `@code_native` of -# `accumulate` with the driver's own `PackedPanel` argument types, the fused -# ops, shuffles, broadcasts and stack spill traffic. -# -# julia --project=. benchmark/probes/fmaddsub_codegen.jl # host ISA -# julia -C znver2 --project=. benchmark/probes/fmaddsub_codegen.jl # AVX2 codegen -# -# `-C znver2` changes LLVM's target only; the counts are a codegen property and -# need no Rome hardware (running the code would need it). Pass -# `--dump` to also print the loop body of each kernel. - -using QuasiStrided -using QuasiStrided: PlanarKernel, OneMKernel, FMAddSubKernel, PackedPanel, zero_accumulator -using InteractiveUtils: code_native -using Printf - -const DUMP = "--dump" in ARGS - -count_re(re, s) = count(re, s) - -# Stack traffic: any vector move to/from [rsp/rbp + ...]. -const SPILL_ST = r"vmov\w*\s+[xyz]mmword ptr \[r[sb]p[^\]]*\]," -const SPILL_LD = r"vmov\w*\s+[xyz]mm\d+,\s*[xyz]mmword ptr \[r[sb]p" - -# The K loop: from the label of the (last) backward conditional branch to that -# branch. Counts are taken over this body only -- the entry/exit copies of the -# accumulator tuple (sret, the `kc == 0` memcpy path) are not per-K-step cost. -function hot_loop(asm::AbstractString) - lines = split(asm, '\n') - labels = Dict{String, Int}() - best = nothing - for (n, l) in enumerate(lines) - m = match(r"^(\.LBB\w+):", l) - m === nothing || (labels[m[1]] = n) - b = match(r"^\s+j(?:ne|nz|b|l|g|a|e)\w*\s+(\.LBB\w+)", l) - if b !== nothing && haskey(labels, b[1]) - best = (labels[b[1]], n) # backward branch: a loop - end - end - best === nothing && return asm - return join(lines[best[1]:best[2]], '\n') -end - -function probe(kernel) - T = QuasiStrided.scalartype(kernel) - R = real(T) - acc = zero_accumulator(kernel) - io = IOBuffer() - code_native( - io, Base.accumulate, - (typeof(kernel), typeof(acc), PackedPanel{R}, PackedPanel{R}, Int); - debuginfo = :none, syntax = :intel - ) - full = String(take!(io)) - asm = hot_loop(full) - return ( - fmaddsub = count_re(r"vfmaddsub\d+p", asm), - fmsubadd = count_re(r"vfmsubadd\d+p", asm), - fma = count_re(r"vfn?madd\d+p", asm), # plain/negated FMA (not addsub) - fms = count_re(r"vfn?msub\d+p", asm), - mul = count_re(r"vmulp", asm), - addsub = count_re(r"v(add|sub)p[sd]", asm), - blend = count_re(r"vblend", asm), - shuf = count_re(r"v(shufp|permilp|perm[0-9a-z]*p|unpck)", asm), - bcast = count_re(r"vbroadcasts", asm), - spill_st = count_re(SPILL_ST, asm), - spill_ld = count_re(SPILL_LD, asm), - asm = asm, full = full, - ) -end - -kernels(::Type{ComplexF64}) = [ - ("planar", PlanarKernel(Val(4), Val(5), ComplexF64, Val(4))), - ("planar", PlanarKernel(Val(24), Val(3), ComplexF64, Val(8))), - ("1m", OneMKernel(Val(4), Val(6), ComplexF64, Val(4))), - ("1m", OneMKernel(Val(12), Val(8), ComplexF64, Val(8))), - [ - ("fmaddsub", FMAddSubKernel(Val(MR), Val(NR), ComplexF64, Val(W))) - for (MR, NR, W) in QuasiStrided.KERNEL_SHAPES_C64_FMADDSUB - ]..., -] -kernels(::Type{ComplexF32}) = [ - ("planar", PlanarKernel(Val(8), Val(5), ComplexF32, Val(8))), - ("planar", PlanarKernel(Val(48), Val(3), ComplexF32, Val(16))), - ("1m", OneMKernel(Val(8), Val(6), ComplexF32, Val(8))), - ("1m", OneMKernel(Val(24), Val(8), ComplexF32, Val(16))), - [ - ("fmaddsub", FMAddSubKernel(Val(MR), Val(NR), ComplexF32, Val(W))) - for (MR, NR, W) in QuasiStrided.KERNEL_SHAPES_C32_FMADDSUB - ]..., -] - -println( - "cpu = ", Sys.CPU_NAME, " julia = ", VERSION, - " (LLVM target: ", Base.JLOptions().cpu_target == C_NULL ? "native" : - unsafe_string(Base.JLOptions().cpu_target), ")" -) -@printf( - "%-10s %-9s %-10s %8s %8s %5s %5s %4s %7s %5s %5s %5s %8s %8s\n", - "T", "method", "shape", "fmaddsub", "fmsubadd", "fma", "fms", "mul", "add/sub", - "shuf", "blend", "bcast", "spill_st", "spill_ld" -) -for T in (ComplexF64, ComplexF32) - for (name, k) in kernels(T) - r = probe(k) - shape = "$(QuasiStrided.mr(k))x$(QuasiStrided.nr(k))/W$(QuasiStrided.lanewidth(k))" - @printf( - "%-10s %-9s %-10s %8d %8d %5d %5d %4d %7d %5d %5d %5d %8d %8d\n", - T, name, shape, r.fmaddsub, r.fmsubadd, r.fma, r.fms, r.mul, r.addsub, - r.shuf, r.blend, r.bcast, r.spill_st, r.spill_ld - ) - if DUMP - println(r.asm) - end - end -end diff --git a/benchmark/probes/probe_call_breakdown.jl b/benchmark/probes/probe_call_breakdown.jl deleted file mode 100644 index b6941af..0000000 --- a/benchmark/probes/probe_call_breakdown.jl +++ /dev/null @@ -1,65 +0,0 @@ -# Where the per-call floor of `tensorcontract!(..., QuasiStridedBackend())` -# goes on a tiny problem: the adapter's prefix (`_qs_prepare`), the pooled -# workspace lookup, planning alone (`_planned` with `identity` as the -# continuation, i.e. `plan_contract`'s body), `execute!` alone on a prebuilt -# plan, and the whole call. Internal API: tracks the current checkout only. -# -# JULIA_NUM_THREADS=1 taskset -c 4-7 \ -# julia --project=benchmark benchmark/probes/probe_call_breakdown.jl - -using TensorOperations, Statistics, Printf, StridedViews -using QuasiStrided -const TO = TensorOperations -const QS = QuasiStrided - -const SAMPLES = 15 -const CALLS = 2000 - -function timeit(f, args...) - for _ in 1:10 - f(args...) - end - bytes = @allocated f(args...) - ts = map(1:SAMPLES) do _ - t = time_ns() - for _ in 1:CALLS - f(args...) - end - (time_ns() - t) / CALLS - end - return median(ts), minimum(ts), bytes -end - -full(C, A, pA, B, pB, pAB) = - TO.tensorcontract!(C, A, pA, false, B, pB, false, pAB, true, false, QuasiStridedBackend()) -prep(C, A, pA, B, pB, pAB) = QS._qs_prepare(C, A, pA, B, pB, pAB, true, false) -wsget(::AbstractArray{T}) where {T} = QS._qs_task_workspace(T) -function plan_only(C, A, pA, B, pB, pAB) - Cv, Av, Bv, iA, iB, iC, _, _ = QS._qs_prepare(C, A, pA, B, pB, pAB, true, false) - return QS._planned( - _ -> nothing, Cv, Av, iA, Bv, iB, iC, nothing, false, false, - nothing, nothing, nothing, QS._qs_task_workspace(eltype(C)), TO.DefaultAllocator(), false - ) -end -exec_only(plan, a, b) = QS.execute!(plan, a, b) - -const CASES = ( - ("8x8x8 gemm", (8, 8), (8, 8), (8, 8), ((1,), (2,)), ((1,), (2,)), ((1, 2), ())), - ("d4 outer", (4, 4), (4, 4), (4, 4, 4, 4), ((1, 2), ()), ((), (1, 2)), ((1, 2, 3, 4), ())), -) - -for T in (Float64, ComplexF64), (name, sa, sb, sc, pA, pB, pAB) in CASES - A, B, C = rand(T, sa), rand(T, sb), zeros(T, sc) - Cv, Av, Bv, iA, iB, iC, _, _ = prep(C, A, pA, B, pB, pAB) - plan = QS.plan_contract(Cv, Av, iA, Bv, iB, iC; workspace = QS._qs_task_workspace(T)) - for (what, f, args) in ( - ("full", full, (C, A, pA, B, pB, pAB)), - ("prepare", prep, (C, A, pA, B, pB, pAB)), - ("workspace", wsget, (C,)), - ("prep+plan", plan_only, (C, A, pA, B, pB, pAB)), - ("execute!", exec_only, (plan, one(T), zero(T))), - ) - med, lo, b = timeit(f, args...) - @printf("%-10s\t%-10s\t%-10s\t%7.1f ns\t(min %7.1f)\t%4d B\n", T, name, what, med, lo, b) - end -end diff --git a/benchmark/probes/probe_case_ab.jl b/benchmark/probes/probe_case_ab.jl deleted file mode 100644 index 0a6f3cd..0000000 --- a/benchmark/probes/probe_case_ab.jl +++ /dev/null @@ -1,63 +0,0 @@ -# Interleaved A/B timing of ONE upstream-suite-style contraction case through -# `TensorOperations.tensorcontract!`, StridedBLAS vs QuasiStridedBackend, -# built exactly as benchmark/bench_to_suite.jl builds it (plain dense arrays in -# the given label orders, `contract_indices` for pA/pB/pAB, alpha=1, beta=0). -# ABAB interleaving over `--rounds` rounds, `--reps` calls per round per -# backend, so a slow patch on the shared machine hits both backends alike. -# -# julia -t 1 --project=benchmark benchmark/probes/probe_case_ab.jl \ -# --expr "abcd bdef>acef" --dim 63 --dtype ComplexF64 --reps 2 --rounds 2 - -using TensorOperations -using TensorOperations: StridedBLAS -using QuasiStrided -using QuasiStrided: QuasiStridedBackend -using Random -using Statistics: median -using Printf -using LinearAlgebra - -include(joinpath(@__DIR__, "..", "harness.jl")) - -const EXPR = argopt("expr", "abcd bdef>acef") -const DIM = argopt("dim", 63) -const T = parse_dtypes(argopt("dtype", "ComplexF64"))[1] -const REPS = argopt("reps", 2) -const ROUNDS = argopt("rounds", 2) - -function parse_expr(expr::AbstractString) - lhs, rhs = split(expr, '>') - a, b = split(strip(lhs), ' ') - return collect(a), collect(b), collect(strip(rhs)) -end -LA, LB, LC = parse_expr(EXPR) -IA = Symbol.(LA); IB = Symbol.(LB); IC = Symbol.(LC) - -rng = MersenneTwister(42) -A = randn(rng, T, ntuple(_ -> DIM, length(LA))...) -B = randn(rng, T, ntuple(_ -> DIM, length(LB))...) -C = zeros(T, ntuple(_ -> DIM, length(LC))...) -pA, pB, pAB = TensorOperations.contract_indices(Tuple(IA), Tuple(IB), Tuple(IC)) -labels = unique(vcat(LA, LB, LC)) -nk = length(intersect(LA, LB)) -flops = 2.0 * DIM^(length(labels)) - -run!(backend) = TensorOperations.tensorcontract!(C, A, pA, false, B, pB, false, pAB, one(T), zero(T), backend) - -println("expr = $EXPR dim = $DIM T = $T flops = $flops bytes = $(sizeof(A) + sizeof(B) + sizeof(C))") -run!(StridedBLAS()); Cref = copy(C) -run!(QuasiStridedBackend()); println("max abs diff QS vs BLAS = ", maximum(abs, C .- Cref)) - -tb = Float64[]; tq = Float64[] -for round in 1:ROUNDS - for _ in 1:REPS - t0 = time_ns(); run!(StridedBLAS()); push!(tb, (time_ns() - t0) / 1.0e9) - end - for _ in 1:REPS - t0 = time_ns(); run!(QuasiStridedBackend()); push!(tq, (time_ns() - t0) / 1.0e9) - end - @printf("round %d: BLAS %s QS %s\n", round, join((@sprintf("%.3f", x) for x in tb[(end - REPS + 1):end]), ","), join((@sprintf("%.3f", x) for x in tq[(end - REPS + 1):end]), ",")) -end -@printf("StridedBLAS median %.4f s (min %.4f, max %.4f) %.2f GF/s\n", median(tb), minimum(tb), maximum(tb), flops / median(tb) / 1.0e9) -@printf("QuasiStrided median %.4f s (min %.4f, max %.4f) %.2f GF/s\n", median(tq), minimum(tq), maximum(tq), flops / median(tq) / 1.0e9) -@printf("QS/BLAS = %.3f\n", median(tq) / median(tb)) diff --git a/benchmark/probes/probe_ccsd_t_layout.jl b/benchmark/probes/probe_ccsd_t_layout.jl deleted file mode 100644 index 7058724..0000000 --- a/benchmark/probes/probe_ccsd_t_layout.jl +++ /dev/null @@ -1,128 +0,0 @@ -# For the four `ccsd_t_*` TCCG quantum-chemistry cases, tallies how many M/N -# micro-tile slivers on the C side are `regular && stride==1` (i.e. would -# satisfy the store fast-path's `_unit_stride_rows` guard) versus not. -# -# Usage: julia --project=. benchmark/probes/probe_ccsd_t_layout.jl [--dim N] -# Writes benchmark/results/-/probes_ccsd_t_layout.txt. - -using QuasiStrided -using QuasiStrided: plan_contract, mr, nr, block_descriptors! -using StridedViews -using Random - -include(joinpath(@__DIR__, "..", "harness.jl")) - -const OUTPATH = joinpath(results_dir(), "probes_ccsd_t_layout.txt") -mkpath(dirname(OUTPATH)) - -io = open(OUTPATH, "w") -out(args...) = (println(io, args...); println(args...)) - -out("git_commit = ", git_commit()) -out("hostname = ", gethostname()) -out("julia_version = ", VERSION) -out("date = ", now()) -out() - -# Manual label assignment matching QuasiStrided's convention (positive = -# open axis matching C's own label numbering in C's order, negative = -# contracted pair). Any consistent labeling is valid; we choose one that -# mirrors C's own axis order 1:1 so indC is just (1,2,...,rank(C)). -# -# a=1 b=2 c=3 i=4 j=5 k=6 (all open, C = C[a,b,c,i,j,k]) -# m=-1 (the one contracted label in every equation) -const CASES = [ - (id = "ccsd_t_1", indA = (4, 5, -1, 1), indB = (-1, 6, 2, 3)), # A=ijma B=mkbc - (id = "ccsd_t_2", indA = (4, 5, -1, 2), indB = (-1, 6, 1, 3)), # A=ijmb B=mkac - (id = "ccsd_t_3", indA = (4, 5, -1, 3), indB = (-1, 6, 1, 2)), # A=ijmc B=mkab - (id = "ccsd_t_4", indA = (4, 6, -1, 2), indB = (-1, 5, 1, 3)), # A=ikmb B=mjac -] -const indC = (1, 2, 3, 4, 5, 6) -const DIM = argopt("dim", 16) - -global refuted = false - -for T in (Float64, Float32) - rng = MersenneTwister(0) - for case in CASES - A = randn(rng, T, DIM, DIM, DIM, DIM) - B = randn(rng, T, DIM, DIM, DIM, DIM) - C = zeros(T, DIM, DIM, DIM, DIM, DIM, DIM) - - plan = plan_contract( - StridedView(C), StridedView(A), case.indA, - StridedView(B), case.indB, indC - ) - - out("case=$(case.id) dtype=$T: typeof(plan.Cstorage) = ", typeof(plan.Cstorage)) - MR, NR = mr(plan.kernel), nr(plan.kernel) - out("case=$(case.id) dtype=$T: mr=$MR nr=$NR") - - Qm = QuasiStrided.axis_length(plan.mgroup) - Qn = QuasiStrided.axis_length(plan.ngroup) - - m_bufs = (zeros(Int, MR), zeros(Int, MR)) - n_unit_rows = 0 - m_total = 0 - mfirst = 0 - while mfirst < Qm - mcount = min(MR, Qm - mfirst) - (dM_A, dM_C) = block_descriptors!(m_bufs, plan.mgroup, mfirst, mcount) - m_total += 1 - if dM_C.regular && dM_C.stride == 1 - n_unit_rows += 1 - end - mfirst += mcount - end - out( - "case=$(case.id) dtype=$T: M-slivers checked=$m_total, ", - "unit-stride (regular && stride==1) = $n_unit_rows" - ) - if n_unit_rows > 0 - out("!!! unit-stride M-sliver found for case=$(case.id) dtype=$T !!!") - global refuted = true - end - - n_bufs = (zeros(Int, NR), zeros(Int, NR)) - n_unit_cols = 0 - n_total = 0 - nfirst = 0 - while nfirst < Qn - ncount = min(NR, Qn - nfirst) - (dN_B, dN_C) = block_descriptors!(n_bufs, plan.ngroup, nfirst, ncount) - n_total += 1 - if dN_C.regular && dN_C.stride == 1 - n_unit_cols += 1 - end - nfirst += ncount - end - out( - "case=$(case.id) dtype=$T: N-slivers checked=$n_total, ", - "unit-stride (regular && stride==1) = $n_unit_cols" - ) - if n_unit_cols > 0 - out("!!! unit-stride N-sliver found for case=$(case.id) dtype=$T !!!") - global refuted = true - end - - # Also print the actual first M-sliver's C-side descriptor verbatim, - # so the raw evidence (not just the pass/fail tally) is on record. - (dM_A0, dM_C0) = block_descriptors!(m_bufs, plan.mgroup, 0, min(MR, Qm)) - out("case=$(case.id) dtype=$T: first M-sliver C descriptor = ", dM_C0) - (dN_B0, dN_C0) = block_descriptors!(n_bufs, plan.ngroup, 0, min(NR, Qn)) - out("case=$(case.id) dtype=$T: first N-sliver C descriptor = ", dN_C0) - out() - end -end - -out("="^72) -if refuted - out("VERDICT: at least one unit-stride sliver found -- the store fast-path") - out("guard could reach some of these cases.") -else - out("VERDICT: zero unit-stride M/N slivers across all cases/dtypes -- the") - out("store fast-path guard cannot help any of them.") -end - -close(io) -println("\nWrote ", OUTPATH) diff --git a/benchmark/probes/probe_julia_versions.sh b/benchmark/probes/probe_julia_versions.sh deleted file mode 100644 index f89ad9a..0000000 --- a/benchmark/probes/probe_julia_versions.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# Reports which Julia versions/toolchains are available on this machine. -# -# Usage: bash benchmark/probes/probe_julia_versions.sh -# Writes benchmark/results/-/probes_julia_versions.txt -set -uo pipefail - -OUTDIR="$(dirname "$0")/../results/$(hostname)-$(date +%Y-%m-%d)" -mkdir -p "$OUTDIR" -OUTPATH="$OUTDIR/probes_julia_versions.txt" - -{ - echo "hostname = $(hostname)" - echo "date = $(date -Iseconds)" - echo - echo "--- which -a julia ---" - which -a julia 2>&1 - echo - echo "--- julia --version ---" - julia --version 2>&1 - echo - echo "--- ls ~/.julia/juliaup ---" - ls ~/.julia/juliaup 2>&1 - echo - echo "--- module avail julia (head -20) ---" - module avail julia 2>&1 | head -20 - echo - echo "--- command -v juliaup ---" - command -v juliaup 2>&1 - echo - echo "--- juliaup list channels (if available) ---" - juliaup list 2>&1 || echo "(juliaup not usable)" -} | tee "$OUTPATH" - -echo -echo "Wrote $OUTPATH" diff --git a/benchmark/probes/probe_korder_changes.jl b/benchmark/probes/probe_korder_changes.jl deleted file mode 100644 index d599051..0000000 --- a/benchmark/probes/probe_korder_changes.jl +++ /dev/null @@ -1,56 +0,0 @@ -# Which upstream-suite :contract cases does `_order_contract_labels` -# (src/planning/labels.jl) actually reorder? For every synthetic/TCCG/batched -# case at one small leg dim, plan it and compare the K composite against the -# pre-rule order (`indA` order, i.e. `_classify_labels`' output). Cases whose -# K group is unchanged run the identical code path before and after the rule -# and cannot have moved in a before/after benchmark except by noise. -# -# The cost model reads the operand sizes (L2 fit, page-sized walks), so run it -# at the leg dims actually benchmarked. -# -# julia --project=benchmark benchmark/probes/probe_korder_changes.jl \ -# [--dim 4] [--sources synthetic,tccg,batched] - -using TensorOperations -using TensorOperationsBenchmarks -using TensorOperationsBenchmarks: ContractSpec, BatchedContractSpec -using QuasiStrided -using QuasiStrided: plan_contract, _classify_labels, _build_pair_group -using StridedViews -using Random - -include(joinpath(@__DIR__, "..", "harness.jl")) - -const TOB = TensorOperationsBenchmarks -const DIM = argopt("dim", 4) -const SOURCES = split(argopt("sources", "synthetic,tccg,batched"), ',') - -gens = Dict("synthetic" => TOB._synthetic_contract_cases, "tccg" => TOB._tccg_cases, "batched" => TOB._batched_contract_cases) -cases = reduce(vcat, [gens[s]((DIM,)) for s in SOURCES]) -changed = String[] -unchanged = 0 -for case in cases - spec = case.spec - spec isa Union{ContractSpec, BatchedContractSpec} || continue - dimsA = ntuple(i -> spec.dims[spec.IA[i]], length(spec.IA)) - dimsB = ntuple(i -> spec.dims[spec.IB[i]], length(spec.IB)) - dimsC = ntuple(i -> spec.dims[spec.IC[i]], length(spec.IC)) - A = StridedView(Array{Float64}(undef, dimsA...)) # planning never reads values - B = StridedView(Array{Float64}(undef, dimsB...)) - C = StridedView(Array{Float64}(undef, dimsC...)) - pA, pB, pAB = TensorOperations.contract_indices(spec.IA, spec.IB, spec.IC) - indA, indB, indC = QuasiStrided._qs_labels(pA, pB, pAB) - plan = plan_contract(C, A, indA, B, indB, indC; oracle = false) - _, _, klabels = _classify_labels(indA, indB, indC) - old = _build_pair_group(klabels, indA, A, indB, B) # pre-rule K group, (A, B) maps - swapped = plan.Astorage === parent(B) - new = plan.kgroup - newstrides = swapped ? (new.strides[2], new.strides[1]) : new.strides - if (new.lengths, newstrides) == (old.lengths, old.strides) - global unchanged += 1 - else - push!(changed, "$(case.id) K old=$(klabels) A/B strides $(old.strides) -> new A/B strides $(newstrides)") - end -end -println("dim = $DIM: $(length(cases)) cases, $(unchanged) with an unchanged K group, $(length(changed)) reordered:") -foreach(println, changed) diff --git a/benchmark/probes/probe_msplit_prototype.jl b/benchmark/probes/probe_msplit_prototype.jl deleted file mode 100644 index cfef2fe..0000000 --- a/benchmark/probes/probe_msplit_prototype.jl +++ /dev/null @@ -1,123 +0,0 @@ -# Prototype: does splitting A's unit-stride M axis into (inner, outer) and -# moving the inner part forward in the M enumeration fix the A-pack line -# amplification of the TCCG intensli cases, and at what store cost? -# -# Builds the plan normally, then rebuilds it with a hand-made M AxisGroup: -# :asis -> the planner's order (C-stride order, A's fastest axis last) -# :pos2 -> C's fastest axis first, then `li` of A's fastest axis, then the rest -# :pos1 -> `li` of A's fastest axis first, then the C order -# -# julia -t 1 --project=benchmark benchmark/probes/probe_msplit_prototype.jl --case intensli_7 --dim 24 --dtype Float64 --li 8 - -using QuasiStrided -using QuasiStrided: plan_contract, execute!, mr, nr, axis_length, AxisGroup, ContractPlan, - _default_kernel, _kernel_from_shape, kernel_shapes, complex_method -using StridedViews -using Random -using Statistics: median -using Printf -using LinearAlgebra - -include(joinpath(@__DIR__, "..", "harness.jl")) - -const KNOWN = Dict( - "intensli_6" => "abcde bf>dcfea", - "intensli_7" => "abcde df>ecbfa", - "intensli_8" => "abcde fb>dfcea", - "ccsd_t_3" => "abcd cefg>fgdabe", -) -const CASE = argopt("case", "intensli_7") -const EXPR = something(argval("expr"), get(KNOWN, CASE, nothing)) -const DIM = argopt("dim", 24) -const T = parse_dtypes(argopt("dtype", "Float64"))[1] -const REPS = argopt("reps", 7) -const LI = argopt("li", 8) - -function parse_expr(expr::AbstractString) - lhs, rhs = split(expr, '>') - a, b = split(strip(lhs), ' ') - return collect(a), collect(b), collect(strip(rhs)) -end -const LA, LB, LC = parse_expr(EXPR) -const LETTERS = unique(vcat(LA, LB, LC)) -label(c) = findfirst(==(c), LETTERS) -const indA = Tuple(label.(LA)); const indB = Tuple(label.(LB)); const indC = Tuple(label.(LC)) - -rng = MersenneTwister(1234) -A = randn(rng, T, ntuple(_ -> DIM, length(LA))...) -B = randn(rng, T, ntuple(_ -> DIM, length(LB))...) -C = zeros(T, ntuple(_ -> DIM, length(LC))...) -Av, Bv, Cv = StridedView(A), StridedView(B), StridedView(C) - -plan0 = plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false) -g = plan0.mgroup -println("case = $CASE dim = $DIM T = $T kernel = ", typeof(plan0.kernel).name.name, " MR=", mr(plan0.kernel), " blocking = ", plan0.blocking) -println("as-is M group: lengths=", g.lengths, " strides=", g.strides) -flops = 2.0 * axis_length(plan0.mgroup) * axis_length(plan0.ngroup) * axis_length(plan0.kgroup) - -# Split dim `du` (1-based, in the group's own order) into (li, L/li) and place -# the inner part at `pos`, the outer part where `du` was. -function split_group(g::AxisGroup{D, P}, du::Int, li::Int, pos::Int) where {D, P} - L = g.lengths[du] - @assert L % li == 0 - dims = [(g.lengths[d], ntuple(p -> g.strides[p][d], P)) for d in 1:D] - inner = (li, ntuple(p -> g.strides[p][du], P)) - outer = (L ÷ li, ntuple(p -> li * g.strides[p][du], P)) - dims[du] = outer - insert!(dims, pos, inner) - lengths = ntuple(i -> dims[i][1], D + 1) - strides = ntuple(p -> ntuple(i -> dims[i][2][p], D + 1), P) - return AxisGroup(lengths, strides) -end - -function with_mgroup(plan::ContractPlan, mg; kernel = plan.kernel) - return ContractPlan( - kernel, mg, plan.ngroup, plan.kgroup, plan.blocking, - plan.Astorage, plan.Abase, plan.Bstorage, plan.Bbase, plan.Cstorage, plan.Cbase, - plan.atransform, plan.btransform, plan.workspace - ) -end - -du = argmin([g.lengths[d] > 1 ? abs(g.strides[1][d]) : typemax(Int) for d in 1:length(g.lengths)]) -println("A's fastest M axis is group dim $du (A stride $(g.strides[1][du]), length $(g.lengths[du]))") - -variants = Pair{String, Any}[] -push!(variants, "asis" => plan0) -if du != 1 - g2 = split_group(g, du, LI, 2) - g1 = split_group(g, du, LI, 1) - push!(variants, "pos2(li=$LI)" => with_mgroup(plan0, g2)) - push!(variants, "pos1(li=$LI)" => with_mgroup(plan0, g1)) - # pos2 with the run-demoted kernel (what `_demote_for_run` would pick for run = L_1) - function demoted_shape(run, kernel) - best = nothing - for shape in kernel_shapes(T, complex_method(kernel)) - if run % shape[1] == 0 && (best === nothing || shape[1] > best[1]) - best = shape - end - end - return best - end - if T <: Real - best = demoted_shape(g.lengths[1], plan0.kernel) - if best !== nothing && best[1] != mr(plan0.kernel) - k2 = _kernel_from_shape(best, T, complex_method(plan0.kernel)) - p2 = plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false, kernel = k2) - push!(variants, "pos2(li=$LI)+MR$(best[1])" => with_mgroup(p2, g2)) - end - end -end - -execute!(plan0, one(T), zero(T)); Cref = copy(C) -for (name, p) in variants - println(" ", name, ": M group lengths=", p.mgroup.lengths, " strides=", p.mgroup.strides) -end -for round in 1:2 - for (name, p) in variants - fill!(C, zero(T)) - execute!(p, one(T), zero(T)) - @assert C ≈ Cref "$name wrong result" - t = median_time_s(() -> execute!(p, one(T), zero(T)); reps = REPS) - @printf("round %d %-22s %9.4f s %7.2f GF/s\n", round, name, t, flops / t / 1.0e9) - end -end diff --git a/benchmark/probes/probe_network_korder.jl b/benchmark/probes/probe_network_korder.jl deleted file mode 100644 index 22a0562..0000000 --- a/benchmark/probes/probe_network_korder.jl +++ /dev/null @@ -1,129 +0,0 @@ -# Which pairwise contractions inside an upstream :network case does the K -# order rule (`_order_contract_labels`, src/planning/labels.jl) change, and -# what does each change cost? Runs the case through `ncon` with -# QuasiStridedBackend, logging every K-order decision (labels, both operands' -# K strides and sizes, the chosen operand), then times the whole network with -# the rule ON and OFF (interleaved), and each changed pairwise contraction on -# its own, rebuilt from the logged shapes. -# -# julia -t 1 --project=benchmark benchmark/probes/probe_network_korder.jl \ -# --case mps_1site --size 32 --dtype ComplexF64 --reps 21 -# -# The rule is switched by redefining `_order_contract_labels` here with a -# mode flag -- a probe-only hack, never to be copied into src/. - -using TensorOperations -using TensorOperations: StridedBLAS -using TensorOperationsBenchmarks -using TensorOperationsBenchmarks: ArrayProvider, randtensor, NetworkSpec -using QuasiStrided -using QuasiStrided: QuasiStridedBackend, _choose_k_order, _k_order_cost -using StridedViews -using Random -using Statistics: median -using Printf -using LinearAlgebra - -include(joinpath(@__DIR__, "..", "harness.jl")) - -const TOB = TensorOperationsBenchmarks -const CASE = argopt("case", "mps_1site") -const SIZE = argopt("size", 32) -const T = parse_dtypes(argopt("dtype", "ComplexF64"))[1] -const REPS = argopt("reps", 21) - -const MODE = Ref(:new) # :new (cost model), :old (indA order), :simple (12ae2b4's rule) - -# 12ae2b4's rule: sort by stride in the operand with the smaller minimum K stride. -function simple_rule(klabels, indA, A, indB, B, Qm, Qn) - minst(ind, v) = minimum((abs(Base.strides(v)[findfirst(==(l), ind)]) for l in klabels if size(v, findfirst(==(l), ind)) > 1); init = typemax(Int)) - mA = minst(indA, A); mB = minst(indB, B) - by_b = mB < mA || (mB == mA && Qn > Qm) - return by_b ? QuasiStrided._sort_labels_by_stride(klabels, indB, B) : QuasiStrided._sort_labels_by_stride(klabels, indA, A) -end -const LOG = Vector{Any}() - -# Replaces the entry point without an explicit `l2bytes` (the one -# `plan_contract` calls); the label lists are NTuples (src/planning/labels.jl). -function QuasiStrided._order_contract_labels( - klabels::NTuple{DK, Int}, indA::NTuple{NA, Int}, A::StridedView, morder::NTuple{DM, Int}, - indB::NTuple{NB, Int}, B::StridedView, norder::NTuple{DN, Int}, Qm::Int, Qn::Int - ) where {DK, NA, NB, DM, DN} - DK <= 1 && return klabels - l2bytes = QuasiStrided._l2_core_bytes() - new = _choose_k_order(klabels, indA, A, morder, indB, B, norder, Qm, Qn, l2bytes) - stA = Base.strides(A); stB = Base.strides(B) - kA = Tuple(stA[findfirst(==(l), indA)] for l in klabels) - kB = Tuple(stB[findfirst(==(l), indB)] for l in klabels) - kL = Tuple(size(A, findfirst(==(l), indA)) for l in klabels) - costs = Tuple((_k_order_cost(o, indA, A, morder, Qm, l2bytes), _k_order_cost(o, indB, B, norder, Qn, l2bytes)) for o in (klabels, new)) - simple = simple_rule(klabels, indA, A, indB, B, Qm, Qn) - push!(LOG, (; klabels, kL, kA, kB, sizeA = size(A), stA, indA, sizeB = size(B), stB, indB, Qm, Qn, costs, changed = new != klabels || simple != klabels, new, simple)) - return MODE[] === :new ? new : MODE[] === :simple ? simple : klabels -end - -gens = Dict("mps_1site" => TOB._mps_cases, "mps_2site" => TOB._mps_cases, "trg_plaquette" => TOB._trg_cases, "ctmrg_corner" => TOB._ctmrg_cases) -cases = gens[CASE]((SIZE,)) -case = cases[findfirst(c -> startswith(c.id, CASE), cases)] -spec = case.spec::NetworkSpec -provider = ArrayProvider{T}() -tensors = map(spec.indexlists) do il - dims = ntuple(i -> spec.dims[abs(il[i])], length(il)) - randtensor(provider, il, dims, T) -end -run(backend) = TensorOperations.ncon(tensors, spec.indexlists, spec.conjlist; order = spec.order, output = spec.output, backend = backend) - -println("case = $(case.id) T = $T indexlists = $(spec.indexlists) order = $(spec.order)") -empty!(LOG); MODE[] = :new -Cnew = run(QuasiStridedBackend()) -decisions = unique(e -> (e.klabels, e.kA, e.kB, e.indA, e.indB, e.sizeA, e.sizeB), LOG) -println("pairwise contractions with >1 K label: $(length(decisions)) distinct (of $(length(LOG)) planned)") -for e in decisions - println( - " K=", e.klabels, " lengths=", e.kL, " A: size=", e.sizeA, " ind=", e.indA, " Kstrides=", e.kA, " B: size=", e.sizeB, " ind=", e.indB, " Kstrides=", e.kB, - " Qm=", e.Qm, " Qn=", e.Qn, " cost(A,B) old=", e.costs[1], " new=", e.costs[2], e.changed ? " CHANGED: new=$(e.new) simple=$(e.simple)" : " (unchanged)" - ) -end -MODE[] = :old -Cold = run(QuasiStridedBackend()) -Cblas = run(StridedBLAS()) -println("max|new-old| = ", maximum(abs, Cnew .- Cold), " max|new-blas| = ", maximum(abs, Cnew .- Cblas)) - -# Whole network, interleaved. -function tmed(f) - f() - ts = [(t0 = time_ns(); f(); (time_ns() - t0) / 1.0e9) for _ in 1:REPS] - return median(ts) -end -for round in 1:2 - MODE[] = :old; told = tmed(() -> run(QuasiStridedBackend())) - MODE[] = :new; tnew = tmed(() -> run(QuasiStridedBackend())) - MODE[] = :simple; tsim = tmed(() -> run(QuasiStridedBackend())) - @printf("round %d network: simple rule %.3e s (simple/old %.3f)\n", round, tsim, tsim / told) - tblas = tmed(() -> run(StridedBLAS())) - @printf("round %d network: QS old %.3e s QS new %.3e s (new/old %.3f) BLAS %.3e s\n", round, told, tnew, tnew / told, tblas) -end - -# Each CHANGED pairwise contraction on its own, rebuilt dense at the logged -# shapes (the network's intermediates are dense, so this is the same layout). -for e in decisions - e.changed || continue - A = randn(T, e.sizeA...); B = randn(T, e.sizeB...) - mlab = [l for l in e.indA if !(l in e.klabels)] - nlab = [l for l in e.indB if !(l in e.klabels)] - indC = Tuple(vcat(mlab, nlab)) - C = zeros(T, Tuple(l in e.indA ? size(A, findfirst(==(l), e.indA)) : size(B, findfirst(==(l), e.indB)) for l in indC)...) - Av, Bv, Cv = StridedView(A), StridedView(B), StridedView(C) - f() = QuasiStrided.contract!(Cv, one(T), Av, e.indA, Bv, e.indB, zero(T), indC) - MODE[] = :old; told = tmed(f) - MODE[] = :new; tnew = tmed(f) - MODE[] = :old; told2 = tmed(f) - MODE[] = :new; tnew2 = tmed(f) - MODE[] = :simple; tsim = tmed(f); MODE[] = :simple; tsim2 = tmed(f) - @printf(" pairwise K=%s: simple %.3e,%.3e s simple/old %.3f\n", e.klabels, tsim, tsim2, min(tsim, tsim2) / min(told, told2)) - flops = 2.0 * prod(size(C)) * prod(e.kL) - @printf( - " pairwise K=%s A%s B%s: old %.3e,%.3e s new %.3e,%.3e s new/old %.3f (%.1f GF/s new)\n", - e.klabels, e.sizeA, e.sizeB, told, told2, tnew, tnew2, min(tnew, tnew2) / min(told, told2), flops / min(tnew, tnew2) / 1.0e9 - ) -end diff --git a/benchmark/probes/probe_pack_order.jl b/benchmark/probes/probe_pack_order.jl deleted file mode 100644 index 7632f6a..0000000 --- a/benchmark/probes/probe_pack_order.jl +++ /dev/null @@ -1,116 +0,0 @@ -# Pack-A walk in isolation: gather every A element of a contraction into -# `mr`-row sliver panels, mc rows x kc K steps at a time, exactly as -# `_execute_nest!` visits them, with the leaf loop in two orders: -# -# sliver for r in slivers, for p in 0:kc-1, for t in 0:mr-1 (shipped: -# one sliver's whole panel, then the next sliver) -# kouter for p in 0:kc-1, for r in slivers, for t in 0:mr-1 (each K step -# across all slivers of the block before the next K step) -# -# When the sliver rows' and the K steps' byte strides in A are multiples of -# 4 KB (intensli_7 at dims 16/24/32: A strides dim^3, dim^4 elements), every -# load of one sliver's panel lands in the same L1 set; `kouter` interleaves -# slivers from other C-order coordinates (other sets). Plain scalar loads -# through precomputed offsets -- no QuasiStrided packer -- so only the walk -# differs. -# -# julia -t 1 --project=benchmark benchmark/probes/probe_pack_order.jl \ -# --case intensli_7 --dim 24 [--shape 8,6,4] [--pad 1] [--reps 7] - -using QuasiStrided -using QuasiStrided: plan_contract, mr, axis_length, _kernel_from_shape -using StridedViews -using Statistics: median -using Printf - -include(joinpath(@__DIR__, "..", "harness.jl")) - -const KNOWN = Dict("intensli_6" => "abcde bf>dcfea", "intensli_7" => "abcde df>ecbfa", "intensli_8" => "abcde fb>dfcea") -const CASE = argopt("case", "intensli_7") -const EXPR = something(argval("expr"), get(KNOWN, CASE, nothing)) -const DIM = argopt("dim", 16) -const PAD = argopt("pad", 0) -const REPS = argopt("reps", 7) -const SHAPE = Tuple(parse.(Int, split(argopt("shape", "8,6,4"), ',')))::NTuple{3, Int} -const T = Float64 - -lhs, rhs = split(EXPR, '>') -la, lb = split(strip(lhs), ' ') -letters = unique(vcat(collect(la), collect(lb), collect(rhs))) -lab(s) = Tuple(findfirst(==(c), letters) for c in s) -indA, indB, indC = lab(la), lab(lb), lab(rhs) - -# `--pad P` pads EVERY axis of A's parent to DIM + P, so no stride is a -# multiple of 4 KB (padding only the first axis leaves dim^3 * (dim+P) * 8 -# a 4 KB multiple at dim 32). -Apar = randn(T, ntuple(_ -> DIM + PAD, length(la))...) -A = view(Apar, ntuple(_ -> 1:DIM, length(la))...) -B = randn(T, ntuple(_ -> DIM, length(lb))...) -C = zeros(T, ntuple(_ -> DIM, length(rhs))...) -plan = plan_contract( - StridedView(C), StridedView(A), indA, StridedView(B), indB, indC; - oracle = false, kernel = _kernel_from_shape(SHAPE, T) -) -pointer(plan.Astorage) == pointer(Apar) || error("swapped plan: A does not feed M") -const Ast = plan.Astorage # the flat storage the engine indexes (0-based offsets) - -# Offsets of every M row and K step into A's parent (0-based), in the plan's -# enumeration order (first axis fastest). -function offsets(lengths, strides) - out = Int[0] - for (L, s) in zip(lengths, strides) - out = vec([o + i * s for o in out, i in 0:(L - 1)]) - end - return out -end -mo = offsets(plan.mgroup.lengths, plan.mgroup.strides[1]) -ko = offsets(plan.kgroup.lengths, plan.kgroup.strides[1]) -base = plan.Abase -MR = mr(plan.kernel) -mc, kc = plan.blocking.mc, plan.blocking.kc -println("case = $CASE dim = $DIM pad = $PAD MR = $MR mc = $mc kc = $kc") -println("A strides (bytes) = ", strides(A) .* sizeof(T), " M strides(A) = ", plan.mgroup.strides[1], " K strides(A) = ", plan.kgroup.strides[1]) - -function pack_all!(buf, Apar, mo, ko, base, MR, mc, kc, ::Val{ORDER}) where {ORDER} - Qm, Qk = length(mo), length(ko) - acc = 0.0 - @inbounds for pc in 0:kc:(Qk - 1) - kb = min(kc, Qk - pc) - for ic in 0:mc:(Qm - 1) - mb = min(mc, Qm - ic) - ns = cld(mb, MR) - if ORDER === :sliver - for r in 0:(ns - 1), p in 0:(kb - 1), t in 0:(MR - 1) - i = r * MR + t - i < mb || continue - buf[r * MR * kb + p * MR + t + 1] = Apar[base + mo[ic + i + 1] + ko[pc + p + 1]] - end - else - for p in 0:(kb - 1), r in 0:(ns - 1), t in 0:(MR - 1) - i = r * MR + t - i < mb || continue - buf[r * MR * kb + p * MR + t + 1] = Apar[base + mo[ic + i + 1] + ko[pc + p + 1]] - end - end - acc += buf[1] # keep the block's pack live - end - end - return acc -end - -buf = zeros(T, (mc + MR) * kc) -n = length(mo) * length(ko) -res = Dict{Symbol, Vector{Float64}}(:sliver => Float64[], :kouter => Float64[]) -for o in (:sliver, :kouter) - pack_all!(buf, Ast, mo, ko, base + 1, MR, mc, kc, Val(o)) # warm-up/compile -end -# `base` (plan.Abase) is 0-based into the parent; Julia indexing is 1-based. -for rep in 1:REPS, o in (:sliver, :kouter) # interleaved - t0 = time_ns() - pack_all!(buf, Ast, mo, ko, base + 1, MR, mc, kc, Val(o)) - push!(res[o], (time_ns() - t0) / n) -end -for o in (:sliver, :kouter) - v = res[o] - @printf(" %-7s %6.2f ns/elem (min %.2f, max %.2f)\n", o, median(v), minimum(v), maximum(v)) -end diff --git a/benchmark/probes/probe_run_demote_kdepth.jl b/benchmark/probes/probe_run_demote_kdepth.jl deleted file mode 100644 index c8e9f06..0000000 --- a/benchmark/probes/probe_run_demote_kdepth.jl +++ /dev/null @@ -1,304 +0,0 @@ -# Probe: cost of the run-length kernel-shape demotion (`_demote_for_run`, -# src/planning/kernel_selection.jl, called from `plan_contract` in -# src/planning/plan.jl) across a range of `Qk` ("K-depth", the contracted -# extent `m` in the fixture below). `r = t_demoted / t_default` per point is -# the evidence behind the K-depth guard `_RUN_DEMOTE_KMAX_F64/_F32`: demotion -# wins at shallow K and loses past the crossover. -# -# On this fixture family the run-length predicate is broken at every -# `(T, a, m)`, so demotion is eligible everywhere; the guard alone decides. -# Hence `auto` matches `forced-demoted` for `Qk <= kmax(T)` and -# `forced-default` above it, and `kernel_type_matches_first_m` is expected to -# be false for `Qk > kmax(T)`. -# -# julia -t 1 --project=benchmark benchmark/probes/probe_run_demote_kdepth.jl -# -# Fixture: A is (i,j,m,a), B is (m,k,b,c), C is (a,b,c,i,j,k), with -# i=j=k=b=c=6 fixed and (a, m) swept. Free extents seen by `plan_contract` -# are therefore Qm = a*36 (i*j=36) and Qn = 216 (b*c*k=6*6*6=216), both -# independent of `m` -- only Qk = m depends on `m`. That in turn means the -# swap decision (`_prefer_swap`) and the demoted shape are themselves -# independent of `m` for a fixed `(T, a)` (only whether the guard lets -# demotion fire depends on `m`); this is checked per point by re-deriving -# `plan.kernel`'s type at every `m` and comparing it against the type -# recorded at the first (shallowest) `m` of the sweep. -# -# Three arms per `(T, a, m)` point, timed interleaved (not -# all-of-arm-1-then-all-of-arm-2), each on its own warmed `ContractPlan` via -# `execute!` (never `plan_contract` itself, which is comparatively cheap and -# would just add noise): -# 1. auto -- plan_contract(...), no `kernel=`; the planner decides. -# 2. forced-default -- plan_contract(...; kernel = _default_kernel(T, Qm, -# Qn)); explicit `kernel=` makes both call sites use -# this kernel verbatim (see `plan_contract` in -# src/planning/plan.jl), bypassing demotion entirely. -# 3. forced-demoted -- plan_contract(...; kernel = ), -# i.e. the demotion's own pick, forced so its cost can be -# measured without re-deciding the swap/demotion. - -include(joinpath(@__DIR__, "..", "harness.jl")) - -using QuasiStrided: _default_kernel -using StridedViews: StridedView -using Statistics: median -using Printf -using Random - -const REPS = 15 -const OUTDIR = results_dir() -mkpath(OUTDIR) -const CSV_PATH = joinpath(OUTDIR, "run_demote_kdepth.csv") -const PROV_PATH = joinpath(OUTDIR, "run_demote_kdepth_PROVENANCE.txt") - -# --------------------------------------------------------------------------- -# Fixture -# --------------------------------------------------------------------------- - -function fixture(::Type{T}, a::Int, m::Int, rng) where {T} - i = j = k = b = c = 6 - A = rand(rng, T, i, j, m, a) - B = rand(rng, T, m, k, b, c) - C = zeros(T, a, b, c, i, j, k) - indA = (1, 2, 3, 4) # i,j,m,a - indB = (3, 5, 6, 7) # m,k,b,c - indC = (4, 6, 7, 1, 2, 5) # a,b,c,i,j,k - return StridedView(C), StridedView(A), indA, StridedView(B), indB, indC -end - -kernel_shape_str(kernel) = "($(mr(kernel)),$(nr(kernel)),$(lanewidth(kernel)))" - -# --------------------------------------------------------------------------- -# One measured point: (T, a, m). `default_kernel`/`demoted_kernel` are passed -# in (computed once per (T,a), reused across the m-sweep, per the header -# comment); `verify_kernel_type` is the type recorded for `demoted_kernel` at -# the first m of the sweep, used to check m-independence at every point. -# --------------------------------------------------------------------------- - -function measure_point( - T::Type, a::Int, m::Int, default_kernel, demoted_kernel, - verify_kernel_type, rng; check_numeric::Bool = false - ) - Cv, Av, indA, Bv, indB, indC = fixture(T, a, m, rng) - - plan_auto = plan_contract(Cv, Av, indA, Bv, indB, indC) - plan_default = plan_contract(Cv, Av, indA, Bv, indB, indC; kernel = default_kernel) - plan_demoted = plan_contract(Cv, Av, indA, Bv, indB, indC; kernel = demoted_kernel) - - kernel_type_ok = typeof(plan_auto.kernel) === verify_kernel_type - matches_default = typeof(plan_auto.kernel) === typeof(plan_default.kernel) - matches_demoted = typeof(plan_auto.kernel) === typeof(plan_demoted.kernel) - - # Warm up (compile + discard) before any timing. - execute!(plan_auto, 1.0, 0.0) - execute!(plan_default, 1.0, 0.0) - execute!(plan_demoted, 1.0, 0.0) - - numeric_ok = true - if check_numeric - C1 = copy(parent(Cv)) - execute!(plan_auto, 1.0, 0.0) - Cref = copy(parent(Cv)) - execute!(plan_default, 1.0, 0.0) - Cdef = copy(parent(Cv)) - execute!(plan_demoted, 1.0, 0.0) - Cdem = copy(parent(Cv)) - numeric_ok = isapprox(Cref, Cdef) && isapprox(Cref, Cdem) - end - - ts_auto = Vector{Float64}(undef, REPS) - ts_default = Vector{Float64}(undef, REPS) - ts_demoted = Vector{Float64}(undef, REPS) - for r in 1:REPS - t0 = time_ns() - execute!(plan_auto, 1.0, 0.0) - t1 = time_ns() - execute!(plan_default, 1.0, 0.0) - t2 = time_ns() - execute!(plan_demoted, 1.0, 0.0) - t3 = time_ns() - ts_auto[r] = (t1 - t0) / 1.0e9 - ts_default[r] = (t2 - t1) / 1.0e9 - ts_demoted[r] = (t3 - t2) / 1.0e9 - end - - t_auto = median(ts_auto) - t_default = median(ts_default) - t_demoted = median(ts_demoted) - r_ratio = t_demoted / t_default - matched_arm = abs(t_auto - t_default) < abs(t_auto - t_demoted) ? "default" : "demoted" - - return ( - dtype = string(T), a = a, m = m, Qm = a * 36, Qn = 216, Qk = m, - default_shape = kernel_shape_str(default_kernel), - demoted_shape = kernel_shape_str(demoted_kernel), - kernel_type_matches_first_m = kernel_type_ok, - auto_matches_default_type = matches_default, - auto_matches_demoted_type = matches_demoted, - t_auto = t_auto, t_default = t_default, t_demoted = t_demoted, - r = r_ratio, auto_matches = matched_arm, - numeric_checked = check_numeric, numeric_ok = numeric_ok, - ) -end - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -function main() - print_env_header(stdout, "probes/probe_run_demote_kdepth.jl") - println("REPS = ", REPS) - - csv = open(CSV_PATH, "w") - println( - csv, - "dtype,a,m,Qm,Qn,Qk,default_shape,demoted_shape,kernel_type_matches_first_m," * - "auto_matches_default_type,auto_matches_demoted_type,t_auto_s,t_default_s," * - "t_demoted_s,r,auto_matches,numeric_checked,numeric_ok" - ) - - canaries = Float64[] - crng = Random.MersenneTwister(0x0F2_1CADE) - push!(canaries, run_canary(crng, "start")) - - ms = [8, 16, 32, 64, 128, 256, 512] - dtype_specs = [ - (Float64, [8, 40]), - (Float32, [16, 80]), - ] - - all_rows = NamedTuple[] - extra_rows = NamedTuple[] # the 2 extra reps of Float64,a=8,m=512 - - rng = Random.MersenneTwister(0x0F2B00B5) - - for (T, avals) in dtype_specs - for a in avals - Qm = a * 36 - Qn = 216 - default_kernel = _default_kernel(T, Qm, Qn) - - # Reference (shallowest) m to establish the demoted kernel + its - # type, per the m-independence claim in the header comment. - m0 = ms[1] - Cv0, Av0, indA0, Bv0, indB0, indC0 = fixture(T, a, m0, rng) - plan0 = plan_contract(Cv0, Av0, indA0, Bv0, indB0, indC0) - demoted_kernel = plan0.kernel - verify_kernel_type = typeof(plan0.kernel) - - println( - "\n=== T=$T a=$a Qm=$Qm Qn=$Qn default_shape=$(kernel_shape_str(default_kernel))" * - " demoted_shape=$(kernel_shape_str(demoted_kernel)) ===" - ) - if kernel_shape_str(default_kernel) == kernel_shape_str(demoted_kernel) - @warn "T=$T a=$a: auto's kernel at m=$m0 has the SAME shape as " * - "_default_kernel(T,Qm,Qn) -- run-length demotion did NOT fire " * - "at the shallowest m, so the forced-demoted arm is the default." - end - - for (i, m) in enumerate(ms) - check_numeric = i == 1 # once per (T,a) - row = measure_point( - T, a, m, default_kernel, demoted_kernel, verify_kernel_type, rng; - check_numeric = check_numeric - ) - if !row.kernel_type_matches_first_m - @warn "T=$T a=$a m=$m: auto's kernel TYPE differs from the " * - "one recorded at m=$m0 (expected once Qk exceeds the " * - "K-depth guard; unexpected below it)." - end - if check_numeric && !row.numeric_ok - @warn "T=$T a=$a m=$m: numeric mismatch between arms!" - end - @printf( - " m=%-4d t_auto=%.3e t_default=%.3e t_demoted=%.3e r=%.4f auto~%s num_ok=%s\n", - m, row.t_auto, row.t_default, row.t_demoted, row.r, - row.auto_matches, check_numeric ? string(row.numeric_ok) : "n/a" - ) - push!(all_rows, row) - println( - csv, - "$(row.dtype),$(row.a),$(row.m),$(row.Qm),$(row.Qn),$(row.Qk)," * - "$(row.default_shape),$(row.demoted_shape),$(row.kernel_type_matches_first_m)," * - "$(row.auto_matches_default_type),$(row.auto_matches_demoted_type)," * - "$(row.t_auto),$(row.t_default),$(row.t_demoted),$(row.r)," * - "$(row.auto_matches),$(row.numeric_checked),$(row.numeric_ok)" - ) - - # The deep-K regression point (Float64, a=8, m=512): 2 EXTRA - # independent reps. - if T === Float64 && a == 8 && m == 512 - for rep in 1:2 - erow = measure_point( - T, a, m, default_kernel, demoted_kernel, verify_kernel_type, rng; - check_numeric = false - ) - push!(extra_rows, erow) - @printf( - " [extra rep %d] m=%-4d t_auto=%.3e t_default=%.3e t_demoted=%.3e r=%.4f auto~%s\n", - rep, m, erow.t_auto, erow.t_default, erow.t_demoted, erow.r, - erow.auto_matches - ) - println( - csv, - "$(erow.dtype),$(erow.a),$(erow.m)_extra$(rep),$(erow.Qm),$(erow.Qn),$(erow.Qk)," * - "$(erow.default_shape),$(erow.demoted_shape),$(erow.kernel_type_matches_first_m)," * - "$(erow.auto_matches_default_type),$(erow.auto_matches_demoted_type)," * - "$(erow.t_auto),$(erow.t_default),$(erow.t_demoted),$(erow.r)," * - "$(erow.auto_matches),$(erow.numeric_checked),$(erow.numeric_ok)" - ) - end - end - end - push!(canaries, run_canary(crng, "after-$(T)-a$(a)")) - end - end - - close(csv) - - spread = relative_spread(canaries) - @printf("\ncanary spread: %.1f%% %s\n", 100spread, canaries) - if spread > 0.15 - @warn "canary spread exceeds 15%: the machine was not quiet enough for " * - "a confident ratio; re-run before trusting these numbers as final." - end - - machine_load = try - strip(read(`uptime`, String)) - catch - "unknown (uptime failed)" - end - println("machine load (uptime): ", machine_load) - - open(PROV_PATH, "w") do io - print_env_header(io, "probes/probe_run_demote_kdepth.jl") - println(io, "reps = ", REPS) - println(io, "canaries = ", canaries) - @printf(io, "canary spread = %.2f%%\n", 100spread) - println(io, "machine load (uptime, at provenance-write time) = ", machine_load) - println(io, "\n--- table (see run_demote_kdepth.csv for the machine-readable form) ---") - for row in all_rows - @printf( - io, - "%-10s a=%-4d m=%-4d Qm=%-5d Qn=%-4d default=%-12s demoted=%-12s t_auto=%.4e t_default=%.4e t_demoted=%.4e r=%.4f auto~%s kernel_type_ok=%s numeric_checked=%s numeric_ok=%s\n", - row.dtype, row.a, row.m, row.Qm, row.Qn, row.default_shape, row.demoted_shape, - row.t_auto, row.t_default, row.t_demoted, row.r, row.auto_matches, - row.kernel_type_matches_first_m, row.numeric_checked, row.numeric_ok - ) - end - println(io, "\n--- extra reps: Float64, a=8, m=512 (the deep-K regression point) ---") - for row in extra_rows - @printf( - io, - "%-10s a=%-4d m=%-4d t_auto=%.4e t_default=%.4e t_demoted=%.4e r=%.4f auto~%s\n", - row.dtype, row.a, row.m, row.t_auto, row.t_default, row.t_demoted, row.r, - row.auto_matches - ) - end - end - - println("\nWrote ", CSV_PATH) - println("Wrote ", PROV_PATH) - return nothing -end - -main() diff --git a/benchmark/probes/probe_stage_breakdown.jl b/benchmark/probes/probe_stage_breakdown.jl deleted file mode 100644 index ecec113..0000000 --- a/benchmark/probes/probe_stage_breakdown.jl +++ /dev/null @@ -1,302 +0,0 @@ -# Stage breakdown of one contraction under `execute!`: an instrumented copy of -# `_execute_nest!` (src/execution/execute.jl) with `time_ns` brackets around -# the B pack, the A pack and the micro-kernel+store loop, so that a slow case -# can be attributed to the stage that eats the time. Also prints the plan's -# resolved axis groups (each label's stride in both operands) so the memory -# walk each pack performs can be read off directly. -# -# julia -t 1 --project=benchmark benchmark/probes/probe_stage_breakdown.jl \ -# --case scrambled_1_3_1 --dim 48 --dtype Float64 --reps 5 -# -# Cases are einsum-style letter strings `A B>C` (TensorOperationsBenchmarks' -# `expr` column of bench_to_suite.csv); `--expr "abcd dcbe>ae"` accepts any. -# Every axis gets extent `--dim`. Times are medians over `--reps` calls after -# one warm-up; the stage times come from the instrumented nest, the "execute!" -# row from the shipped driver (they should agree to within the timer overhead). - -using QuasiStrided -using QuasiStrided: plan_contract, execute!, mr, nr, axis_length, affine_ramp, - fill_offsets!, describe_block, descriptor_offset_range, checked_span_bounds, - unsafe_pack_a!, unsafe_pack_b!, packed_a_per_k, packed_b_per_k, - _ramp_slivers!, _ramp_descriptor, _ramp_offset_range, _classify_slivers!, - _axis_of, _sliver_panel, _pack_sliver!, unsafe_execute_micro_tile! -using StridedViews -using Random -using Statistics: median -using Printf -using LinearAlgebra - -include(joinpath(@__DIR__, "..", "harness.jl")) - -const KNOWN = Dict( - "scrambled_1_3_1" => "abcd dcbe>ae", - "gemm_ready_1_3_1" => "abcd bcde>ae", - "b_permuted_1_3_1" => "abcd becd>ae", - "a_permuted_2_2_2" => "abcd bdef>acef", - "b_permuted_2_2_2" => "abcd cedf>abef", - "gemm_ready_2_2_2" => "abcd cdef>abef", - "intensli_6" => "abcde bf>dcfea", - "intensli_7" => "abcde df>ecbfa", - "intensli_8" => "abcde fb>dfcea", - "ao2mo_1" => "ab acde>bcde", - "ccsd_t_3" => "abcd cefg>fgdabe", -) - -const CASE = argopt("case", "scrambled_1_3_1") -const EXPR = something(argval("expr"), get(KNOWN, CASE, nothing)) -EXPR === nothing && error("unknown --case $CASE and no --expr given") -const DIM = argopt("dim", 48) -const T = parse_dtypes(argopt("dtype", "Float64"))[1] -const REPS = argopt("reps", 5) - -function parse_expr(expr::AbstractString) - lhs, rhs = split(expr, '>') - a, b = split(strip(lhs), ' ') - return collect(a), collect(b), collect(strip(rhs)) -end - -const LA, LB, LC = parse_expr(EXPR) -const LETTERS = unique(vcat(LA, LB, LC)) -label(c) = findfirst(==(c), LETTERS) -const indA = Tuple(label.(LA)) -const indB = Tuple(label.(LB)) -const indC = Tuple(label.(LC)) - -# `--nothp 1` backs A and C by 4 KB pages only (anonymous mmap + -# madvise(MADV_NOHUGEPAGE)), to emulate a node whose transparent huge pages -# are off (or not granted) on a host where they are on for Julia's arrays. -const NOTHP = argopt("nothp", 0) == 1 -function smallpage_array(::Type{T}, dims::Dims) where {T} - n = prod(dims) * sizeof(T) - len = cld(n, 4096) * 4096 - PROT_RW, MAP_PRIVATE_ANON, MADV_NOHUGEPAGE = Cint(3), Cint(0x22), Cint(15) - p = ccall(:mmap, Ptr{Cvoid}, (Ptr{Cvoid}, Csize_t, Cint, Cint, Cint, Int64), C_NULL, len, PROT_RW, MAP_PRIVATE_ANON, -1, 0) - p == Ptr{Cvoid}(-1) && error("mmap failed") - ccall(:madvise, Cint, (Ptr{Cvoid}, Csize_t, Cint), p, len, MADV_NOHUGEPAGE) == 0 || error("madvise failed") - return unsafe_wrap(Array, Ptr{T}(p), dims; own = false) # leaked on purpose: probe lifetime -end -rng = MersenneTwister(1234) -A = randn(rng, T, ntuple(_ -> DIM, length(LA))...) -B = randn(rng, T, ntuple(_ -> DIM, length(LB))...) -C = zeros(T, ntuple(_ -> DIM, length(LC))...) -if NOTHP - A = copyto!(smallpage_array(T, size(A)), A) - C = fill!(smallpage_array(T, size(C)), zero(T)) -end -# `--pad P` stores A (and C) in a parent with EVERY axis padded to `DIM + P`, -# so no stride is a multiple of 4 KB (at a power-of-two or 24 `--dim` the -# plain layout puts A's outer strides at 4 KB multiples: cache-set / slice / -# DRAM-bank aliasing test; padding one axis leaves the outer strides aligned). -const PAD = argopt("pad", 0) -function padded(X::Array{T}) where {T} - pdims = size(X) .+ PAD - P = NOTHP ? fill!(smallpage_array(T, pdims), zero(T)) : zeros(T, pdims) - V = view(P, map(n -> 1:n, size(X))...) - copyto!(V, X) - return V -end -if PAD > 0 - A = padded(A) - C = padded(C) -end -Av, Bv, Cv = StridedView(A), StridedView(B), StridedView(C) - -# `--shape MR,NR,W` names a menu kernel shape (e.g. `8,6,4`, the AVX2 Float64 -# shape) instead of the host default, to reason about another ISA's register -# tile on this host (its cache geometry is still this host's, of course). -const SHAPE = argval("shape") -using QuasiStrided: _kernel_from_shape -plan = if SHAPE === nothing - plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false) -else - shape = Tuple(parse.(Int, split(SHAPE, ',')))::NTuple{3, Int} - plan_contract(Cv, Av, indA, Bv, indB, indC; oracle = false, kernel = _kernel_from_shape(shape, T)) -end - -# Optional hand-made M split (see probe_msplit_prototype.jl): `--msplit-pos P -# --msplit-li L` splits A's fastest M axis into (L, rest) and moves the inner -# part to position P of the M enumeration. -const MSPLIT_POS = argopt("msplit-pos", 0) -const MSPLIT_LI = argopt("msplit-li", 8) -if MSPLIT_POS > 0 - using QuasiStrided: AxisGroup, ContractPlan - g = plan.mgroup - D = length(g.lengths) - du = argmin([g.lengths[d] > 1 ? abs(g.strides[1][d]) : typemax(Int) for d in 1:D]) - dims = [(g.lengths[d], ntuple(p -> g.strides[p][d], 2)) for d in 1:D] - inner = (MSPLIT_LI, dims[du][2]) - outer = (g.lengths[du] ÷ MSPLIT_LI, MSPLIT_LI .* dims[du][2]) - dims[du] = outer - insert!(dims, MSPLIT_POS, inner) - mg = AxisGroup(ntuple(i -> dims[i][1], D + 1), ntuple(p -> ntuple(i -> dims[i][2][p], D + 1), 2)) - global plan = ContractPlan( - plan.kernel, mg, plan.ngroup, plan.kgroup, plan.blocking, - plan.Astorage, plan.Abase, plan.Bstorage, plan.Bbase, plan.Cstorage, plan.Cbase, - plan.atransform, plan.btransform, plan.workspace - ) -end - -# --------------------------------------------------------------------------- -# Plan report -# --------------------------------------------------------------------------- -println("case = $CASE expr = $EXPR dim = $DIM T = $T nothp = $NOTHP pad = $PAD") -println("A labels ", join(LA), " strides ", strides(Av)) -println("B labels ", join(LB), " strides ", strides(Bv)) -println("C labels ", join(LC), " strides ", strides(Cv)) -println("kernel = ", typeof(plan.kernel).name.name, " MR=", mr(plan.kernel), " NR=", nr(plan.kernel)) -println("blocking = ", plan.blocking) -println("swapped = ", plan.Astorage !== parent(Av)) -for (name, g, ops) in (("M", plan.mgroup, "(A,C)"), ("N", plan.ngroup, "(B,C)"), ("K", plan.kgroup, "(A,B)")) - println("$name group $ops: lengths=", g.lengths, " strides=", g.strides, " Q=", axis_length(g), " ramp=", affine_ramp(g)[1]) -end -flops = 2.0 * axis_length(plan.mgroup) * axis_length(plan.ngroup) * axis_length(plan.kgroup) -println("flops = ", flops, " bytes(A+B+C) = ", (length(A) + length(B) + length(C)) * sizeof(T)) - -# --------------------------------------------------------------------------- -# Instrumented nest (copy of `_execute_nest!`, timers added, nothing else) -# --------------------------------------------------------------------------- -mutable struct StageTimes - packb::Int - packa::Int - kern::Int - total::Int -end - -function timed_nest!(plan, alphaT::T, betaT::T, st::StageTimes) where {T} - ws = plan.workspace - kernel = plan.kernel - MRk = mr(kernel) - NRk = nr(kernel) - Qm = axis_length(plan.mgroup) - Qn = axis_length(plan.ngroup) - Qk = axis_length(plan.kgroup) - mc_eff = plan.blocking.mc - kc_eff = plan.blocking.kc - nc_eff = plan.blocking.nc - MRp = packed_a_per_k(kernel) - NRp = packed_b_per_k(kernel) - atransform = plan.atransform - btransform = plan.btransform - (m_ramp, m_step) = affine_ramp(plan.mgroup) - (n_ramp, n_step) = affine_ramp(plan.ngroup) - (k_ramp, k_step) = affine_ramp(plan.kgroup) - lenA = length(plan.Astorage) - lenB = length(plan.Bstorage) - lenC = length(plan.Cstorage) - t_start = time_ns() - GC.@preserve ws begin - jc = 0 - while jc < Qn - nblock = min(nc_eff, Qn - jc) - n_slivers = cld(nblock, NRk) - (rng_nB, rng_nC) = if n_ramp - _ramp_slivers!(ws.n_desc_B, ws.n_desc_C, n_step[1], n_step[2], jc, nblock, NRk, n_slivers) - else - fill_offsets!((ws.n_buf_B, ws.n_buf_C), plan.ngroup, jc, nblock) - _classify_slivers!(ws.n_desc_B, ws.n_desc_C, ws.n_buf_B, ws.n_buf_C, nblock, NRk, n_slivers) - end - pc = 0 - firstpanel = true - while pc < Qk - kblock = min(kc_eff, Qk - pc) - dK_A, dK_B, rng_kA, rng_kB = if k_ramp - ( - _ramp_descriptor(k_step[1], pc, kblock), _ramp_descriptor(k_step[2], pc, kblock), - _ramp_offset_range(k_step[1], pc, kblock), _ramp_offset_range(k_step[2], pc, kblock), - ) - else - fill_offsets!((ws.k_buf_A, ws.k_buf_B), plan.kgroup, pc, kblock) - dA = describe_block(ws.k_buf_A, 0, kblock) - dB = describe_block(ws.k_buf_B, 0, kblock) - (dA, dB, descriptor_offset_range(dA, ws.k_buf_A, 0), descriptor_offset_range(dB, ws.k_buf_B, 0)) - end - colsA_k = _axis_of(dK_A, ws.k_buf_A, 0) - rowsB_k = _axis_of(dK_B, ws.k_buf_B, 0) - checked_span_bounds(plan.Bbase, rng_kB, rng_nB, lenB) - beta_eff = firstpanel ? betaT : one(T) - - t0 = time_ns() - for s in 0:(n_slivers - 1) - sfirst = s * NRk - colsB = _axis_of(ws.n_desc_B[s + 1], ws.n_buf_B, sfirst) - bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) - _pack_sliver!(unsafe_pack_b!, bpanel, plan.Bstorage, plan.Bbase, rowsB_k, colsB, kernel, btransform) - end - st.packb += time_ns() - t0 - - ic = 0 - while ic < Qm - mblock = min(mc_eff, Qm - ic) - m_slivers = cld(mblock, MRk) - (rng_mA, rng_mC) = if m_ramp - _ramp_slivers!(ws.m_desc_A, ws.m_desc_C, m_step[1], m_step[2], ic, mblock, MRk, m_slivers) - else - fill_offsets!((ws.m_buf_A, ws.m_buf_C), plan.mgroup, ic, mblock) - _classify_slivers!(ws.m_desc_A, ws.m_desc_C, ws.m_buf_A, ws.m_buf_C, mblock, MRk, m_slivers) - end - checked_span_bounds(plan.Abase, rng_mA, rng_kA, lenA) - checked_span_bounds(plan.Cbase, rng_mC, rng_nC, lenC) - - t0 = time_ns() - for r in 0:(m_slivers - 1) - rfirst = r * MRk - rowsA = _axis_of(ws.m_desc_A[r + 1], ws.m_buf_A, rfirst) - apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) - _pack_sliver!(unsafe_pack_a!, apanel, plan.Astorage, plan.Abase, rowsA, colsA_k, kernel, atransform) - end - t1 = time_ns() - st.packa += t1 - t0 - - for s in 0:(n_slivers - 1) - sfirst = s * NRk - colsC = _axis_of(ws.n_desc_C[s + 1], ws.n_buf_C, sfirst) - bpanel = _sliver_panel(ws.packed_b, NRp, kblock, s) - for r in 0:(m_slivers - 1) - rfirst = r * MRk - rowsC = _axis_of(ws.m_desc_C[r + 1], ws.m_buf_C, rfirst) - apanel = _sliver_panel(ws.packed_a, MRp, kblock, r) - unsafe_execute_micro_tile!(kernel, plan.Cstorage, plan.Cbase, rowsC, colsC, apanel, bpanel, kblock, alphaT, beta_eff) - end - end - st.kern += time_ns() - t1 - ic += mblock - end - firstpanel = false - pc += kblock - end - jc += nblock - end - end - st.total += time_ns() - t_start - return nothing -end - -# --------------------------------------------------------------------------- -# Timing -# --------------------------------------------------------------------------- -execute!(plan, one(T), zero(T)) # warm-up -Cref = copy(C) -t_exec = median_time_s(() -> execute!(plan, one(T), zero(T)); reps = REPS) - -st = StageTimes(0, 0, 0, 0) -timed_nest!(plan, one(T), zero(T), st) # warm-up -@assert C ≈ Cref -rows = StageTimes[] -for r in 1:REPS - s = StageTimes(0, 0, 0, 0) - timed_nest!(plan, one(T), zero(T), s) - push!(rows, s) -end -med(f) = median([f(s) for s in rows]) / 1.0e9 - -t_total = med(s -> s.total) -t_packb = med(s -> s.packb) -t_packa = med(s -> s.packa) -t_kern = med(s -> s.kern) -println() -@printf("execute! median %9.4f s %7.2f GF/s\n", t_exec, flops / t_exec / 1.0e9) -@printf("nest total %9.4f s %7.2f GF/s\n", t_total, flops / t_total / 1.0e9) -@printf(" pack B %9.4f s (%5.1f%%) %7.2f ns/elem of B read\n", t_packb, 100 * t_packb / t_total, 1.0e9 * t_packb / (length(B) * cld(axis_length(plan.mgroup), plan.blocking.mc) / cld(axis_length(plan.mgroup), plan.blocking.mc))) -@printf(" pack A %9.4f s (%5.1f%%) %7.2f ns/elem of A read (x%d N-blocks)\n", t_packa, 100 * t_packa / t_total, 1.0e9 * t_packa / (length(A) * cld(axis_length(plan.ngroup), plan.blocking.nc)), cld(axis_length(plan.ngroup), plan.blocking.nc)) -@printf(" kernel+store %9.4f s (%5.1f%%) %7.2f GF/s kernel-only\n", t_kern, 100 * t_kern / t_total, flops / t_kern / 1.0e9) -@printf(" other %9.4f s (%5.1f%%)\n", t_total - t_packb - t_packa - t_kern, 100 * (t_total - t_packb - t_packa - t_kern) / t_total) diff --git a/benchmark/probes/probe_storage_type.jl b/benchmark/probes/probe_storage_type.jl deleted file mode 100644 index a700c61..0000000 --- a/benchmark/probes/probe_storage_type.jl +++ /dev/null @@ -1,75 +0,0 @@ -# Checks whether `parent(StridedView(::Array{T}))` is a `Vector{T}` (would -# satisfy the store fast-path guard) or a `Memory{T}` (never satisfies it), -# and whether `SIMD.vload`/`vstore` work correctly and allocation-free on it. -# -# Usage: julia --project=. benchmark/probes/probe_storage_type.jl -# Writes benchmark/results/-/probes_storage_type.txt - -using StridedViews -using SIMD - -include(joinpath(@__DIR__, "..", "harness.jl")) - -const OUTPATH = joinpath(results_dir(), "probes_storage_type.txt") -mkpath(dirname(OUTPATH)) - -io = open(OUTPATH, "w") -out(args...) = (println(io, args...); println(args...)) - -out("git_commit = ", git_commit()) -out("hostname = ", gethostname()) -out("julia_version = ", VERSION) -out("date = ", now()) -out("isdefined(Core, :Memory) = ", isdefined(Core, :Memory)) -out() - -for T in (Float64, Float32) - for nd in (2, 6) - p = parent(StridedView(zeros(T, ntuple(_ -> 4, nd)))) - out("T=$T nd=$nd: typeof(parent(StridedView(...))) = ", typeof(p)) - out("T=$T nd=$nd: isa Vector{$T} = ", p isa Vector{T}) - out("T=$T nd=$nd: isa DenseVector{$T} = ", p isa DenseVector{T}) - end -end -out() - -for T in (Float64, Float32) - W = 64 ÷ sizeof(T) - m = parent(StridedView(collect(T, 1:(4 * W)))) - out("T=$T: W=$W, storage type for vload/vstore probe = ", typeof(m)) - - @noinline function do_vload(m, W::Int, ::Type{T}) where {T} - return SIMD.vload(SIMD.Vec{W, T}, m, W + 1) - end - @noinline function alloc_of_vload(m, W::Int, ::Type{T}) where {T} - return @allocated do_vload(m, W, T) - end - v = do_vload(m, W, T) # warm up (compile) - v = do_vload(m, W, T) # measured - expected = SIMD.Vec{W, T}(ntuple(i -> T(W + i), W)) - out("T=$T: vload result == expected (all lanes): ", all(SIMD.Vec{W, Bool}(Tuple(v == expected)))) - alloc_of_vload(m, W, T) # warm up the measuring function itself - alloc_load = alloc_of_vload(m, W, T) - out("T=$T: @allocated do_vload (post-warmup, measured inside a compiled function) = ", alloc_load) - - @noinline function do_vstore!(m, v, W::Int) - SIMD.vstore(v, m, 1) - return nothing - end - @noinline function alloc_of_vstore(m, v, W::Int) - return @allocated do_vstore!(m, v, W) - end - newvals = SIMD.Vec{W, T}(ntuple(i -> T(1000 + i), W)) - do_vstore!(m, newvals, W) # warm up - alloc_of_vstore(m, newvals, W) # warm up the measuring function itself - alloc_store = alloc_of_vstore(m, newvals, W) - readback = [m[i] for i in 1:W] - out("T=$T: vstore readback == written values: ", readback == [T(1000 + i) for i in 1:W]) - out("T=$T: @allocated do_vstore! (post-warmup, measured inside a compiled function) = ", alloc_store) - - out("T=$T: pointer(m,3) == pointer(m) + 2*sizeof(T): ", pointer(m, 3) == pointer(m) + 2 * sizeof(T)) - out() -end - -close(io) -println("\nWrote ", OUTPATH) diff --git a/benchmark/profile_buckets.jl b/benchmark/profile_buckets.jl deleted file mode 100644 index ccf1276..0000000 --- a/benchmark/profile_buckets.jl +++ /dev/null @@ -1,250 +0,0 @@ -# Bucketed cost-attribution helper for benchmark/profile_to_suite.jl. -# `include`d, not a module, consistent with benchmark/harness.jl. -# -# Parses `Profile.fetch()`'s raw backtrace data directly (rather than relying -# on `Profile.print`'s text output) so we can compute, per frame, which named -# cost bucket it belongs to and tally sample counts ourselves. - -using Profile -using Printf - -# Bucket definitions per backend "family". Each entry is -# `(bucketname, [substring, ...])`; a frame matches a bucket if ANY of its -# substrings appears in either the frame's function name or file path -# (case-sensitive, as the source identifiers themselves are). -# -# Two tiers, checked in two separate passes over a whole sample's stack (see -# `_classify_backtrace`), NOT a single first-match-wins pass over one bucket -# list: `SPECIFIC_QS_BUCKETS` matches on function names (unambiguous: a -# frame named `pack_a!` is packing, wherever it lives), `FALLBACK_QS_BUCKETS` -# matches on file-path substrings (the `src/` stage folders: `"microkernels/"`, -# `"planning/"`, ...), -# used only if NO frame anywhere in the sample's stack matched a specific -# name. This two-pass split matters: a stack's leaf is very often a generic -# or third-party frame (an inlined `SIMD.jl` intrinsic, a `macro expansion` -# thunk, `Base.range`'s `iterate`) that itself matches nothing specific, and -# that leaf's *file* can be misleading -- e.g. a `macro expansion` frame -# inside `_store_tile_vector!`'s generated body lives in the same file -# (`src/microkernels/simd.jl`) as the FMA microkernel, so checking file -# substrings before walking further up the stack to the actual -# `_store_tile_vector!` frame would misclassify the whole sample as -# "microkernel" instead of "store". Running the specific-name pass across -# the ENTIRE stack first, before ever consulting a file-level fallback, -# avoids that: the walk keeps going past unmatched generic/third-party -# frames until it either finds a named function anywhere in the sample, or -# exhausts the stack and falls back to file-level matching. -const SPECIFIC_QS_BUCKETS = [ - ("gc/alloc", ["jl_gc", "gc_pool_alloc", "tensoralloc"]), - ("adapter/prepare", ["_qs_prepare", "_qs_eligible", "_qs_throw", "argcheck", "dimcheck", "mightalias"]), - ( - "planning", - [ - "plan_contract", "_planned", "_plan_contract", "_plan_with_kernel", - "_plan_resolved", "_menu_val", "_classify_labels", "_order_free_labels", - "_prefer_swap", "_default_kernel", "_default_shape", "_resolved_defaults", - "_kernel_from_shape", "default_blocking", - # These would land in "planning" through the ancestor walk anyway - # (`plan_contract` is their only caller); naming them makes the - # attribution explicit and cannot change any classification. - "_build_pair_group", "_candidate_mrs", - ], - ), - ("packing", ["pack_a!", "pack_b!", "_pack_panel!", "_pack_a_contiguous!", "_pack_sliver!", "_pack_emit", "_check_pack_a", "_check_pack_b"]), - # "store_tile" (no trailing "!") deliberately catches all three store - # entry points -- `store_tile!`, `_store_tile_scattered!`, and - # `_store_tile_vector!` -- the last of which does NOT contain the - # substring "store_tile!" (the "!" lands after "vector", not "tile"). - ("store", ["store_tile", "scale_tile!", "_axpby_tile!", "_axpby_at!", "_store_prologue!"]), - ("microkernel", ["accumulate", "_accumulate_step", "_execute_micro_tile!", "execute_tile!", "zero_accumulator"]), - # The BLIS macro-blocking five/six-loop nest and its per-block - # bookkeeping -- distinct from "planning" (the one-time, per-call - # `plan_contract` construction): this is the *executed* loop nest. - ( - "driver_loop", - [ - "_execute_nest!", "execute!", "_axis_of", "_classify_slivers!", - "_sliver_panel", "_scale_micro_tile!", "_scale_all_of_C!", - "fill_offsets!", "describe_block", "block_descriptors!", "checked_tile_storage_bounds", - # The closed-form affine block path and the hoisted once-per-macro-block bounds check. - # Same note as under "planning" -- `_execute_nest!` is the only - # caller of each, so these names change no classification; the - # `unsafe_pack_*!`/`unsafe_execute_*!` entry points need no entry - # at all, since they already contain the `pack_a!`/`execute_tile!` - # substrings their buckets match on. - "affine_ramp", "_ramp_slivers!", "_ramp_descriptor", "_ramp_offset_range", - "descriptor_offset_range", "checked_span_bounds", - ], - ), -] -const FALLBACK_QS_BUCKETS = [ - ("gc/alloc", ["gc"]), - ("adapter/prepare", ["integrations/", "StridedView"]), - ("packing", ["packing/"]), - ("microkernel", ["microkernels/"]), - ("driver_loop", ["execution/", "layout/"]), - ("planning", ["planning/"]), -] - -const SPECIFIC_BLAS_BUCKETS = [ - ("gc/alloc", ["jl_gc", "gc_pool_alloc", "tensoralloc"]), - ("blas", ["dgemm", "sgemm", "gemm", "openblas", "cblas"]), - ("permute/copy", ["_mapreduce", "permutedims"]), -] -const FALLBACK_BLAS_BUCKETS = [ - ("gc/alloc", ["gc"]), - ("permute/copy", ["Strided", "copy"]), -] - -const BUCKET_ORDER_QS = ["adapter/prepare", "planning", "driver_loop", "packing", "microkernel", "store", "gc/alloc", "TO overhead", "other"] -const BUCKET_ORDER_BLAS = ["blas", "permute/copy", "gc/alloc", "TO overhead", "other"] - -# Cache StackTraces lookups: the same instruction pointer recurs across many -# backtraces/samples, and `StackTraces.lookup` is not free. -const _LOOKUP_CACHE = Dict{UInt, Vector{Base.StackTraces.StackFrame}}() - -function _lookup_cached(ip) - key = UInt(ip) - return get!(_LOOKUP_CACHE, key) do - Base.StackTraces.lookup(convert(Ptr{Cvoid}, ip)) - end -end - -_frame_haystack(frame) = string(frame.func) * "|" * string(frame.file) - -# `occursin("promote", ...)` alone would match ordinary `Base.promote`/ -# `promote_type` frames that can appear inlined deep inside the kernel; -# restrict to TensorOperations' own promotion entry point. -function _is_tensoroperations_path(frame) - f = string(frame.file) - fn = string(frame.func) - return occursin("TensorOperations", f) || occursin("tensorcontract", fn) || - occursin("ncon", fn) || occursin("promote_contract", fn) -end - -function _match_bucket(hay::AbstractString, buckets) - for (name, substrs) in buckets - for s in substrs - occursin(s, hay) && return name - end - end - return nothing -end - -# Two-pass, single-frame classification: specific (function-name) match -# first, generic (file-path) fallback second. Kept separate from the -# per-sample walk in `_classify_backtrace` (which must run the *specific* -# pass across the WHOLE stack before considering any *fallback* match) -- -# this function alone would still be wrong to call leaf-only for the same -# reason leaf-only self-time was wrong (see module docstring above). -function _classify_frame_or_nothing(frame, backendname::AbstractString; fallback::Bool) - hay = _frame_haystack(frame) - specific, generic = backendname == "StridedBLAS" ? - (SPECIFIC_BLAS_BUCKETS, FALLBACK_BLAS_BUCKETS) : (SPECIFIC_QS_BUCKETS, FALLBACK_QS_BUCKETS) - name = _match_bucket(hay, specific) - name === nothing || return name - fallback || return nothing - name = _match_bucket(hay, generic) - name === nothing || return name - # "TO overhead" fallback: anything in the TensorOperations package path - # (or the generic entry-point function names) not already caught above. - _is_tensoroperations_path(frame) && return "TO overhead" - return nothing -end - -# Walk one sample's IPs leaf-to-root (as `Profile.fetch` stores them), -# checking every inlined frame at each IP innermost-first. Pass 1 checks -# ONLY specific (function-name) matches across the entire stack; only if -# that whole-stack pass finds nothing does pass 2 re-walk the same stack -# allowing file-level fallback matches. This ordering is what makes -# `_store_tile_vector!`'s generated `macro expansion` leaf classify -# correctly: that leaf frame itself matches nothing specific, but walking -# further up the SAME sample's stack (still within pass 1) reaches the -# `_store_tile_vector!` frame itself, which does. A single-pass "first -# frame with any match, specific-or-fallback" walk would instead stop at -# the leaf's own file-level "kernels/" match and call it "microkernel". -function _classify_backtrace(ips, backendname::AbstractString) - for fallback in (false, true) - for ip in ips - for frame in _lookup_cached(ip) - name = _classify_frame_or_nothing(frame, backendname; fallback) - name === nothing || return name - end - end - end - return "other" -end - -""" - compute_buckets(data, backendname) -> Dict{String,Int} - -`data` must be the raw backtrace stream from `Profile.fetch(include_meta = -false)` -- metadata (threadid/taskid/etc, included by `fetch`'s default -`include_meta = true`) is NOT instruction-pointer data and must be stripped -first, or `Base.StackTraces.lookup` will resolve garbage frames from it. - -Backtraces are stored leaf-first (the actively-executing frame comes first) -and each per-sample backtrace is terminated by a `0` sentinel. This function -splits on those sentinels and classifies each sample via `_classify_backtrace` --- "nearest named ancestor, specific names preferred over file-level -fallbacks", not strict leaf-only self-time, and not inclusive/"Count"-column -semantics either: each sample still contributes to exactly one bucket, so -generic runtime/dispatch frames sitting between named layers cannot -spuriously dominate every bucket the way naive inclusive counting would. - -Returns raw sample counts per bucket name (NOT yet normalized to -percentages); callers should divide by the sum of all bucket counts (not by -`length(data)`, which also counts sentinels) to get percentages that sum to -100%. -""" -function compute_buckets(data, backendname::AbstractString) - order = backendname == "StridedBLAS" ? BUCKET_ORDER_BLAS : BUCKET_ORDER_QS - counts = Dict{String, Int}(name => 0 for name in order) - sample_ips = UInt[] - for ip in data - if iszero(ip) - # Sentinel: end of one sample's backtrace. - if !isempty(sample_ips) - name = _classify_backtrace(sample_ips, backendname) - counts[name] = get(counts, name, 0) + 1 - empty!(sample_ips) - end - continue - end - push!(sample_ips, UInt(ip)) - end - if !isempty(sample_ips) # tolerate a stream not ending on a sentinel - name = _classify_backtrace(sample_ips, backendname) - counts[name] = get(counts, name, 0) + 1 - end - return counts -end - -""" - print_bucket_table(io, caseid, backendname, buckets, total_samples, allocated_bytes, reps) - -Print a percentage-of-samples table for one (case, backend) pair. Percentages -are computed against the sum of all bucket counts (not the raw `data` -length, which also includes sentinel/zero entries), so they sum to ~100% -regardless of how many samples were sentinels. -""" -function print_bucket_table(io, caseid, backendname, buckets, total_samples, allocated_bytes, reps) - denom = sum(values(buckets)) - println(io, "="^72) - @printf( - io, "case=%s backend=%s reps=%d total_raw_samples=%d classified_samples=%d\n", - caseid, backendname, reps, total_samples, denom - ) - @printf(io, "allocated (representative single call): %d bytes\n", allocated_bytes) - println(io, "-"^72) - order = backendname == "StridedBLAS" ? BUCKET_ORDER_BLAS : BUCKET_ORDER_QS - pct_sum = 0.0 - for name in order - c = get(buckets, name, 0) - pct = denom == 0 ? 0.0 : 100.0 * c / denom - pct_sum += pct - @printf(io, " %-18s %8d samples %6.2f%%\n", name, c, pct) - end - @printf(io, " %-18s %8s %6.2f%% (sanity check, should be ~100.00%%)\n", "TOTAL", "", pct_sum) - println(io) - return nothing -end diff --git a/benchmark/profile_to_suite.jl b/benchmark/profile_to_suite.jl deleted file mode 100644 index 884b2ac..0000000 --- a/benchmark/profile_to_suite.jl +++ /dev/null @@ -1,517 +0,0 @@ -# Profiles QuasiStridedBackend() vs StridedBLAS() on tensor-contraction -# cases drawn from the upstream TensorOperationsBenchmarks suite plus this -# project's own benchmark/harness.jl shapes (plain square GEMM, small-skewed, -# and the permuted/negative-stride/sliced "scattered" fixture), bucketing -# sampled cost into adapter/planning/packing/microkernel/store/blas/etc. -# -# Covers the full MAIN_SHAPES/SMALL_SHAPES/EXTRA_SHAPES x 4-dtype grid (38 -# CASES + 5 DIRECT_CASES, including two explicit OneMKernel entries) -- see -# the "Full-grid extension" comment below for the additive rules. -# -# julia --project=benchmark benchmark/profile_to_suite.jl [caseid ...] -# -# With no case ids, profiles every case in CASES and DIRECT_CASES below; edit -# those lists to add/replace cases. CASES goes through the TensorOperations -# adapter, profiled under both QuasiStridedBackend() and StridedBLAS(); -# DIRECT_CASES calls `plan_contract`/`execute!` directly (no adapter, no -# backend choice) for fixtures `harness.jl` builds directly rather than as -# plain labels/dims. Writes flat/tree profiles, an optional ProfileCanvas -# HTML flamegraph, and a bucket table per (case, backend) to -# benchmark/results/-/profiles/. -# -# CAVEAT: with `C=true`, Julia's profiler also samples idle helper threads -# (GC/IO, sitting in `__futex_abstimed_wait_common`). `profile_buckets.jl`'s -# `compute_buckets` classifies by walking each sample's whole backtrace (not -# just its leaf frame -- see that file's module docstring for why), so an -# idle thread's samples still land in "other" rather than stealing from a -# named bucket, but they are not filtered out by thread id, so a busy -# machine can inflate "other" here. Re-run on a quiet machine if "other" is -# large and unexplained. -# -# NOTE: each run OVERWRITES the previous one's artefacts for the same case -# (fixed filenames under `results_dir()`); pass `--tag