From abdd2882330fc16aa31b28976466f7a2876441f8 Mon Sep 17 00:00:00 2001 From: Joachim Brand Date: Thu, 27 Aug 2026 01:24:02 +1200 Subject: [PATCH 01/13] Use identity comparison during type mangling (#910) (cherry picked from commit 1c5bef9f46a491b1118201ed240def03e05d9140) --- src/mangling.jl | 2 +- test/utils.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/mangling.jl b/src/mangling.jl index 504b1f294..f681c7f98 100644 --- a/src/mangling.jl +++ b/src/mangling.jl @@ -34,7 +34,7 @@ safe_name(@nospecialize(x)) = safe_name(repr(x)) # profilers, support them (grouping different instantiations of the same kernel together). function mangle_param(@nospecialize(t), substitutions = Any[], top = false) - t == Nothing && return "v" + t === Nothing && return "v" # Manual lookup instead of `findfirst(isequal(x), substitutions)`: the latter # builds a `Base.Fix{2, isequal, typeof(x)}` that specializes `findfirst` per diff --git a/test/utils.jl b/test/utils.jl index 2ab8f7210..7bc858dd6 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -64,6 +64,10 @@ end @test mangle(identity) == "identity" @test mangle(identity, Nothing) == "identity()" + # `missing` as a type parameter: comparing against `Nothing` with `==` returns + # `missing` instead of `false`, which used to throw a `TypeError` in a boolean context. + @test mangle(identity, Val{missing}) == "identity(Val)" + # primitive types @test mangle(identity, Int32) == "identity(Int32)" @test mangle(identity, Int64) == "identity(Int64)" From e16a2ceb6eda46b99aaaae51377ddd7e565d282f Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sun, 6 Sep 2026 09:40:47 +0200 Subject: [PATCH 02/13] Mangle a `Vararg` whose length is still a `TypeVar` (#921) (cherry picked from commit 370be42184026abc113688a31006e8da77fb6ec5) --- src/mangling.jl | 8 ++++++-- test/utils.jl | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/mangling.jl b/src/mangling.jl index f681c7f98..9272f05a4 100644 --- a/src/mangling.jl +++ b/src/mangling.jl @@ -120,10 +120,14 @@ function mangle_param(@nospecialize(t), substitutions = Any[], top = false) mangle_param(Base.unwrap_unionall(t), substitutions) elseif isa(t, Core.TypeofVararg) T = isdefined(t, :T) ? t.T : Any - if isdefined(t, :N) + # `N` is a `TypeVar` when the length is still parametric, which is what unwrapping a + # `UnionAll` such as `NTuple{N,Int}` leaves behind. There is no count to repeat then, + # so the type is mangled like the unbounded `Vararg{T}` it is equal to. + N = isdefined(t, :N) && isa(t.N, Integer) ? t.N : nothing + if N !== nothing # For NTuple, repeat the type as needed str = "" - for _ in 1:t.N + for _ in 1:N str *= mangle_param(T, substitutions) end str diff --git a/test/utils.jl b/test/utils.jl index 7bc858dd6..a19ab99ec 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -97,6 +97,10 @@ end @test mangle(identity, Tuple{1, 2}, Tuple{}, Tuple) == "identity(Tuple<1, 2>, Tuple<>, Tuple)" @test mangle(identity, NTuple{2, Int}) == "identity(Tuple)" @test mangle(identity, Tuple{Vararg{Int}}) == "identity(Tuple<>)" + # A `Vararg` whose length is still a `TypeVar`, which is what unwrapping a `UnionAll` + # leaves behind. It is the same type as the unbounded `Vararg` above, so it mangles alike. + @test mangle(identity, NTuple{N, Int} where {N}) == "identity(Tuple<>)" + @test mangle(identity, Val{NTuple{N, Int} where {N}}) == "identity(Val>)" # many substitutions @test mangle(identity, Val{1}, Val{2}, Val{3}, Val{4}, Val{5}, Val{6}, Val{7}, Val{8}, From 248f22cbd3056f005f42ba74ebb2bd40a7ea2cbc Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 25 Sep 2026 15:22:40 +0200 Subject: [PATCH 03/13] Throw a KernelError for non-singleton type arguments (#954) (cherry picked from commit 87076e07d172224dec6608d2450e951de74ba7ed) --- src/validation.jl | 10 ++++++++++ test/native.jl | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/validation.jl b/src/validation.jl index 600bf73d6..762bdbf4e 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -98,6 +98,16 @@ function check_invocation(@nospecialize(job::CompilerJob)) This is a CPU-only object not supported by GPUCompiler.""")) end + # Before `Core.TypeEgal`, `Type{T}` is only a singleton when `T` has a unique + # representation, so e.g. `Type{Union{Missing, Int}}` or `Type{Vector}` would be + # passed as a boxed host pointer. + if Base.isType(dt) + throw(KernelError(job, "passing a non-singleton type argument", + """Argument $arg_i to your kernel function is the type $(dt.parameters[1]), which + cannot be passed to a GPU kernel on this version of Julia. + Pass `Val($(dt.parameters[1]))` instead, or a value of that type.""")) + end + # If an object doesn't have fields, it can only be used by identity, so we can allow # them to be passed to the GPU (this also applies to e.g. Symbols). if fieldcount(dt) == 0 diff --git a/test/native.jl b/test/native.jl index d06eace7d..64e8999c6 100644 --- a/test/native.jl +++ b/test/native.jl @@ -625,6 +625,30 @@ end end end +@testset "non-singleton type arguments" begin + mod = @eval module $(gensym()) + import ..sink + foo(::Type{T}) where {T} = (sink(Int(Missing <: T)); return) + bar(::Val{T}) where {T} = (sink(Int(Missing <: T)); return) + end + + for T in (Union{Missing, Int}, Vector) + if isdefined(Core, :TypeEgal) + # `Core.TypeEgal` makes every closed type argument a singleton + Native.code_execution(mod.foo, Tuple{Type{T}}) + else + @test_throws_message(KernelError, + Native.code_execution(mod.foo, Tuple{Type{T}})) do msg + occursin("passing a non-singleton type argument", msg) && + occursin(string(T), msg) + end + end + + # the suggested alternative + Native.code_execution(mod.bar, Tuple{Val{T}}) + end +end + @testset "invalid LLVM IR" begin mod = @eval module $(gensym()) foobar(i) = println(i) From e30b738a0e78a2143c9ddda9aba5e4ed5936d995 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 5 Oct 2026 13:07:43 +0200 Subject: [PATCH 04/13] Look up the global logger's level in the latest world (#977) (cherry picked from commit c2f106b857e1520c5038365545105e012fa0c129) --- src/utils.jl | 7 ++++--- test/utils.jl | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index e8938a8fb..637b19328 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -74,10 +74,11 @@ using Logging const STDERR_HAS_COLOR = Ref{Bool}(false) -# Prevent invalidation when packages define custom loggers -# Using invoke in combination with @nospecialize eliminates backedges to these methods +# Call into the latest world: custom loggers are usually defined after the compiler, +# which may run in an older world (e.g. through `invoke_in_world`). As a dynamic call, +# combined with @nospecialize, it also avoids invalidation by recording no backedges. function _invoked_min_enabled_level(@nospecialize(logger)) - return invoke(Logging.min_enabled_level, Tuple{typeof(logger)}, logger)::LogLevel + return Base.invokelatest(Logging.min_enabled_level, logger)::LogLevel end # define safe loggers for use in generated functions (where task switches are not allowed) diff --git a/test/utils.jl b/test/utils.jl index a19ab99ec..551e51c7a 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -52,6 +52,26 @@ end end end +using Logging + +safe_log() = GPUCompiler.@safe_debug "safe_log" + +@testset "safe logging" begin + # the compiler may run in an older world than the global logger's methods + # (e.g., when invoked through `invoke_in_world` to reuse precompiled code) + world = Base.get_world_counter() + @eval struct NewerLogger <: AbstractLogger end + @eval Logging.min_enabled_level(::NewerLogger) = Logging.Info + @eval Logging.shouldlog(::NewerLogger, args...) = true + @eval Logging.handle_message(::NewerLogger, args...; kwargs...) = nothing + old_logger = global_logger(NewerLogger()) + try + @test Base.invoke_in_world(world, safe_log) === nothing + finally + global_logger(old_logger) + end +end + @testset "mangling" begin using demumble_jll From 4223c349073c38f6bbb1f66c3a262bc20336e7a1 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 06:57:55 +0200 Subject: [PATCH 05/13] PTX: Restore FMA contraction. (#881) (cherry picked from commit 96fb6f6e58dbe592d7449981fd14c8bb987c75f2) --- src/ptx.jl | 2 ++ test/ptx.jl | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/ptx.jl b/src/ptx.jl index dfde75ad3..12636bda2 100644 --- a/src/ptx.jl +++ b/src/ptx.jl @@ -356,10 +356,12 @@ end output = tempname(cleanup=false) * (filetype == "asm" ? ".ptx" : ".cubin") write(input, mod) + # Match nvcc by leaving eligible multiply-add pairs contractible for ptxas. cmd = `$(NVPTX_LLVM_Backend_jll.llc()) $input -mtriple=$(llvm_triple(target)) -mcpu=$(cpu_name(target)) -mattr=+ptx$(target.ptx.major)$(target.ptx.minor) + -nvptx-fma-level=1 -filetype=$filetype -o $output` out = Pipe() diff --git a/test/ptx.jl b/test/ptx.jl index bbc18aeb7..f4833089a 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -430,6 +430,21 @@ end PTX.code_native(devnull, mod.kernel, Tuple{Float32,Ptr{Float32}}) end +@testset "FMA contraction" begin + function contract(x) + for _ in 1:8 + x = 2.001f0 * x - x + end + return x + end + + ptx = sprint(PTX.code_native, contract, Tuple{Float32}) + @test occursin("mul.f32", ptx) + @test occursin("sub.f32", ptx) + @test !occursin("mul.rn.f32", ptx) + @test !occursin("sub.rn.f32", ptx) +end + @testset "fastmath" begin # `fastmath=true` on the target calls `apply_fastmath!` from # `finish_linked_module!`, stamping `unsafe-fp-math` + fast-math flags on From 9817ef65d9cc363b80d75b2246e9db2c59a918d2 Mon Sep 17 00:00:00 2001 From: Simeon David Schaub Date: Wed, 15 Jul 2026 08:58:47 +0200 Subject: [PATCH 06/13] [SPIRV] add workaround for khronos translator (#879) Co-authored-by: Tim Besard (cherry picked from commit 081c8925653fc802d7c9f3ab98df3925568b611d) --- src/spirv.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/spirv.jl b/src/spirv.jl index 72ee4576a..2d25c6f8e 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -360,5 +360,13 @@ function wrap_byval(@nospecialize(job::CompilerJob), mod::LLVM.Module, f::LLVM.F # remove the old function # NOTE: if we ever have legitimate uses of the old function, create a shim instead replace_function!(f, new_f) + + # XXX: work around KhronosGroup/SPIRV-LLVM-Translator#3389 + if job.config.target.backend === :khronos + @dispose pb=NewPMPassBuilder() begin + add!(pb, SimplifyCFGPass()) + run!(pb, new_f, llvm_machine(job.config.target)) + end + end return new_f end From e281c4185de37180470483d403abfb1fd0b5a75a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 4 Sep 2026 09:21:34 +0200 Subject: [PATCH 07/13] Handle allocation failure when boxing (#919) (cherry picked from commit ea53814c105de8cdca724fe81e0f9af0b886de71) --- src/runtime.jl | 5 +++++ test/spirv.jl | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/runtime.jl b/src/runtime.jl index b7ebfc6f1..916e35b3d 100644 --- a/src/runtime.jl +++ b/src/runtime.jl @@ -243,6 +243,11 @@ end quote ptr = malloc($(Csize_t(allocsz))) + if ptr == C_NULL + report_oom($(Csize_t(allocsz))) + throw(OutOfMemoryError()) + end + # store the type tag ptr = convert(Ptr{tag_type}, ptr) Core.Intrinsics.pointerset(ptr, $tag | $gc_bits, #=index=# 1, #=align=# $tag_size) diff --git a/test/spirv.jl b/test/spirv.jl index b0dba4d24..3658af863 100644 --- a/test/spirv.jl +++ b/test/spirv.jl @@ -45,6 +45,32 @@ end SPIRV.code_llvm(mod.kernel, Tuple{}; backend, kernel=true) end end + +@testset "failed runtime boxing" begin + mod = @eval module $(gensym()) + import ..GPUCompiler + + function kernel(x::Float32) + throw(GPUCompiler.Runtime.box_float32(x)) + end + end + + # Call the helper directly because Julia's lowering of boxed exception fields varies by + # version. The allocator-less SPIR-V runtime should take a real OOM path before the stores. + @test @filecheck begin + @check_label "define spir_kernel void @_Z6kernel" + @check "gpu_malloc" + @check "icmp eq" + @check "br i1" + @check "gpu_report_oom" + @check "gpu_signal_exception" + @check "store" + @check "store" + @check "gpu_report_exception" + @check "gpu_signal_exception" + SPIRV.code_llvm(mod.kernel, Tuple{Float32}; backend, kernel=true, dump_module=true) + end +end end @testset "unsupported type detection" begin From 4d47eaa0e0c629ab1a26068d1f791fbd5a41155a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 2 Oct 2026 11:26:12 +0200 Subject: [PATCH 08/13] Fix invalid IR from small type tags on Julia 1.10 and 1.11 (#972) Co-authored-by: Claude Opus 5.5 (cherry picked from commit b0dd255eb3e79eb76dd7cce276ff1db261ea0d6e) --- src/jlgen.jl | 54 +++++++++++++++++++++++++++++++++++++++++++ test/spirv.jl | 20 ++++++++++++++++ test/spirv/imaging.jl | 24 +++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 test/spirv/imaging.jl diff --git a/src/jlgen.jl b/src/jlgen.jl index 8db65c899..e325a8a6c 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -763,6 +763,56 @@ const HAS_LLVM_GET_CIS = ( ) ) +# Julia before 1.12 casts pointers into `jl_small_typeof` to the generic address space with a +# `bitcast`: for constant references to small-tagged types in imaging mode, as used during +# precompilation (fixed in 1.11 by JuliaLang/julia#51517), and when looking up the type of a +# boxed value (fixed in 1.12). That's invalid IR on targets that put globals in another +# address space, like SPIR-V, so turn those bitcasts into address space casts. +function fix_small_typeof_casts!(mod::LLVM.Module) + haskey(globals(mod), "jl_small_typeof") || return false + + function isaddrspacechange(val) + isa(val, LLVM.ConstantExpr) && opcode(val) == LLVM.API.LLVMBitCast || + isa(val, LLVM.BitCastInst) || return false + src_typ = value_type(operands(val)[1]) + dst_typ = value_type(val) + return isa(src_typ, LLVM.PointerType) && isa(dst_typ, LLVM.PointerType) && + addrspace(src_typ) != addrspace(dst_typ) + end + + # find the bad casts first, looking through the GEPs that index the table + casts = LLVM.Value[] + worklist = LLVM.Value[globals(mod)["jl_small_typeof"]] + while !isempty(worklist) + val = pop!(worklist) + for use in uses(val) + usr = user(use) + if isaddrspacechange(usr) + push!(casts, usr) + elseif isa(usr, LLVM.GetElementPtrInst) || isa(usr, LLVM.BitCastInst) || + isa(usr, LLVM.ConstantExpr) && + opcode(usr) in (LLVM.API.LLVMGetElementPtr, LLVM.API.LLVMBitCast) + push!(worklist, usr) + end + end + end + + for cast in unique(casts) + src = operands(cast)[1] + typ = value_type(cast) + if isa(cast, LLVM.ConstantExpr) + replace_uses!(cast, const_addrspacecast(src, typ)) + else + @dispose builder=IRBuilder() begin + position!(builder, cast) # also picks up its debug location + replace_uses!(cast, addrspacecast!(builder, src, typ)) + end + erase!(cast) + end + end + return !isempty(casts) +end + function compile_method_instance(@nospecialize(job::CompilerJob)) if job.source.def.primary_world > job.world error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") @@ -867,6 +917,10 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) cache_gbl = nothing end + @static if VERSION < v"1.12-" + fix_small_typeof_casts!(llvm_mod) + end + # Since Julia 1.13, the caller is responsible for initializing global variables that # point to global values or bindings with their address in memory. # Similarly on previous versions when imaging=true, it is also the caller's responsibility diff --git a/test/spirv.jl b/test/spirv.jl index 3658af863..2995907d4 100644 --- a/test/spirv.jl +++ b/test/spirv.jl @@ -71,6 +71,26 @@ end SPIRV.code_llvm(mod.kernel, Tuple{Float32}; backend, kernel=true, dump_module=true) end end + +@testset "small type tags" begin + mod = @eval module $(gensym()) + function kernel(out::Core.LLVMPtr{UInt,1}, p::Ptr{Cvoid}) + x = unsafe_pointer_to_objref(p) + T = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), typeof(x)) + Base.unsafe_store!(out, UInt(T)) + return + end + end + + # Julia looks up the type of a boxed value in `jl_small_typeof`, a global in the + # cross-workgroup address space, which older versions cast with a `bitcast`. + job, _ = SPIRV.create_job(mod.kernel, (Core.LLVMPtr{UInt,1}, Ptr{Cvoid}); + backend, kernel=true) + JuliaContext() do ctx + ir, _ = GPUCompiler.compile(:llvm, job) + @test (verify(ir); true) + end +end end @testset "unsupported type detection" begin diff --git a/test/spirv/imaging.jl b/test/spirv/imaging.jl new file mode 100644 index 000000000..6ad010117 --- /dev/null +++ b/test/spirv/imaging.jl @@ -0,0 +1,24 @@ +# Julia generates code in imaging mode when precompiling a package, which on 1.10 refers to +# `Bool` through `jl_small_typeof`, a global in the cross-workgroup address space. Force that +# mode in a separate process, because code coverage disables package images, and with them +# imaging mode during precompilation. +script = """ + using GPUCompiler, LLVM + include($(repr(joinpath(@__DIR__, "..", "helpers", "runtime.jl")))) + include($(repr(joinpath(@__DIR__, "..", "helpers", "spirv.jl")))) + + function kernel(out::Core.LLVMPtr{UInt,1}) + T = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), Bool) + Base.unsafe_store!(out, UInt(T)) + return + end + + job, _ = SPIRV.create_job(kernel, (Core.LLVMPtr{UInt,1},); backend=:llvm, kernel=true) + JuliaContext() do ctx + ir, _ = GPUCompiler.compile(:llvm, job) + verify(ir) + end + """ +# Base.julia_cmd() inherits coverage from the test runner, which conflicts with imaging mode. +cmd = `$(Base.julia_cmd()) --code-coverage=none --image-codegen --project=$(Base.active_project()) -e $script` +@test success(pipeline(cmd; stdout, stderr)) From 0a225f872e73a91c5d8b84b664478b018c4487ae Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 23 Sep 2026 17:22:20 +0200 Subject: [PATCH 09/13] SPIR-V: lower llvm.minimum and llvm.maximum (#937) (cherry picked from commit 8eecfc8abf4f188478a83e447a479f6e0886573e) --- src/spirv.jl | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/spirv.jl | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) diff --git a/src/spirv.jl b/src/spirv.jl index 2d25c6f8e..f70ad2252 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -107,6 +107,9 @@ function finish_ir!(job::CompilerJob{SPIRVCompilerTarget}, mod::LLVM.Module, # `unordered` heap-reference accesses become; the orderings serve no purpose on device demote_unordered_atomics!(mod) + # the SPIR-V back-ends lower `llvm.minimum`/`llvm.maximum` to NaN-ignoring `fmin`/`fmax` + lower_minimum_maximum!(mod) + # convert the kernel state argument to a byval reference if job.config.kernel state = kernel_state_type(job) @@ -271,6 +274,81 @@ function rm_freeze!(@nospecialize(job::CompilerJob), mod::LLVM.Module) return changed end +# expand `llvm.minimum` and `llvm.maximum`, which Julia uses for `min` and `max` of +# floating-point numbers. these return NaN when either operand is NaN, and order -0.0 before +# +0.0, but both SPIR-V back-ends translate them to OpenCL's `fmin` and `fmax`, which return +# the other operand when one is NaN, and may return either zero. so use `llvm.minnum` and +# `llvm.maxnum` (which translate to the same `fmin` and `fmax`) and fix up those cases, +# unless the call's fast-math flags say they don't occur. +function lower_minimum_maximum!(mod::LLVM.Module) + changed = false + @tracepoint "lower minimum/maximum" begin + + for f in collect(functions(mod)) + isdeclaration(f) || continue + fn = LLVM.name(f) + is_minimum = startswith(fn, "llvm.minimum.") + is_minimum || startswith(fn, "llvm.maximum.") || continue + + typ = return_type(function_type(f)) + eltyp = typ isa LLVM.VectorType ? eltype(typ) : typ + bits = if eltyp == LLVM.HalfType() + 16 + elseif eltyp == LLVM.FloatType() + 32 + elseif eltyp == LLVM.DoubleType() + 64 + else + continue + end + ityp = LLVM.IntType(bits) + if typ isa LLVM.VectorType + ityp = LLVM.VectorType(ityp, length(typ)) + end + num = LLVM.Function(mod, LLVM.Intrinsic(is_minimum ? "llvm.minnum" : "llvm.maxnum"), + LLVMType[typ]) + + for use in collect(uses(f)) + call = user(use) + call isa LLVM.CallInst || continue + x, y = arguments(call) + flags = LLVM.fast_math(call) + @dispose builder=IRBuilder() begin + position!(builder, call) + debuglocation!(builder, call) + + res = call!(builder, function_type(num), num, LLVM.Value[x, y]) + fast_math!(res; flags...) + + # if both operands are zero, combine their sign bits + if !flags.nsz + zero = LLVM.null(typ) + both_zero = and!(builder, fcmp!(builder, LLVM.API.LLVMRealOEQ, x, zero), + fcmp!(builder, LLVM.API.LLVMRealOEQ, y, zero)) + xi = bitcast!(builder, x, ityp) + yi = bitcast!(builder, y, ityp) + zi = is_minimum ? or!(builder, xi, yi) : and!(builder, xi, yi) + res = select!(builder, both_zero, bitcast!(builder, zi, typ), res) + end + + # if either operand is NaN, return a NaN + if !flags.nnan + either_nan = fcmp!(builder, LLVM.API.LLVMRealUNO, x, y) + res = select!(builder, either_nan, fadd!(builder, x, y), res) + end + + replace_uses!(call, res) + erase!(call) + end + changed = true + end + isempty(uses(f)) && erase!(f) + end + + end + return changed +end + # convert alloca [N x i128] to alloca [N x <2 x i64>] # SPIR-V doesn't support i128 types, but we can represent them as vectors function convert_i128_allocas!(mod::LLVM.Module) diff --git a/test/spirv.jl b/test/spirv.jl index 2995907d4..6a0fee705 100644 --- a/test/spirv.jl +++ b/test/spirv.jl @@ -155,6 +155,53 @@ end end +@testset "min and max" begin + # Julia's `min` and `max` propagate NaNs and order -0.0 before +0.0, which the `fmin` and + # `fmax` the SPIR-V back-ends translate `llvm.minimum` and `llvm.maximum` to don't do + # (call the intrinsics directly: on Julia 1.10 and 1.11, `min` and `max` only use them + # on aarch64 hosts) + mod = @eval module $(gensym()) + const V = NTuple{4, VecElement{Float32}} + smin(x, y) = ccall("llvm.minimum.f32", llvmcall, Float32, (Float32, Float32), x, y) + smax(x, y) = ccall("llvm.maximum.f32", llvmcall, Float32, (Float32, Float32), x, y) + vmin(x, y) = ccall("llvm.minimum.v4f32", llvmcall, V, (V, V), x, y) + function kernel(p::Ptr{Float32}, q::Ptr{V}, x::Float32, y::Float32, v::V, w::V) + unsafe_store!(p, smin(x, y), 1) + unsafe_store!(p, smax(x, y), 2) + unsafe_store!(q, vmin(v, w)) + return + end + fast_kernel(p::Ptr{Float32}, x::Float32, y::Float32) = + (unsafe_store!(p, Base.FastMath.min_fast(x, y)); return) + end + V = mod.V + + @test @filecheck begin + @check_label "define {{.*}} @{{(julia|j)_kernel_[0-9]+}}" + @check "call float @llvm.minnum.f32" + @check "fcmp uno float" + @check "call float @llvm.maxnum.f32" + @check "fcmp uno float" + @check "call <4 x float> @llvm.minnum.v4f32" + @check "fcmp uno <4 x float>" + @check_not "llvm.minimum" + @check_not "llvm.maximum" + SPIRV.code_llvm(mod.kernel, Tuple{Ptr{Float32}, Ptr{V}, Float32, Float32, V, V}; backend) + end + + # with fast-math flags, NaNs and signed zeros don't need to be handled + # (Julia 1.12+ implements `min_fast` using `llvm.minimum`) + if VERSION >= v"1.12" + @test @filecheck begin + @check_label "define {{.*}} @{{(julia|j)_fast_kernel_[0-9]+}}" + @check "call fast float @llvm.minnum.f32" + @check_not "fcmp" + @check_not "select" + SPIRV.code_llvm(mod.fast_kernel, Tuple{Ptr{Float32}, Float32, Float32}; backend) + end + end +end + ############################################################################################ @testset "asm" begin @@ -218,6 +265,23 @@ end end end +@testset "min and max" begin + mod = @eval module $(gensym()) + smin(x, y) = ccall("llvm.minimum.f32", llvmcall, Float32, (Float32, Float32), x, y) + function kernel(p::Ptr{Float32}, x::Float32, y::Float32) + unsafe_store!(p, smin(x, y)) + return + end + end + + @test @filecheck begin + @check "OpEntryPoint Kernel %[[KERNEL:[^ ]+]]" + @check "OpExtInst %float %{{[0-9]+}} fmin" + @check "OpUnordered" + SPIRV.code_native(mod.kernel, Tuple{Ptr{Float32}, Float32, Float32}; backend, kernel=true) + end +end + @testset "inlining of throwing callees" begin mod = @eval module $(gensym()) @noinline function guard(x) From b3283ac60db8047765fb1c04014062593934745d Mon Sep 17 00:00:00 2001 From: Tim Esselens Date: Tue, 29 Sep 2026 09:15:50 +0200 Subject: [PATCH 10/13] Metal: lower `llvm.sin` and `llvm.cos` to AIR (#967) Co-authored-by: Claude Opus 5.5 (1M context) (cherry picked from commit cc26075717e5df6af43a7b2992f8a33dae4f1634) --- src/metal.jl | 6 ++++++ test/metal.jl | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/metal.jl b/src/metal.jl index b7cb5bf60..a505c33f4 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -1563,6 +1563,12 @@ function lower_math_intrinsics!(fun::LLVM.Function) LLVM.Intrinsic("llvm.ceil") => ("air.ceil", "air.fast_ceil"), LLVM.Intrinsic("llvm.trunc") => ("air.trunc", "air.fast_trunc"), LLVM.Intrinsic("llvm.rint") => ("air.rint", "air.fast_rint"), + # Julia doesn't emit these (Metal.jl calls `air.sin`/`air.cos` directly), but Enzyme's + # derivatives of those calls do, and Apple's back-end crashes on them. The f16 builtins + # are less accurate than rounding the f32 ones on some GPUs (M1), but match what Apple's + # frontend and Metal.jl use for half. + LLVM.Intrinsic("llvm.sin") => ("air.sin", "air.fast_sin"), + LLVM.Intrinsic("llvm.cos") => ("air.cos", "air.fast_cos"), ) worklist = Tuple{LLVM.CallBase, String, Union{String,Nothing}}[] diff --git a/test/metal.jl b/test/metal.jl index 10dc91718..3e5923674 100644 --- a/test/metal.jl +++ b/test/metal.jl @@ -881,6 +881,10 @@ end declare float @llvm.fma.f32(float, float, float) declare half @llvm.fma.f16(half, half, half) declare <4 x float> @llvm.sqrt.v4f32(<4 x float>) + declare float @llvm.sin.f32(float) + declare half @llvm.sin.f16(half) + declare float @llvm.cos.f32(float) + declare half @llvm.cos.f16(half) define void @f(float %x, half %h, <4 x float> %v) { %a = call float @llvm.sqrt.f32(float %x) %b = call afn float @llvm.sqrt.f32(float %x) @@ -893,6 +897,12 @@ end %j = call float @llvm.fma.f32(float %x, float %x, float %x) %l = call half @llvm.fma.f16(half %h, half %h, half %h) %k = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %v) + %m = call float @llvm.sin.f32(float %x) + %m2 = call afn float @llvm.sin.f32(float %x) + %m3 = call half @llvm.sin.f16(half %h) + %n = call float @llvm.cos.f32(float %x) + %n2 = call afn float @llvm.cos.f32(float %x) + %n3 = call afn half @llvm.cos.f16(half %h) ret void } """ @@ -917,6 +927,14 @@ end @test "air.fma.f32" in names @test "air.fma.f16" in names @test !("air.fast_fma.f32" in names) + # sin and cos (emitted by Enzyme's derivatives, not by Julia) follow the same rules + @test "air.sin.f32" in names + @test "air.fast_sin.f32" in names + @test "air.sin.f16" in names + @test "air.cos.f32" in names + @test "air.fast_cos.f32" in names + @test "air.cos.f16" in names + @test !("air.fast_cos.f16" in names) # no scalar llvm.* math intrinsics survive; the vector one is left (no air..v4f32) @test !any(n -> startswith(n, "llvm.") && !endswith(n, "v4f32"), names) @test "llvm.sqrt.v4f32" in names From 64c0b5e4988c7102c6d15082494fda04e79ce211 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 16 Sep 2026 07:07:39 +0200 Subject: [PATCH 11/13] Metal: lower bare LLVM fences to air.atomic.fence (#932) (cherry picked from commit cae55dd8b5cb7602a394c49aca105113ab15c429) --- src/metal.jl | 75 +++++++++++++++++++++++++++++++++++++++++++++++++ test/gcn.jl | 2 +- test/metal.jl | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++ test/native.jl | 4 +-- 4 files changed, 154 insertions(+), 3 deletions(-) diff --git a/src/metal.jl b/src/metal.jl index a505c33f4..15ce5bf30 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -506,6 +506,9 @@ function lower_air!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), mod::L # AIR does not support LLVM atomic load/store instructions (see `demote_unordered_atomics!`) demote_unordered_atomics!(mod) + # the macOS 27 back-end rejects bare LLVM fences (Metal.jl#968) + lower_fences!(job, mod) + # strip device-side `trap`s and rewrite `unreachable` into clean returns (#433, #370). this # runs post-`optimize!`, after the trap has finished serving as the optimizer guard; the pass # force-inlines throwing functions into the kernel first so the rewrite is sound, then scrubs @@ -557,6 +560,78 @@ function lower_air!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), mod::L return end +# Before LLVM 18, `ordering(inst)` calls `LLVMGetOrdering`, which incorrectly casts fences +# to AtomicRMWInst. Use the stable textual form on all versions to keep this workaround tested. +function fence_ordering(inst::LLVM.FenceInst) + # Scope names escape embedded quotes as \22; metadata follows the ordering. + m = match(r"^\s*fence(?:\s+syncscope\(\"[^\"]*\"\))?\s+(acquire|release|acq_rel|seq_cst)\b", + string(inst)) + m === nothing && error("Unexpected fence instruction: $inst") + return m.captures[1] == "acquire" ? LLVM.API.LLVMAtomicOrderingAcquire : + m.captures[1] == "release" ? LLVM.API.LLVMAtomicOrderingRelease : + m.captures[1] == "acq_rel" ? LLVM.API.LLVMAtomicOrderingAcquireRelease : + LLVM.API.LLVMAtomicOrderingSequentiallyConsistent +end + +# Lower LLVM fences to air.atomic.fence(flags, order, scope), as MSL's atomic_thread_fence +# does. Bare fences from Julia's atomic_fence crash the macOS 27 AGX back-end (Metal.jl#968). +# +# MSL memory_order values are acquire=2, release=3, acq_rel=4, seq_cst=5. Metal 3.2-4.0 +# only supports relaxed/seq_cst fences, so strengthen other orderings to seq_cst. Before +# Metal 3.2 the intrinsic is unavailable; retain the bare fence. Such targets still +# require a back-end that accepts bare fences. +# +# Cover device and threadgroup memory (mem_flags=1|2), the shared writable LLVM address spaces. +# Use thread scope (0) for singlethread and device scope (2) otherwise. Metal has no +# system-wide scope: this only synchronizes threads on the same device. +function lower_fences!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), mod::LLVM.Module) + metal = job.config.target.metal + metal >= v"3.2" || return false + + worklist = LLVM.FenceInst[] + for f in functions(mod), bb in blocks(f), inst in instructions(bb) + inst isa LLVM.FenceInst && push!(worklist, inst) + end + isempty(worklist) && return false + + T_int32 = LLVM.Int32Type() + fence_ft = LLVM.FunctionType(LLVM.VoidType(), [T_int32, T_int32, T_int32]) + fence_fn = if haskey(functions(mod), "air.atomic.fence") + functions(mod)["air.atomic.fence"] + else + LLVM.Function(mod, "air.atomic.fence", fence_ft) + end + + for inst in worklist + order = if metal < v"4.1" + 5 # seq_cst + else + ord = fence_ordering(inst) + if ord == LLVM.API.LLVMAtomicOrderingAcquire + 2 + elseif ord == LLVM.API.LLVMAtomicOrderingRelease + 3 + elseif ord == LLVM.API.LLVMAtomicOrderingAcquireRelease + 4 + else + 5 # seq_cst (the only other ordering LLVM allows on a fence) + end + end + scope = syncscope(inst) == SyncScope("singlethread") ? 0 : 2 + flags = 1 | 2 # device | threadgroup + + @dispose builder=IRBuilder() begin + position!(builder, inst) + debuglocation!(builder, inst) + call!(builder, fence_ft, fence_fn, + [ConstantInt(T_int32, flags), ConstantInt(T_int32, order), + ConstantInt(T_int32, scope)]) + end + erase!(inst) + end + return true +end + @unlocked function mcgen(job::CompilerJob{MetalCompilerTarget}, mod::LLVM.Module, format=LLVM.API.LLVMObjectFile) # lower LLVM constructs that the AIR back-end does not support; this takes the place diff --git a/test/gcn.jl b/test/gcn.jl index e91c75795..e2d1cb5be 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -510,7 +510,7 @@ end # a zero-byte allocation lowers to a null pointer rather than a degenerate alloca. @test @filecheck begin @check_label "define {{.*}}@{{(julia|j)_empty_scratch_[0-9]+}}" - @check_not "alloca" + @check_not "= alloca" @check_not "julia.gpu.alloca" GCN.code_llvm(mod.empty_scratch, Tuple{}) end diff --git a/test/metal.jl b/test/metal.jl index 3e5923674..9e3c40c50 100644 --- a/test/metal.jl +++ b/test/metal.jl @@ -1340,6 +1340,82 @@ end @test occursin(r"store i64", air) end +@testset "fence lowering" begin + orders = [("acquire", 2), ("release", 3), ("acq_rel", 4), ("seq_cst", 5)] + scopes = [("", 2), ("syncscope(\"singlethread\") ", 0), + ("syncscope(\"workgroup\") ", 2), + (raw"syncscope(\"fence acquire\22\5C\0A\") ", 2)] + metadata = ("", ", !dbg !3, !annotation !4") + fences_ir = join(["fence $scope$order$md" for (order, _) in orders + for (scope, _) in scopes for md in metadata], "\n") + ir = """ + define void @f() !dbg !2 {\n$fences_ir\nret void\n} + !llvm.dbg.cu = !{!0} + !llvm.module.flags = !{!5} + !0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "GPUCompiler", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug) + !1 = !DIFile(filename: "fence.jl", directory: "/") + !2 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, spFlags: DISPFlagDefinition, unit: !0) + !3 = !DILocation(line: 2, column: 1, scope: !2) + !4 = !{!"fence seq_cst"} + !5 = !{i32 2, !"Debug Info Version", i32 3} + !6 = !DISubroutineType(types: !7) + !7 = !{} + """ + fence_job(metal) = let + source = methodinstance(typeof(identity), Tuple{Int}, Base.get_world_counter()) + target = MetalCompilerTarget(; macos=v"27", metal, air=v"2.9") + CompilerJob(source, CompilerConfig(target, Metal.CompilerParams(); kernel=true)) + end + fences(mod) = [i for f in functions(mod) for bb in blocks(f) for i in instructions(bb) + if i isa LLVM.FenceInst] + + @testset "Metal $metal, existing declaration: $declared" for + metal in (v"3.1", v"3.2", v"4.0", v"4.1"), declared in (false, true) + Context() do ctx + decl = declared ? "declare void @air.atomic.fence(i32, i32, i32)\n" : "" + mod = parse(LLVM.Module, decl * ir) + original = string(mod) + @test length(fences(mod)) == length(orders) * length(scopes) * length(metadata) + @test GPUCompiler.lower_fences!(fence_job(metal), mod) == (metal >= v"3.2") + @test (verify(mod); true) + if metal < v"3.2" + @test string(mod) == original + else + @test isempty(fences(mod)) + calls = [m.match for m in eachmatch(r"call void @air.atomic.fence\([^\n]+\)", string(mod))] + expected = ["call void @air.atomic.fence(i32 3, i32 $(metal < v"4.1" ? 5 : order), i32 $scope)" + for (_, order) in orders for (_, scope) in scopes for _ in metadata] + @test calls == expected + @test count("declare void @air.atomic.fence(", string(mod)) == 1 + end + @test !GPUCompiler.lower_fences!(fence_job(metal), mod) + end + end + + # end-to-end: Julia's `atomic_fence` intrinsic must not reach the AIR as a bare `fence` + # (Julia 1.14 added a syncscope argument to the intrinsic, JuliaLang/julia#60311) + function kernel(p::Core.LLVMPtr{Int,1}) + Core.Intrinsics.atomic_pointerset(reinterpret(Ptr{Int}, p), 1, :monotonic) + @static if VERSION >= v"1.14.0-DEV.1371" + Core.Intrinsics.atomic_fence(:release, :system) + else + Core.Intrinsics.atomic_fence(:release) + end + return + end + source = methodinstance(typeof(kernel), Tuple{Core.LLVMPtr{Int,1}}, Base.get_world_counter()) + target = MetalCompilerTarget(; macos=v"27", metal=v"4.1", air=v"2.9") + config = CompilerConfig(target, Metal.CompilerParams(); kernel=true) + job = CompilerJob(source, config) + + llvm_ir = sprint(io->GPUCompiler.code_llvm(io, job; dump_module=true)) + @test occursin("fence release", llvm_ir) + + air = sprint(io->GPUCompiler.code_native(io, job; dump_module=true)) + @test !occursin(r"^\s*fence "m, air) + @test occursin("call void @air.atomic.fence(i32 3, i32 3, i32 2)", air) +end + # byval lowering must strip the (non-IPO-safe) Julia const-region metadata off loads derived # from the materialized argument; check the helper walks gep/addrspacecast chains and removes it. @testset "const-region metadata stripping for materialized args" begin diff --git a/test/native.jl b/test/native.jl index 64e8999c6..5c0e34b02 100644 --- a/test/native.jl +++ b/test/native.jl @@ -885,7 +885,7 @@ end # once optimized the slot is promoted away entirely (result is x + x). @test @filecheck begin @check_label "define float @{{(julia|j)_scratch_[0-9]+}}" - @check_not "alloca" + @check_not "= alloca" @check_not "julia.gpu.alloca" Native.code_llvm(mod.scratch, Tuple{Float32}) end @@ -893,7 +893,7 @@ end # a zero-byte allocation lowers to a null pointer rather than a degenerate alloca. @test @filecheck begin @check_label "define {{.*}}@{{(julia|j)_empty_scratch_[0-9]+}}" - @check_not "alloca" + @check_not "= alloca" @check_not "julia.gpu.alloca" Native.code_llvm(mod.empty_scratch, Tuple{}) end From cf557447ecc87cc2af4c0350f1d8aaf01033dece Mon Sep 17 00:00:00 2001 From: Tim Esselens Date: Fri, 25 Sep 2026 13:08:44 +0200 Subject: [PATCH 12/13] Metal: expand `llvm.powi` into multiplies (#946) [1.x: re-expressed in 1.23's worklist style, without #941's helpers.] Co-authored-by: Claude Opus 5.5 (1M context) (cherry picked from commit 0d06ee45e22825c48f94d022ac0e93cd75588fc8) --- src/metal.jl | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/metal.jl | 42 +++++++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/src/metal.jl b/src/metal.jl index 15ce5bf30..a14d14438 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -1738,6 +1738,65 @@ function fuse_minmax3!(fun::LLVM.Function) return changed end +# 1.0 of a floating-point type, splat across the lanes of a vector type +fp_one(typ::LLVMType) = LLVM.Value(LLVM.API.LLVMConstReal(typ, 1.0)) + +# `x^n` for a constant `n`, unrolled into multiplies like SelectionDAG's `ExpandPowI` +function expand_powi!(builder::IRBuilder, x::LLVM.Value, n::Int) + m = unsigned(abs(n)) # also for `typemin(n)`, which `abs` returns as is + res = nothing # 1.0, until the first set bit + sq = x # x^(2^i) for the current bit i + while m != 0 + isodd(m) && (res = res === nothing ? sq : fmul!(builder, res, sq)) + m >>= 1 + m != 0 && (sq = fmul!(builder, sq, sq)) + end + res = something(res, fp_one(value_type(x))) + return n < 0 ? fdiv!(builder, fp_one(value_type(x)), res) : res +end + +# `x^n` for any other `n`, as a loop over the exponent's bits like compiler-rt's `__powisf2` +function build_powi!(mod::LLVM.Module, fn::String, typ::LLVMType, ntyp::LLVMType) + f = LLVM.Function(mod, fn, LLVM.FunctionType(typ, LLVMType[typ, ntyp])) + linkage!(f, LLVM.API.LLVMInternalLinkage) + push!(function_attributes(f), EnumAttribute("alwaysinline")) + x, n = parameters(f) + one = fp_one(typ) + zero = LLVM.ConstantInt(ntyp, 0) + + bb_entry = BasicBlock(f, "entry") + bb_loop = BasicBlock(f, "loop") + bb_done = BasicBlock(f, "done") + @dispose builder=IRBuilder() begin + position!(builder, bb_entry) + br!(builder, icmp!(builder, LLVM.API.LLVMIntEQ, n, zero), bb_done, bb_loop) + + # multiply the squares selected by the bits of `n`, least significant first. like + # `__powisf2`, shift the signed `n` by halving it (rounding towards zero), so as not + # to take `abs(n)`, which InstCombine would turn into an `llvm.abs` after that has + # already been lowered. + position!(builder, bb_loop) + acc = phi!(builder, typ, "acc") + sq = phi!(builder, typ, "sq") + rest = phi!(builder, ntyp, "rest") + bit = trunc!(builder, rest, LLVM.Int1Type()) + acc′ = select!(builder, bit, fmul!(builder, acc, sq), acc) + sq′ = fmul!(builder, sq, sq) + rest′ = sdiv!(builder, rest, LLVM.ConstantInt(ntyp, 2)) + br!(builder, icmp!(builder, LLVM.API.LLVMIntEQ, rest′, zero), bb_done, bb_loop) + append!(incoming(acc), [(one, bb_entry), (acc′, bb_loop)]) + append!(incoming(sq), [(x, bb_entry), (sq′, bb_loop)]) + append!(incoming(rest), [(n, bb_entry), (rest′, bb_loop)]) + + position!(builder, bb_done) + pow = phi!(builder, typ, "pow") + append!(incoming(pow), [(one, bb_entry), (acc′, bb_loop)]) + negative = icmp!(builder, LLVM.API.LLVMIntSLT, n, zero) + ret!(builder, select!(builder, negative, fdiv!(builder, one, pow), pow)) + end + return f +end + # replace LLVM intrinsics with AIR equivalents function lower_llvm_intrinsics!(@nospecialize(job::CompilerJob), fun::LLVM.Function) isdeclaration(fun) && return false @@ -2147,6 +2206,42 @@ function lower_llvm_intrinsics!(@nospecialize(job::CompilerJob), fun::LLVM.Funct changed = true end end + + # integer power, which AIR lacks (MSL has no `pown`, and `pow` is undefined for negative + # bases), by exponentiation by squaring as LLVM's back-ends expand it: a constant + # exponent is unrolled into multiplies, any other calls a loop over the exponent's bits. + # A negative exponent takes the reciprocal, and `x^0` is 1 (even for NaN). The exponent + # of a vector `powi` is a scalar. + if intr == LLVM.Intrinsic("llvm.powi") + x, n = arguments(call) + + @dispose builder=IRBuilder() begin + position!(builder, call) + debuglocation!(builder, call) + + new_value = if n isa LLVM.ConstantInt && width(value_type(n)) <= 64 + expand_powi!(builder, x, convert(Int, n)) + else + # the loop halves the exponent, which doesn't work for an `i1` (where 2 + # wraps to 0) + if width(value_type(n)) < 32 + n = sext!(builder, n, LLVM.Int32Type()) + end + + # keep the mangled value type, e.g. llvm.powi.v2f32.i16 -> air.powi.v2f32.i32 + fn = "air.powi.$(split(LLVM.name(call_fun), '.')[3]).i$(width(value_type(n)))" + new_intr = if haskey(functions(mod), fn) + functions(mod)[fn] + else + build_powi!(mod, fn, value_type(x), value_type(n)) + end + call!(builder, function_type(new_intr), new_intr, LLVM.Value[x, n]) + end + replace_uses!(call, new_value) + erase!(call) + changed = true + end + end end # fuse chained integer min/max (now lowered to air.min/max) into AIR's 3-way builtins diff --git a/test/metal.jl b/test/metal.jl index 9e3c40c50..005be23f2 100644 --- a/test/metal.jl +++ b/test/metal.jl @@ -960,6 +960,48 @@ end end end +@testset "integer power lowering" begin + # AIR has no integer power, so `llvm.powi` (as emitted for `@fastmath x^n`) is expanded + # into multiplies: unrolled for a constant exponent, and a loop over its bits otherwise. + mod = @eval module $(gensym()) + pow_const(x) = @fastmath x^-5 + pow_var(x, n) = @fastmath x^n + pow_vec(x, n) = ccall("llvm.powi.v2f32.i64", llvmcall, NTuple{2, VecElement{Float32}}, + (NTuple{2, VecElement{Float32}}, Int64), x, n) + end + + # x^-5 = 1 / (x * (x^2)^2) + @test @filecheck begin + @check_label "define float @{{(julia|j)_pow_const_[0-9]+}}" + @check_not "@llvm.powi" + @check_count 3 "fmul float" + @check "fdiv float 1.000000e+00" + @check_not "fmul" + @check "ret float" + Metal.code_native(mod.pow_const, Tuple{Float32}) + end + + @test @filecheck begin + @check_label "define float @{{(julia|j)_pow_var_[0-9]+}}" + @check_not "@llvm.powi" + @check "phi float" + @check "fmul float" + @check "fdiv float 1.000000e+00" + @check_not "@{{(llvm|air)\\.[a-z_]*pow}}" + Metal.code_native(mod.pow_var, Tuple{Float32, Int32}) + end + + @test @filecheck begin + @check_label "define <2 x float> @{{(julia|j)_pow_vec_[0-9]+}}" + @check_not "@llvm.powi" + @check "phi <2 x float>" + @check "fmul <2 x float>" + @check "fdiv <2 x float>" + @check_not "@{{(llvm|air)\\.[a-z_]*pow}}" + Metal.code_native(mod.pow_vec, Tuple{NTuple{2, VecElement{Float32}}, Int64}) + end +end + @testset "integer intrinsic lowering" begin # The integer ops Julia emits as llvm.* are lowered to their AIR builtins, so Metal.jl need # not wrap them. Names/signatures verified against Apple's frontend: From 4401de34df41b0506209afaf28551e8cafa6ce23 Mon Sep 17 00:00:00 2001 From: Tim Esselens Date: Mon, 28 Sep 2026 21:56:21 +0200 Subject: [PATCH 13/13] Metal: expand vector reductions into scalar chains (#962) [1.x: re-expressed in 1.23's worklist style, without #941's helpers.] Co-authored-by: Claude Opus 5.5 (1M context) (cherry picked from commit 9f0d0b87aeafd33f626aae6335ca7504e0a412c0) --- src/metal.jl | 89 +++++++++++++++++++++++++++++++++++++++++++++------ test/metal.jl | 49 ++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 10 deletions(-) diff --git a/src/metal.jl b/src/metal.jl index a14d14438..939850e84 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -543,15 +543,6 @@ function lower_air!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), mod::L end end - # perform codegen passes that would normally run during machine code emission - if LLVM.has_oldpm() - # XXX: codegen passes don't seem available in the new pass manager yet - @dispose pm=ModulePassManager() begin - expand_reductions!(pm) - run!(pm, mod) - end - end - # flatten chained byte GEPs that the AGX back-end miscompiles for 1-byte accesses on a # 2-threadgroup grid (see `merge_byte_gep_chains!`). run last, after the intrinsic-lowering # cleanup above, so the merged form is what reaches the AIR downgrader / back-end. @@ -1616,6 +1607,82 @@ function scalarize_vector_minmax!(fun::LLVM.Function) return true end +# AIR has no vector reductions either, and Apple's back-end rejects `llvm.vector.reduce.*` or +# crashes its compiler service on it. Expand each reduction into a chain of its scalar operation +# over the lanes, in lane order, as SelectionDAG does for targets without vector operations +# (`TargetLowering::expandVecReduce` and `expandVecReduceSeq`): that is how the ordered +# `fadd`/`fmul` reductions are defined, and a valid order for the others. LLVM's IR-level +# `ExpandReductions` pass doesn't fit here: it leaves non-power-of-2 vectors and `fmin`/`fmax` +# without `nnan` for SelectionDAG to handle, it turns `and`/`or` over `i1` lanes into an `iN` +# bitcast that Apple's back-end rejects, and the new pass manager doesn't have it before LLVM 21. +function expand_vector_reductions!(fun::LLVM.Function) + # llvm reduction => IRBuilder function, or the scalar intrinsic to chain + reductions = Dict{LLVM.Intrinsic,Any}( + LLVM.Intrinsic("llvm.vector.reduce.add") => add!, + LLVM.Intrinsic("llvm.vector.reduce.mul") => mul!, + LLVM.Intrinsic("llvm.vector.reduce.and") => and!, + LLVM.Intrinsic("llvm.vector.reduce.or") => or!, + LLVM.Intrinsic("llvm.vector.reduce.xor") => xor!, + LLVM.Intrinsic("llvm.vector.reduce.fadd") => fadd!, + LLVM.Intrinsic("llvm.vector.reduce.fmul") => fmul!, + LLVM.Intrinsic("llvm.vector.reduce.smax") => "llvm.smax", + LLVM.Intrinsic("llvm.vector.reduce.smin") => "llvm.smin", + LLVM.Intrinsic("llvm.vector.reduce.umax") => "llvm.umax", + LLVM.Intrinsic("llvm.vector.reduce.umin") => "llvm.umin", + LLVM.Intrinsic("llvm.vector.reduce.fmax") => "llvm.maxnum", + LLVM.Intrinsic("llvm.vector.reduce.fmin") => "llvm.minnum", + ) + if LLVM.version() >= v"17" + reductions[LLVM.Intrinsic("llvm.vector.reduce.fmaximum")] = "llvm.maximum" + reductions[LLVM.Intrinsic("llvm.vector.reduce.fminimum")] = "llvm.minimum" + end + # AIR has no integer min/max on `i1`, where they are logic operations (true is -1 when signed) + i1_minmax = Dict("llvm.smax" => and!, "llvm.smin" => or!, + "llvm.umax" => or!, "llvm.umin" => and!) + + worklist = Tuple{LLVM.CallBase, Any}[] + for bb in blocks(fun), inst in instructions(bb) + inst isa LLVM.CallBase || continue + callee = called_operand(inst) + (callee isa LLVM.Function && LLVM.isintrinsic(callee)) || continue + op = get(reductions, LLVM.Intrinsic(callee), nothing) + op === nothing && continue + push!(worklist, (inst, op)) + end + isempty(worklist) && return false + + mod = LLVM.parent(fun) + for (call, op) in worklist + args = collect(LLVM.Value, arguments(call)) + vec = last(args) # `fadd`/`fmul` take a start value first + elty = eltype(value_type(vec)) + if op isa String && elty == LLVM.Int1Type() + op = i1_minmax[op] + elseif op isa String + f = LLVM.Function(mod, LLVM.Intrinsic(op), LLVMType[elty]) + op = (builder, a, b) -> call!(builder, function_type(f), f, LLVM.Value[a, b]) + end + # each step carries the reduction's fast-math flags, as in SelectionDAG; min/max rely + # on them to select the relaxed AIR builtins (see the minimum/maximum lowering) + fmf = elty isa LLVM.FloatingPointType ? LLVM.fast_math(call) : nothing + @dispose builder=IRBuilder() begin + position!(builder, call) + debuglocation!(builder, call) + res = length(args) == 2 ? args[1] : nothing + for i in 0:Int(length(value_type(vec)))-1 + lane = extract_element!(builder, vec, ConstantInt(LLVM.Int32Type(), i)) + res === nothing && (res = lane; continue) + res = op(builder, res, lane) + # (steps on constants fold to constants, which carry no flags) + fmf !== nothing && res isa LLVM.Instruction && LLVM.fast_math!(res; fmf...) + end + replace_uses!(call, res) + erase!(call) + end + end + return true +end + # floating-point math intrinsics that Julia emits as plain `llvm.*` and that Metal exposes as # AIR device functions. Each has a precise `air.` for f16/f32; some additionally have a # relaxed, f32-only `air.fast_` that we select when the call is `afn`-flagged — set per-op @@ -1804,8 +1871,10 @@ function lower_llvm_intrinsics!(@nospecialize(job::CompilerJob), fun::LLVM.Funct mod = LLVM.parent(fun) changed = false - # AIR lacks vector min/max intrinsics; scalarize so the per-call lowering below applies. + # AIR lacks vector min/max intrinsics and vector reductions; scalarize them so the per-call + # lowering below applies. changed |= scalarize_vector_minmax!(fun) + changed |= expand_vector_reductions!(fun) # lower the floating-point math intrinsics Julia emits (sqrt, fma, floor, ...) to their # AIR device functions, picking the relaxed `air.fast_*` variant for `afn`-flagged calls. diff --git a/test/metal.jl b/test/metal.jl index 005be23f2..134d3a543 100644 --- a/test/metal.jl +++ b/test/metal.jl @@ -1002,6 +1002,55 @@ end end end +@testset "vector reduction lowering" begin + # AIR has no vector reductions: each is expanded into a chain of its scalar operation over the + # lanes, in lane order, and min/max into the scalar intrinsics, which are lowered to AIR. + km = @eval module $(gensym()) + f() = return + prod4(v) = ccall("llvm.vector.reduce.mul.v4i64", llvmcall, Int64, (NTuple{4, VecElement{Int64}},), v) + end + job, _ = Metal.create_job(km.f, Tuple{}) + ops = ["add", "mul", "and", "or", "xor", "smax", "smin", "umax", "umin", "fadd", "fmul", "fmax", "fmin"] + LLVM.version() >= v"17" && append!(ops, ["fmaximum", "fminimum"]) + Context() do ctx + ir = IOBuffer() + for op in ops, n in (3, 4) + t, s = op[1] == 'f' ? ("float", "f32") : ("i32", "i32") + args = op in ("fadd", "fmul") ? "$t %s, <$n x $t> %v" : "<$n x $t> %v" + flags = op[1] == 'f' && n == 3 ? "nnan " : "" + println(ir, "declare $t @llvm.vector.reduce.$op.v$n$s($args)") + println(ir, "define $t @$op$n($args) {\n %r = call $flags$t @llvm.vector.reduce.$op.v$n$s($args)\n ret $t %r\n}") + end + println(ir, "declare i1 @llvm.vector.reduce.smax.v3i1(<3 x i1>)") + println(ir, "define i1 @smax_i1(<3 x i1> %v) {\n %r = call i1 @llvm.vector.reduce.smax.v3i1(<3 x i1> %v)\n ret i1 %r\n}") + mod = parse(LLVM.Module, String(take!(ir))) + insts(f) = [i for bb in blocks(f) for i in instructions(bb)] + callees(f) = [LLVM.name(called_operand(i)) for i in insts(f) if i isa LLVM.CallBase] + for f in functions(mod) + isdeclaration(f) && continue + GPUCompiler.lower_llvm_intrinsics!(job, f) + @test !any(startswith("llvm."), callees(f)) + end + # the ordered reduction continues from its start value, with the call's fast-math flags + f = functions(mod)["fadd3"] + adds = filter(i -> i isa LLVM.FAddInst, insts(f)) + @test length(adds) == 3 + @test operands(first(adds))[1] == parameters(f)[1] + @test all(i -> LLVM.fast_math(i).nnan, adds) + @test "air.fmin.f32" in callees(functions(mod)["fmin4"]) + @test isempty(callees(functions(mod)["smax_i1"])) # `and` on i1 lanes + @test (verify(mod); true) + end + + # end to end + @test @filecheck begin + @check_label "define i64 @{{(julia|j)_prod4_[0-9]+}}" + @check_not "@llvm.vector.reduce" + @check_count 3 "mul i64" + Metal.code_native(km.prod4, Tuple{NTuple{4, VecElement{Int64}}}) + end +end + @testset "integer intrinsic lowering" begin # The integer ops Julia emits as llvm.* are lowered to their AIR builtins, so Metal.jl need # not wrap them. Names/signatures verified against Apple's frontend: