From 150b19c8ae93a1d920ec7d335ba7d899fe4704c6 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 6 Oct 2026 17:01:20 +0200 Subject: [PATCH 1/4] Validate atomics for the GCN target Atomics reach the GCN back-end from AMDGPU.jl, UnsafeAtomics and Atomix, Enzyme and Julia's intrinsics, so check them on the IR like the PTX target does. Julia's LLVM compiles atomics wider than 64 bits or under-aligned ones to libatomic calls, which only fail when the code object is loaded (hipErrorNoBinaryForGpu); the external back-end rejects them without pointing at the Julia code that performs them. Atomics on constant, region or buffer resource memory and unknown synchronization scopes fail in the back-end or abort it. The cluster scope is accepted when the LLVM that generates code knows it (LLVM 22 and later), which for the external back-end is the version of AMDGPU_LLVM_Backend_jll rather than Julia's. --- src/gcn.jl | 80 ++++++++++++++++++++++++++++++++ test/gcn.jl | 108 ++++++++++++++++++++++++++++++++++++++++++++ test/helpers/gcn.jl | 5 +- 3 files changed, 191 insertions(+), 2 deletions(-) diff --git a/src/gcn.jl b/src/gcn.jl index cb4146f59..0655678e9 100644 --- a/src/gcn.jl +++ b/src/gcn.jl @@ -243,6 +243,86 @@ const AMDGPUObjectFile = Cint(1) return String(code) end +# the version of the LLVM that generates code for the target, which can differ from the one +# GPUCompiler runs in. only the release matters, so JLL rebuilds are ignored. +function backend_llvm_version(target::GCNCompilerTarget) + target.backend === :external || return LLVM.version() + jll = get(Base.loaded_modules, getfield(AMDGPU_LLVM_Backend_jll, :pkg), nothing) + # (without the back-end, `mcgen` reports a clearer error) + jll === nothing && return LLVM.version() + version = pkgversion(jll) + return VersionNumber(version.major, version.minor, version.patch) +end + + +## validation + +# Atomic operations and fences come from many front-ends (AMDGPU.jl's atomic functions, +# UnsafeAtomics and Atomix, Enzyme, Julia's atomic intrinsics), so they are validated here, +# on the IR, rather than in any of them. The AMDGPU back-end does not reject everything the +# target cannot run: Julia's LLVM emits calls to libatomic for oversized or misaligned +# atomics, which only fail when loading the code object, some atomics abort instruction +# selection, and the errors it does report don't point at the Julia code. +# Validation runs after `lower_syncscopes!`, so the scopes are the ones AMDGPU knows. +function validate_ir(job::CompilerJob{GCNCompilerTarget}, mod::LLVM.Module) + errors = IRError[] + dl = mod.datalayout + for f in mod.functions, bb in f.blocks, inst in bb.instructions + reason = if is_atomic_memop(inst) + gcn_atomic_error(job, dl, inst) + elseif inst isa LLVM.FenceInst + gcn_syncscope_error(job.config.target, inst, "fence") + else + nothing + end + reason === nothing || push!(errors, (reason, backtrace(inst), string(inst))) + end + return errors +end + +# the synchronization scopes of AMDGPU, as `lower_syncscopes!` leaves them, and their +# variants that only order the address space of the operation (`one-as`) +const GCN_SYNCSCOPES = ("singlethread", "wavefront", "workgroup", "agent", "system", + "singlethread-one-as", "wavefront-one-as", "workgroup-one-as", + "agent-one-as", "one-as") + +function gcn_syncscope_error(target::GCNCompilerTarget, inst::LLVM.Instruction, + what="atomic operation") + name = inst.syncscope.name + name in GCN_SYNCSCOPES && return nothing + # (a cluster is the agent on targets without workgroup clusters) + name in ("cluster", "cluster-one-as") && backend_llvm_version(target) >= v"22" && + return nothing + return "$what with synchronization scope $(repr(name))" +end + +# Why the target cannot run the atomic memory operation `inst`, or `nothing` if it can. +# Operations without an instruction (e.g. 8- and 16-bit ones, or ones on private memory) +# are fine: the back-end expands them. +function gcn_atomic_error(@nospecialize(job::CompilerJob{GCNCompilerTarget}), + dl::LLVM.DataLayout, inst::LLVM.Instruction) + reason = gcn_syncscope_error(job.config.target, inst) + reason === nothing || return reason + + # (constant memory isn't writable, and the region (GDS) and buffer resource address + # spaces abort instruction selection) + as = inst.pointer_operand.value_type.addrspace + if !(as in (0, 1, 3, 5, 7)) + return "atomic operation in address space $as (GCN only supports atomics on flat, global, local, private and buffer memory)" + end + + # (this includes packed floating-point values, which some targets support natively) + bits = Int(LLVM.bit_size(dl, atomic_value_type(inst))) + if bits > 64 + return "$bits-bit atomic operation (GCN supports atomics of at most 64 bits)" + end + if inst.alignment < bits ÷ 8 + return "atomic operation with alignment $(inst.alignment) (requires at least $(bits ÷ 8)-byte alignment)" + end + + return nothing +end + ## LLVM passes diff --git a/test/gcn.jl b/test/gcn.jl index 90ce90a77..9e865826e 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -84,6 +84,83 @@ end end end +@testset "atomic validation" begin + # validate textual IR, with AMDGPU's synchronization scopes, for a GCN target + function validate_atomics(body; backend=:inprocess) + source = methodinstance(typeof(identity), Tuple{Int}, Base.get_world_counter()) + target = GCNCompilerTarget(; dev_isa="gfx90a", backend) + job = CompilerJob(source, CompilerConfig(target, GCN.CompilerParams(); kernel=true)) + datalayout = @dispose dl=GPUCompiler.llvm_datalayout(target) begin + string(dl) + end + Context(; opaque_pointers=true) do ctx + mod = parse(LLVM.Module, """ + target datalayout = "$datalayout" + define void @kernel(ptr addrspace(1) %p, ptr addrspace(3) %s, + ptr addrspace(4) %c, ptr addrspace(5) %l, ptr %g) { + $body + ret void + }""") + join(first.(GPUCompiler.validate_ir(job, mod)), "\n") + end + end + + @testset "unsupported" begin + for (body, reason) in ( + ("%a = atomicrmw xchg ptr addrspace(1) %p, i128 1 syncscope(\"agent\") monotonic, align 16", + "128-bit atomic operation (GCN supports atomics of at most 64 bits)"), + ("%a = cmpxchg ptr %g, i128 0, i128 1 syncscope(\"agent\") monotonic monotonic, align 16", + "128-bit atomic operation"), + ("%a = load atomic i128, ptr addrspace(1) %p syncscope(\"agent\") acquire, align 16", + "128-bit atomic operation"), + ("store atomic fp128 0xL0, ptr addrspace(3) %s syncscope(\"workgroup\") release, align 16", + "128-bit atomic operation"), + ("%a = atomicrmw add ptr addrspace(1) %p, i64 1 syncscope(\"agent\") monotonic, align 4", + "atomic operation with alignment 4 (requires at least 8-byte alignment)"), + ("%a = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope(\"device\") monotonic, align 4", + "atomic operation with synchronization scope \"device\""), + ("%a = load atomic i32, ptr %g syncscope(\"system-one-as\") monotonic, align 4", + "atomic operation with synchronization scope \"system-one-as\""), + ("fence syncscope(\"subgroup\") seq_cst", + "fence with synchronization scope \"subgroup\""), + ("%a = load atomic i32, ptr addrspace(4) %c syncscope(\"agent\") monotonic, align 4", + "atomic operation in address space 4"), + ) + @test occursin(reason, validate_atomics(body)) + end + end + + @testset "supported" begin + for body in ( + "%a = atomicrmw add ptr addrspace(1) %p, i64 1 syncscope(\"agent\") monotonic, align 8", + "%a = cmpxchg ptr %g, i64 0, i64 1 seq_cst seq_cst, align 8", + "%a = load atomic ptr, ptr addrspace(1) %p unordered, align 8", + # operations the back-end expands + "%a = atomicrmw nand ptr addrspace(3) %s, i8 1 syncscope(\"workgroup\") monotonic, align 1", + "%a = atomicrmw fmax ptr %g, half 1.0 syncscope(\"wavefront\") monotonic, align 2", + # private memory, where atomics are plain memory operations + "%a = atomicrmw add ptr addrspace(5) %l, i32 1 syncscope(\"agent\") monotonic, align 4", + # scopes that only order the address space of the operation + "%a = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope(\"agent-one-as\") monotonic, align 4", + "fence syncscope(\"one-as\") seq_cst\nfence syncscope(\"singlethread-one-as\") acquire", + ) + @test validate_atomics(body) == "" + end + + # packed floating-point values (LLVM 19 added vector `atomicrmw fadd`) + if LLVM.version() >= v"19" + @test validate_atomics("%a = atomicrmw fadd ptr addrspace(1) %p, <2 x half> zeroinitializer syncscope(\"agent\") monotonic, align 4") == "" + end + end + + # the cluster scope (the agent on targets without clusters) is only known from LLVM 22 + cluster = "%a = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope(\"cluster\") monotonic, align 4\n" * + "fence syncscope(\"cluster-one-as\") acquire" + @test occursin("synchronization scope \"cluster\"", validate_atomics(cluster)) == + (LLVM.version() < v"22") + @test validate_atomics(cluster; backend=:external) == "" +end + @testset "kernel calling convention" begin mod = @eval module $(gensym()) kernel() = return @@ -296,6 +373,37 @@ end end end +@testset "atomic validation" begin + # atomics in Julia code are reported with the frame that performs them, for both + # back-ends (Julia's LLVM would emit a libatomic call, failing only at load time) + function add_kernel(T) + ptr = typed_ptrs ? "$T addrspace(1)*" : "ptr addrspace(1)" + ir = """ + define void @entry($ptr %p, $T %x) #0 { + %old = atomicrmw add $ptr %p, $T %x syncscope("agent") monotonic, align 16 + ret void + } + attributes #0 = { alwaysinline }""" + JT = T == "i128" ? Int128 : Int64 + mod = @eval module $(gensym()) + kernel(p::Core.LLVMPtr{$JT,1}, x::$JT) = + (Base.llvmcall(($ir, "entry"), Nothing, Tuple{Core.LLVMPtr{$JT,1},$JT}, p, x); + return) + end + Base.invokelatest(getfield, mod, :kernel), Tuple{Core.LLVMPtr{JT,1},JT} + end + f, tt = add_kernel("i128") + for backend in (:inprocess, :external) + @test_throws_message(InvalidIRError, + Base.invokelatest(GCN.code_execution, f, tt; backend)) do msg + occursin("Reason: unsupported 128-bit atomic operation", msg) && + occursin(r"\[\d+\] kernel", msg) + end + end + asm, _ = Base.invokelatest(GCN.code_execution, add_kernel("i64")...) + @test occursin("global_atomic_add_x2", asm) +end + @testset "s_load for kernarg struct access" begin mod = @eval module $(gensym()) struct MyStruct diff --git a/test/helpers/gcn.jl b/test/helpers/gcn.jl index 1af2dd362..c152dc2ac 100644 --- a/test/helpers/gcn.jl +++ b/test/helpers/gcn.jl @@ -14,12 +14,13 @@ GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime GPUCompiler.relocation_lowering(@nospecialize(job::CompilerJob{<:Any,CompilerParams})) = job.config.params.patch ? :patch : :bake -function create_job(@nospecialize(func), @nospecialize(types); backend::Symbol=:external, +function create_job(@nospecialize(func), @nospecialize(types); dev_isa="gfx900", + backend::Symbol=:external, minthreads=nothing, maxthreads=nothing, patch::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) - target = GCNCompilerTarget(dev_isa="gfx900"; backend, minthreads, maxthreads) + target = GCNCompilerTarget(; dev_isa, backend, minthreads, maxthreads) params = CompilerParams(patch) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs From 1442fa4043030e771dc0c13b8b79a22e9318b859 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 6 Oct 2026 17:13:46 +0200 Subject: [PATCH 2/4] Work around LLVM 22 failing to select atomic usub_sat on GCN Since LLVM 22, the AMDGPU back-end selects 32-bit `atomicrmw usub_sat` natively on gfx10.3 and later, but it has no pattern for flat memory, and on local memory only for gfx12, so it fails with "Cannot select" (llvm/llvm-project#229442). This breaks UnsafeAtomics' saturating subtraction on the external back-end (LLVM 23): on local memory for gfx10.3 and gfx11, and on flat memory for gfx12, or for gfx10.3 and gfx11 once the operation carries !amdgpu.no.fine.grained.memory. Expand these operations to compare-exchange loops after optimization, keyed on the version of the LLVM that generates code. Flat memory is expanded on every target, where LLVM otherwise chooses between the instruction and a loop based on scope and metadata. This runs whether or not the IR is validated. --- src/gcn.jl | 40 +++++++++++++++++++++++++++++++++++++++ test/gcn.jl | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/src/gcn.jl b/src/gcn.jl index 0655678e9..0fe553e32 100644 --- a/src/gcn.jl +++ b/src/gcn.jl @@ -114,6 +114,10 @@ function finish_ir!( end end end + + # after optimization, which can change address spaces, and also without validation + expand_atomics!(job, mod) + return entry end @@ -326,6 +330,42 @@ end ## LLVM passes +# Expand atomic read-modify-writes that the back-end cannot compile to compare-exchange +# loops. These work around bugs in the back-end, and should be revisited when updating it. +function expand_atomics!(@nospecialize(job::CompilerJob{GCNCompilerTarget}), + mod::LLVM.Module) + target = job.config.target + version = backend_llvm_version(target) + # (expansion changes the control flow, so collect the instructions first) + insts = LLVM.AtomicRMWInst[] + for f in mod.functions, bb in f.blocks, inst in bb.instructions + if inst isa LLVM.AtomicRMWInst && needs_cmpxchg_expansion(target, version, inst) + push!(insts, inst) + end + end + foreach(expand_to_cmpxchg!, insts) + return !isempty(insts) +end + +function needs_cmpxchg_expansion(target::GCNCompilerTarget, version::VersionNumber, + inst::LLVM.AtomicRMWInst) + as = inst.pointer_operand.value_type.addrspace + T = inst.value_operand.value_type + + # LLVM 22 selects 32-bit `usub_sat` on gfx10.3 and later, but lacks the pattern for + # flat memory, and for local memory before gfx12, failing with "Cannot select" + # (llvm/llvm-project#229442, unfixed as of LLVM 23). Expand it on flat memory on every + # target, where LLVM otherwise expands it depending on the scope and metadata. + if version >= v"22" && inst.binop == LLVM.AtomicRMWBinOp.USubSat && + T isa LLVM.IntegerType && T.width == 32 + as == 0 && return true + as == 3 && occursin(r"^gfx(103\d|11\d\d|10-3-generic|11(-\d+)?-generic)$", + target.dev_isa) && return true + end + + return false +end + function lower_throw_extra!(@nospecialize(job::CompilerJob), mod::LLVM.Module) changed = false @tracepoint "lower throw (extra)" begin diff --git a/test/gcn.jl b/test/gcn.jl index 9e865826e..655c86142 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -404,6 +404,60 @@ end @test occursin("global_atomic_add_x2", asm) end +LLVM.version() >= v"20" && @testset "usub_sat atomics" begin + # LLVM 22 and 23 fail to select 32-bit `usub_sat` on flat memory, and on local memory + # before gfx12 (llvm/llvm-project#229442), so it's expanded to a compare-exchange loop + function rmw_kernel(as, op="usub_sat") + ir = """ + define void @entry(ptr addrspace($as) %p, i32 %v) #0 { + %r = atomicrmw $op ptr addrspace($as) %p, i32 %v syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !0 + ret void + } + attributes #0 = { alwaysinline } + !0 = !{}""" + mod = @eval module $(gensym()) + kernel(p::Core.LLVMPtr{UInt32,$as}, v::UInt32) = + (Base.llvmcall(($ir, "entry"), Nothing, Tuple{Core.LLVMPtr{UInt32,$as}, UInt32}, + p, v); return) + end + Base.invokelatest(getfield, mod, :kernel), Tuple{Core.LLVMPtr{UInt32,as},UInt32} + end + + # (flat pointers in kernel arguments are global pointers to the back-end) + for (dev_isa, as, expanded, instruction) in ( + ("gfx1030", 3, true, "ds_cmpst_rtn_b32"), + ("gfx1100", 3, true, "ds_cmpstore_rtn_b32"), + ("gfx10-3-generic", 3, true, "ds_cmpst_rtn_b32"), + ("gfx11-generic", 3, true, "ds_cmpstore_rtn_b32"), + ("gfx1200", 3, false, "ds_sub_clamp_u32"), + ("gfx1030", 0, true, "flat_atomic_cmpswap"), + ("gfx1100", 0, true, "flat_atomic_cmpswap_b32"), + ("gfx1200", 0, true, "flat_atomic_cmpswap_b32"), + ("gfx90a", 0, true, "flat_atomic_cmpswap"), + ("gfx1100", 1, false, "global_atomic_csub_u32"), + ("gfx1200", 1, false, "global_atomic_sub_clamp_u32"), + ) + f, tt = rmw_kernel(as) + kernel = as != 0 + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel)) + @test occursin("atomicrmw usub_sat", ir) != expanded + asm = sprint(io->GCN.code_native(io, f, tt; dev_isa, kernel)) + @test occursin(instruction, asm) + + # Julia's LLVM, and other operations, are unaffected + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel, backend=:inprocess)) + @test occursin("atomicrmw usub_sat", ir) == (LLVM.version() < v"22" || !expanded) + f, tt = rmw_kernel(as, "usub_cond") + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel)) + @test occursin("atomicrmw usub_cond", ir) + end + + # the expansion is a workaround, not part of validation + f, tt = rmw_kernel(3) + asm = sprint(io->GCN.code_native(io, f, tt; dev_isa="gfx1030", kernel=true, validate=false)) + @test occursin("ds_cmpst_rtn_b32", asm) +end + @testset "s_load for kernarg struct access" begin mod = @eval module $(gensym()) struct MyStruct From d1d554689ceb07e1da28b179bac98b9d992a93a4 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 6 Oct 2026 17:51:02 +0200 Subject: [PATCH 3/4] Work around the GCN atomic optimizer crashing on sub-word atomics Before LLVM 21, AMDGPUAtomicOptimizer broadcasts the result of an 8- or 16-bit atomic read-modify-write on a uniform address with an illegal readfirstlane (fixed by llvm/llvm-project#128388). With Julia's own LLVM this aborts the process (LLVM 20, Julia 1.13) or crashes it (LLVM 18, Julia 1.12), e.g. for an Atomix `+=` on an Int8 array element whose result is used. The optimizer can only be disabled through a process-wide option, so expand the operations it handles to compare-exchange loops when their result is used, without predicting uniformity. This only affects the in-process back-end; the external one uses LLVM 23. --- src/gcn.jl | 14 +++++++++++++ test/gcn.jl | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/gcn.jl b/src/gcn.jl index 0fe553e32..76500a0c5 100644 --- a/src/gcn.jl +++ b/src/gcn.jl @@ -347,6 +347,11 @@ function expand_atomics!(@nospecialize(job::CompilerJob{GCNCompilerTarget}), return !isempty(insts) end +# the operations AMDGPUAtomicOptimizer combines across a wavefront +const OPTIMIZED_ATOMICRMW_OPS = let Op = LLVM.AtomicRMWBinOp + (Op.Add, Op.Sub, Op.And, Op.Or, Op.Xor, Op.Min, Op.Max, Op.UMin, Op.UMax) +end + function needs_cmpxchg_expansion(target::GCNCompilerTarget, version::VersionNumber, inst::LLVM.AtomicRMWInst) as = inst.pointer_operand.value_type.addrspace @@ -363,6 +368,15 @@ function needs_cmpxchg_expansion(target::GCNCompilerTarget, version::VersionNumb target.dev_isa) && return true end + # Before LLVM 21, the atomic optimizer broadcasts the result of 8- and 16-bit operations + # on uniform addresses with an illegal `readfirstlane`, crashing the back-end + # (llvm/llvm-project#128388). Expand the operations it optimizes when their result is + # used, without trying to predict whether the address is uniform. + if version < v"21" && T isa LLVM.IntegerType && T.width in (8, 16) && as in (0, 1, 3) && + inst.binop in OPTIMIZED_ATOMICRMW_OPS && !isempty(inst.uses) + return true + end + return false end diff --git a/test/gcn.jl b/test/gcn.jl index 655c86142..f0b6043e0 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -458,6 +458,65 @@ LLVM.version() >= v"20" && @testset "usub_sat atomics" begin @test occursin("ds_cmpst_rtn_b32", asm) end +@testset "sub-word atomics with a used result" begin + # before LLVM 21, the back-end crashes on 8- and 16-bit atomic operations on uniform + # addresses whose result is used (llvm/llvm-project#128388), so they are expanded to a + # compare-exchange loop + jltype(T) = T == "i8" ? Int8 : T == "i16" ? Int16 : Int32 + function rmw_source(T, op; used=true, as=1) + align = sizeof(jltype(T)) + ptr = typed_ptrs ? "$T addrspace($as)*" : "ptr addrspace($as)" + """ + define void @entry($ptr %p, $T %x) #0 { + %old = atomicrmw $op $ptr %p, $T %x syncscope("agent") monotonic, align $align + $(used ? "%q = getelementptr inbounds $T, $ptr %p, i64 1" : "") + $(used ? "store $T %old, $ptr %q, align $align" : "") + ret void + } + attributes #0 = { alwaysinline }""" + end + function rmw_ir(T, op; used=true, as=1, backend) + JT, ir = jltype(T), rmw_source(T, op; used, as) + mod = @eval module $(gensym()) + kernel(p::Core.LLVMPtr{$JT,$as}, x::$JT) = + (Base.llvmcall(($ir, "entry"), Nothing, Tuple{Core.LLVMPtr{$JT,$as},$JT}, p, x); + return) + end + kernel = Base.invokelatest(getfield, mod, :kernel) + Base.invokelatest(sprint, io->GCN.code_llvm(io, kernel, Tuple{Core.LLVMPtr{JT,as},JT}; + dev_isa="gfx1030", kernel=true, backend)) + end + + for (T, op, as) in (("i8", "add", 1), ("i16", "max", 3), ("i8", "umin", 0)) + @test occursin("atomicrmw $op", rmw_ir(T, op; as, backend=:inprocess)) == + (LLVM.version() >= v"21") + @test occursin("atomicrmw $op", rmw_ir(T, op; as, backend=:external)) + end + # other operations, unused results and wider atomics are unaffected + @test occursin("atomicrmw xchg", rmw_ir("i8", "xchg"; backend=:inprocess)) + @test occursin("atomicrmw add", rmw_ir("i8", "add"; used=false, backend=:inprocess)) + @test occursin("atomicrmw add", rmw_ir("i32", "add"; backend=:inprocess)) + + # the back-end crashes the process, so generate code in another one + script = """ + using GPUCompiler, LLVM + include($(repr(joinpath(@__DIR__, "helpers", "runtime.jl")))) + include($(repr(joinpath(@__DIR__, "helpers", "gcn.jl")))) + kernel_i8(p::Core.LLVMPtr{Int8,1}, x::Int8) = + (Base.llvmcall(($(repr(rmw_source("i8", "add"; as=1))), "entry"), Nothing, + Tuple{Core.LLVMPtr{Int8,1},Int8}, p, x); return) + kernel_i16(p::Core.LLVMPtr{Int16,3}, x::Int16) = + (Base.llvmcall(($(repr(rmw_source("i16", "max"; as=3))), "entry"), Nothing, + Tuple{Core.LLVMPtr{Int16,3},Int16}, p, x); return) + for (f, tt) in ((kernel_i8, Tuple{Core.LLVMPtr{Int8,1},Int8}), + (kernel_i16, Tuple{Core.LLVMPtr{Int16,3},Int16})) + GCN.code_native(devnull, f, tt; dev_isa="gfx1030", kernel=true, backend=:inprocess) + end + """ + cmd = `$(Base.julia_cmd()) --project=$(Base.active_project()) -e $script` + @test success(pipeline(cmd; stdout, stderr)) +end + @testset "s_load for kernarg struct access" begin mod = @eval module $(gensym()) struct MyStruct From 37e1d20a0d3aa4d0786d6a4e72e2fd648b998599 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 7 Oct 2026 07:06:10 +0200 Subject: [PATCH 4/4] Only work around the GCN usub_sat selection failure with Julia's LLVM. AMDGPU_LLVM_Backend_jll 23.1.3 carries the fix for llvm/llvm-project#229442, so require it and keep expanding 32-bit usub_sat to a compare-exchange loop only for the in-process back-end. On gfx12 flat memory this now selects the native flat_atomic_sub_clamp_u32. --- Project.toml | 2 +- src/gcn.jl | 9 +++++---- test/gcn.jl | 33 ++++++++++++++++++++++----------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/Project.toml b/Project.toml index 25a026e63..bcd9fc8c8 100644 --- a/Project.toml +++ b/Project.toml @@ -37,7 +37,7 @@ SPIRV_LLVM_Translator_jll = "4a5d46fc-d8cf-5151-a261-86b458210efb" HighlightsExt = "Highlights" [compat] -AMDGPU_LLVM_Backend_jll = "23" +AMDGPU_LLVM_Backend_jll = "23.1.3" CompilerCaching = "0.4, 0.5" ExprTools = "0.1" Highlights = "0.6" diff --git a/src/gcn.jl b/src/gcn.jl index 76500a0c5..f67955024 100644 --- a/src/gcn.jl +++ b/src/gcn.jl @@ -359,10 +359,11 @@ function needs_cmpxchg_expansion(target::GCNCompilerTarget, version::VersionNumb # LLVM 22 selects 32-bit `usub_sat` on gfx10.3 and later, but lacks the pattern for # flat memory, and for local memory before gfx12, failing with "Cannot select" - # (llvm/llvm-project#229442, unfixed as of LLVM 23). Expand it on flat memory on every - # target, where LLVM otherwise expands it depending on the scope and metadata. - if version >= v"22" && inst.binop == LLVM.AtomicRMWBinOp.USubSat && - T isa LLVM.IntegerType && T.width == 32 + # (llvm/llvm-project#229442). Expand it on flat memory on every target, where LLVM + # otherwise expands it depending on the scope and metadata. AMDGPU_LLVM_Backend_jll + # carries the fix since 23.1.3, so only Julia's own LLVM needs this. + if target.backend === :inprocess && version >= v"22" && + inst.binop == LLVM.AtomicRMWBinOp.USubSat && T isa LLVM.IntegerType && T.width == 32 as == 0 && return true as == 3 && occursin(r"^gfx(103\d|11\d\d|10-3-generic|11(-\d+)?-generic)$", target.dev_isa) && return true diff --git a/test/gcn.jl b/test/gcn.jl index f0b6043e0..49cae0350 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -405,8 +405,9 @@ end end LLVM.version() >= v"20" && @testset "usub_sat atomics" begin - # LLVM 22 and 23 fail to select 32-bit `usub_sat` on flat memory, and on local memory - # before gfx12 (llvm/llvm-project#229442), so it's expanded to a compare-exchange loop + # LLVM 22 fails to select 32-bit `usub_sat` on flat memory, and on local memory before + # gfx12 (llvm/llvm-project#229442). AMDGPU_LLVM_Backend_jll carries the fix, but with + # Julia's own LLVM the operation is expanded to a compare-exchange loop function rmw_kernel(as, op="usub_sat") ir = """ define void @entry(ptr addrspace($as) %p, i32 %v) #0 { @@ -424,7 +425,7 @@ LLVM.version() >= v"20" && @testset "usub_sat atomics" begin end # (flat pointers in kernel arguments are global pointers to the back-end) - for (dev_isa, as, expanded, instruction) in ( + for (dev_isa, as, affected, instruction) in ( ("gfx1030", 3, true, "ds_cmpst_rtn_b32"), ("gfx1100", 3, true, "ds_cmpstore_rtn_b32"), ("gfx10-3-generic", 3, true, "ds_cmpst_rtn_b32"), @@ -432,30 +433,40 @@ LLVM.version() >= v"20" && @testset "usub_sat atomics" begin ("gfx1200", 3, false, "ds_sub_clamp_u32"), ("gfx1030", 0, true, "flat_atomic_cmpswap"), ("gfx1100", 0, true, "flat_atomic_cmpswap_b32"), - ("gfx1200", 0, true, "flat_atomic_cmpswap_b32"), + ("gfx1200", 0, true, "flat_atomic_sub_clamp_u32"), ("gfx90a", 0, true, "flat_atomic_cmpswap"), ("gfx1100", 1, false, "global_atomic_csub_u32"), ("gfx1200", 1, false, "global_atomic_sub_clamp_u32"), ) f, tt = rmw_kernel(as) kernel = as != 0 + + # the back-end compiles the operation, expanding it where the hardware lacks it ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel)) - @test occursin("atomicrmw usub_sat", ir) != expanded + @test occursin("atomicrmw usub_sat", ir) asm = sprint(io->GCN.code_native(io, f, tt; dev_isa, kernel)) @test occursin(instruction, asm) - # Julia's LLVM, and other operations, are unaffected + # Julia's LLVM needs the workaround from version 22 on ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel, backend=:inprocess)) - @test occursin("atomicrmw usub_sat", ir) == (LLVM.version() < v"22" || !expanded) + @test occursin("atomicrmw usub_sat", ir) == (LLVM.version() < v"22" || !affected) + + # other operations are unaffected f, tt = rmw_kernel(as, "usub_cond") - ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel)) + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel, backend=:inprocess)) @test occursin("atomicrmw usub_cond", ir) end # the expansion is a workaround, not part of validation - f, tt = rmw_kernel(3) - asm = sprint(io->GCN.code_native(io, f, tt; dev_isa="gfx1030", kernel=true, validate=false)) - @test occursin("ds_cmpst_rtn_b32", asm) + if LLVM.version() >= v"22" + f, tt = rmw_kernel(3) + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa="gfx1030", kernel=true, + backend=:inprocess, validate=false)) + @test !occursin("atomicrmw usub_sat", ir) + asm = sprint(io->GCN.code_native(io, f, tt; dev_isa="gfx1030", kernel=true, + backend=:inprocess, validate=false)) + @test occursin("ds_cmpst_rtn_b32", asm) + end end @testset "sub-word atomics with a used result" begin