diff --git a/Project.toml b/Project.toml index 25a026e6..bcd9fc8c 100644 --- a/Project.toml +++ b/Project.toml @@ -37,7 +37,7 @@ SPIRV_LLVM_Translator_jll = "4a5d46fc-d8cf-5151-a261-86b458210efb" HighlightsExt = "Highlights" [compat] -AMDGPU_LLVM_Backend_jll = "23" +AMDGPU_LLVM_Backend_jll = "23.1.3" CompilerCaching = "0.4, 0.5" ExprTools = "0.1" Highlights = "0.6" diff --git a/src/gcn.jl b/src/gcn.jl index cb4146f5..f6795502 100644 --- a/src/gcn.jl +++ b/src/gcn.jl @@ -114,6 +114,10 @@ function finish_ir!( end end end + + # after optimization, which can change address spaces, and also without validation + expand_atomics!(job, mod) + return entry end @@ -243,9 +247,140 @@ const AMDGPUObjectFile = Cint(1) return String(code) end +# the version of the LLVM that generates code for the target, which can differ from the one +# GPUCompiler runs in. only the release matters, so JLL rebuilds are ignored. +function backend_llvm_version(target::GCNCompilerTarget) + target.backend === :external || return LLVM.version() + jll = get(Base.loaded_modules, getfield(AMDGPU_LLVM_Backend_jll, :pkg), nothing) + # (without the back-end, `mcgen` reports a clearer error) + jll === nothing && return LLVM.version() + version = pkgversion(jll) + return VersionNumber(version.major, version.minor, version.patch) +end + + +## validation + +# Atomic operations and fences come from many front-ends (AMDGPU.jl's atomic functions, +# UnsafeAtomics and Atomix, Enzyme, Julia's atomic intrinsics), so they are validated here, +# on the IR, rather than in any of them. The AMDGPU back-end does not reject everything the +# target cannot run: Julia's LLVM emits calls to libatomic for oversized or misaligned +# atomics, which only fail when loading the code object, some atomics abort instruction +# selection, and the errors it does report don't point at the Julia code. +# Validation runs after `lower_syncscopes!`, so the scopes are the ones AMDGPU knows. +function validate_ir(job::CompilerJob{GCNCompilerTarget}, mod::LLVM.Module) + errors = IRError[] + dl = mod.datalayout + for f in mod.functions, bb in f.blocks, inst in bb.instructions + reason = if is_atomic_memop(inst) + gcn_atomic_error(job, dl, inst) + elseif inst isa LLVM.FenceInst + gcn_syncscope_error(job.config.target, inst, "fence") + else + nothing + end + reason === nothing || push!(errors, (reason, backtrace(inst), string(inst))) + end + return errors +end + +# the synchronization scopes of AMDGPU, as `lower_syncscopes!` leaves them, and their +# variants that only order the address space of the operation (`one-as`) +const GCN_SYNCSCOPES = ("singlethread", "wavefront", "workgroup", "agent", "system", + "singlethread-one-as", "wavefront-one-as", "workgroup-one-as", + "agent-one-as", "one-as") + +function gcn_syncscope_error(target::GCNCompilerTarget, inst::LLVM.Instruction, + what="atomic operation") + name = inst.syncscope.name + name in GCN_SYNCSCOPES && return nothing + # (a cluster is the agent on targets without workgroup clusters) + name in ("cluster", "cluster-one-as") && backend_llvm_version(target) >= v"22" && + return nothing + return "$what with synchronization scope $(repr(name))" +end + +# Why the target cannot run the atomic memory operation `inst`, or `nothing` if it can. +# Operations without an instruction (e.g. 8- and 16-bit ones, or ones on private memory) +# are fine: the back-end expands them. +function gcn_atomic_error(@nospecialize(job::CompilerJob{GCNCompilerTarget}), + dl::LLVM.DataLayout, inst::LLVM.Instruction) + reason = gcn_syncscope_error(job.config.target, inst) + reason === nothing || return reason + + # (constant memory isn't writable, and the region (GDS) and buffer resource address + # spaces abort instruction selection) + as = inst.pointer_operand.value_type.addrspace + if !(as in (0, 1, 3, 5, 7)) + return "atomic operation in address space $as (GCN only supports atomics on flat, global, local, private and buffer memory)" + end + + # (this includes packed floating-point values, which some targets support natively) + bits = Int(LLVM.bit_size(dl, atomic_value_type(inst))) + if bits > 64 + return "$bits-bit atomic operation (GCN supports atomics of at most 64 bits)" + end + if inst.alignment < bits ÷ 8 + return "atomic operation with alignment $(inst.alignment) (requires at least $(bits ÷ 8)-byte alignment)" + end + + return nothing +end + ## LLVM passes +# Expand atomic read-modify-writes that the back-end cannot compile to compare-exchange +# loops. These work around bugs in the back-end, and should be revisited when updating it. +function expand_atomics!(@nospecialize(job::CompilerJob{GCNCompilerTarget}), + mod::LLVM.Module) + target = job.config.target + version = backend_llvm_version(target) + # (expansion changes the control flow, so collect the instructions first) + insts = LLVM.AtomicRMWInst[] + for f in mod.functions, bb in f.blocks, inst in bb.instructions + if inst isa LLVM.AtomicRMWInst && needs_cmpxchg_expansion(target, version, inst) + push!(insts, inst) + end + end + foreach(expand_to_cmpxchg!, insts) + return !isempty(insts) +end + +# the operations AMDGPUAtomicOptimizer combines across a wavefront +const OPTIMIZED_ATOMICRMW_OPS = let Op = LLVM.AtomicRMWBinOp + (Op.Add, Op.Sub, Op.And, Op.Or, Op.Xor, Op.Min, Op.Max, Op.UMin, Op.UMax) +end + +function needs_cmpxchg_expansion(target::GCNCompilerTarget, version::VersionNumber, + inst::LLVM.AtomicRMWInst) + as = inst.pointer_operand.value_type.addrspace + T = inst.value_operand.value_type + + # LLVM 22 selects 32-bit `usub_sat` on gfx10.3 and later, but lacks the pattern for + # flat memory, and for local memory before gfx12, failing with "Cannot select" + # (llvm/llvm-project#229442). Expand it on flat memory on every target, where LLVM + # otherwise expands it depending on the scope and metadata. AMDGPU_LLVM_Backend_jll + # carries the fix since 23.1.3, so only Julia's own LLVM needs this. + if target.backend === :inprocess && version >= v"22" && + inst.binop == LLVM.AtomicRMWBinOp.USubSat && T isa LLVM.IntegerType && T.width == 32 + as == 0 && return true + as == 3 && occursin(r"^gfx(103\d|11\d\d|10-3-generic|11(-\d+)?-generic)$", + target.dev_isa) && return true + end + + # Before LLVM 21, the atomic optimizer broadcasts the result of 8- and 16-bit operations + # on uniform addresses with an illegal `readfirstlane`, crashing the back-end + # (llvm/llvm-project#128388). Expand the operations it optimizes when their result is + # used, without trying to predict whether the address is uniform. + if version < v"21" && T isa LLVM.IntegerType && T.width in (8, 16) && as in (0, 1, 3) && + inst.binop in OPTIMIZED_ATOMICRMW_OPS && !isempty(inst.uses) + return true + end + + return false +end + function lower_throw_extra!(@nospecialize(job::CompilerJob), mod::LLVM.Module) changed = false @tracepoint "lower throw (extra)" begin diff --git a/test/gcn.jl b/test/gcn.jl index 90ce90a7..49cae035 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -84,6 +84,83 @@ end end end +@testset "atomic validation" begin + # validate textual IR, with AMDGPU's synchronization scopes, for a GCN target + function validate_atomics(body; backend=:inprocess) + source = methodinstance(typeof(identity), Tuple{Int}, Base.get_world_counter()) + target = GCNCompilerTarget(; dev_isa="gfx90a", backend) + job = CompilerJob(source, CompilerConfig(target, GCN.CompilerParams(); kernel=true)) + datalayout = @dispose dl=GPUCompiler.llvm_datalayout(target) begin + string(dl) + end + Context(; opaque_pointers=true) do ctx + mod = parse(LLVM.Module, """ + target datalayout = "$datalayout" + define void @kernel(ptr addrspace(1) %p, ptr addrspace(3) %s, + ptr addrspace(4) %c, ptr addrspace(5) %l, ptr %g) { + $body + ret void + }""") + join(first.(GPUCompiler.validate_ir(job, mod)), "\n") + end + end + + @testset "unsupported" begin + for (body, reason) in ( + ("%a = atomicrmw xchg ptr addrspace(1) %p, i128 1 syncscope(\"agent\") monotonic, align 16", + "128-bit atomic operation (GCN supports atomics of at most 64 bits)"), + ("%a = cmpxchg ptr %g, i128 0, i128 1 syncscope(\"agent\") monotonic monotonic, align 16", + "128-bit atomic operation"), + ("%a = load atomic i128, ptr addrspace(1) %p syncscope(\"agent\") acquire, align 16", + "128-bit atomic operation"), + ("store atomic fp128 0xL0, ptr addrspace(3) %s syncscope(\"workgroup\") release, align 16", + "128-bit atomic operation"), + ("%a = atomicrmw add ptr addrspace(1) %p, i64 1 syncscope(\"agent\") monotonic, align 4", + "atomic operation with alignment 4 (requires at least 8-byte alignment)"), + ("%a = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope(\"device\") monotonic, align 4", + "atomic operation with synchronization scope \"device\""), + ("%a = load atomic i32, ptr %g syncscope(\"system-one-as\") monotonic, align 4", + "atomic operation with synchronization scope \"system-one-as\""), + ("fence syncscope(\"subgroup\") seq_cst", + "fence with synchronization scope \"subgroup\""), + ("%a = load atomic i32, ptr addrspace(4) %c syncscope(\"agent\") monotonic, align 4", + "atomic operation in address space 4"), + ) + @test occursin(reason, validate_atomics(body)) + end + end + + @testset "supported" begin + for body in ( + "%a = atomicrmw add ptr addrspace(1) %p, i64 1 syncscope(\"agent\") monotonic, align 8", + "%a = cmpxchg ptr %g, i64 0, i64 1 seq_cst seq_cst, align 8", + "%a = load atomic ptr, ptr addrspace(1) %p unordered, align 8", + # operations the back-end expands + "%a = atomicrmw nand ptr addrspace(3) %s, i8 1 syncscope(\"workgroup\") monotonic, align 1", + "%a = atomicrmw fmax ptr %g, half 1.0 syncscope(\"wavefront\") monotonic, align 2", + # private memory, where atomics are plain memory operations + "%a = atomicrmw add ptr addrspace(5) %l, i32 1 syncscope(\"agent\") monotonic, align 4", + # scopes that only order the address space of the operation + "%a = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope(\"agent-one-as\") monotonic, align 4", + "fence syncscope(\"one-as\") seq_cst\nfence syncscope(\"singlethread-one-as\") acquire", + ) + @test validate_atomics(body) == "" + end + + # packed floating-point values (LLVM 19 added vector `atomicrmw fadd`) + if LLVM.version() >= v"19" + @test validate_atomics("%a = atomicrmw fadd ptr addrspace(1) %p, <2 x half> zeroinitializer syncscope(\"agent\") monotonic, align 4") == "" + end + end + + # the cluster scope (the agent on targets without clusters) is only known from LLVM 22 + cluster = "%a = atomicrmw add ptr addrspace(1) %p, i32 1 syncscope(\"cluster\") monotonic, align 4\n" * + "fence syncscope(\"cluster-one-as\") acquire" + @test occursin("synchronization scope \"cluster\"", validate_atomics(cluster)) == + (LLVM.version() < v"22") + @test validate_atomics(cluster; backend=:external) == "" +end + @testset "kernel calling convention" begin mod = @eval module $(gensym()) kernel() = return @@ -296,6 +373,161 @@ end end end +@testset "atomic validation" begin + # atomics in Julia code are reported with the frame that performs them, for both + # back-ends (Julia's LLVM would emit a libatomic call, failing only at load time) + function add_kernel(T) + ptr = typed_ptrs ? "$T addrspace(1)*" : "ptr addrspace(1)" + ir = """ + define void @entry($ptr %p, $T %x) #0 { + %old = atomicrmw add $ptr %p, $T %x syncscope("agent") monotonic, align 16 + ret void + } + attributes #0 = { alwaysinline }""" + JT = T == "i128" ? Int128 : Int64 + mod = @eval module $(gensym()) + kernel(p::Core.LLVMPtr{$JT,1}, x::$JT) = + (Base.llvmcall(($ir, "entry"), Nothing, Tuple{Core.LLVMPtr{$JT,1},$JT}, p, x); + return) + end + Base.invokelatest(getfield, mod, :kernel), Tuple{Core.LLVMPtr{JT,1},JT} + end + f, tt = add_kernel("i128") + for backend in (:inprocess, :external) + @test_throws_message(InvalidIRError, + Base.invokelatest(GCN.code_execution, f, tt; backend)) do msg + occursin("Reason: unsupported 128-bit atomic operation", msg) && + occursin(r"\[\d+\] kernel", msg) + end + end + asm, _ = Base.invokelatest(GCN.code_execution, add_kernel("i64")...) + @test occursin("global_atomic_add_x2", asm) +end + +LLVM.version() >= v"20" && @testset "usub_sat atomics" begin + # LLVM 22 fails to select 32-bit `usub_sat` on flat memory, and on local memory before + # gfx12 (llvm/llvm-project#229442). AMDGPU_LLVM_Backend_jll carries the fix, but with + # Julia's own LLVM the operation is expanded to a compare-exchange loop + function rmw_kernel(as, op="usub_sat") + ir = """ + define void @entry(ptr addrspace($as) %p, i32 %v) #0 { + %r = atomicrmw $op ptr addrspace($as) %p, i32 %v syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !0 + ret void + } + attributes #0 = { alwaysinline } + !0 = !{}""" + mod = @eval module $(gensym()) + kernel(p::Core.LLVMPtr{UInt32,$as}, v::UInt32) = + (Base.llvmcall(($ir, "entry"), Nothing, Tuple{Core.LLVMPtr{UInt32,$as}, UInt32}, + p, v); return) + end + Base.invokelatest(getfield, mod, :kernel), Tuple{Core.LLVMPtr{UInt32,as},UInt32} + end + + # (flat pointers in kernel arguments are global pointers to the back-end) + for (dev_isa, as, affected, instruction) in ( + ("gfx1030", 3, true, "ds_cmpst_rtn_b32"), + ("gfx1100", 3, true, "ds_cmpstore_rtn_b32"), + ("gfx10-3-generic", 3, true, "ds_cmpst_rtn_b32"), + ("gfx11-generic", 3, true, "ds_cmpstore_rtn_b32"), + ("gfx1200", 3, false, "ds_sub_clamp_u32"), + ("gfx1030", 0, true, "flat_atomic_cmpswap"), + ("gfx1100", 0, true, "flat_atomic_cmpswap_b32"), + ("gfx1200", 0, true, "flat_atomic_sub_clamp_u32"), + ("gfx90a", 0, true, "flat_atomic_cmpswap"), + ("gfx1100", 1, false, "global_atomic_csub_u32"), + ("gfx1200", 1, false, "global_atomic_sub_clamp_u32"), + ) + f, tt = rmw_kernel(as) + kernel = as != 0 + + # the back-end compiles the operation, expanding it where the hardware lacks it + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel)) + @test occursin("atomicrmw usub_sat", ir) + asm = sprint(io->GCN.code_native(io, f, tt; dev_isa, kernel)) + @test occursin(instruction, asm) + + # Julia's LLVM needs the workaround from version 22 on + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel, backend=:inprocess)) + @test occursin("atomicrmw usub_sat", ir) == (LLVM.version() < v"22" || !affected) + + # other operations are unaffected + f, tt = rmw_kernel(as, "usub_cond") + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa, kernel, backend=:inprocess)) + @test occursin("atomicrmw usub_cond", ir) + end + + # the expansion is a workaround, not part of validation + if LLVM.version() >= v"22" + f, tt = rmw_kernel(3) + ir = sprint(io->GCN.code_llvm(io, f, tt; dev_isa="gfx1030", kernel=true, + backend=:inprocess, validate=false)) + @test !occursin("atomicrmw usub_sat", ir) + asm = sprint(io->GCN.code_native(io, f, tt; dev_isa="gfx1030", kernel=true, + backend=:inprocess, validate=false)) + @test occursin("ds_cmpst_rtn_b32", asm) + end +end + +@testset "sub-word atomics with a used result" begin + # before LLVM 21, the back-end crashes on 8- and 16-bit atomic operations on uniform + # addresses whose result is used (llvm/llvm-project#128388), so they are expanded to a + # compare-exchange loop + jltype(T) = T == "i8" ? Int8 : T == "i16" ? Int16 : Int32 + function rmw_source(T, op; used=true, as=1) + align = sizeof(jltype(T)) + ptr = typed_ptrs ? "$T addrspace($as)*" : "ptr addrspace($as)" + """ + define void @entry($ptr %p, $T %x) #0 { + %old = atomicrmw $op $ptr %p, $T %x syncscope("agent") monotonic, align $align + $(used ? "%q = getelementptr inbounds $T, $ptr %p, i64 1" : "") + $(used ? "store $T %old, $ptr %q, align $align" : "") + ret void + } + attributes #0 = { alwaysinline }""" + end + function rmw_ir(T, op; used=true, as=1, backend) + JT, ir = jltype(T), rmw_source(T, op; used, as) + mod = @eval module $(gensym()) + kernel(p::Core.LLVMPtr{$JT,$as}, x::$JT) = + (Base.llvmcall(($ir, "entry"), Nothing, Tuple{Core.LLVMPtr{$JT,$as},$JT}, p, x); + return) + end + kernel = Base.invokelatest(getfield, mod, :kernel) + Base.invokelatest(sprint, io->GCN.code_llvm(io, kernel, Tuple{Core.LLVMPtr{JT,as},JT}; + dev_isa="gfx1030", kernel=true, backend)) + end + + for (T, op, as) in (("i8", "add", 1), ("i16", "max", 3), ("i8", "umin", 0)) + @test occursin("atomicrmw $op", rmw_ir(T, op; as, backend=:inprocess)) == + (LLVM.version() >= v"21") + @test occursin("atomicrmw $op", rmw_ir(T, op; as, backend=:external)) + end + # other operations, unused results and wider atomics are unaffected + @test occursin("atomicrmw xchg", rmw_ir("i8", "xchg"; backend=:inprocess)) + @test occursin("atomicrmw add", rmw_ir("i8", "add"; used=false, backend=:inprocess)) + @test occursin("atomicrmw add", rmw_ir("i32", "add"; backend=:inprocess)) + + # the back-end crashes the process, so generate code in another one + script = """ + using GPUCompiler, LLVM + include($(repr(joinpath(@__DIR__, "helpers", "runtime.jl")))) + include($(repr(joinpath(@__DIR__, "helpers", "gcn.jl")))) + kernel_i8(p::Core.LLVMPtr{Int8,1}, x::Int8) = + (Base.llvmcall(($(repr(rmw_source("i8", "add"; as=1))), "entry"), Nothing, + Tuple{Core.LLVMPtr{Int8,1},Int8}, p, x); return) + kernel_i16(p::Core.LLVMPtr{Int16,3}, x::Int16) = + (Base.llvmcall(($(repr(rmw_source("i16", "max"; as=3))), "entry"), Nothing, + Tuple{Core.LLVMPtr{Int16,3},Int16}, p, x); return) + for (f, tt) in ((kernel_i8, Tuple{Core.LLVMPtr{Int8,1},Int8}), + (kernel_i16, Tuple{Core.LLVMPtr{Int16,3},Int16})) + GCN.code_native(devnull, f, tt; dev_isa="gfx1030", kernel=true, backend=:inprocess) + end + """ + cmd = `$(Base.julia_cmd()) --project=$(Base.active_project()) -e $script` + @test success(pipeline(cmd; stdout, stderr)) +end + @testset "s_load for kernarg struct access" begin mod = @eval module $(gensym()) struct MyStruct diff --git a/test/helpers/gcn.jl b/test/helpers/gcn.jl index 1af2dd36..c152dc2a 100644 --- a/test/helpers/gcn.jl +++ b/test/helpers/gcn.jl @@ -14,12 +14,13 @@ GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime GPUCompiler.relocation_lowering(@nospecialize(job::CompilerJob{<:Any,CompilerParams})) = job.config.params.patch ? :patch : :bake -function create_job(@nospecialize(func), @nospecialize(types); backend::Symbol=:external, +function create_job(@nospecialize(func), @nospecialize(types); dev_isa="gfx900", + backend::Symbol=:external, minthreads=nothing, maxthreads=nothing, patch::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) - target = GCNCompilerTarget(dev_isa="gfx900"; backend, minthreads, maxthreads) + target = GCNCompilerTarget(; dev_isa, backend, minthreads, maxthreads) params = CompilerParams(patch) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs