diff --git a/Project.toml b/Project.toml index 8f6a97cb..d4ad4bda 100644 --- a/Project.toml +++ b/Project.toml @@ -1,12 +1,13 @@ name = "GPUCompiler" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "1.22.7" +version = "2.0.0" authors = ["Tim Besard "] [workspace] projects = ["test"] [deps] +CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" ExprTools = "e2ba6199-217a-4e67-a87a-7c52f15ade04" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" @@ -15,8 +16,6 @@ Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Preferences = "21216c6a-2e73-6563-6e65-726566657250" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -Scratch = "6c6a2e73-6563-6170-7368-637461726353" -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tracy = "e689c965-62c8-4b79-b2c5-8359227902fd" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -26,8 +25,10 @@ AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444" NVPTX_LLVM_Backend_jll = "ef6e0fe3-e6ef-59c0-bde6-4989574699e0" + [compat] AMDGPU_LLVM_Backend_jll = "22" +CompilerCaching = "0.3" ExprTools = "0.1" InteractiveUtils = "1" LLVM = "9.9" @@ -38,8 +39,6 @@ NVPTX_LLVM_Backend_jll = "22" PrecompileTools = "1" Preferences = "1" REPL = "1" -Scratch = "1" -Serialization = "1" TOML = "1" Tracy = "0.1.4" UUIDs = "1" diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index 2e646267..49017261 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -8,8 +8,6 @@ using ExprTools: splitdef, combinedef using Libdl -using Serialization -using Scratch: @get_scratch! using Preferences const ENABLE_TRACY = parse(Bool, @load_preference("tracy", "false")) @@ -35,8 +33,21 @@ end const CC = Core.Compiler using Core: MethodInstance, CodeInstance, CodeInfo -compile_cache = nothing # set during __init__() -const pkgver = Base.pkgversion(GPUCompiler) +# `HAS_INTEGRATED_CACHE` distinguishes 1.11+ (owner-keyed `Core.Compiler.InternalCodeCache`) +# from 1.10 (per-interpreter `CodeCache` IdDict + invalidation callbacks). The two have +# disjoint shapes; everything cache-related fans on this flag. +const HAS_INTEGRATED_CACHE = VERSION >= v"1.11.0-DEV.1552" + +# Loads as an empty shell on 1.10; on 1.11+ provides `CacheView`, `typeinf!`, +# `get_codeinfos`, `lookup`, `results`, etc. We `import` the module name (not its +# exports) to avoid clashing with `LLVM.lookup`; internal call sites qualify with +# `CompilerCaching.`. +import CompilerCaching + +# Optional callback invoked from `compile(...)` / `cached_compilation(...)` before +# compilation runs. Set by `@device_code_*` reflection macros. Defined here (early) +# so the legacy `cached_compilation` in deprecated.jl can reference it. +const compile_hook = Ref{Union{Nothing,Function}}(nothing) include("utils.jl") include("mangling.jl") @@ -54,6 +65,7 @@ include("metal.jl") include("runtime.jl") # compiler implementation +include("deprecated.jl") include("jlgen.jl") include("irgen.jl") include("optim.jl") @@ -67,24 +79,19 @@ include("driver.jl") include("execution.jl") include("reflection.jl") -include("deprecated.jl") - include("precompile.jl") function __init__() STDERR_HAS_COLOR[] = get(stderr, :color, false) - dir = @get_scratch!("compiled") - ## add the Julia version - dir = joinpath(dir, "v$(VERSION.major).$(VERSION.minor)") - ## also add the package version - if pkgver !== nothing - # XXX: `Base.pkgversion` is buggy and sometimes returns `nothing`, see e.g. - # JuliaLang/PackageCompiler.jl#896 and JuliaGPU/GPUCompiler.jl#593 - dir = joinpath(dir, "v$(pkgver.major).$(pkgver.minor)") + @static if !HAS_INTEGRATED_CACHE + # session-local results keyed by objectid+world; entries serialized during + # GPUCompiler's own precompilation can never be valid in a later session + empty!(job_results) + # ditto for the in-process CodeCaches: CIs deposited by our own precompile + # workload carry world ages from the precompilation process + empty!(GLOBAL_CI_CACHES) end - mkpath(dir) - global compile_cache = dir @static if ENABLE_TRACY Tracy.@register_tracepoints() diff --git a/src/bpf.jl b/src/bpf.jl index 80070d6d..ff75ea23 100644 --- a/src/bpf.jl +++ b/src/bpf.jl @@ -26,8 +26,6 @@ end ## job -runtime_slug(job::CompilerJob{BPFCompilerTarget}) = "bpf" - const bpf_intrinsics = () # TODO isintrinsic(::CompilerJob{BPFCompilerTarget}, fn::String) = in(fn, bpf_intrinsics) diff --git a/src/deprecated.jl b/src/deprecated.jl index c5848854..5b5e0627 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -1,45 +1,327 @@ -# Deprecations scheduled for removal in the next major release. +# Legacy and deprecated functionality. +# +# This file houses code paths that are kept for Julia 1.10 (which lacks the integrated +# CodeInstance cache and `analysis_results`) plus older interfaces retained for the 2.0 +# breaking release. Everything here is expected to go away once 1.10 falls off the LTS +# line and downstream packages have migrated. + +# `WorldView` is used by the integrated-cache JIT lookup callback (1.11–1.13) and by the +# legacy `CodeCache` interface (1.10). Moved out of `Core.Compiler` on 1.14+. +@static if VERSION < v"1.14-" + using Core.Compiler: WorldView +end + + +## Julia 1.10: in-process `CodeCache` + WorldView + +@static if !HAS_INTEGRATED_CACHE + +""" + CodeCache + +Per-job CodeInstance store used on Julia 1.10, where there's no integrated cache and no +`cache_owner` partition on `CodeInstance`. Inference deposits CIs via a `WorldView` over +this store; invalidation callbacks attached to each `MethodInstance` clip `max_world` on +stale CIs when methods are redefined. +""" +struct CodeCache + dict::IdDict{MethodInstance, Vector{CodeInstance}} + + CodeCache() = new(IdDict{MethodInstance, Vector{CodeInstance}}()) +end + +function Base.show(io::IO, ::MIME"text/plain", cc::CodeCache) + print(io, "CodeCache with $(mapreduce(length, +, values(cc.dict); init=0)) entries") + if !isempty(cc.dict) + print(io, ": ") + for (mi, cis) in cc.dict + println(io) + print(io, " ") + show(io, mi) + for ci in cis + println(io) + worldstr = if ci.min_world == typemax(UInt) + "empty world range" + elseif ci.max_world == typemax(UInt) + "worlds $(Int(ci.min_world))+" + else + "worlds $(Int(ci.min_world)) to $(Int(ci.max_world))" + end + print(io, " CodeInstance for ", worldstr) + end + end + end +end + +Base.empty!(cc::CodeCache) = empty!(cc.dict) + +# Session-local registry of `CodeCache`s, keyed by the configuration that produced them. +# 1.10 has no per-CI partitioning, so we partition by `cache_owner(job)` ourselves. +const GLOBAL_CI_CACHES = Dict{Any, CodeCache}() +const GLOBAL_CI_CACHES_LOCK = ReentrantLock() + +function get_code_cache(@nospecialize(job::CompilerJob)) + key = cache_owner(job) + Base.@lock GLOBAL_CI_CACHES_LOCK get!(GLOBAL_CI_CACHES, key) do + CodeCache() + end +end + +# Invalidation: keep `max_world` honest when callees in a CI's edge graph change. +struct CodeCacheCallback + cache::CodeCache +end + +function CC.setindex!(cache::CodeCache, ci::CodeInstance, mi::MethodInstance) + add_codecache_callback!(cache, mi) + cis = get!(cache.dict, mi, CodeInstance[]) + push!(cis, ci) +end + +@static if VERSION ≥ v"1.11.0-DEV.798" + +function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) + callback = CodeCacheCallback(cache) + CC.add_invalidation_callback!(callback, mi) +end +function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32) + cis = get(callback.cache.dict, replaced, nothing) + cis === nothing && return + for ci in cis + if ci.max_world == ~0 % Csize_t + @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" +@static if VERSION >= v"1.11.0-DEV.1390" + @atomic ci.max_world = max_world +else + ci.max_world = max_world +end + end + @assert ci.max_world <= max_world + end +end + +else + +function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) + callback = CodeCacheCallback(cache) + if !isdefined(mi, :callbacks) + mi.callbacks = Any[callback] + elseif !in(callback, mi.callbacks) + push!(mi.callbacks, callback) + end +end +function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32, + seen::Set{MethodInstance}=Set{MethodInstance}()) + push!(seen, replaced) + + cis = get(callback.cache.dict, replaced, nothing) + if cis !== nothing + for ci in cis + if ci.max_world == ~0 % Csize_t + @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" + ci.max_world = max_world + end + @assert ci.max_world <= max_world + end + end + + # recurse into backedges to update their valid range too + if isdefined(replaced, :backedges) + backedges = filter(replaced.backedges) do @nospecialize(mi) + if mi isa MethodInstance + mi ∉ seen + elseif mi isa Type + # an `invoke` call, which is a `(sig, MethodInstance)` pair. + # let's ignore the `sig` and process the `MethodInstance` next. + false + else + error("invalid backedge") + end + end + + # Don't touch/empty backedges — `invalidate_method_instance` in C does that later. + + for mi in backedges + callback(mi::MethodInstance, max_world, seen) + end + end +end + +end -function defs(mod::LLVM.Module) - safe_depwarn("`GPUCompiler.defs(mod)` is deprecated; inline `filter(f -> !isdeclaration(f), collect(functions(mod)))`.", - :defs) - filter(f -> !isdeclaration(f), collect(functions(mod))) +## 1.10 cache view interface + +function CC.haskey(wvc::WorldView{CodeCache}, mi::MethodInstance) + CC.get(wvc, mi, nothing) !== nothing +end + +function CC.get(wvc::WorldView{CodeCache}, mi::MethodInstance, default) + for ci in get!(wvc.cache.dict, mi, CodeInstance[]) + if ci.min_world <= wvc.worlds.min_world && wvc.worlds.max_world <= ci.max_world + return ci + end + end + return default +end + +function CC.getindex(wvc::WorldView{CodeCache}, mi::MethodInstance) + r = CC.get(wvc, mi, nothing) + r === nothing && throw(KeyError(mi)) + return r::CodeInstance end -function decls(mod::LLVM.Module) - safe_depwarn("`GPUCompiler.decls(mod)` is deprecated; inline `filter(f -> isdeclaration(f) && !LLVM.isintrinsic(f), collect(functions(mod)))`.", - :decls) - filter(f -> isdeclaration(f) && !LLVM.isintrinsic(f), collect(functions(mod))) +function CC.setindex!(wvc::WorldView{CodeCache}, ci::CodeInstance, mi::MethodInstance) + CC.setindex!(wvc.cache, ci, mi) end -link_library!(mod::LLVM.Module, lib::LLVM.Module) = link_library!(mod, [lib]) -function link_library!(mod::LLVM.Module, libs::Vector{LLVM.Module}) - safe_depwarn("`GPUCompiler.link_library!` is deprecated; call `LLVM.link!(mod, copy(lib))` directly, or `LLVM.link!(mod, lib; only_needed=true)` with a freshly-parsed library.", - :link_library!) - libs = [copy(lib) for lib in libs] - for lib in libs - link!(mod, lib) + +## 1.10 StackedMethodTable +# +# Priority-lookup method table. The 1.11+ version lives in CompilerCaching; here we keep +# the 1.10 copy, which has to produce the older `MethodMatchResult` shape from +# `CC.findall` / `CC.findsup`. +struct StackedMethodTable{MTV<:CC.MethodTableView} <: CC.MethodTableView + world::UInt + mt::Core.MethodTable + parent::MTV +end +StackedMethodTable(world::UInt, mt::Core.MethodTable) = + StackedMethodTable(world, mt, CC.InternalMethodTable(world)) +StackedMethodTable(world::UInt, mt::Core.MethodTable, parent::Core.MethodTable) = + StackedMethodTable(world, mt, StackedMethodTable(world, parent)) + +CC.isoverlayed(::StackedMethodTable) = true + +function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) + result = CC._findall(sig, table.mt, table.world, limit) + result === nothing && return nothing # too many matches + nr = CC.length(result) + if nr ≥ 1 && CC.getindex(result, nr).fully_covers + return CC.MethodMatchResult(result, true) end + + parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodMatchResult} + parent_result === nothing && return nothing # too many matches + + overlayed = parent_result.overlayed | !CC.isempty(result) + parent_result = parent_result.matches::CC.MethodLookupResult + + return CC.MethodMatchResult( + CC.MethodLookupResult( + CC.vcat(result.matches, parent_result.matches), + CC.WorldRange( + CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), + CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), + result.ambig | parent_result.ambig), + overlayed) +end + +function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) + match, valid_worlds = CC._findsup(sig, table.mt, table.world) + match !== nothing && return match, valid_worlds, true + parent_match, parent_valid_worlds, overlayed = CC.findsup(sig, table.parent) + return ( + parent_match, + CC.WorldRange( + max(valid_worlds.min_world, parent_valid_worlds.min_world), + min(valid_worlds.max_world, parent_valid_worlds.max_world)), + overlayed) end -# no-op 3-arg fallback so downstream overrides that chain via -# `invoke(GPUCompiler.link_libraries!, Tuple{CompilerJob, Module, -# Vector{String}}, ...)` still resolve. -link_libraries!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - undefined_fns::Vector{String}) = return -# `true` when a downstream package has defined a 3-arg `link_libraries!` -# override for `job`, i.e. the dispatched method isn't our fallback above. +## 1.10 `cached_results` # -# Uses the same `jl_gf_invoke_lookup` path as `Core._hasmethod` rather than -# `which`, so it's safe to call from generated-function-adjacent contexts -# where `Base.get_world_counter()` returns `typemax(UInt)` and reflection -# queries like `which` / `methods` fail (see JuliaLang/julia#48611). -# All this because Enzyme.jl calls GPUCompiler.jl from a generated function. -function has_legacy_link_libraries(@nospecialize(job::CompilerJob)) - tt = Tuple{typeof(link_libraries!), typeof(job), - LLVM.Module, Vector{String}} - world = ccall(:jl_get_tls_world_age, UInt, ()) - m = ccall(:jl_gf_invoke_lookup, Any, (Any, Any, UInt), tt, nothing, world) - return m !== nothing && (m::Method).module !== @__MODULE__ +# Session-local storage for the per-job results structs; on 1.11+ these live on the +# `CodeInstance`s of Julia's integrated cache instead (see `interface.jl`). The key +# mirrors the 1.11+ semantics — the full job identity, i.e. method instance, world, +# and entire config — but nothing persists across sessions, and redefinition +# protection comes from the world age in the key rather than from CI invalidation. +const job_results = Dict{Any,Any}() +const job_results_lock = ReentrantLock() + +# specialized for the launch hot path, mirroring the 1.11+ implementation +function cached_results(::Type{V}, job::CompilerJob) where {V} + # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive + # boxing allocation; the MI is kept alive by its method specializations. + key = (V, objectid(job.source), job.world, job.config) + Base.@lock job_results_lock begin + get!(job_results, key) do + V() + end::V + end end + + +## 1.10 session-dependent results +# +# Nothing to wipe: `job_results` never persists meaningfully across sessions (its +# `objectid`-based keys are session-specific, and our own image's entries are cleared +# in `__init__`; entries written by a downstream package's workload don't make it into +# that package's image at all, cf. cross-image mutation loss). +mark_session_dependent!(@nospecialize(job::CompilerJob)) = nothing + +end # !HAS_INTEGRATED_CACHE + + +## Legacy `cached_compilation` (1.10+) + +# A session-local, MI-keyed kernel cache modeled after the pre-CompilerCaching API. Used +# by back-ends that haven't migrated to the `CacheView`-based flow (and by all back-ends +# on Julia 1.10, where the new flow doesn't apply because there's no integrated cache / +# `analysis_results`). + +""" + cached_compilation(cache::AbstractDict, src::MethodInstance, cfg::CompilerConfig, + compiler, linker) + +Compile a method instance `src` with configuration `cfg`, by invoking `compiler` and +`linker` and storing the result in `cache`. + +The `cache` argument should be a dictionary that can be indexed using any value and store +whatever the `linker` function returns. The `compiler` function should take a +`CompilerJob` and return data that the `linker` function then turns into a session-local +artifact (e.g. a `CuModule`). + +This is the legacy caching API used before GPUCompiler 2.0. New code on Julia 1.11+ +should prefer `CompilerCaching.CacheView`-based caching (see the package extension). +""" +function cached_compilation(cache::AbstractDict{<:Any,V}, + src::MethodInstance, cfg::CompilerConfig, + compiler::Function, linker::Function) where {V} + world = tls_world_age() + key = (objectid(src), world, cfg) + # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive boxing + # allocation. Base does this with a multi-level lookup; we use a single-level dict. + + Base.@lock cached_compilation_lock begin + obj = get(cache, key, nothing) + end + + if obj === nothing || compile_hook[] !== nothing + obj = actual_compilation(cache, src, world, cfg, compiler, linker)::V + Base.@lock cached_compilation_lock begin + cache[key] = obj + end + end + obj::V +end + +const cached_compilation_lock = ReentrantLock() + +@noinline function actual_compilation(cache::AbstractDict, src::MethodInstance, world::UInt, + cfg::CompilerConfig, compiler::Function, + linker::Function) + job = CompilerJob(src, cfg, world) + asm = nothing + + # provide a hook to use in tools / debuggers + if compile_hook[] !== nothing + Base.invokelatest(compile_hook[], job) + end + + asm = compiler(job) + obj = linker(job, asm) + obj +end + +@public cached_compilation diff --git a/src/driver.jl b/src/driver.jl index 30529c70..8c52e884 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -42,42 +42,23 @@ end ## compiler entrypoint export compile - -# (::CompilerJob) -const compile_hook = Ref{Union{Nothing,Function}}(nothing) +# `compile_hook` is declared in `GPUCompiler.jl` so files included earlier +# (notably `deprecated.jl`'s `cached_compilation`) can reference it too. """ compile(target::Symbol, job::CompilerJob) Compile a `job` to one of the following formats as specified by the `target` argument: -`:julia` for Julia IR, `:llvm` for LLVM IR and `:asm` for machine code. +`:llvm` for LLVM IR, `:asm` for assembly, or `:obj` for object code. """ -function compile(target::Symbol, @nospecialize(job::CompilerJob); kwargs...) - # XXX: remove on next major version - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `compile` API does not take keyword arguments anymore. Use CompilerConfig instead.", :compile) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - +function compile(target::Symbol, @nospecialize(job::CompilerJob)) if compile_hook[] !== nothing Base.invokelatest(compile_hook[], job) end - return compile_unhooked(target, job) end -# XXX: remove on next major version -function codegen(output::Symbol, @nospecialize(job::CompilerJob); kwargs...) - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `codegen` function is an internal API. Use `GPUCompiler.compile` (with any kwargs passed to `CompilerConfig`) instead.", :codegen) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - compile_unhooked(output, job) -end - -function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob); kwargs...) +function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) if context(; throw_error=false) === nothing error("No active LLVM context. Use `JuliaContext()` do-block syntax to create one.") end @@ -188,14 +169,7 @@ end const __llvm_initialized = Ref(false) -@locked function emit_llvm(@nospecialize(job::CompilerJob); kwargs...) - # XXX: remove on next major version - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `emit_llvm` function is an internal API. Use `GPUCompiler.compile` (with any kwargs passed to `CompilerConfig`) instead.", :emit_llvm) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - +@locked function emit_llvm(@nospecialize(job::CompilerJob)) if !__llvm_initialized[] InitializeAllTargets() InitializeAllTargetInfos() @@ -259,7 +233,7 @@ const __llvm_initialized = Ref(false) target = nest_target(dyn_job.config.target, job.config.target) params = nest_params(dyn_job.config.params, job.config.params) config = CompilerConfig(dyn_job.config; toplevel=false, target, params) - return codegen(:llvm, CompilerJob(dyn_job; config)) + return compile_unhooked(:llvm, CompilerJob(dyn_job; config)) end # compile and link @@ -323,23 +297,7 @@ const __llvm_initialized = Ref(false) @tracepoint "Library linking" begin # target-specific libraries - @tracepoint "target libraries" begin - if has_legacy_link_libraries(job) - safe_depwarn( - "3-arg `link_libraries!(job, mod, undefined_fns)` is deprecated; " * - "migrate your override to the 2-arg form `link_libraries!(job, mod)`. " * - "Instead of inspecting `undefined_fns` to decide what to link, " * - "parse the library lazily with `parse(LLVM.Module, bytes; lazy=true)` " * - "and link it with `LLVM.link!(mod, lib; only_needed=true)` — " * - "the linker will then materialize only the referenced symbols.", - :link_libraries!) - undefined_fns = [LLVM.name(f) for f in functions(ir) - if isdeclaration(f) && !LLVM.isintrinsic(f)] - link_libraries!(job, ir, undefined_fns) - else - link_libraries!(job, ir) - end - end + @tracepoint "target libraries" link_libraries!(job, ir) # GPU run-time library if !uses_julia_runtime(job) @@ -376,6 +334,12 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) + relocate_gvs!(ir, gv_to_value) + + # the IR (and any artifact derived from it) now embeds session-absolute + # addresses; keep results for this job out of a package image + isempty(gv_to_value) || mark_session_dependent!(job) + if job.config.optimize @tracepoint "optimization" begin optimize!(job, ir; job.config.opt_level) diff --git a/src/execution.jl b/src/execution.jl index 9b4940a7..dcdd4236 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -66,224 +66,3 @@ function assign_args!(code, _args) return vars, var_exprs end - - -## cached compilation - -### Notes on interactions with package images and disk cache. -# Julia uses package images (pkgimg) to cache both the result of inference, -# and the result of native code emissions. Up until Julia v1.11 neither the -# inferred nor the nativce code of foreign abstract interpreters was cached -# across sessions. Julia v1.11 allows for caching of inference results across -# sessions as long as those inference results are created during precompilation. -# -# Julia cache hierarchy is roughly as follows: -# Function (name of a thing) -# -> Method (particular piece of code to dispatch to with a signature) -# -> MethodInstance (A particular Method + particular signature) -# -> CodeInstance (A MethodInstance compiled for a world) -# -# In order to cache code across sessions we need to insert CodeInstance(owner=GPUCompilerCacheToken) -# into the internal cache. Once we have done so we know that a particular CodeInstance is unique in -# the system. (During pkgimg loading conflicts will be resolved). -# -# When a pkgimg is loaded we check it's validity, this means checking that all depdencies are the same, -# the pkgimg was created for the right set of compiler flags, and that all source code that was used -# to create this pkgimg is the same. When a CodeInstance is inside a pkgimg we can extend the chain of -# validity even for GPU code, we cannot verify a "runtime" CodeInstance in the same way. -# -# Therefore when we see a compilation request for a CodeInstance that is originating from a pkgimg -# we can use it as part of the hash for the on-disk cache. (see `cache_file`) - -""" - disk_cache_enabled() - -Query if caching to disk is enabled. -""" -disk_cache_enabled() = parse(Bool, @load_preference("disk_cache", "false")) - -""" - enable_disk_cache!(state::Bool=true) - -Activate the GPUCompiler disk cache in the current environment. -You will need to restart your Julia environment for it to take effect. - -!!! note - The cache functionality requires Julia 1.11 -""" -function enable_disk_cache!(state::Bool=true) - @set_preferences!("disk_cache"=>string(state)) -end - -disk_cache_path() = @get_scratch!("disk_cache") -clear_disk_cache!() = rm(disk_cache_path(); recursive=true, force=true) - -const cache_lock = ReentrantLock() - -""" - cached_compilation(cache::Dict{Any}, src::MethodInstance, cfg::CompilerConfig, - compiler, linker) - -Compile a method instance `src` with configuration `cfg`, by invoking `compiler` and -`linker` and storing the result in `cache`. - -The `cache` argument should be a dictionary that can be indexed using any value and store -whatever the `linker` function returns. The `compiler` function should take a `CompilerJob` -and return data that can be cached across sessions (e.g., LLVM IR). This data is then -forwarded, along with the `CompilerJob`, to the `linker` function which is allowed to create -session-dependent objects (e.g., a `CuModule`). -""" -function cached_compilation(cache::AbstractDict{<:Any,V}, - src::MethodInstance, cfg::CompilerConfig, - compiler::Function, linker::Function) where {V} - # NOTE: we index the cach both using (mi, world, cfg) keys, for the fast look-up, - # and using CodeInfo keys for the slow look-up. we need to cache both for - # performance, but cannot use a separate private cache for the ci->obj lookup - # (e.g. putting it next to the CodeInfo's in the CodeCache) because some clients - # expect to be able to wipe the cache (e.g. CUDA.jl's `device_reset!`) - - # fast path: index the cache directly for the *current* world + compiler config - - world = tls_world_age() - key = (objectid(src), world, cfg) - # NOTE: we store the MethodInstance's objectid to avoid an expensive allocation. - # Base does this with a multi-level lookup, first keyed on the mi, - # then a linear scan over the (typically few) entries. - - # NOTE: no use of lock(::Function)/@lock/get! to avoid try/catch and closure overhead - lock(cache_lock) - obj = get(cache, key, nothing) - unlock(cache_lock) - - if obj === nothing || compile_hook[] !== nothing - obj = actual_compilation(cache, src, world, cfg, compiler, linker)::V - lock(cache_lock) - cache[key] = obj - unlock(cache_lock) - end - return obj::V -end - -@noinline function cache_file(ci::CodeInstance, cfg::CompilerConfig) - h = hash(Base.objectid(ci)) - @static if isdefined(Base, :object_build_id) - bid = Base.object_build_id(ci) - if bid === nothing # CI is from a runtime compilation, not worth caching on disk - return nothing - else - bid = bid % UInt64 # The upper 64bit are a checksum, unavailable during precompilation - end - h = hash(bid, h) - end - h = hash(cfg, h) - - gpucompiler_buildid = Base.module_build_id(@__MODULE__) - if (gpucompiler_buildid >> 64) % UInt64 == 0xffffffffffffffff - return nothing # Don't cache during precompilation of GPUCompiler - end - - return joinpath( - disk_cache_path(), - # bifurcate the cache by build id of GPUCompiler - string(gpucompiler_buildid), - string(h, ".jls")) -end - -struct DiskCacheEntry - src::Type # Originally MethodInstance, but upon deserialize they were not uniqued... - cfg::CompilerConfig - asm -end - -@noinline function actual_compilation(cache::AbstractDict, src::MethodInstance, world::UInt, - cfg::CompilerConfig, compiler::Function, linker::Function) - job = CompilerJob(src, cfg, world) - obj = nothing - - # fast path: find an applicable CodeInstance and see if we have compiled it before - ci = ci_cache_lookup(ci_cache(job), src, world, world)::Union{Nothing,CodeInstance} - if ci !== nothing - key = (ci, cfg) - obj = get(cache, key, nothing) - end - - # slow path: compile and link - if obj === nothing || compile_hook[] !== nothing - asm = nothing - path = nothing - ondisk_hit = false - @static if VERSION >= v"1.11.0-" - # Don't try to hit the disk cache if we are for a *compile* hook - # TODO: - # - Sould we hit disk cache if Base.generating_output() - # - Should we allow backend to opt out? - if ci !== nothing && obj === nothing && disk_cache_enabled() - path = cache_file(ci, cfg) - @debug "Looking for on-disk cache" job path - if path !== nothing && isfile(path) - ondisk_hit = true - try - @debug "Loading compiled kernel" job path - # The MI we deserialize here didn't get uniqued... - entry = deserialize(path)::DiskCacheEntry - if entry.src == src.specTypes && entry.cfg == cfg - asm = entry.asm - else - @show entry.src == src.specTypes - @show entry.cfg == cfg - @warn "Cache missmatch" src.specTypes cfg entry.src entry.cfg - end - catch ex - @warn "Failed to load compiled kernel" job path exception=(ex, catch_backtrace()) - end - end - end - end - - if asm === nothing || compile_hook[] !== nothing - # Run the compiler in-case we need to hook it. - asm = compiler(job) - end - if obj !== nothing - # we got here because of a *compile* hook; don't bother linking - return obj - end - - @static if VERSION >= v"1.11.0-" - if !ondisk_hit && path !== nothing && disk_cache_enabled() - @debug "Writing out on-disk cache" job path - mkpath(dirname(path)) - entry = DiskCacheEntry(src.specTypes, cfg, asm) - - # atomic write to disk - tmppath, io = mktemp(dirname(path); cleanup=false) - serialize(io, entry) - close(io) - @static if VERSION >= v"1.12.0-DEV.1023" - mv(tmppath, path; force=true) - else - Base.rename(tmppath, path, force=true) - end - end - end - - obj = linker(job, asm) - - if ci === nothing - ci = ci_cache_lookup(ci_cache(job), src, world, world) - if ci === nothing - error("""Did not find CodeInstance for $job. - - Pleaase make sure that the `compiler` function passed to `cached_compilation` - invokes GPUCompiler with exactly the same configuration as passed to the API. - - Note that you should do this by calling `GPUCompiler.compile`, and not by - using reflection functions (which alter the compiler configuration).""") - end - key = (ci, cfg) - end - cache[key] = obj - end - - return obj -end diff --git a/src/gcn.jl b/src/gcn.jl index be047143..dc06c8b3 100644 --- a/src/gcn.jl +++ b/src/gcn.jl @@ -39,10 +39,6 @@ end ## job -# TODO: encode debug build or not in the compiler job -# https://github.com/JuliaGPU/CUDAnative.jl/issues/368 -runtime_slug(job::CompilerJob{GCNCompilerTarget}) = "gcn-$(job.config.target.dev_isa)$(job.config.target.features)-$(job.config.target.backend)" - const gcn_intrinsics = () # TODO: ("vprintf", "__assertfail", "malloc", "free") isintrinsic(::CompilerJob{GCNCompilerTarget}, fn::String) = in(fn, gcn_intrinsics) diff --git a/src/interface.jl b/src/interface.jl index 6aec9f1e..c563a74d 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -261,17 +261,23 @@ runtime_module(@nospecialize(job::CompilerJob)) = error("Not implemented") isintrinsic(@nospecialize(job::CompilerJob), fn::String) = false # provide a specific interpreter to use. -if VERSION >= v"1.11.0-DEV.1552" -get_interpreter(@nospecialize(job::CompilerJob)) = - GPUInterpreter(job.world; method_table_view=maybe_cached(method_table_view(job)), - token=ci_cache_token(job), inf_params=inference_params(job), +@static if HAS_INTEGRATED_CACHE +function get_interpreter(@nospecialize(job::CompilerJob)) + GPUInterpreter(job.world; + method_table_view=maybe_cached(method_table_view(job)), + owner=cache_owner(job), + inf_params=inference_params(job), opt_params=optimization_params(job)) +end else -get_interpreter(@nospecialize(job::CompilerJob)) = - GPUInterpreter(job.world; method_table_view=maybe_cached(method_table_view(job)), - code_cache=ci_cache(job), inf_params=inference_params(job), +function get_interpreter(@nospecialize(job::CompilerJob)) + GPUInterpreter(job.world; + method_table_view=maybe_cached(method_table_view(job)), + code_cache=get_code_cache(job), + inf_params=inference_params(job), opt_params=optimization_params(job)) end +end # does this target support throwing Julia exceptions with jl_throw? # if not, calls to throw will be replaced with calls to the GPU runtime @@ -281,11 +287,6 @@ can_throw(@nospecialize(job::CompilerJob)) = uses_julia_runtime(job) # if not, safepoints at function entry will not be emitted can_safepoint(@nospecialize(job::CompilerJob)) = uses_julia_runtime(job) -# generate a string that represents the type of compilation, for selecting a compiled -# instance of the runtime library. this slug should encode everything that affects -# the generated code of this compiler job (with exception of the function source) -runtime_slug(@nospecialize(job::CompilerJob)) = error("Not implemented") - # the type of the kernel state object, or Nothing if this back-end doesn't need one. # # the generated code will be rewritten to include an object of this type as the first @@ -305,34 +306,175 @@ pass_by_ref(@nospecialize(job::CompilerJob)) = false # whether pointer is a valid call target valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false +# Cache partitioning. On Julia 1.11+, the owner is stored on every `CodeInstance` and +# compared via `jl_egal`, so it (and every field) must be immutable for cross-session +# matches (e.g. via package precompilation); custom `target` / `params` types must be +# `struct`s, not `mutable struct`s. +# +# On Julia 1.10, where there's no per-CI owner field, this token only identifies the +# session-local cache partition (see `cached_compilation` / `GLOBAL_CI_CACHES` in +# `deprecated.jl`); the immutability requirement is still useful for stable hashing. +# # Care is required for anything that impacts: # - method_table # - inference_params # - optimization_params -# By default that is just always_inline -# the cache token is compared with jl_egal -struct GPUCompilerCacheToken - target_type::Type +# The default covers the full target+params instances (so backends with version- or +# arch-specific knobs partition cleanly), `always_inline` (which feeds optimization_params), +# and the method table. +struct GPUCompilerCacheToken{T<:AbstractCompilerTarget, P<:AbstractCompilerParams} + target::T + params::P always_inline::Bool method_table::Core.MethodTable end -ci_cache_token(@nospecialize(job::CompilerJob)) = - GPUCompilerCacheToken(typeof(job.config.target), job.config.always_inline, method_table(job)) +# NOTE: deliberately specialized (one instantiation per back-end): this runs on the +# kernel launch hot path, and constructing the token dynamically is an order of +# magnitude slower. +cache_owner(job::CompilerJob) = + GPUCompilerCacheToken(job.config.target, job.config.params, + job.config.always_inline, method_table(job)) -# the codeinstance cache to use -- should only be used for the constructor -if VERSION >= v"1.11.0-DEV.1552" - # Soft deprecated user should use `CC.code_cache(get_interpreter(job))` - ci_cache(@nospecialize(job::CompilerJob)) = CC.code_cache(get_interpreter(job)) -else -function ci_cache(@nospecialize(job::CompilerJob)) - lock(GLOBAL_CI_CACHES_LOCK) do - cache = get!(GLOBAL_CI_CACHES, job.config) do - CodeCache() +""" + cached_results(::Type{V}, job::CompilerJob) -> V + +Retrieve the compilation-results struct of type `V` for `job`, creating an empty one on +first access. `V` must be a `mutable struct` with a zero-arg constructor; back-ends +define one such struct holding the artifacts of each compilation stage, check whether +the relevant fields are populated, and fill them in (running the compiler) when not: + +```julia +mutable struct MetalResults + metallib::Union{Nothing,Vector{UInt8}} + entry::Union{Nothing,String} + pipelines::Vector{Tuple{MTLDevice,MTLComputePipelineState}} # session-local + MetalResults() = new(nothing, nothing, []) +end + +res = GPUCompiler.cached_results(MetalResults, job) +if res.metallib === nothing || GPUCompiler.compile_hook[] !== nothing + ...compile, populate res... +end +``` + +Results are keyed by the *full* compiler job: its method instance, world age, and entire +`CompilerConfig` (two jobs differing only in, say, the kernel `name` get distinct +structs). Storage differs per Julia version, transparently to the back-end: + +- On Julia 1.11+, the struct lives on the `CodeInstance`'s `analysis_results` chain in + Julia's integrated code cache, partitioned by [`cache_owner`](@ref) and keyed by + config within the per-CI [`JobResults`](@ref) container. Method redefinition + invalidates the CI — and with it the attached results. When no CI exists yet, + inference is run to create one. Artifacts stored during precompilation are serialized + into the package image along with the CI: populate only session-portable values + (bytes, strings) during precompile workloads, and keep session-local handles (device + modules, pipeline objects) in fields that remain empty until first use at run time. + +- On Julia 1.10, the struct lives in a session-local Dict; redefinition protection + comes from the world age in the key, and nothing persists across sessions. + +Thread safety: concurrent calls for the same job return the same struct, but +GPUCompiler does not serialize back-end *compilation*; take a back-end lock around +the check-and-compile sequence (all back-ends already do, e.g. `mtlfunction_lock`). +""" +function cached_results end + +@static if HAS_INTEGRATED_CACHE + +""" + JobResults + +Per-CodeInstance container mapping a `CompilerConfig` to the back-end's results struct. + +A `CodeInstance` is shared by every job whose [`cache_owner`](@ref) matches — the owner +token only covers what affects *inference* (target, params, `always_inline`, method +table). The remaining `CompilerConfig` fields (`kernel`, `name`, `entry_abi`, +`opt_level`, …) do affect *codegen*, so artifacts must not be shared across them. +Entries are matched by `===` (`jl_egal`): `CompilerConfig` is an immutable struct, so +structurally identical configs compare equal, including against configs deserialized +from package images. +""" +mutable struct JobResults + entries::Vector{Tuple{CompilerConfig,Any}} + JobResults() = new(Tuple{CompilerConfig,Any}[]) +end + +const cached_results_lock = ReentrantLock() + +# NOTE: like `cache_owner`, specialized for the launch hot path (bounded number of +# instantiations: one per back-end and results type). +function cached_results(::Type{V}, job::CompilerJob) where {V} + owner = cache_owner(job) + cache = CompilerCaching.CacheView{JobResults}(owner, job.world) + + # get a CI for the job, running inference to create one if needed + ci = get(cache, job.source, nothing) + if ci === nothing + interp = get_interpreter(job) + ci = CompilerCaching.typeinf!(interp, job.source) + ci === nothing && error("Inference of $(job.source) failed") + end + + jr = CompilerCaching.results(JobResults, ci) + Base.@lock cached_results_lock begin + for (config, v) in jr.entries + config === job.config && v isa V && return v::V + end + v = V() + push!(jr.entries, (job.config, v)) + return v + end +end + +## session-dependent results +# +# Some compilation results embed session-specific data: `relocate_gvs!` bakes absolute +# pointers into the IR of toplevel jobs that reference `julia.constgv` globals, and any +# artifact a back-end derives from that IR (metallib, SPIR-V, ...) inherits them. Such +# results must not survive into a package image, while remaining available for +# within-session lookups during the precompilation process itself. Julia wipes its own +# session-dependent CodeInstance state during serialization (staticdata.c); we +# approximate that with an `atexit` hook, which the runtime invokes *before* +# `jl_write_compiler_output`: right before the image is written, the entries of jobs +# marked session-dependent are deleted from their `JobResults` container, so a later +# session simply recompiles them. + +const session_dependent_jobs = Vector{CompilerJob}() +const session_dependent_lock = ReentrantLock() + +function mark_session_dependent!(@nospecialize(job::CompilerJob)) + ccall(:jl_generating_output, Cint, ()) == 1 || return + Base.@lock session_dependent_lock begin + if isempty(session_dependent_jobs) + atexit(wipe_session_dependent_results) end - return cache + push!(session_dependent_jobs, job) end + return end + +function wipe_session_dependent_results() + Base.@lock session_dependent_lock begin + for job in session_dependent_jobs + cache = CompilerCaching.CacheView{JobResults}(cache_owner(job), job.world) + ci = get(cache, job.source, nothing) + ci === nothing && continue + jr = CompilerCaching.results(JobResults, ci) + Base.@lock cached_results_lock begin + filter!(entry -> entry[1] !== job.config, jr.entries) + end + end + empty!(session_dependent_jobs) + end + return +end + +end # HAS_INTEGRATED_CACHE + +@public GPUCompilerCacheToken, cache_owner, cached_results +@static if HAS_INTEGRATED_CACHE + @public JobResults end # the method table to use @@ -401,9 +543,3 @@ finish_ir!(@nospecialize(job::CompilerJob), mod::LLVM.Module, entry::LLVM.Functi # whether an LLVM function is valid for this back-end validate_ir(@nospecialize(job::CompilerJob), mod::LLVM.Module) = IRError[] - -# deprecated -struct DeprecationMarker end -process_module!(@nospecialize(job::CompilerJob), mod::LLVM.Module) = DeprecationMarker() -process_entry!(@nospecialize(job::CompilerJob), mod::LLVM.Module, entry::LLVM.Function) = - DeprecationMarker() diff --git a/src/irgen.jl b/src/irgen.jl index 80dac84d..9622e1f3 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -43,11 +43,6 @@ function irgen(@nospecialize(job::CompilerJob)) end end - deprecation_marker = process_module!(job, mod) - if deprecation_marker != DeprecationMarker() - safe_depwarn("GPUCompiler.process_module! is deprecated; implement GPUCompiler.finish_module! instead", :process_module) - end - # sanitize global values (Julia doesn't when using the external codegen policy) for val in [collect(globals(mod)); collect(functions(mod))] isdeclaration(val) && continue @@ -69,11 +64,6 @@ function irgen(@nospecialize(job::CompilerJob)) elseif job.config.kernel LLVM.name!(entry, mangle_sig(job.source.specTypes)) end - deprecation_marker = process_entry!(job, mod, entry) - if deprecation_marker != DeprecationMarker() - safe_depwarn("GPUCompiler.process_entry! is deprecated; implement GPUCompiler.finish_module! instead", :process_entry) - entry = deprecation_marker - end if job.config.entry_abi === :specfunc func = compiled[job.source].func specfunc = LLVM.name(entry) diff --git a/src/jlgen.jl b/src/jlgen.jl index 8db65c89..1eb26039 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -3,15 +3,13 @@ ## world age lookups -# `tls_world_age` should be used to look up the current world age. in most cases, this is -# what you should use to invoke the compiler with. - -if isdefined(Base, :tls_world_age) +@static if isdefined(Base, :tls_world_age) import Base: tls_world_age else tls_world_age() = ccall(:jl_get_tls_world_age, UInt, ()) end + ## looking up method instances export methodinstance, generic_methodinstance @@ -38,11 +36,6 @@ function MethodError(ft::Type{<:Function}, tt::Type, world::Integer=typemax(UInt end MethodError(ft, tt, world=typemax(UInt)) = Base.MethodError(ft, tt, world) -# generate a LineInfoNode for the current source code location -macro LineInfoNode(method) - Core.LineInfoNode(__module__, method, __source__.file, Int32(__source__.line), Int32(0)) -end - """ methodinstance(ft::Type, tt::Type, [world::UInt]) @@ -56,7 +49,6 @@ This function is highly optimized, and results do not need to be cached addition Only use this function with concrete signatures, i.e., using the types of values you would pass at run time. For non-concrete signatures, use `generic_methodinstance` instead. - """ methodinstance @@ -72,11 +64,10 @@ function generic_methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type) return mi::MethodInstance end -# on 1.11 (JuliaLang/julia#52572, merged as part of JuliaLang/julia#52233) we can use -# Julia's cached method lookup to simply look up method instances at run time. +# On 1.11+ (JuliaLang/julia#52572 / JuliaLang/julia#52233) `jl_method_lookup_by_tt` returns +# a usable `MethodInstance` directly, so we just call it. @static if VERSION >= v"1.11.0-DEV.1552" -# XXX: version of Base.method_instance that uses a function type @inline function methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), world::Integer=tls_world_age()) sig = signature_type_by_tt(ft, tt) @@ -96,294 +87,23 @@ end return mi end -# on older versions of Julia, we always need to use the generic lookup +# On 1.10 we always use the generic lookup. (A @generated fast path used to exist, +# but it was unreachable: the (ft::Type, tt::Type) method above it always won dispatch.) else - const methodinstance = generic_methodinstance - -function methodinstance_generator(world::UInt, source, self, ft::Type, tt::Type) - @nospecialize - @assert CC.isType(ft) && CC.isType(tt) - ft = ft.parameters[1] - tt = tt.parameters[1] - - stub = Core.GeneratedFunctionStub(identity, Core.svec(:methodinstance, :ft, :tt), Core.svec()) - - # look up the method match - method_error = :(throw(MethodError(ft, tt, $world))) - sig = Tuple{ft, tt.parameters...} - min_world = Ref{UInt}(typemin(UInt)) - max_world = Ref{UInt}(typemax(UInt)) - match = ccall(:jl_gf_invoke_lookup_worlds, Any, - (Any, Any, Csize_t, Ref{Csize_t}, Ref{Csize_t}), - sig, #=mt=# nothing, world, min_world, max_world) - match === nothing && return stub(world, source, method_error) - - # look up the method and code instance - mi = ccall(:jl_specializations_get_linfo, Ref{MethodInstance}, - (Any, Any, Any), match.method, match.spec_types, match.sparams) - ci = CC.retrieve_code_info(mi, world) - - # prepare a new code info - new_ci = copy(ci) - empty!(new_ci.code) - empty!(new_ci.codelocs) - empty!(new_ci.linetable) - empty!(new_ci.ssaflags) - new_ci.ssavaluetypes = 0 - - # propagate edge metadata - new_ci.min_world = min_world[] - new_ci.max_world = max_world[] - new_ci.edges = Any[mi] - - # prepare the slots - new_ci.slotnames = Symbol[Symbol("#self#"), :ft, :tt] - new_ci.slotflags = UInt8[0x00 for i = 1:3] - - # return the method instance - push!(new_ci.code, CC.ReturnNode(mi)) - push!(new_ci.ssaflags, 0x00) - push!(new_ci.linetable, @LineInfoNode(methodinstance)) - push!(new_ci.codelocs, 1) - new_ci.ssavaluetypes += 1 - - return new_ci -end - -@eval function methodinstance(ft, tt) - $(Expr(:meta, :generated_only)) - $(Expr(:meta, :generated, methodinstance_generator)) -end - -end - - -## code instance cache -const HAS_INTEGRATED_CACHE = VERSION >= v"1.11.0-DEV.1552" - -if !HAS_INTEGRATED_CACHE -struct CodeCache - dict::IdDict{MethodInstance,Vector{CodeInstance}} - - CodeCache() = new(IdDict{MethodInstance,Vector{CodeInstance}}()) -end - -function Base.show(io::IO, ::MIME"text/plain", cc::CodeCache) - print(io, "CodeCache with $(mapreduce(length, +, values(cc.dict); init=0)) entries") - if !isempty(cc.dict) - print(io, ": ") - for (mi, cis) in cc.dict - println(io) - print(io, " ") - show(io, mi) - - function worldstr(min_world, max_world) - if min_world == typemax(UInt) - "empty world range" - elseif max_world == typemax(UInt) - "worlds $(Int(min_world))+" - else - "worlds $(Int(min_world)) to $(Int(max_world))" - end - end - - for (i,ci) in enumerate(cis) - println(io) - print(io, " CodeInstance for ", worldstr(ci.min_world, ci.max_world)) - end - end - end -end - -Base.empty!(cc::CodeCache) = empty!(cc.dict) - -const GLOBAL_CI_CACHES = Dict{CompilerConfig, CodeCache}() -const GLOBAL_CI_CACHES_LOCK = ReentrantLock() - - -## method invalidations - -function CC.setindex!(cache::CodeCache, ci::CodeInstance, mi::MethodInstance) - # make sure the invalidation callback is attached to the method instance - add_codecache_callback!(cache, mi) - cis = get!(cache.dict, mi, CodeInstance[]) - push!(cis, ci) -end - -# invalidation (like invalidate_method_instance, but for our cache) -struct CodeCacheCallback - cache::CodeCache -end - -@static if VERSION ≥ v"1.11.0-DEV.798" - -function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) - callback = CodeCacheCallback(cache) - CC.add_invalidation_callback!(callback, mi) -end -function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32) - cis = get(callback.cache.dict, replaced, nothing) - if cis === nothing - return - end - for ci in cis - if ci.max_world == ~0 % Csize_t - @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" -@static if VERSION >= v"1.11.0-DEV.1390" - @atomic ci.max_world = max_world -else - ci.max_world = max_world -end - end - @assert ci.max_world <= max_world - end -end - -else - -function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) - callback = CodeCacheCallback(cache) - if !isdefined(mi, :callbacks) - mi.callbacks = Any[callback] - elseif !in(callback, mi.callbacks) - push!(mi.callbacks, callback) - end -end -function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32, - seen::Set{MethodInstance}=Set{MethodInstance}()) - push!(seen, replaced) - - cis = get(callback.cache.dict, replaced, nothing) - if cis === nothing - return - end - for ci in cis - if ci.max_world == ~0 % Csize_t - @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" - ci.max_world = max_world - end - @assert ci.max_world <= max_world - end - - # recurse to all backedges to update their valid range also - if isdefined(replaced, :backedges) - backedges = filter(replaced.backedges) do @nospecialize(mi) - if mi isa MethodInstance - mi ∉ seen - elseif mi isa Type - # an `invoke` call, which is a `(sig, MethodInstance)` pair. - # let's ignore the `sig` and process the `MethodInstance` next. - false - else - error("invalid backedge") - end - end - - # Don't touch/empty backedges `invalidate_method_instance` in C will do that later - # replaced.backedges = Any[] - - for mi in backedges - callback(mi::MethodInstance, max_world, seen) - end - end end -end -end # !HAS_INTEGRATED_CACHE - ## method overrides Base.Experimental.@MethodTable(GLOBAL_METHOD_TABLE) -# Implements a priority lookup for method tables, where the first match in the stack get's returned. -# An alternative to this would be to use a "Union" where we would query the parent method table and -# do a most-specific match. -struct StackedMethodTable{MTV<:CC.MethodTableView} <: CC.MethodTableView - world::UInt - mt::Core.MethodTable - parent::MTV +# Priority-lookup method table. On 1.11+ we re-export CompilerCaching's; the 1.10 copy +# (with the older `MethodMatchResult`-returning `findall` signature) lives in deprecated.jl. +@static if HAS_INTEGRATED_CACHE + using CompilerCaching: StackedMethodTable end -StackedMethodTable(world::UInt, mt::Core.MethodTable) = StackedMethodTable(world, mt, CC.InternalMethodTable(world)) -StackedMethodTable(world::UInt, mt::Core.MethodTable, parent::Core.MethodTable) = StackedMethodTable(world, mt, StackedMethodTable(world, parent)) - -CC.isoverlayed(::StackedMethodTable) = true - -@static if VERSION >= v"1.11.0-DEV.363" - # https://github.com/JuliaLang/julia/pull/51078 - # same API as before but without returning isoverlayed flag - function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # to many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - # no need to fall back to the parent method view - return result - end - - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodLookupResult} - parent_result === nothing && return nothing #too many matches - # merge the parent match results with the internal method table - return CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig) - end - - function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds - parent_match, parent_valid_worlds = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)) - ) - end -else - function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # to many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - # no need to fall back to the parent method view - return CC.MethodMatchResult(result, true) - end - - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodMatchResult} - parent_result === nothing && return nothing #too many matches - - overlayed = parent_result.overlayed | !CC.isempty(result) - parent_result = parent_result.matches::CC.MethodLookupResult - - # merge the parent match results with the internal method table - return CC.MethodMatchResult( - CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig), - overlayed) - end - - function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds, true - parent_match, parent_valid_worlds, overlayed = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)), - overlayed) - end -end ## interpreter @@ -394,7 +114,6 @@ else import Core.Compiler: get_world_counter, get_world_counter as get_inference_world end -const MTType = Core.MethodTable if isdefined(Core.Compiler, :CachedMethodTable) using Core.Compiler: CachedMethodTable maybe_cached(mtv::CC.MethodTableView) = CachedMethodTable(mtv) @@ -407,17 +126,28 @@ get_method_table_view(world::UInt, mt::CC.MethodTable) = CC.OverlayMethodTable(w # VERSION >= v"1.14.0-DEV.1691" const INFERENCE_CACHE_TYPE = isdefined(CC, :InferenceCache) ? CC.InferenceCache : Vector{CC.InferenceResult} +""" + GPUInterpreter + +Foreign abstract interpreter that drives Julia inference for GPU compilation. + +The interpreter is partitioned by an `owner` token (1.11+) or an in-process +`CodeCache` (1.10, see [`deprecated.jl`](deprecated.jl)). The owner is what +makes per-job CIs distinct in the integrated cache; together with the world +age it is all CompilerCaching needs to drive inference into the integrated +cache. Compilation results are not attached during inference at all — they +are attached lazily when looked up (see [`cached_results`](@ref)). +""" struct GPUInterpreter{MTV<:CC.MethodTableView} <: CC.AbstractInterpreter world::UInt method_table_view::MTV @static if HAS_INTEGRATED_CACHE - token::Any + owner::Any else code_cache::CodeCache end inf_cache::INFERENCE_CACHE_TYPE - inf_params::CC.InferenceParams opt_params::CC.OptimizationParams end @@ -425,31 +155,30 @@ end @static if HAS_INTEGRATED_CACHE function GPUInterpreter(world::UInt=Base.get_world_counter(); method_table_view::CC.MethodTableView, - token::Any, + owner::Any, inf_params::CC.InferenceParams, opt_params::CC.OptimizationParams) @assert world <= Base.get_world_counter() - - inf_cache = INFERENCE_CACHE_TYPE() - - return GPUInterpreter(world, method_table_view, - token, inf_cache, - inf_params, opt_params) + return GPUInterpreter{typeof(method_table_view)}( + world, method_table_view, owner, INFERENCE_CACHE_TYPE(), + inf_params, opt_params) end function GPUInterpreter(interp::GPUInterpreter; world::UInt=interp.world, method_table_view::CC.MethodTableView=interp.method_table_view, - token::Any=interp.token, + owner::Any=interp.owner, inf_cache::INFERENCE_CACHE_TYPE=interp.inf_cache, inf_params::CC.InferenceParams=interp.inf_params, opt_params::CC.OptimizationParams=interp.opt_params) - return GPUInterpreter(world, method_table_view, - token, inf_cache, - inf_params, opt_params) + return GPUInterpreter{typeof(method_table_view)}( + world, method_table_view, owner, inf_cache, + inf_params, opt_params) end -else +CC.cache_owner(interp::GPUInterpreter) = interp.owner + +else # 1.10: in-process CodeCache function GPUInterpreter(world::UInt=Base.get_world_counter(); method_table_view::CC.MethodTableView, @@ -457,12 +186,9 @@ function GPUInterpreter(world::UInt=Base.get_world_counter(); inf_params::CC.InferenceParams, opt_params::CC.OptimizationParams) @assert world <= Base.get_world_counter() - - inf_cache = Vector{CC.InferenceResult}() - - return GPUInterpreter(world, method_table_view, - code_cache, inf_cache, - inf_params, opt_params) + return GPUInterpreter{typeof(method_table_view)}( + world, method_table_view, code_cache, Vector{CC.InferenceResult}(), + inf_params, opt_params) end function GPUInterpreter(interp::GPUInterpreter; @@ -472,21 +198,19 @@ function GPUInterpreter(interp::GPUInterpreter; inf_cache::Vector{CC.InferenceResult}=interp.inf_cache, inf_params::CC.InferenceParams=interp.inf_params, opt_params::CC.OptimizationParams=interp.opt_params) - return GPUInterpreter(world, method_table_view, - code_cache, inf_cache, - inf_params, opt_params) + return GPUInterpreter{typeof(method_table_view)}( + world, method_table_view, code_cache, inf_cache, + inf_params, opt_params) end + +CC.code_cache(interp::GPUInterpreter) = WorldView(interp.code_cache, interp.world) + end # HAS_INTEGRATED_CACHE CC.InferenceParams(interp::GPUInterpreter) = interp.inf_params CC.OptimizationParams(interp::GPUInterpreter) = interp.opt_params #=CC.=#get_inference_world(interp::GPUInterpreter) = interp.world CC.get_inference_cache(interp::GPUInterpreter) = interp.inf_cache -@static if HAS_INTEGRATED_CACHE - CC.cache_owner(interp::GPUInterpreter) = interp.token -else - CC.code_cache(interp::GPUInterpreter) = WorldView(interp.code_cache, interp.world) -end # No need to do any locking since we're not putting our results into the runtime cache CC.lock_mi_inference(interp::GPUInterpreter, mi::MethodInstance) = nothing @@ -507,8 +231,6 @@ CC.method_table(interp::GPUInterpreter) = interp.method_table_view # semi-concrete interepretation is broken with overlays (JuliaLang/julia#47349) function CC.concrete_eval_eligible(interp::GPUInterpreter, @nospecialize(f), result::CC.MethodCallResult, arginfo::CC.ArgInfo, sv::CC.InferenceState) - # NOTE it's fine to skip overloading with `sv::IRInterpretationState` since we disables - # semi-concrete interpretation anyway. ret = @invoke CC.concrete_eval_eligible(interp::CC.AbstractInterpreter, f::Any, result::CC.MethodCallResult, arginfo::CC.ArgInfo, sv::CC.InferenceState) if ret === :semi_concrete_eval @@ -525,163 +247,46 @@ function CC.concrete_eval_eligible(interp::GPUInterpreter, end -## world view of the cache -@static if VERSION < v"1.14-" - using Core.Compiler: WorldView -end - -if !HAS_INTEGRATED_CACHE - -function CC.haskey(wvc::WorldView{CodeCache}, mi::MethodInstance) - CC.get(wvc, mi, nothing) !== nothing -end - -function CC.get(wvc::WorldView{CodeCache}, mi::MethodInstance, default) - # check the cache - for ci in get!(wvc.cache.dict, mi, CodeInstance[]) - if ci.min_world <= wvc.worlds.min_world && wvc.worlds.max_world <= ci.max_world - # TODO: if (code && (code == jl_nothing || jl_ir_flag_inferred((jl_array_t*)code))) - src = if ci.inferred isa Vector{UInt8} - ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), - mi.def, C_NULL, ci.inferred) - else - ci.inferred - end - return ci - end - end - - return default -end +## driving inference and walking callees -function CC.getindex(wvc::WorldView{CodeCache}, mi::MethodInstance) - r = CC.get(wvc, mi, nothing) - r === nothing && throw(KeyError(mi)) - return r::CodeInstance -end - -function CC.setindex!(wvc::WorldView{CodeCache}, ci::CodeInstance, mi::MethodInstance) - CC.setindex!(wvc.cache, ci, mi) -end - -end # HAS_INTEGRATED_CACHE - -## codegen/inference integration - -function ci_cache_populate(interp, cache, mi, min_world, max_world) - codeinfos = Pair{CodeInstance, CodeInfo}[] - @static if VERSION >= v"1.12.0-DEV.1434" - # see typeinfer.jl: typeinf_ext_toplevel - has_compilequeue = VERSION >= v"1.13.0-DEV.499" || v"1.12-beta3" <= VERSION < v"1.13-" - ci = CC.typeinf_ext(interp, mi, CC.SOURCE_MODE_NOT_REQUIRED) - if has_compilequeue - workqueue = CC.CompilationQueue(; interp) - push!(workqueue, ci) - else - workqueue = CodeInstance[ci] - inspected = IdSet{CodeInstance}() - end - while !isempty(workqueue) - callee = pop!(workqueue) - if has_compilequeue - CC.isinspected(workqueue, callee) && continue - CC.markinspected!(workqueue, callee) - else - callee in inspected && continue - push!(inspected, callee) - end - - # now make sure everything has source code, if desired - mi = CC.get_ci_mi(callee) - if CC.use_const_api(callee) - if VERSION >= v"1.13.0-DEV.1121" - src = CC.codeinfo_for_const(interp, mi, CC.WorldRange(callee.min_world, callee.max_world), callee.edges, callee.rettype_const) - else - src = CC.codeinfo_for_const(interp, mi, callee.rettype_const) - end - else - # TODO: typeinf_code could return something with different edges/ages/owner/abi (needing an update to callee), which we don't handle here - src = CC.typeinf_code(interp, mi, true) - end - if src isa CodeInfo - if has_compilequeue - sptypes = CC.sptypes_from_meth_instance(mi) - CC.collectinvokes!(workqueue, src, sptypes) - else - CC.collectinvokes!(workqueue, src) - end - push!(codeinfos, callee => src) - end - end - elseif VERSION >= v"1.12.0-DEV.15" - inferred_ci = CC.typeinf_ext_toplevel(interp, mi, CC.SOURCE_MODE_FORCE_SOURCE) - @assert inferred_ci !== nothing "Inference of $mi failed" - - # inference should have populated our cache - wvc = WorldView(cache, min_world, max_world) - @assert CC.haskey(wvc, mi) "GPUCompiler: Failed to compile method for $mi, between worlds $min_world and $max_world" - ci = CC.getindex(wvc, mi) - - # if ci is rettype_const, the inference result won't have been cached - # (because it is normally not supposed to be used ever again). - # to avoid the need to re-infer, set that field here. - if ci.inferred === nothing - CC.setindex!(wvc, inferred_ci, mi) - ci = CC.getindex(wvc, mi) - end - else +# Drive type inference on `mi` using `interp`. On 1.11+ this is `CompilerCaching.typeinf!`, +# which (on 1.12+) recursively walks callees so their CIs carry stored source for +# `CompilerCaching.get_codeinfos` to read back into the `jl_emit_native` payload. Returns +# the root `CodeInstance` (or `nothing` if inference failed). +@static if HAS_INTEGRATED_CACHE + drive_inference!(interp::GPUInterpreter, mi::MethodInstance) = + CompilerCaching.typeinf!(interp, mi) +else + # 1.10: an inline copy that talks to the per-interpreter `CodeCache` instead of the + # integrated one. Returns `nothing` (no integrated CI to hand back; the caller + # fetches via `code_cache(interp)`). + function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) src = CC.typeinf_ext_toplevel(interp, mi) + @assert src !== nothing "Inference of $mi failed" - # inference should have populated our cache - wvc = WorldView(cache, min_world, max_world) - - @assert CC.haskey(wvc, mi) "GPUCompiler: Failed to compile method for $mi, between worlds $min_world and $max_world" - ci = CC.getindex(wvc, mi) - - # if ci is rettype_const, the inference result won't have been cached - # (because it is normally not supposed to be used ever again). - # to avoid the need to re-infer, set that field here. - if ci.inferred === nothing - @atomic ci.inferred = src + # For const-return CIs the inference result wasn't recorded — set it from the + # returned source so callers re-using the CI don't need to re-infer. + wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) + if CC.haskey(wvc, mi) + ci = CC.getindex(wvc, mi) + if ci.inferred === nothing + @atomic ci.inferred = src + end end - end - - return codeinfos -end - -@static if VERSION >= v"1.14-" -function ci_cache_lookup(cache, mi, min_world, max_world) - # In Julia 1.14+, WorldView was replaced by InternalCodeCache with WorldRange - # cache is OverlayCodeCache{InternalCodeCache}, extract owner from globalcache - owner = cache.globalcache.owner - wvc = CC.InternalCodeCache(owner, CC.WorldRange(min_world, max_world)) - ci = CC.get(wvc, mi, nothing) - return ci -end -else -function ci_cache_lookup(cache, mi, min_world, max_world) - wvc = WorldView(cache, min_world, max_world) - ci = CC.get(wvc, mi, nothing) - if VERSION < v"1.12.0-DEV.1434" && ci !== nothing && ci.inferred === nothing - # if for some reason we did end up with a codeinfo without inferred source, e.g., - # because of calling `Base.return_types` which only sets rettyp, pretend we didn't - # run inference so that we re-infer now and not during codegen (which is disallowed) return nothing end - return ci end -end # @static if -## interface +## codegen/inference integration -# for platforms without @cfunction-with-closure support -const _method_instances = Ref{Any}() -const _cache = Ref{Any}() -function _lookup_fun(mi, min_world, max_world) - push!(_method_instances[], mi) - ci_cache_lookup(_cache[], mi, min_world, max_world) -end +const HAS_LLVM_GET_CIS = ( + VERSION >= v"1.13.0-DEV.1120" || ( + Libdl.dlsym( + unsafe_load(cglobal(:jl_libjulia_handle, Ptr{Cvoid})), :jl_get_llvm_cis, throw_error = false + ) !== nothing + ) +) @enum CompilationPolicy::Cint begin CompilationPolicyDefault = 0 @@ -698,11 +303,8 @@ function Base.precompile(@nospecialize(job::CompilerJob)) if job.source.def.primary_world > job.world error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") end - - # populate the cache interp = get_interpreter(job) - cache = CC.code_cache(interp) - ci_cache_populate(interp, cache, job.source, job.world, job.world) + drive_inference!(interp, job.source) return true end @@ -755,29 +357,66 @@ function record_coverage(mi::MethodInstance, src::CodeInfo) return end -const HAS_LLVM_GET_CIS = ( - VERSION >= v"1.13.0-DEV.1120" || ( - Libdl.dlsym( - unsafe_load(cglobal(:jl_libjulia_handle, Ptr{Cvoid})), :jl_get_llvm_cis, throw_error = false - ) !== nothing - ) -) +# for platforms without @cfunction-with-closure support, used pre-1.12 (and on 1.12 paths +# that still require `cgparams.lookup`). +const _method_instances = Ref{Any}() +const _lookup_cache = Ref{Any}() +function _lookup_fun(mi, min_world, max_world) + push!(_method_instances[], mi) + lookup_ci(_lookup_cache[], mi, min_world, max_world) +end + +# Resolve a `(MI, world)` to a CodeInstance, on whatever cache shape we have: +# `WorldView{CodeCache}` on 1.10, an `owner` token on 1.11+. +@static if HAS_INTEGRATED_CACHE +function lookup_ci(owner, mi::MethodInstance, min_world::UInt, max_world::UInt) + @static if VERSION >= v"1.14-" + wvc = CC.InternalCodeCache(owner, CC.WorldRange(min_world, max_world)) + else + wvc = WorldView(CC.InternalCodeCache(owner), CC.WorldRange(min_world, max_world)) + end + return CC.get(wvc, mi, nothing) +end +else +function lookup_ci(cache::CodeCache, mi::MethodInstance, min_world::UInt, max_world::UInt) + wvc = WorldView(cache, min_world, max_world) + ci = CC.get(wvc, mi, nothing) + if ci !== nothing && ci.inferred === nothing + # rettype-only CI without source — pretend we don't have it so we re-infer + return nothing + end + return ci +end +end function compile_method_instance(@nospecialize(job::CompilerJob)) if job.source.def.primary_world > job.world error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") end - # populate the cache + # drive inference interp = get_interpreter(job) - cache = CC.code_cache(interp) - populated = ci_cache_populate(interp, cache, job.source, job.world, job.world) + root_ci = drive_inference!(interp, job.source) + + # the cache handle for callback lookups: an owner token on 1.11+, a CodeCache on 1.10 + cache_handle = @static if HAS_INTEGRATED_CACHE + cache_owner(job) + else + interp.code_cache + end + + # gather (CI, CodeInfo) pairs for jl_emit_native (1.12+) + codeinfo_pairs = if VERSION >= v"1.12.0-DEV.1823" && root_ci !== nothing + CompilerCaching.get_codeinfos(root_ci) + else + nothing + end # record line coverage of all compiled code (on older versions of Julia, where - # `ci_cache_populate` does not return sources, this happens after codegen instead) - if Base.JLOptions().code_coverage != 0 - for (ci, src) in populated - record_coverage(ci.def::MethodInstance, src) + # inference does not return sources, this happens after codegen instead) + if Base.JLOptions().code_coverage != 0 && codeinfo_pairs !== nothing + for (ci′, src) in codeinfo_pairs + record_coverage(ci′.def::MethodInstance, src::CodeInfo) end end @@ -787,11 +426,11 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if Sys.ARCH == :x86 || Sys.ARCH == :x86_64 function lookup_fun(mi, min_world, max_world) push!(method_instances, mi) - ci_cache_lookup(cache, mi, min_world, max_world) + lookup_ci(cache_handle, mi, min_world, max_world) end lookup_cb = @cfunction($lookup_fun, Any, (Any, UInt, UInt)) else - _cache[] = cache + _lookup_cache[] = cache_handle _method_instances[] = method_instances lookup_cb = @cfunction(_lookup_fun, Any, (Any, UInt, UInt)) end @@ -831,10 +470,10 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) native_code = if VERSION >= v"1.12.0-DEV.1823" codeinfos = Any[] - for (ci, src) in populated - # each item in the list should be a CodeInstance followed by a CodeInfo + for (ci′, src) in codeinfo_pairs + # each item in the list is a CodeInstance followed by a CodeInfo # indicating something to compile - push!(codeinfos, ci::CodeInstance) + push!(codeinfos, ci′::CodeInstance) push!(codeinfos, src::CodeInfo) end @ccall jl_emit_native(codeinfos::Vector{Any}, ts_mod::LLVM.API.LLVMOrcThreadSafeModuleRef, Ref(params)::Ptr{Base.CodegenParams}, #=extern linkage=# false::Cint)::Ptr{Cvoid} @@ -856,22 +495,17 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # XXX: this is wrong; we can't expose the underlying LLVM module, but should # instead always go through the callback in order to unlock it properly. - # rework this once we depend on Julia 1.9 or later. llvm_ts_mod = LLVM.ThreadSafeModule(llvm_mod_ref) llvm_mod = nothing llvm_ts_mod() do mod llvm_mod = mod end end - if !(Sys.ARCH == :x86 || Sys.ARCH == :x86_64) - cache_gbl = nothing - end # Since Julia 1.13, the caller is responsible for initializing global variables that - # point to global values or bindings with their address in memory. - # Similarly on previous versions when imaging=true, it is also the caller's responsibility - # (see https://github.com/JuliaGPU/GPUCompiler.jl/issues/753), but we can support this on versions - # that have HAS_LLVM_GVS_GLOBALS. + # point to global values or bindings with their address in memory. Similarly on earlier + # versions where `HAS_LLVM_GVS_GLOBALS` is true (see + # https://github.com/JuliaGPU/GPUCompiler.jl/issues/753). gvs = nothing inits = nothing @static if VERSION >= v"1.13.0-DEV.623" @@ -907,19 +541,14 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end end - # Maintain a map from global variables to their initialized Julia values. - # The objects pointed to are perma-rooted, during codegen. - # It is legal to call `Base.unsafe_pointer_to_objref` on `values(gv_to_value)`, - # but x->pointer_from_objref(Base.unsafe_pointer_to_objref(x)) is not idempotent, - # thus we store raw pointers here. - # Currently GVs are privatized, so users may have to handle embedded pointers, - # but this dictionary provides a clear indication that the embedded pointer is - # indeed avalid Julia object. + # Maintain a map from global variables to their initialized Julia values. The + # objects pointed to are perma-rooted during codegen. We *don't* bake these + # addresses into the IR yet, so that we can cache it across sessions. gv_to_value = Dict{String, Ptr{Cvoid}}() - - # On certain version of Julia we have no reliable way to match the `gvs` to their initializers `inits`. if gvs === nothing - # global variables here properly. + # No reliable GV table on this Julia — best-effort discovery from the module. + # On these older versions Julia's own emit may have already baked in absolute + # pointer values; we recover them by reading existing initializers. for gv in globals(llvm_mod) if !haskey(metadata(gv), "julia.constgv") continue @@ -945,22 +574,20 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) for (gv_ref, init) in zip(gvs, inits) gv = GlobalVariable(gv_ref) gv_to_value[LLVM.name(gv)] = init - # set the initializer - # TODO(vc): To enable full relocation we should actually strip out the initializers here. - if LLVM.isnull(initializer(gv)) - val = const_inttoptr(ConstantInt(Int64(init)), value_type(initializer(gv))) - initializer!(gv, val) - end + # Mirror what `jl_emit_native_impl` does on 1.13+ (aotcompile.cpp:865): + # reset the initializer to null so the IR we hand back is session-portable, + # then `relocate_gvs!` at the toplevel link step writes the session-current + # value in. On 1.13+ this is a no-op (Julia already nulled them); on 1.12, + # where `jl_emit_native_impl` bakes pointers via `literal_static_pointer_val`, + # this is what makes the bitcode we cache safe across sessions. + initializer!(gv, LLVM.null(value_type(initializer(gv)))) end end - code_instances = Core.CodeInstance[] + code_instances = CodeInstance[] if HAS_LLVM_GET_CIS # on sufficiently recent versions of Julia, we can query the CIs compiled. - # this is required after the move to `invoke(::CodeInstance)`, because our - # lookup function (used to populate method_instances) isn't always called then. - num_cis = Ref{Csize_t}(0) @ccall jl_get_llvm_cis(native_code::Ptr{Cvoid}, num_cis::Ptr{Csize_t}, C_NULL::Ptr{Cvoid})::Nothing @@ -970,7 +597,6 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) )::Nothing elseif VERSION >= v"1.12.0-DEV.1703" # slightly older versions of Julia used MIs directly - num_mis = Ref{Csize_t}(0) @ccall jl_get_llvm_mis(native_code::Ptr{Cvoid}, num_mis::Ptr{Csize_t}, C_NULL::Ptr{Cvoid})::Nothing @@ -981,53 +607,48 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if !HAS_LLVM_GET_CIS for mi in method_instances - ci = ci_cache_lookup(cache, mi, job.world, job.world) - ci === nothing && continue + ci′ = lookup_ci(cache_handle, mi, job.world, job.world) + ci′ === nothing && continue llvm_func_idx = Ref{Int32}(-1) llvm_specfunc_idx = Ref{Int32}(-1) ccall( :jl_get_function_id, Nothing, (Ptr{Cvoid}, Any, Ptr{Int32}, Ptr{Int32}), - native_code, ci, llvm_func_idx, llvm_specfunc_idx + native_code, ci′, llvm_func_idx, llvm_specfunc_idx ) - # Suppose we have two nested interpreters in use at the same time. - # Looking up a ci from the cache is not unique for a given mi. - # Consequently its possible we may not have compiled the ci found - # by the cache (instead having compiled the ci from the other interp). if llvm_func_idx[] == -1 continue end - push!(code_instances, ci) + push!(code_instances, ci′) end else # `jl_get_llvm_cis` can report stale CIs that no longer cover the world # this job was compiled in. The explicit cache lookup path already filters # by world; do the same here before de-duplicating by MI. - filter!(code_instances) do ci - ci.min_world <= job.world <= ci.max_world + filter!(code_instances) do ci′ + ci′.min_world <= job.world <= ci′.max_world end - # To avoid a clash in the compiled cache containing both with an interpreter token (like GPUCompiler.GPUCompilerCacheToken) and native, - # prefer the non-native code-instance. - # TODO: in the future we should migrate compiled to have the ci as the key, not the mi. + # `jl_get_llvm_cis` may return both our owner-token CI and a native + # owner-less CI for the same MI. Prefer ours when present, but keep + # owner-less CIs that are the only compiled entry for their method. owned_mis = Set{MethodInstance}() - for ci in code_instances - if ci.owner !== nothing - push!(owned_mis, ci.def::MethodInstance) + for ci′ in code_instances + if ci′.owner !== nothing + push!(owned_mis, ci′.def::MethodInstance) end end - filter!(code_instances) do ci - return ci.owner !== nothing || ci.def ∉ owned_mis + filter!(code_instances) do ci′ + return ci′.owner !== nothing || ci′.def ∉ owned_mis end end - # Avoid redundant code_instances. This is necessary to avoid false positives trying to add the same key'd mi to the compiled Dict. unique!(code_instances) # record line coverage of all compiled code (on newer versions of Julia, this happens - # based on the sources returned by `ci_cache_populate` instead) - if Base.JLOptions().code_coverage != 0 && isempty(populated) + # based on the sources returned by inference instead) + if Base.JLOptions().code_coverage != 0 && codeinfo_pairs === nothing for ci in code_instances src = ci.inferred if src isa String @@ -1040,21 +661,22 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end resize!(method_instances, length(code_instances)) - for (i, ci) in enumerate(code_instances) - method_instances[i] = ci.def::MethodInstance + for (i, ci′) in enumerate(code_instances) + method_instances[i] = ci′.def::MethodInstance end # process all compiled method instances compiled = Dict() - for (ci, mi) in zip(code_instances, method_instances) - - # get the function index + function compiled_entry(ci′::CodeInstance; required::Bool=true) llvm_func_idx = Ref{Int32}(-1) llvm_specfunc_idx = Ref{Int32}(-1) ccall(:jl_get_function_id, Nothing, (Ptr{Cvoid}, Any, Ptr{Int32}, Ptr{Int32}), - native_code, ci, llvm_func_idx, llvm_specfunc_idx) - @assert llvm_func_idx[] != -1 || llvm_specfunc_idx[] != -1 "Static compilation failed" + native_code, ci′, llvm_func_idx, llvm_specfunc_idx) + if llvm_func_idx[] == -1 && llvm_specfunc_idx[] == -1 + required && error("Static compilation failed") + return nothing + end # get the function llvm_func = if llvm_func_idx[] >= 1 @@ -1075,11 +697,24 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) nothing end + (; ci=ci′, func=llvm_func, specfunc=llvm_specfunc) + end + + for (ci′, mi) in zip(code_instances, method_instances) @assert !haskey(compiled, mi) # NOTE: it's not safe to store raw LLVM functions here, since those may get # removed or renamed during optimization, so we store their name instead. - compiled[mi] = (; ci, func=llvm_func, specfunc=llvm_specfunc) + compiled[mi] = compiled_entry(ci′) + end + + if !haskey(compiled, job.source) && root_ci !== nothing + # On Julia 1.14-dev, `jl_get_llvm_cis` may omit the root CI. That root + # can also be less-specific than `job.source` when Julia deliberately + # reuses unspecialized code, so recover the entry that codegen actually + # emitted and expose it under the requested source. + entry = compiled_entry(root_ci; required=false) + entry !== nothing && (compiled[job.source] = entry) end # ensure that the requested method instance was compiled @@ -1088,7 +723,30 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) return llvm_mod, compiled, gv_to_value end -# partially revert JuliaLangjulia#49391 +""" + relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + +Bake absolute pointer values into the initializers of `julia.constgv`-tagged +global variables, matching them by name against `gv_to_value`. Only GVs whose +initializer is currently null are touched: Preserving existing initializers +covers the older-Julia path where Julia itself emits pointer values directly. + +GVs present in `mod` but missing from `gv_to_value` keep a null initializer. +""" +function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + mod_gvs = globals(mod) + for (name, init) in gv_to_value + haskey(mod_gvs, name) || continue + gv = mod_gvs[name] + if LLVM.isnull(initializer(gv)) + val = const_inttoptr(ConstantInt(Int64(init)), value_type(initializer(gv))) + initializer!(gv, val) + end + end + return +end + +# partially revert JuliaLang/julia#49391 — see #527 @static if v"1.11.0-DEV.1603" <= VERSION < v"1.12.0-DEV.347" && # reverted on master !(v"1.11-beta2" <= VERSION < v"1.12") # reverted on 1.11-beta2 function CC.typeinf(interp::GPUInterpreter, frame::CC.InferenceState) diff --git a/src/metal.jl b/src/metal.jl index 0c7dd6aa..413e4a5b 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -106,12 +106,6 @@ have_fma(@nospecialize(target::MetalCompilerTarget), T::Type) = true ## job -# the debug-info level is part of the slug (as on PTX): it changes the debug metadata emitted -# into the runtime library, and the device-exception reporters branch on it, so a library -# built at one level must not be reused at another. -runtime_slug(job::CompilerJob{MetalCompilerTarget}) = - "metal-macos$(job.config.target.macos)-debuginfo=$(Int(llvm_debug_info(job)))" - isintrinsic(@nospecialize(job::CompilerJob{MetalCompilerTarget}), fn::String) = return startswith(fn, "air.") diff --git a/src/native.jl b/src/native.jl index fdd880ec..536c31d4 100644 --- a/src/native.jl +++ b/src/native.jl @@ -33,6 +33,5 @@ end ## job -runtime_slug(job::CompilerJob{NativeCompilerTarget}) = "native_$(job.config.target.cpu)-$(hash(job.config.target.features))$(job.config.target.jlruntime ? "-jlrt" : "")" uses_julia_runtime(job::CompilerJob{NativeCompilerTarget}) = job.config.target.jlruntime can_vectorize(job::CompilerJob{NativeCompilerTarget}) = true diff --git a/src/optim.jl b/src/optim.jl index 7a07d200..b0b339a3 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -358,6 +358,8 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op end add!(fpm, GCInvariantVerifierPass()) add!(fpm, LateLowerGCPass()) + # FinalLowerGCPass moved from a module pass to a function pass in + # JuliaLang/julia#51081 (1.11.0-DEV.208). if uses_julia_runtime(job) && VERSION >= v"1.11.0-DEV.208" add!(fpm, FinalLowerGCPass()) end @@ -501,13 +503,7 @@ function (self::LinkLibraries)(mod::LLVM.Module) # references after the normal library-linking phase has run. Give back-ends a # second chance to resolve those references before they lower their own runtime # intrinsics. This is a no-op for back-ends without a link_libraries! override. - if has_legacy_link_libraries(self.job) - undefined_fns = [LLVM.name(f) for f in functions(mod) - if isdeclaration(f) && !LLVM.isintrinsic(f)] - link_libraries!(self.job, mod, undefined_fns) - else - link_libraries!(self.job, mod) - end + link_libraries!(self.job, mod) return true end GPULinkLibrariesPass(job) = NewPMModulePass("GPULinkLibraries", LinkLibraries(job)) diff --git a/src/ptx.jl b/src/ptx.jl index dfde75ad..3c22f5c9 100644 --- a/src/ptx.jl +++ b/src/ptx.jl @@ -138,21 +138,14 @@ isintrinsic(@nospecialize(job::CompilerJob{PTXCompilerTarget}), fn::String) = # libdevice's __CUDA_ARCH dispatch, are still supported by the external back-end. startswith(fn, "llvm.nvvm.") -# XXX: the debuginfo part should be handled by GPUCompiler as it applies to all back-ends. -runtime_slug(@nospecialize(job::CompilerJob{PTXCompilerTarget})) = - "ptx$(job.config.target.ptx.major)$(job.config.target.ptx.minor)" * - "-$(cpu_name(job.config.target))" * - "-debuginfo=$(Int(llvm_debug_info(job)))" - function finish_module!(@nospecialize(job::CompilerJob{PTXCompilerTarget}), mod::LLVM.Module, entry::LLVM.Function) # tell NVVMReflect whether to flush denormals; this mirrors what Clang does # for `-fcuda-flush-denormals-to-zero` and is the only `__nvvm_reflect` key # LLVM's NVVMReflectPass honors besides `__CUDA_ARCH`. only emit it on the # toplevel module that runs through `optimize!`, as sub-modules (the cached - # runtime, deferred jobs) don't need it, and the cached runtime in - # particular would otherwise conflict on link if it was built with a - # different `fastmath` setting (which isn't part of `runtime_slug`). + # runtime, deferred jobs) don't need it and should not get module-level + # flags that can collide when linked into the toplevel module. if job.config.toplevel flags(mod)["nvvm-reflect-ftz", LLVM.API.LLVMModuleFlagBehaviorOverride] = Metadata(ConstantInt(Int32(job.config.target.fastmath ? 1 : 0))) diff --git a/src/rtlib.jl b/src/rtlib.jl index d9149d34..d0757c43 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,10 +60,33 @@ end ## functionality to build the runtime library +# Per-function compilation results for the GPU runtime library, cached through the +# same `cached_results` mechanism back-ends use for kernels. On 1.11+ the bitcode +# thus lives on the runtime function's `CodeInstance` — possibly alongside a +# back-end's own results struct — and persists through precompilation, so sessions +# loading a back-end that compiled its runtime during precompile skip codegen +# entirely. On 1.10 it is cached for the duration of the session. +mutable struct RuntimeFunctionResults + bitcode::Union{Nothing,Vector{UInt8}} + RuntimeFunctionResults() = new(nothing) +end + +# Compile a single runtime function and link it into `mod`. The renamed bitcode is +# memoized through `RuntimeFunctionResults`; the session-local `runtime_libs` cache +# below additionally avoids repeating the parse-and-link work within a session. function emit_function!(mod, config::CompilerConfig, f, method) tt = Base.to_tuple_type(method.types) source = generic_methodinstance(f, tt) - new_mod, meta = compile_unhooked(:llvm, CompilerJob(source, config)) + name = method.llvm_name + rt_job = CompilerJob(source, config) + + res = cached_results(RuntimeFunctionResults, rt_job) + if res.bitcode !== nothing + link!(mod, parse(LLVM.Module, MemoryBuffer(res.bitcode))) + return + end + + new_mod, meta = compile_unhooked(:llvm, rt_job) ft = function_type(meta.entry) expected_ft = convert(LLVM.FunctionType, method) if return_type(ft) != return_type(expected_ft) @@ -73,28 +96,36 @@ function emit_function!(mod, config::CompilerConfig, f, method) # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) - temp_name = LLVM.name(meta.entry) - link!(mod, new_mod) - entry = functions(mod)[temp_name] - - # if a declaration already existed, replace it with the function to avoid aliasing - # (and getting function names like gpu_signal_exception1) - name = method.llvm_name - if haskey(functions(mod), name) - decl = functions(mod)[name] - @assert value_type(decl) == value_type(entry) - replace_uses!(decl, entry) + # rename to the final `gpu_*` name on the per-function module, so the cached bitcode + # is immediately link-ready (no per-session rename pass on a cache hit). + if haskey(functions(new_mod), name) && functions(new_mod)[name] !== meta.entry + decl = functions(new_mod)[name] + @assert value_type(decl) == value_type(meta.entry) + replace_uses!(decl, meta.entry) erase!(decl) end - LLVM.name!(entry, name) + LLVM.name!(meta.entry, name) + + io = IOBuffer() + write(io, new_mod) + res.bitcode = take!(io) + + link!(mod, new_mod) end -function build_runtime(@nospecialize(job::CompilerJob)) - mod = LLVM.Module("GPUCompiler run-time library") +# the compiler job passed into here identifies the job that requires the runtime. +# derive a config that represents the runtime itself (notably with kernel=false). +# fields that identify the *kernel* job, like its entry-point name, are reset so +# that runtime artifacts are keyed (and persisted) identically for all kernels +# sharing the remaining codegen-relevant settings, instead of once per cosmetic +# config variation. +function runtime_config(@nospecialize(job::CompilerJob)) + CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, + strip=false, name=nothing) +end - # the compiler job passed into here is identifies the job that requires the runtime. - # derive a job that represents the runtime itself (notably with kernel=false). - config = CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, strip=false) +function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) + mod = LLVM.Module("GPUCompiler run-time library") for method in values(Runtime.methods) def = if isa(method.def, Symbol) @@ -114,50 +145,30 @@ function build_runtime(@nospecialize(job::CompilerJob)) mod end -@static if VERSION >= v"1.11.0" - import Core.Compiler: is_asserts -else - is_asserts() = false -end +# Session-local cache of assembled runtime libraries, keyed by +# `(runtime_config(job), opaque_pointers)`: the derived runtime config covers every +# codegen-relevant setting (e.g. the debug level, which is baked into the runtime IR +# as a constant), while cosmetic kernel-job fields are normalized away. Cross-session +# persistence happens at the per-function level (see `RuntimeFunctionResults`): +# reassemble on first use of each session, then reuse within the session. +# +# NOTE: unlike the per-function bitcode, whose CodeInstances Julia invalidates on +# method redefinition, the assembled library is world-blind: redefining a runtime +# function mid-session leaves a stale assembly behind. That only affects interactive +# development of a back-end's runtime module (restart, or `empty!` this cache). +const runtime_libs = Dict{Tuple{CompilerConfig, Bool}, Vector{UInt8}}() +const runtime_libs_lock = ReentrantLock() @locked function load_runtime(@nospecialize(job::CompilerJob)) - global compile_cache - if compile_cache === nothing # during precompilation - return build_runtime(job) - end + config = runtime_config(job) + key = (config, !supports_typed_pointers(context())) - slug = runtime_slug(job) - if !supports_typed_pointers(context()) - slug *= "-opaque" - end - - # Julia codegen changes metadata in modules when `FORCE_ASSERTIONS=1` - if is_asserts() - slug *= "-asserts" - end - - name = "runtime_$(slug).bc" - path = joinpath(compile_cache, name) - - if !ispath(path) - @debug "Building the GPU runtime library at $path" - mkpath(compile_cache) - lib = build_runtime(job) - - # atomic write to disk - temp_path, io = mktemp(dirname(path); cleanup=false) + bytes = Base.@lock runtime_libs_lock get!(runtime_libs, key) do + lib = build_runtime(job, config) + io = IOBuffer() write(io, lib) - close(io) - @static if VERSION >= v"1.12.0-DEV.1023" - mv(temp_path, path; force=true) - else - Base.rename(temp_path, path, force=true) - end + take!(io) end - return parse(LLVM.Module, MemoryBufferFile(path); lazy=true) + return parse(LLVM.Module, MemoryBuffer(bytes); lazy=true) end - -# remove the existing cache -# NOTE: call this function from global scope, so any change triggers recompilation. -reset_runtime() = rm(compile_cache; recursive=true, force=true) diff --git a/src/spirv.jl b/src/spirv.jl index 3400fef6..f0af2f20 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -24,7 +24,13 @@ export SPIRVCompilerTarget Base.@kwdef struct SPIRVCompilerTarget <: AbstractCompilerTarget version::Union{Nothing,VersionNumber} = nothing - extensions::Vector{String} = [] + # SPIR-V extensions, as the comma-separated specifier string passed verbatim to the + # translator/back-end via `--spirv-ext`, e.g. "+SPV_EXT_shader_atomic_float_add,+SPV_KHR_expect_assume" + # (LLVM feature-string style, cf. `GCNCompilerTarget.features`). Kept as a plain + # `String` -- not a `Vector` -- because target fields must be immutable for the + # `jl_egal`-based cache lookups (`cache_owner`, `JobResults`) to match across + # sessions; see the §2.2 rule in CACHING-REDESIGN.md. + extensions::String = "" supports_fp16::Bool = true supports_fp64::Bool = true supports_bfloat16::Bool = false @@ -58,11 +64,6 @@ llvm_datalayout(::SPIRVCompilerTarget) = Int===Int64 ? ## job -# TODO: encode debug build or not in the compiler job -# https://github.com/JuliaGPU/CUDAnative.jl/issues/368 -runtime_slug(job::CompilerJob{SPIRVCompilerTarget}) = - "spirv-" * String(job.config.target.backend) - function finish_module!(job::CompilerJob{SPIRVCompilerTarget}, mod::LLVM.Module, entry::LLVM.Function) # update calling convention @@ -154,8 +155,7 @@ end cmd = `$(SPIRV_LLVM_Backend_jll.llc()) $input -filetype=obj -o $translated` if !isempty(job.config.target.extensions) - str = join(map(ext->"+$ext", job.config.target.extensions), ",") - cmd = `$(cmd) -spirv-ext=$str` + cmd = `$(cmd) -spirv-ext=$(job.config.target.extensions)` end elseif job.config.target.backend === :khronos translator = if isavailable(SPIRV_LLVM_Translator_jll) @@ -168,8 +168,7 @@ end cmd = `$translator -o $translated $input --spirv-debug-info-version=ocl-100` if !isempty(job.config.target.extensions) - str = join(map(ext->"+$ext", job.config.target.extensions), ",") - cmd = `$(cmd) --spirv-ext=$str` + cmd = `$(cmd) --spirv-ext=$(job.config.target.extensions)` end if job.config.target.version !== nothing diff --git a/src/utils.jl b/src/utils.jl index e8938a8f..db2fc7d7 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -6,6 +6,28 @@ else __has_internal_julia_change(version_or::VersionNumber, feature::Symbol) = false end + + +## `public` keyword compat + +""" + @public foo, bar + +Declare `foo, bar` as public API. Lowers to `public foo, bar` on 1.11+ (where `public` +is keyword syntax) and to a no-op on 1.10. +""" +macro public(symbols_expr) + syms = symbols_expr isa Symbol ? [symbols_expr] : + symbols_expr.head === :tuple ? [a isa Symbol ? a : a.args[1] for a in symbols_expr.args] : + [symbols_expr.args[1]] + if VERSION >= v"1.11.0-DEV.469" + esc(Expr(:public, syms...)) + else + nothing + end +end + + ## debug verification should_verify() = ccall(:jl_is_debugbuild, Cint, ()) == 1 || diff --git a/src/validation.jl b/src/validation.jl index c1a86549..5760afc7 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -14,18 +14,14 @@ function method_matches(@nospecialize(tt::Type{<:Tuple}); world::Integer) end function typeinf_type(mi::MethodInstance; interp::CC.AbstractInterpreter) - @static if VERSION < v"1.11.0" - code = Core.Compiler.get(Core.Compiler.code_cache(interp), mi, nothing) - if code isa Core.Compiler.CodeInstance - return code.rettype - end - result = Core.Compiler.InferenceResult(mi, Core.Compiler.typeinf_lattice(interp)) - Core.Compiler.typeinf(interp, result, :global) - Core.Compiler.is_inferred(result) || return Any - Core.Compiler.widenconst(Core.Compiler.ignorelimited(result.result)) + @static if hasmethod(Core.Compiler.typeinf_type, Tuple{CC.AbstractInterpreter, MethodInstance}) + rt = Core.Compiler.typeinf_type(interp, mi) else - something(Core.Compiler.typeinf_type(interp, mi), Any) + # Julia 1.10: only the 4-arg form exists; reconstruct it from the MI. + method = mi.def::Method + rt = Core.Compiler.typeinf_type(interp, method, mi.specTypes, mi.sparam_vals) end + return something(rt, Any) end function check_method(@nospecialize(job::CompilerJob)) diff --git a/test/Project.toml b/test/Project.toml index d01c6e6e..c05c17b8 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" FileCheck = "4e644321-382b-4b05-b0b6-5d23c3d944fb" GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" diff --git a/test/gcn.jl b/test/gcn.jl index 99dae742..9e47236d 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -20,12 +20,14 @@ sink_gcn(i) = sink(i, Val(5)) kernel() = return end - # the backend participates in the runtime slug, so different back-ends don't share a cache + # the backend participates in the cache owner, so different back-ends don't share a cache job_ext, _ = GCN.create_job(mod.kernel, Tuple{}; backend=:external) job_inp, _ = GCN.create_job(mod.kernel, Tuple{}; backend=:inprocess) - @test endswith(GPUCompiler.runtime_slug(job_ext), "-external") - @test endswith(GPUCompiler.runtime_slug(job_inp), "-inprocess") - @test GPUCompiler.runtime_slug(job_ext) != GPUCompiler.runtime_slug(job_inp) + owner_ext = GPUCompiler.cache_owner(job_ext) + owner_inp = GPUCompiler.cache_owner(job_inp) + @test owner_ext.target.backend === :external + @test owner_inp.target.backend === :inprocess + @test owner_ext != owner_inp # the explicit :external backend generates machine code through the external llc @test (GCN.code_native(devnull, mod.kernel, Tuple{}; backend=:external); true) diff --git a/test/helpers/metal.jl b/test/helpers/metal.jl index 0000a022..8bea2a25 100644 --- a/test/helpers/metal.jl +++ b/test/helpers/metal.jl @@ -25,8 +25,6 @@ end ThreadedRuntimeCompilerJob = CompilerJob{MetalCompilerTarget,ThreadedRuntimeCompilerParams} GPUCompiler.runtime_module(::ThreadedRuntimeCompilerJob) = ThreadedRuntime -GPUCompiler.runtime_slug(job::ThreadedRuntimeCompilerJob) = - "metal-threadedruntime-macos$(job.config.target.macos)-debuginfo=$(Int(GPUCompiler.llvm_debug_info(job)))" function create_job(@nospecialize(func), @nospecialize(types); kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 656028f4..63d8a0f3 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -69,22 +69,4 @@ function code_execution(@nospecialize(func), @nospecialize(types); kwargs...) end end -const runtime_cache = Dict{Any, Any}() - -function compiler(job) - JuliaContext() do ctx - GPUCompiler.compile(:asm, job) - end -end - -function linker(job, asm) - asm -end - -# simulates cached codegen -function cached_execution(@nospecialize(func), @nospecialize(types); kwargs...) - job, kwargs = create_job(func, types; validate=false, kwargs...) - GPUCompiler.cached_compilation(runtime_cache, job.source, job.config, compiler, linker) -end - end diff --git a/test/native.jl b/test/native.jl index 6aebba91..62bf9130 100644 --- a/test/native.jl +++ b/test/native.jl @@ -1,5 +1,8 @@ @testset "reflection" begin - job, _ = Native.create_job(identity, (Int,)) + mod = @eval module $(gensym()) + f(x::Int) = x + end + job, _ = Native.create_job(mod.f, (Int,)) @test only(GPUCompiler.code_lowered(job)) isa Core.CodeInfo @@ -7,17 +10,17 @@ @test rt === Int @test @filecheck begin - @check "MethodInstance for identity" + @check "MethodInstance for {{.*}}f" GPUCompiler.code_warntype(job) end @test @filecheck begin - @check "@{{(julia|j)_identity_[0-9]+}}" + @check "@{{(julia|j)_f_[0-9]+}}" GPUCompiler.code_llvm(job) end @test @filecheck begin - @check "@{{(julia|j)_identity_[0-9]+}}" + @check "@{{(julia|j)_f_[0-9]+}}" GPUCompiler.code_native(job) end end @@ -118,97 +121,69 @@ end @check "add i64 %{{[0-9]+}}, 2" GPUCompiler.code_llvm(job) end + end - # cached_compilation interface - invocations = Ref(0) - function compiler(job) - invocations[] += 1 - JuliaContext() do ctx - ir, ir_meta = GPUCompiler.compile(:llvm, job) - string(ir) + @testset "cached results" begin + mod = @eval module $(gensym()) + mutable struct Results + asm::Union{Nothing,String} + Results() = new(nothing) + end + mutable struct OtherResults + data::Any + OtherResults() = new(nothing) end - end - linker(job, compiled) = compiled - cache = Dict() - ft = typeof(mod.kernel) - tt = Tuple{Int64} - - # initial compilation - source = methodinstance(ft, tt, Base.get_world_counter()) - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 2" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 1 - - # cached compilation - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 2" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 1 - - # redefinition - @eval mod kernel(i) = child(i)+3 - source = methodinstance(ft, tt, Base.get_world_counter()) - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 2 - # cached compilation - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) + @noinline child(i) = i + kernel(i) = child(i)+1 end - @test invocations[] == 2 - # redefinition of an unrelated function - @eval mod unrelated(i) = 42 - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 2 - - # redefining child functions - @eval mod @noinline child(i) = i+1 - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 3 - - # cached compilation - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 3 - - # change in configuration - config = CompilerConfig(job.config; name="foobar") - @test @filecheck begin - @check "define i64 @foobar" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, config, compiler, linker) - end - @test invocations[] == 4 - - # tasks running in the background should keep on using the old version - c1, c2 = Condition(), Condition() - function background(job) - local_source = methodinstance(ft, tt, Base.get_world_counter()) - notify(c1) - wait(c2) # wait for redefinition - GPUCompiler.cached_compilation(cache, local_source, job.config, compiler, linker) - end - t = @async Base.invokelatest(background, job) - wait(c1) # make sure the task has started - @eval mod kernel(i) = child(i)+4 - source = methodinstance(ft, tt, Base.get_world_counter()) - ir = Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test contains(ir, r"add i64 %\d+, 4") - notify(c2) # wake up the task - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - fetch(t) + # get-or-create: first access yields an empty struct, later accesses the same one + job, _ = Native.create_job(mod.kernel, (Int64,)) + res = GPUCompiler.cached_results(mod.Results, job) + @test res isa mod.Results + @test res.asm === nothing + res.asm = "compiled" + @test GPUCompiler.cached_results(mod.Results, job) === res + + # independent consumers get independent structs for the same job + other = GPUCompiler.cached_results(mod.OtherResults, job) + @test other isa mod.OtherResults + @test GPUCompiler.cached_results(mod.Results, job) === res + + # results are keyed by the full config: a job differing only in codegen-level + # settings (here: the kernel name) must not share artifacts + named_job, _ = Native.create_job(mod.kernel, (Int64,); name="custom") + @test named_job.source === job.source + named_res = GPUCompiler.cached_results(mod.Results, named_job) + @test named_res !== res + @test named_res.asm === nothing + + # ... but an equal config constructed from scratch resolves to the same struct + job2, _ = Native.create_job(mod.kernel, (Int64,)) + @test GPUCompiler.cached_results(mod.Results, job2) === res + + # redefinition invalidates: a job in the new world gets a fresh struct + @eval mod kernel(i) = child(i)+2 + new_job, _ = Native.create_job(mod.kernel, (Int64,)) + new_res = GPUCompiler.cached_results(mod.Results, new_job) + @test new_res !== res + @test new_res.asm === nothing + + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # session-dependent results (e.g. artifacts with relocated GVs) are wiped + # before image serialization; emulate the atexit-driven wipe directly + new_res.asm = "session-dependent" + other_job, _ = Native.create_job(mod.kernel, (Int64,); name="other") + other_res = GPUCompiler.cached_results(mod.Results, other_job) + push!(GPUCompiler.session_dependent_jobs, new_job) + GPUCompiler.wipe_session_dependent_results() + @test isempty(GPUCompiler.session_dependent_jobs) + wiped_res = GPUCompiler.cached_results(mod.Results, new_job) + @test wiped_res !== new_res + @test wiped_res.asm === nothing + # ... without affecting other configs on the same CI + @test GPUCompiler.cached_results(mod.Results, other_job) === other_res end end @@ -216,12 +191,14 @@ end # when types have no fields, we should always allow them mod = @eval module $(gensym()) struct Empty end + accept_empty(::Empty) = nothing + accept_symbol(::Symbol) = nothing end - Native.code_execution(Returns(nothing), (mod.Empty,)) + Native.code_execution(mod.accept_empty, (mod.Empty,)) # this also applies to Symbols - Native.code_execution(Returns(nothing), (Symbol,)) + Native.code_execution(mod.accept_symbol, (Symbol,)) end @testset "code coverage" begin @@ -416,19 +393,23 @@ end end @testset "function entry safepoint emission" begin + mod = @eval module $(gensym()) + f(::Nothing) = nothing + end + @test @filecheck begin - @check_label "define void @{{(julia|j)_identity_[0-9]+}}" + @check_label "define void @{{(julia|j)_f_[0-9]+}}" @check_not "%safepoint" - Native.code_llvm(identity, Tuple{Nothing}; entry_safepoint=false, optimize=false, dump_module=true) + Native.code_llvm(mod.f, Tuple{Nothing}; entry_safepoint=false, optimize=false, dump_module=true) end # XXX: broken by JuliaLang/julia#57010, # see https://github.com/JuliaLang/julia/pull/57010/files#r2079576894 if VERSION < v"1.13.0-DEV.533" @test @filecheck begin - @check_label "define void @{{(julia|j)_identity_[0-9]+}}" + @check_label "define void @{{(julia|j)_f_[0-9]+}}" @check "%safepoint" - Native.code_llvm(identity, Tuple{Nothing}; entry_safepoint=true, optimize=false, dump_module=true) + Native.code_llvm(mod.f, Tuple{Nothing}; entry_safepoint=true, optimize=false, dump_module=true) end end end diff --git a/test/native/precompile.jl b/test/native/precompile.jl index f08a561f..b48ed86e 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -14,6 +14,9 @@ precompile_test_harness("Inference caching") do load_path return end + # a kernel that makes no calls, so its CodeInstance has no inference edges + leaf_kernel(A) = nothing + function kernel_w_global(A, x, sym) if sym == :A A[1] = x @@ -34,6 +37,11 @@ precompile_test_harness("Inference caching") do load_path precompile(job) end + let + job, _ = NativeCompiler.Native.create_job(leaf_kernel, (Vector{Int},)) + precompile(job) + end + let job, _ = NativeCompiler.Native.create_job(kernel_w_global, (Vector{Int}, Int, Symbol)) precompile(job) @@ -66,13 +74,17 @@ precompile_test_harness("Inference caching") do load_path import NativeCompiler # Check that no cached entry is present - identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + identity_mis = Any[ + GPUCompiler.methodinstance(typeof(identity), Tuple{Int}), + GPUCompiler.CompilerCaching.method_instance(identity, (Int,)), + ] + unique!(identity_mis) token = let job, _ = NativeCompiler.Native.create_job(identity, (Int,)) - GPUCompiler.ci_cache_token(job) + GPUCompiler.cache_owner(job) end - @test !check_presence(identity_mi, token) + @test all(!check_presence(mi, token) for mi in identity_mis) using NativeBackend @@ -83,11 +95,28 @@ precompile_test_harness("Inference caching") do load_path kernel_w_global_mi = GPUCompiler.methodinstance(typeof(NativeBackend.kernel_w_global), Tuple{Vector{Int}, Int, Symbol}) @test check_presence(kernel_w_global_mi, token) + # a CodeInstance without inference edges (a kernel making no calls) is + # serialized with the revalidation sentinel but excluded from the edge + # verification list on Julia 1.12 (jl_record_edges skips empty-edge CIs), + # so it deserializes permanently invalid. Fixed by the serialization + # rework in 1.13; 1.11 uses the old scheme and is unaffected. + leaf_edges_lost = v"1.12-" <= VERSION < v"1.13-" + leaf_kernel_mi = GPUCompiler.methodinstance(typeof(NativeBackend.leaf_kernel), Tuple{Vector{Int}}) + @test check_presence(leaf_kernel_mi, token) broken=leaf_edges_lost + square_mi = GPUCompiler.methodinstance(typeof(NativeBackend.square), Tuple{Float64}) @test check_presence(square_mi, token) # check that identity survived - @test check_presence(identity_mi, token) broken=(v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || v"1.13.0-" <= VERSION < v"1.13.0-beta3"|| v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843") + # NOTE: external CIs from the workload survive only flakily on 1.13 + # (the 1.13.0-beta3 backport did not fully fix this), so skip the + # check there. + ext_cis_lost = v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || + v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843" + ext_cis_flaky = v"1.13.0-" <= VERSION < v"1.14-" + if !ext_cis_flaky + @test any(mi -> check_presence(mi, token), identity_mis) broken=ext_cis_lost + end # Recompiling a foreign method after loading precompiled owner-token CIs # may also surface a native owner-less CI for the same MethodInstance. @@ -103,27 +132,5 @@ precompile_test_harness("Inference caching") do load_path _, meta = GPUCompiler.compile(:llvm, job) @test haskey(meta.compiled, job.source) end - - GPUCompiler.clear_disk_cache!() - @test GPUCompiler.disk_cache_enabled() == false - - GPUCompiler.enable_disk_cache!() - @test GPUCompiler.disk_cache_enabled() == true - - job, _ = NativeCompiler.Native.create_job(NativeBackend.kernel, (Vector{Int}, Int); validate=false) - @assert job.source == kernel_mi - ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world) - @assert ci !== nothing - @assert ci.inferred !== nothing - path = GPUCompiler.cache_file(ci, job.config) - @test path !== nothing - @test !ispath(path) - NativeCompiler.Native.cached_execution(NativeBackend.kernel, (Vector{Int}, Int)) - @test ispath(path) - GPUCompiler.clear_disk_cache!() - @test !ispath(path) - - GPUCompiler.enable_disk_cache!(false) - @test GPUCompiler.disk_cache_enabled() == false end end diff --git a/test/ptx.jl b/test/ptx.jl index 2d93f3d5..d1769cb6 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -10,23 +10,19 @@ end end -@testset "kernel state survives a runtime rebuild during optimization" begin - # Clearing the cache forces the relink pass inside `optimize!` to rebuild the runtime - # (nested compilation); the kernel must still get its state argument afterwards. +@testset "kernel state survives a runtime rebuild" begin + # Clearing the runtime cache forces the library link inside `emit_llvm` to rebuild + # the runtime (nested compilation); the kernel must still get its state argument + # afterwards. mod = @eval module $(gensym()) function kernel(x) x < 1 && throw(DivideError()) return end end - old_cache = GPUCompiler.compile_cache - ir = try - GPUCompiler.compile_cache = nothing - sprint() do io - PTX.code_llvm(io, mod.kernel, Tuple{Int}; kernel=true, dump_module=true) - end - finally - GPUCompiler.compile_cache = old_cache + Base.@lock GPUCompiler.runtime_libs_lock empty!(GPUCompiler.runtime_libs) + ir = sprint() do io + PTX.code_llvm(io, mod.kernel, Tuple{Int}; kernel=true, dump_module=true) end @test occursin("gpu_report_exception", ir) @test occursin("[1 x i64] %state", ir) @@ -527,8 +523,8 @@ end @test GPUCompiler.cpu_name(PTXCompilerTarget(cap=v"10.0", feature_set=:family)) == "sm_100f" @test_throws ErrorException GPUCompiler.cpu_name(PTXCompilerTarget(cap=v"9.0", feature_set=:bogus)) - # hash must discriminate, otherwise two configs differing only on feature_set - # would share the same on-disk runtime slug and collide in the compiler cache. + # hash must discriminate, otherwise two targets differing only on feature_set + # could compare equal inside cache-owner keys. @test hash(PTXCompilerTarget(cap=v"9.0", feature_set=:baseline)) != hash(PTXCompilerTarget(cap=v"9.0", feature_set=:arch)) diff --git a/test/ptx/precompile.jl b/test/ptx/precompile.jl index 290c60be..84c10abd 100644 --- a/test/ptx/precompile.jl +++ b/test/ptx/precompile.jl @@ -30,16 +30,25 @@ precompile_test_harness("Inference caching") do load_path import PTXCompiler # Check that no cached entry is present - identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + identity_mis = Any[ + GPUCompiler.methodinstance(typeof(identity), Tuple{Int}), + GPUCompiler.CompilerCaching.method_instance(identity, (Int,)), + ] + unique!(identity_mis) + # NOTE: use the standalone package's module to construct the token — the + # sandbox's own PTX helper module defines a distinct CompilerParams + # type, whose token can never match the precompiled CIs. token = let - job, _ = PTX.create_job(identity, (Int,)) - GPUCompiler.ci_cache_token(job) + job, _ = PTXCompiler.PTX.create_job(identity, (Int,)) + GPUCompiler.cache_owner(job) end - ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing - while ci !== nothing - @test ci.owner !== token - ci = isdefined(ci, :next) ? ci.next : nothing + for identity_mi in identity_mis + ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing + while ci !== nothing + @test ci.owner !== token + ci = isdefined(ci, :next) ? ci.next : nothing + end end using PTXBackend @@ -49,6 +58,14 @@ precompile_test_harness("Inference caching") do load_path @test check_presence(kernel_mi, token) # check that identity survived - @test check_presence(identity_mi, token) broken=(v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || v"1.13.0-" <= VERSION < v"1.13.0-beta3"|| v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843") + # NOTE: external CIs from the workload survive only flakily on 1.13 + # (the 1.13.0-beta3 backport did not fully fix this), so skip the + # check there. + ext_cis_lost = v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || + v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843" + ext_cis_flaky = v"1.13.0-" <= VERSION < v"1.14-" + if !ext_cis_flaky + @test any(mi -> check_presence(mi, token), identity_mis) broken=ext_cis_lost + end end end diff --git a/test/spirv.jl b/test/spirv.jl index b5006194..a100f4f1 100644 --- a/test/spirv.jl +++ b/test/spirv.jl @@ -182,7 +182,7 @@ end """, NTuple{N, Core.VecElement{Float32}}, NTuple{2, NTuple{N, Core.VecElement{Float32}}}, x.data, y.data)) end end - kernel(x...) = @noinline fadd(x...) + kernel(x, y) = @noinline fadd(x, y) end @test @filecheck begin @@ -191,7 +191,8 @@ end @check_label "define void @{{(julia|j)_kernel_[0-9]+}}" @check cond=(v"1.12" <= VERSION < v"1.12.5") "alloca <2 x i64>, align 16" @check cond=(VERSION >= v"1.12.5") "alloca [2 x i64], align 16" - SPIRV.code_llvm(mod.kernel, NTuple{2, mod.Vec{4, Float32}}; backend, dump_module=true) + SPIRV.code_llvm(mod.kernel, Tuple{mod.Vec{4, Float32}, mod.Vec{4, Float32}}; + backend, dump_module=true) end @test @filecheck begin @@ -199,7 +200,8 @@ end @check_label "define void @{{(julia|j)_kernel_[0-9]+}}" @check cond=(v"1.12" <= VERSION < v"1.12.5") "alloca [2 x <2 x i64>], align 16" @check cond=(VERSION >= v"1.12.5") "alloca [4 x i64], align 16" - SPIRV.code_llvm(mod.kernel, NTuple{2, mod.Vec{8, Float32}}; backend, dump_module=true) + SPIRV.code_llvm(mod.kernel, Tuple{mod.Vec{8, Float32}, mod.Vec{8, Float32}}; + backend, dump_module=true) end end