diff --git a/CLAUDE.md b/CLAUDE.md index 6496343..cdd9dca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,6 +40,7 @@ Keep Markdown prose to one sentence per line. - `@cached` preserves the method signature and moves the body to `MemoizationKit.implementation`. - The default global cache shares one size limit across a function's cached methods, with a separate cache per container type. - Preserve inferred return types and allocation-free RAM hits for concrete keys with the built-in containers. +- `ClockCache` hits are lock-free; keep the rules listed in `src/containers/clock.jl`. Task-local caches use the locked `LocalClockCache`. - Built-in RAM containers distinguish key types and use `hash` and `isequal`. - `MemoizationKit.cachekey` may merge equivalent inputs; merged calls must accept the same result and return type. - Compute misses outside cache locks so recursion works; concurrent misses may compute twice. diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 66ef505..f5ac05c 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -4,6 +4,7 @@ MemoizationKit = "1b238080-9255-4fe9-b224-89eb24efe93b" LRUCache = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [sources] MemoizationKit = {path = ".."} diff --git a/docs/src/eviction.md b/docs/src/eviction.md index c548435..21ab6f7 100644 --- a/docs/src/eviction.md +++ b/docs/src/eviction.md @@ -34,6 +34,7 @@ Unlike LRU, Clock does not track the exact order of hits. A recently inserted entry that has never been hit again can be evicted before an older entry that has been reused. Choose Clock for approximate recency with little bookkeeping on hits: a hit updates a bit rather than a recency list. +A shared `ClockCache` is read without a lock, so hits from many threads do not wait on each other. Eviction may scan several slots to find a victim. ## LRU: exact recency @@ -59,7 +60,7 @@ sort!(collect(keys(cache))) # B was evicted Unlike Clock, LRU always retains the exact recency order, even after several hits between evictions. Choose LRU when that ordering suits the workload. -Maintaining the order requires more bookkeeping on hits, but choosing the next victim does not require a scan. +Maintaining the order requires more bookkeeping on hits, under a lock, but choosing the next victim does not require a scan. Neither policy guarantees a better hit rate for every workload; use the [dashboard](dashboard.md) or [timing](timing.md) to evaluate yours. ## Selecting a policy @@ -83,5 +84,5 @@ Custom eviction policies can be supplied through the [cache interface](interface Both policies evict as many entries as needed to fit a new value or a reduced limit. A value larger than the entire limit is returned without being stored. Byte limits measure cached values, not keys or container overhead. -Shared caches use locks under either policy; computations run outside those locks. +Shared caches take a lock to insert and evict entries, and `LRU` also on hits; computations run outside those locks. Disk caches have no eviction policy; see [Disk caching](disk.md) for cleanup. diff --git a/docs/src/index.md b/docs/src/index.md index 65d76bc..22f662f 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -167,6 +167,7 @@ See [Configuration](configuration.md) for persistent defaults and custom size me Changes to external state or method definitions (including through Revise) do not automatically invalidate results. Clear the global cache with `empty_caches!`; for disk results, update [`MemoizationKit.diskversion`](@ref). - Built-in shared caches are thread-safe. + [`ClockCache`](@ref) hits take no lock, so they scale with the number of threads. Computations run outside the cache lock, allowing recursion; simultaneous misses may compute the same key more than once. Exceptions are propagated without storing a result. @@ -183,7 +184,7 @@ Strategies, settings, persistence, and monitoring work together under one functi | [Memoize.jl](https://github.com/JuliaCollections/Memoize.jl#usage) | You want a small `@memoize` API with a dictionary of your choice. | Limits and statistics depend on the container; persistence, a dashboard, and package settings require additional integration. | | [Memoization.jl](https://github.com/marius311/Memoization.jl#readme) | You need to memoize individual calls or closures, as well as method definitions. | Its default cache is unbounded and not thread-safe; custom containers can address those needs, but management and persistence tools require additional integration. | | [LRUCache.jl](https://github.com/JuliaCollections/LRUCache.jl#readme) | You want a thread-safe cache dictionary with limits, statistics, resizing, and eviction callbacks. | You write the function wrapper and management code, or combine it with a memoization macro. | -| MemoizationKit.jl | You want bounded caches, per-type strategies, disk persistence, and live monitoring under one API. | The macro requires a method definition; shared caches use locks, and disk results need explicit versioning and cleanup. | +| MemoizationKit.jl | You want bounded caches, per-type strategies, disk persistence, and live monitoring under one API. | The macro requires a method definition; shared caches lock to insert entries (and `LRU` also on hits), and disk results need explicit versioning and cleanup. | Memoize.jl and Memoization.jl both accept LRUCache.jl containers, so bounded memory and statistics are available without MemoizationKit. Memoization.jl also preserves return-type inference. diff --git a/src/MemoizationKit.jl b/src/MemoizationKit.jl index b353060..29005ee 100644 --- a/src/MemoizationKit.jl +++ b/src/MemoizationKit.jl @@ -26,6 +26,7 @@ include("containers/interface.jl") include("containers/slots.jl") include("containers/lru.jl") include("containers/clock.jl") +include("containers/localclock.jl") include("registry.jl") include("preferences.jl") diff --git a/src/cachestyle.jl b/src/cachestyle.jl index 50df162..b12e6f4 100644 --- a/src/cachestyle.jl +++ b/src/cachestyle.jl @@ -54,7 +54,9 @@ Strategy that stores results in caches local to the current task, of container t `C` is either an `AbstractCache` or an `AbstractDict` type such as `Dict`; if it is not already concrete, it is completed to `C{K, V}`. Without `C`, the container is the one set by the `container` preference. Task-local caches need no locking but are not shared, not bounded -unless `C` is, and are not visible to [`cache_info`](@ref). +unless `C` is, and are not visible to [`cache_info`](@ref). `TaskLocalCache{ClockCache}` uses +an internal variant of [`ClockCache`](@ref) whose hits take the lock, which costs nothing +uncontended and saves the memory of the per-thread counters. """ struct TaskLocalCache{C} <: CacheStyle end diff --git a/src/call.jl b/src/call.jl index 7d994a0..d07349f 100644 --- a/src/call.jl +++ b/src/call.jl @@ -80,7 +80,10 @@ end # One untyped cache per function for the containers of MemoizationKit; other dictionaries (e.g. `Dict`) # would box untyped keys, so they get one typed dictionary per key and value type. -_localtype(C, K, V) = C isa UnionAll ? (C <: AbstractCache ? C{Any, Any} : C{K, V}) : C +_localtype(C, K, V) = C isa UnionAll ? (C <: AbstractCache ? _localcache(C){Any, Any} : C{K, V}) : C +# A task-local cache is used by one task only, so lock-free hits gain nothing there, while the +# per-thread counters of `ClockCache` would cost memory in every task. +_localcache(C) = C === ClockCache ? LocalClockCache : C # Task-local caches follow the size settings of `f`, but are not registered. function _newlocal(f, ::Type{T}) where {T <: AbstractCache} fc = @lock REGISTRY.lock _functioncaches!(f) diff --git a/src/containers/clock.jl b/src/containers/clock.jl index ac257f7..c3f73f3 100644 --- a/src/containers/clock.jl +++ b/src/containers/clock.jl @@ -1,3 +1,36 @@ +# An entry of a `ClockCache`; only its reference bit changes after construction. +mutable struct Entry{K, V} + const hash::UInt # the fields read on a hit come first + const key::K + const val::V + @atomic used::Bool # the reference bit + const size::Int + const position::Int # in `ring` +end + +struct Tombstone end +const TOMBSTONE = Tombstone() + +# The table of entries is an open-addressing hash table with linear probing, at most half full, +# read without the lock. This is safe because: +# 1. entries do not change after construction, apart from their reference bit; +# 2. a published table never changes length, but is replaced as a whole; +# 3. a slot only changes, under the lock, from `nothing` or `TOMBSTONE` to an entry, from an entry +# to `TOMBSTONE`, or to `nothing` if the next slot is `nothing`. Entries never move, so a probe +# finds every entry that is present. +# Slots are loaded with acquire and stored with release ordering. Before Julia 1.12, which added +# atomic indexing, `setindex!` into a `Vector{Any}` is a release store with a GC write barrier +# (checked by the tests); storing through a pointer would skip the barrier. +@static if VERSION >= v"1.12" + const SlotTable = AtomicMemory{Any} + loadslot(t::SlotTable, i::Int) = @atomic :acquire t[i] + storeslot!(t::SlotTable, i::Int, x) = (@atomic :release t[i] = x; t) +else + const SlotTable = Vector{Any} + loadslot(t::SlotTable, i::Int) = GC.@preserve t unsafe_load(pointer(t, i), :acquire) + storeslot!(t::SlotTable, i::Int, x) = (t[i] = x; t) +end + """ ClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) @@ -8,37 +41,225 @@ never hit again are evicted first. `maxsize` limits the number of entries, or, when `by` is given, the sum of `by(value)` over all entries. + +Lookups take no lock and write no shared memory, so hits scale with the number of threads that +share the cache. Insertions, evictions and the other changes take a lock. A lookup that races +with a removal may still return the removed value, and one that races with an insertion may +miss it; a returned value always belongs to its key. """ -mutable struct ClockCache{K, V} <: SlotCache{K, V} - const slots::Slots{V} - const ref::Vector{Bool} - const live::Vector{Bool} - hand::Int # last inspected slot +mutable struct ClockCache{K, V} <: DictCache{K, V} + @atomic table::SlotTable + occupied::Int # slots of `table` holding an entry or a tombstone + const ring::Vector{Entry{K, V}} # the clock; free positions are unassigned + const freepositions::Vector{Int} + hand::Int # last inspected position of `ring` + count::Int + currentsize::Int + maxsize::Int + const by::Any + const counts::Vector{Int} # hits and misses per thread, see `countlookup!` + const lock::ReentrantLock + + function ClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) where {K, V} + maxsize >= 0 || throw(ArgumentError("maxsize must be non-negative")) + counts = zeros(Int, 8 * nextpow(2, Threads.maxthreadid())) + return new{K, V}(emptytable(), 0, Entry{K, V}[], Int[], 0, 0, 0, maxsize, by, counts, ReentrantLock()) + end +end + +function emptytable(n::Int = 16) + t = SlotTable(undef, n) + foreach(i -> storeslot!(t, i, nothing), 1:n) + return t end -ClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) where {K, V} = - ClockCache{K, V}(Slots{V}(maxsize, by), Bool[], Bool[], 0) +# The slot after `i`, wrapping around (tables have a power-of-two length); a probe for `hash` +# starts at `nextslot(t, hash % Int)`. +nextslot(t::SlotTable, i::Int) = i & (length(t) - 1) + 1 -function admit!(c::ClockCache, i::Int) - i > length(c.ref) && (push!(c.ref, false); push!(c.live, false)) - c.ref[i] = false - c.live[i] = true +# The entry for the probe `p` (see `Key`), or `nothing`. +findentry(::ClockCache, ::Nothing) = nothing +@inline function findentry(c::ClockCache{K, V}, p::Key{P}) where {K, V, P} + t = @atomic :acquire c.table + i = nextslot(t, p.hash % Int) + for _ in 1:length(t) # bounds the probe also while slots change + e = loadslot(t, i) + e === nothing && return nothing + e isa Entry{K, V} && e.hash == p.hash && e.key isa P && isequal(p.key, e.key::P) && return e + i = nextslot(t, i) + end + return nothing +end + +# Sets the reference bit only if it is clear, so hot entries are not written to. +@inline function usevalue!(e::Entry) + (@atomic :monotonic e.used) || (@atomic :monotonic e.used = true) + return e.val +end + +# Each thread counts in its own cache line (8 `Int`s), since a shared counter would bounce +# between cores on every hit; a power of two of lines avoids a division. Tasks on one thread +# cannot interleave between the load and the store, as neither yields. +@enum LookupOutcome HIT = 1 MISS = 2 +@inline function countlookup!(c::ClockCache, outcome::LookupOutcome) + counts = c.counts + i = 8 * ((Threads.threadid() - 1) & (length(counts) รท 8 - 1)) + Int(outcome) + GC.@preserve counts begin + p = pointer(counts, i) + unsafe_store!(p, unsafe_load(p, :monotonic) + 1, :monotonic) + end + return nothing +end + +function Base.get!(default::F, c::ClockCache{K, V}, key) where {F <: Base.Callable, K, V} + probe = _newkey(c, key) + e = findentry(c, probe) + if e !== nothing + countlookup!(c, HIT) + return usevalue!(e) + end + countlookup!(c, MISS) + v = convert(V, default())::V + sz = _entrysize(c, v) + @lock c.lock begin + # another task may have filled the key while we were computing + e = findentry(c, probe) + e === nothing || return e.val + addentry!(c, probe, v, sz) + end + return v +end + +function Base.get(c::ClockCache, key, default) + e = findentry(c, _probe(c, key)) + return e === nothing ? default : usevalue!(e) +end +Base.haskey(c::ClockCache, key) = findentry(c, _probe(c, key)) !== nothing +Base.length(c::ClockCache) = @lock c.lock c.count +Base.isempty(c::ClockCache) = length(c) == 0 + +# The mutations below are called with the lock held. + +# Insert a key that is not present, evicting until it fits; a value larger than the whole cache +# is not stored. +function addentry!(c::ClockCache{K, V}, p::Key, v, sz::Int) where {K, V} + sz > c.maxsize && return c + while c.currentsize + sz > c.maxsize && c.count > 0 + evict!(c) + end + if 2 * (c.occupied + 1) > length(c.table) # publish a new table without tombstones + t = emptytable(max(16, nextpow(2, 4 * (c.count + 1)))) + for i in eachindex(c.ring) + isassigned(c.ring, i) && placeentry!(t, c.ring[i]) + end + c.occupied = c.count + @atomic :release c.table = t + end + position = isempty(c.freepositions) ? length(resize!(c.ring, length(c.ring) + 1)) : pop!(c.freepositions) + e = c.ring[position] = Entry{K, V}(p.hash, p.key, v, false, sz, position) + placeentry!(c.table, e) === nothing && (c.occupied += 1) + c.count += 1 + c.currentsize += sz return c end -touch!(c::ClockCache, i::Int) = (c.ref[i] = true; c) +# Store `e` in the first slot of its probe without an entry, and return what was there. +function placeentry!(t::SlotTable, e::Entry) + i = nextslot(t, e.hash % Int) + while (old = loadslot(t, i)) isa Entry + i = nextslot(t, i) + end + storeslot!(t, i, e) + return old +end -forget!(c::ClockCache, i::Int) = (c.live[i] = false; c) +function removeentry!(c::ClockCache, e::Entry) + t = c.table + i = nextslot(t, e.hash % Int) + while loadslot(t, i) !== e + i = nextslot(t, i) + end + if loadslot(t, nextslot(t, i)) === nothing + # no probe passes this slot or the tombstones before it, so they can all become empty + while true + storeslot!(t, i, nothing) + c.occupied -= 1 + i = nextslot(t, i - 2) + loadslot(t, i) === TOMBSTONE || break + end + else + storeslot!(t, i, TOMBSTONE) + end + Base._unsetindex!(c.ring, e.position) # internal + push!(c.freepositions, e.position) + c.count -= 1 + c.currentsize -= e.size + return c +end -# Terminates within two sweeps, since the first sweep clears every reference bit. -function victim(c::ClockCache) - n = length(c.ref) +# The hand clears set reference bits until it finds a clear one. Readers may set bits again +# meanwhile, so after two sweeps the hand evicts the next entry anyway. Requires `c.count > 0`. +function evict!(c::ClockCache) + n = length(c.ring) + steps = 0 while true c.hand = c.hand >= n ? 1 : c.hand + 1 - i = c.hand - c.live[i] || continue - c.ref[i] || return i - c.ref[i] = false + steps += 1 + isassigned(c.ring, c.hand) || continue + e = c.ring[c.hand] + (steps > 2n || !(@atomic :monotonic e.used)) && return removeentry!(c, e) + @atomic :monotonic e.used = false end return end + +function Base.setindex!(c::ClockCache{K, V}, v, key) where {K, V} + probe = _newkey(c, key) + val = convert(V, v)::V + sz = _entrysize(c, val) + @lock c.lock begin + e = findentry(c, probe) + e === nothing || removeentry!(c, e) + addentry!(c, probe, val, sz) + end + return c +end + +function Base.delete!(c::ClockCache, key) + @lock c.lock begin + e = findentry(c, _probe(c, key)) + e === nothing || removeentry!(c, e) + end + return c +end + +function Base.resize!(c::ClockCache; maxsize::Integer) + maxsize >= 0 || throw(ArgumentError("maxsize must be non-negative")) + @lock c.lock begin + c.maxsize = maxsize + while c.currentsize > c.maxsize && c.count > 0 + evict!(c) + end + end + return c +end + +function Base.empty!(c::ClockCache) + @lock c.lock begin + @atomic :release c.table = emptytable() + empty!(c.ring) + empty!(c.freepositions) + c.occupied = c.hand = c.count = c.currentsize = 0 + end + return c +end + +function cache_stats(c::ClockCache) + counts = c.counts + total(outcome) = GC.@preserve counts sum(i -> unsafe_load(pointer(counts, i), :monotonic), Int(outcome):8:length(counts)) + return @lock c.lock (; hits = total(HIT), misses = total(MISS), length = c.count, c.currentsize, c.maxsize, c.by) +end + +function snapshot(c::ClockCache{K, V}) where {K, V} + return @lock c.lock Pair{K, V}[c.ring[i].key => c.ring[i].val for i in eachindex(c.ring) if isassigned(c.ring, i)] +end diff --git a/src/containers/localclock.jl b/src/containers/localclock.jl new file mode 100644 index 0000000..3f5ed9b --- /dev/null +++ b/src/containers/localclock.jl @@ -0,0 +1,36 @@ +# `ClockCache` for a single task, where lock-free hits gain nothing: every operation takes the +# (uncontended) lock, and the hit and miss counters are plain fields. `TaskLocalCache{ClockCache}` +# uses it, see `_localtype`; it evicts in the same order as `ClockCache`. +mutable struct LocalClockCache{K, V} <: SlotCache{K, V} + const slots::Slots{V} + const ref::Vector{Bool} + const live::Vector{Bool} + hand::Int # last inspected slot +end + +LocalClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) where {K, V} = + LocalClockCache{K, V}(Slots{V}(maxsize, by), Bool[], Bool[], 0) + +function admit!(c::LocalClockCache, i::Int) + i > length(c.ref) && (push!(c.ref, false); push!(c.live, false)) + c.ref[i] = false + c.live[i] = true + return c +end + +touch!(c::LocalClockCache, i::Int) = (c.ref[i] = true; c) + +forget!(c::LocalClockCache, i::Int) = (c.live[i] = false; c) + +# Terminates within two sweeps, since the first sweep clears every reference bit. +function victim(c::LocalClockCache) + n = length(c.ref) + while true + c.hand = c.hand >= n ? 1 : c.hand + 1 + i = c.hand + c.live[i] || continue + c.ref[i] || return i + c.ref[i] = false + end + return +end diff --git a/src/containers/slots.jl b/src/containers/slots.jl index 0abd341..ce2467f 100644 --- a/src/containers/slots.jl +++ b/src/containers/slots.jl @@ -12,10 +12,21 @@ Base.hash(k::Key, h::UInt) = hash(k.hash, h) Base.isequal(p::Key{K}, s::Key{Any}) where {K} = p.hash == s.hash && s.key isa K && isequal(p.key, s.key::K) Base.isequal(a::Key{Any}, b::Key{Any}) = a.hash == b.hash && typeof(a.key) === typeof(b.key) && isequal(a.key, b.key) -# The shared implementation of `LRU` and `ClockCache`: every method of a dict-like +# Caches that iterate over `snapshot(c)`, the entries copied under the lock, so that iteration is +# safe while other tasks use the cache (holding the lock across `iterate` calls would deadlock on +# an early `break`). The length may change meanwhile, so `collect` must not rely on it. +abstract type DictCache{K, V} <: AbstractCache{K, V} end +Base.IteratorSize(::Type{<:DictCache}) = Base.SizeUnknown() +function Base.iterate(c::DictCache, (entries, i) = (snapshot(c), 1)) + return i > length(entries) ? nothing : (entries[i], (entries, i + 1)) +end +# The entries below the compact header. +Base.show(io::IO, m::MIME"text/plain", c::DictCache) = invoke(show, Tuple{IO, MIME"text/plain", AbstractDict}, io, m, c) + +# The shared implementation of `LRU` and `LocalClockCache`: every method of a dict-like # `AbstractCache`, on the storage in the field `slots::Slots{V}`. The subtypes are eviction # policies over slot numbers, through the hooks below, which are called with the lock held. -abstract type SlotCache{K, V} <: AbstractCache{K, V} end +abstract type SlotCache{K, V} <: DictCache{K, V} end # The entries in numbered slots (reused once freed), the key index, the size accounting, the # statistics and the lock. @@ -51,20 +62,24 @@ function victim end function forget! end # The probe for a key that is inserted: converted to `K`, which throws if that is not possible. -_newkey(::SlotCache{K}, key) where {K} = Key(key isa K ? key : convert(K, key)::K) +_newkey(::AbstractCache{K}, key) where {K} = Key(key isa K ? key : convert(K, key)::K) -# The slot of `key` in `c`, or 0. A key that cannot be converted to `K` is not in the cache. -function _slot(c::SlotCache{K}, key) where {K} - key isa K && return get(c.slots.index, Key(key), 0) +# The probe for a key that is looked up, or `nothing`: a key that cannot be converted to `K` is +# not in the cache. +function _probe(::AbstractCache{K}, key) where {K} + key isa K && return Key(key) k = try convert(K, key)::K catch - return 0 + return nothing end - return get(c.slots.index, Key(k), 0) + return Key(k) end -_entrysize(s::Slots, v) = s.by === nothing ? 1 : Int(s.by(v))::Int +# The slot of `key` in `c`, or 0. +_slot(c::SlotCache, key) = (p = _probe(c, key); p === nothing ? 0 : get(c.slots.index, p, 0)) + +_entrysize(c, v) = c.by === nothing ? 1 : Int(c.by(v))::Int # `c` is a `Slots` or a `ClockCache` # Insert a new key (not currently present), evicting until it fits. # Values larger than the whole cache are not stored. @@ -91,26 +106,18 @@ function _insert!(c::SlotCache, p::Key, v, sz::Int) end # Free slot `i`: drop it from the index and clear its key and value so they can be -# garbage-collected while the slot sits on the free list. +# garbage-collected while the slot sits on the free list (`Base._unsetindex!` is internal). function _free!(c::SlotCache, i::Int) forget!(c, i) s = c.slots delete!(s.index, s.keys[i]) - _unset!(s.keys, i) - _unset!(s.vals, i) + Base._unsetindex!(s.keys, i) + Base._unsetindex!(s.vals, i) s.currentsize -= s.sizes[i] push!(s.free, i) return c end -# `Base._unsetindex!` is internal, so fall back to keeping the reference if it ever disappears. -function _unset!(v::Vector, i::Int) - @static if isdefined(Base, :_unsetindex!) - Base._unsetindex!(v, i) - end - return v -end - function Base.get!(default::Base.Callable, c::SlotCache{K, V}, key) where {K, V} p = _newkey(c, key) s = c.slots @@ -141,30 +148,13 @@ function Base.get(c::SlotCache, key, default) end end -function Base.getindex(c::SlotCache, key) - return @lock c.slots.lock begin - i = _slot(c, key) - i == 0 && throw(KeyError(key)) - touch!(c, i) - c.slots.vals[i] - end -end - Base.haskey(c::SlotCache, key) = @lock c.slots.lock _slot(c, key) != 0 Base.length(c::SlotCache) = @lock c.slots.lock length(c.slots.index) Base.isempty(c::SlotCache) = @lock c.slots.lock isempty(c.slots.index) -# Iteration walks a snapshot copied under the lock, so it is safe while other tasks use the -# cache (holding the lock across `iterate` calls would deadlock on an early `break`). -# The length may change between `length` and `iterate`, so `collect` must not rely on it. -Base.IteratorSize(::Type{<:SlotCache}) = Base.SizeUnknown() -function Base.iterate(c::SlotCache{K, V}) where {K, V} +function snapshot(c::SlotCache{K, V}) where {K, V} s = c.slots - snapshot = @lock s.lock Pair{K, V}[k.key::K => s.vals[i] for (k, i) in s.index] - return iterate(c, (snapshot, 1)) -end -function Base.iterate(::SlotCache, (snapshot, i)::Tuple{Vector, Int}) - return i > length(snapshot) ? nothing : (snapshot[i], (snapshot, i + 1)) + return @lock s.lock Pair{K, V}[k.key::K => s.vals[i] for (k, i) in s.index] end function Base.setindex!(c::SlotCache{K, V}, v, key) where {K, V} @@ -210,6 +200,3 @@ function cache_stats(c::SlotCache) s = c.slots return @lock s.lock (; s.hits, s.misses, length = length(s.index), s.currentsize, s.maxsize, s.by) end - -# The entries below the compact header. -Base.show(io::IO, m::MIME"text/plain", c::SlotCache) = invoke(show, Tuple{IO, MIME"text/plain", AbstractDict}, io, m, c) diff --git a/test/Project.toml b/test/Project.toml index 3479116..fc5d33e 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,8 +1,10 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" MemoizationKit = "1b238080-9255-4fe9-b224-89eb24efe93b" ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Preferences = "21216c6a-2e73-6563-6e65-726566657250" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Tachikoma = "468859d6-42d8-48b7-8ad9-1d312e0e3b0a" @@ -14,7 +16,9 @@ MemoizationKit = {path = ".."} [compat] Aqua = "0.8.9" +InteractiveUtils = "1.10" ParallelTestRunner = "2.2.0" +Random = "1.10" SQLite = "1.6" Serialization = "1" Tachikoma = "2.7" diff --git a/test/test_concurrency.jl b/test/test_concurrency.jl new file mode 100644 index 0000000..c4c3a44 --- /dev/null +++ b/test/test_concurrency.jl @@ -0,0 +1,128 @@ +using Test +using Random +using InteractiveUtils: code_llvm +using MemoizationKit +using MemoizationKit: cache_stats, LocalClockCache + +# Every value is a function of its key, so any value returned for the wrong key is detected. +expected(k::Int) = 3k +expected(k::NTuple{4, Int}) = (sum(k), k[1]) +expected(k::String) = length(k) +expected(k::Tuple{Vector{Int}, Symbol}) = (sum(k[1]), k[2]) + +function randkey(rng, n) + i = rand(rng, 1:n) + r = rand(rng) + return r < 0.4 ? i : r < 0.7 ? (i, i + 1, -i, 2i) : r < 0.85 ? string(i) : ([i, 1], :s) +end + +# Many tasks doing every operation at once, checking each returned value; returns the number +# of `get!` calls and whether all values were correct. +function stress!(c, ntasks, niter; nkeys = 500, maxsizes = (8, 64, 256)) + gets = Threads.Atomic{Int}(0) + ok = Threads.Atomic{Bool}(true) + @sync for t in 1:ntasks + Threads.@spawn begin + rng = Xoshiro(t) + n = 0 + for _ in 1:niter + k = randkey(rng, nkeys) + r = rand(rng) + if r < 0.8 + n += 1 + isequal(get!(() -> expected(k), c, k), expected(k)) || (ok[] = false) + elseif r < 0.9 + v = get(c, k, nothing) + v === nothing || isequal(v, expected(k)) || (ok[] = false) + elseif r < 0.95 + c[k] = expected(k) + elseif r < 0.99 + delete!(c, k) + elseif r < 0.995 + resize!(c; maxsize = rand(rng, maxsizes)) + elseif r < 0.997 + empty!(c) + else + all(((k, v),) -> isequal(v, expected(k)), c) || (ok[] = false) + end + end + Threads.atomic_add!(gets, n) + end + end + return gets[], ok[] +end + +const NTASKS = 4 * Threads.nthreads() + +@testset "$C: concurrent stress" for C in (LRU, ClockCache, LocalClockCache) + for by in (nothing, x -> 1 + (x isa Tuple ? 1 : 0)) + c = C{Any, Any}(; maxsize = 64, by) + gets, ok = stress!(c, NTASKS, 20_000) + @test ok + s = cache_stats(c) + @test s.hits + s.misses == gets + @test s.currentsize <= s.maxsize + @test length(c) == length(collect(c)) == s.length + @test all(((k, v),) -> isequal(v, expected(k)), c) + @test s.currentsize == (by === nothing ? length(c) : sum(by(v) for (k, v) in c; init = 0)) + end +end + +# Many readers of a few hot keys while a writer churns the rest of the cache, so that the +# lock-free table is rebuilt and entries are evicted under the readers. +@testset "ClockCache: hits during rebuilds" begin + c = ClockCache{Any, Any}(; maxsize = 2_000) + ok = Threads.Atomic{Bool}(true) + done = Threads.Atomic{Bool}(false) + @sync begin + for t in 1:max(1, Threads.nthreads() - 1) + Threads.@spawn while !done[] + k = mod(t, 8) + get!(() -> 3k, c, k) == 3k || (ok[] = false) + v = get(c, (k, k + 1, -k, 2k), nothing) + v === nothing || v == expected((k, k + 1, -k, 2k)) || (ok[] = false) + yield() # lets the writer run when there is a single thread + end + end + Threads.@spawn begin + rng = Xoshiro(0) + for i in 1:100_000 + k = (i, i + 1, -i, 2i) + get!(() -> expected(k), c, k) + i % 25_000 == 0 && empty!(c) + i % 7_919 == 0 && resize!(c; maxsize = rand(rng, 100:3_000)) + end + done[] = true + end + end + @test ok[] + @test all(((k, v),) -> isequal(v, expected(k)), c) +end + +# Without `AtomicMemory`, lock-free readers rely on `setindex!` into a `Vector{Any}` compiling to +# a release store with a GC write barrier. +if VERSION < v"1.12" + @testset "ClockCache: slot stores are release stores" begin + ir = sprint(code_llvm, MemoizationKit.storeslot!, (Vector{Any}, Int, Any)) + @test occursin(r"store atomic .* release", ir) + @test occursin("gc_queue_root", ir) + end +end + +@cached shared(x::Int) = x^2 +MemoizationKit.CacheStyle(::typeof(shared), ::Int) = GlobalCache{ClockCache}() +@cached tasklocal(x::Int) = x^2 +MemoizationKit.CacheStyle(::typeof(tasklocal), ::Int) = TaskLocalCache{ClockCache}() + +@testset "@cached with $f" for f in (shared, tasklocal) + @test @inferred(f(3)) == 9 + f(4) + @test (@allocated f(4)) == 0 +end + +@testset "containers of @cached" begin + c = only(cache_info(shared)).second + @test c isa ClockCache{Any, Any} + @test cache_stats(c).misses == 2 && cache_stats(c).hits >= 1 + @test MemoizationKit._container(tasklocal, TaskLocalCache{ClockCache}(), (1,), Int) isa LocalClockCache{Any, Any} +end diff --git a/test/test_containers.jl b/test/test_containers.jl index 349a10a..2b832f0 100644 --- a/test/test_containers.jl +++ b/test/test_containers.jl @@ -1,8 +1,8 @@ using Test using MemoizationKit -using MemoizationKit: AbstractCache, cache_stats +using MemoizationKit: AbstractCache, cache_stats, LocalClockCache -const CACHETYPES = (LRU, ClockCache) +const CACHETYPES = (LRU, ClockCache, LocalClockCache) @testset "$C: basic interface" for C in CACHETYPES c = C{Int, String}(; maxsize = 3) @@ -64,8 +64,8 @@ end @test !haskey(c, 1) && haskey(c, 2) end -@testset "ClockCache: second chance" begin - c = ClockCache{Int, Int}(; maxsize = 3) +@testset "$C: second chance" for C in (ClockCache, LocalClockCache) + c = C{Int, Int}(; maxsize = 3) for i in 1:3 c[i] = i end