From aa0f35ad48d254ba8b3483789606e0eb3b973593 Mon Sep 17 00:00:00 2001
From: Lincoln Stein <lincoln.stein@gmail.com>
Date: Sun, 30 Aug 2026 22:28:07 -0400
Subject: [PATCH 1/3] fix(model cache): make_room honors the admission grace
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

make_room's eviction loop checked only is_locked, but multi-model
invocations load their whole set before locking any of it (the H3 text
encoder loads text_encoder, then tokenizer, then processor, then locks) —
so a sibling's cold-load make_room runs with a just-admitted entry
unlocked, and under shared-RAM-budget deficit it evicted the 27GB text
encoder admitted one second earlier to make room for a 50MB tokenizer.
Evicting such an entry frees NOTHING (the loader's handle keeps the model
alive) while detaching the record from all cache accounting and turning
every subsequent paced-lock pass into an issue-7513 diagnostic line
(observed: ~25 identical INFO lines per text-encoder load on a dual-GPU
rig). The asynchronous eviction paths (budget reconcile, peer eviction)
already honored awaiting_first_use; the synchronous path now does too.

The keep-alive timeout clear passes spare_awaiting_first_use=False: after
an idle period a surviving grace is abandoned by definition (a healthy
loader locks within seconds, and every lock resets the timer), so the
timeout keeps clearing everything unlocked.

Also demote continue_lock's detached-record diagnostic to DEBUG — lock()
already reports it once at INFO, and a paced stream repeated the same
line dozens of times.

Two existing tests exercised make_room on entries that had never been
locked; they now release the grace first, the way every real loader does.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S4B5exWsbC2z2Uu2tA167A
---
 .../load/model_cache/model_cache.py           | 32 ++++++--
 .../test_model_cache_drop_model.py            |  6 ++
 .../model_cache/test_model_cache_make_room.py | 76 +++++++++++++++++++
 .../test_model_cache_shared_weights.py        |  7 ++
 4 files changed, 113 insertions(+), 8 deletions(-)
 create mode 100644 tests/backend/model_manager/load/model_cache/test_model_cache_make_room.py

diff --git a/invokeai/backend/model_manager/load/model_cache/model_cache.py b/invokeai/backend/model_manager/load/model_cache/model_cache.py
index e737c8b7af9..b9119e61b43 100644
--- a/invokeai/backend/model_manager/load/model_cache/model_cache.py
+++ b/invokeai/backend/model_manager/load/model_cache/model_cache.py
@@ -578,8 +578,10 @@ def _on_timeout(self) -> None:
             )
             # Clear the cache by requesting a very large amount of space.
             # This is the same logic used by the "Clear Model Cache" button.
-            # Using 1000 GB ensures all unlocked models are removed.
-            self._make_room_internal(1000 * GB)
+            # Using 1000 GB ensures all unlocked models are removed. A surviving admission grace
+            # is abandoned by definition here — a healthy loader locks within seconds and every
+            # lock resets the keep-alive timer — so the timeout clear ignores it.
+            self._make_room_internal(1000 * GB, spare_awaiting_first_use=False)
         elif len(self._cached_models) > 0:
             # All models are locked, don't log at info level
             self._logger.debug(
@@ -1085,9 +1087,10 @@ def continue_lock(
         the last pass's.
         """
         if cache_entry.key not in self._cached_models:
-            # Same diagnostic as lock()/unlock() (issue 7513): a detached record's continuation
-            # passes should not run silently.
-            self._logger.info(
+            # Same diagnostic as lock()/unlock() (issue 7513) — but at DEBUG: lock() already said
+            # it once at INFO, and a paced stream repeats this method dozens of times, which turned
+            # one detached record into a page of identical log lines.
+            self._logger.debug(
                 f"Continuing paced lock of model cache entry {cache_entry.key} "
                 f"(Type: {cache_entry.cached_model.model.__class__.__name__}), but it has already been dropped from "
                 "the RAM cache. This is a sign that the model loading order is non-optimal in the invocation code "
@@ -1662,8 +1665,12 @@ def make_room(self, bytes_needed: int) -> CacheClearResult:
         """
         return self._make_room_internal(bytes_needed)
 
-    def _make_room_internal(self, bytes_needed: int) -> CacheClearResult:
-        """Internal implementation of make_room(). Assumes the lock is already held."""
+    def _make_room_internal(self, bytes_needed: int, spare_awaiting_first_use: bool = True) -> CacheClearResult:
+        """Internal implementation of make_room(). Assumes the lock is already held.
+
+        `spare_awaiting_first_use=False` (the keep-alive timeout clear) also evicts entries whose
+        admission grace was never released — abandoned by definition after an idle period.
+        """
         self._logger.debug(f"Making room for {bytes_needed / MB:.2f}MB of RAM.")
         self._log_cache_state(title="Before dropping models:")
 
@@ -1688,7 +1695,16 @@ def _make_room_internal(self, bytes_needed: int) -> CacheClearResult:
             model_key = self._cache_stack[pos]
             cache_entry = self._cached_models[model_key]
 
-            if not cache_entry.is_locked:
+            # awaiting_first_use marks the window between put() and the loader's first lock() —
+            # a window a SINGLE worker thread can re-enter this method inside, because multi-model
+            # invocations load their whole set before locking any of it (e.g. the H3 text encoder
+            # loads text_encoder, then tokenizer, then processor; the tokenizer's cold-load
+            # make_room runs with the 27GB text encoder admitted but not yet locked). Evicting
+            # such an entry frees NOTHING — the loader's handle keeps the model alive — while
+            # detaching the record from all cache accounting and turning every subsequent lock
+            # pass into an issue-7513 diagnostic. The asynchronous eviction paths (budget
+            # reconcile, peer eviction) already honor the grace; the synchronous path must too.
+            if not cache_entry.is_locked and not (spare_awaiting_first_use and cache_entry.awaiting_first_use):
                 ram_bytes_freed += cache_entry.cached_model.total_bytes()
                 self._logger.debug(
                     f"Dropping {model_key} from RAM cache to free {(cache_entry.cached_model.total_bytes() / MB):.2f}MB."
diff --git a/tests/backend/model_manager/load/model_cache/test_model_cache_drop_model.py b/tests/backend/model_manager/load/model_cache/test_model_cache_drop_model.py
index 17c674c235c..047a3c7be93 100644
--- a/tests/backend/model_manager/load/model_cache/test_model_cache_drop_model.py
+++ b/tests/backend/model_manager/load/model_cache/test_model_cache_drop_model.py
@@ -172,6 +172,12 @@ def test_drop_model_updates_stats_and_fires_callbacks(cache: ModelCache):
 def test_make_room_returns_clear_result_and_updates_current_cache_usage(cache: ModelCache):
     cache.put("model-a", torch.nn.Linear(4, 4))
     cache.put("model-b", torch.nn.Linear(4, 4))
+    # Release the admission grace the way a real loader does — make_room spares entries still
+    # awaiting their first lock (the loader's handle keeps them alive; evicting frees nothing).
+    for key in ("model-a", "model-b"):
+        record = cache.get(key)
+        cache.lock(record, None)
+        cache.unlock(record)
     cache.stats = CacheStats()
     before_bytes = cache._get_ram_in_use()
 
diff --git a/tests/backend/model_manager/load/model_cache/test_model_cache_make_room.py b/tests/backend/model_manager/load/model_cache/test_model_cache_make_room.py
new file mode 100644
index 00000000000..c8547d07805
--- /dev/null
+++ b/tests/backend/model_manager/load/model_cache/test_model_cache_make_room.py
@@ -0,0 +1,76 @@
+"""Tests for make_room()'s eviction guards."""
+
+import logging
+from unittest.mock import MagicMock
+
+import torch
+
+from invokeai.backend.model_manager.load.model_cache.model_cache import ModelCache
+from tests.backend.model_manager.load.model_cache.cached_model.utils import DummyModule
+
+
+def _make_cache() -> ModelCache:
+    logger = MagicMock()
+    logger.getEffectiveLevel.return_value = logging.INFO
+    return ModelCache(
+        execution_device_working_mem_gb=1.0,
+        enable_partial_loading=False,
+        keep_ram_copy_of_weights=True,
+        execution_device="cpu",
+        storage_device="cpu",
+        logger=logger,
+    )
+
+
+def test_make_room_spares_entries_awaiting_first_use():
+    """A just-admitted entry (put() done, first lock() pending) must survive make_room: the
+    loader's handle keeps the model alive anyway — multi-model invocations load their whole set
+    before locking any of it, so a sibling's cold-load make_room runs inside this window — and
+    evicting it frees nothing while detaching the record from all cache accounting."""
+    cache = _make_cache()
+    cache.put("fresh", DummyModule())
+    record = cache.get("fresh")
+
+    # Precondition: the admission grace is armed (the deferred worker is running in-process).
+    assert record.awaiting_first_use
+
+    cache.make_room(10**15)
+    assert "fresh" in cache._cached_models, "make_room evicted an entry still awaiting its first lock"
+
+    # After the first lock/unlock cycle the grace is released and the entry is ordinary cache
+    # content again.
+    cache.lock(record, None)
+    cache.unlock(record)
+    assert not record.awaiting_first_use
+
+    cache.make_room(10**15)
+    assert "fresh" not in cache._cached_models
+
+
+def test_make_room_still_evicts_ordinary_unlocked_entries():
+    cache = _make_cache()
+    cache.put("used", DummyModule())
+    record = cache.get("used")
+    cache.lock(record, None)
+    cache.unlock(record)
+
+    cache.make_room(10**15)
+    assert "used" not in cache._cached_models
+
+
+def test_make_room_never_evicts_locked_entries():
+    cache = _make_cache()
+    cache.put("held", DummyModule())
+    record = cache.get("held")
+    cache.lock(record, None)
+    try:
+        cache.make_room(10**15)
+        assert "held" in cache._cached_models
+    finally:
+        cache.unlock(record)
+
+
+def test_dummy_module_is_cpu():
+    # Guard against the fixture silently needing CUDA: everything above must run CPU-only.
+    assert not any(p.is_cuda for p in DummyModule().parameters())
+    assert torch.cuda.is_available() or True
diff --git a/tests/backend/model_manager/load/model_cache/test_model_cache_shared_weights.py b/tests/backend/model_manager/load/model_cache/test_model_cache_shared_weights.py
index 2c75a68b98b..69824536ce3 100644
--- a/tests/backend/model_manager/load/model_cache/test_model_cache_shared_weights.py
+++ b/tests/backend/model_manager/load/model_cache/test_model_cache_shared_weights.py
@@ -53,12 +53,19 @@ def test_two_device_caches_share_one_cpu_copy(mock_logger: MagicMock):
         assert sd_a is sd_b
 
         # Evicting from one device drops only its reference; the weights stay for the other.
+        # (Release the admission grace first: make_room spares entries awaiting their first lock.)
+        record_a = cache_a.get("m")
+        cache_a.lock(record_a, None)
+        cache_a.unlock(record_a)
         cache_a.make_room(10**12)
         assert "m" not in cache_a._cached_models
         assert store.refcount("m") == 1
         assert "m" in store
 
         # Evicting from the last device frees the shared RAM.
+        record_b = cache_b.get("m")
+        cache_b.lock(record_b, None)
+        cache_b.unlock(record_b)
         cache_b.make_room(10**12)
         assert store.refcount("m") == 0
         assert "m" not in store

From 0dc9a51913d49aaafa751fa7d82793ccb403e9b5 Mon Sep 17 00:00:00 2001
From: Lincoln Stein <lincoln.stein@gmail.com>
Date: Sun, 30 Aug 2026 22:43:50 -0400
Subject: [PATCH 2/3] fix(model cache): tie the admission grace to its
 LoadedModel handle
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adversarial review of the previous commit proved its guard incomplete:
put()'s stale-grace sweep cleared EVERY grace on each admission, so only
the most recently admitted sibling was ever protected, and only until the
next put() — both reproduced paths re-created the field failure (a
just-admitted 27GB model evicted by a sibling's cold load) with the guard
in place. The sweep's premise ('a grace surviving to the next admission is
stale') is false in exactly the window the grace exists for: multi-model
invocations load their whole set before locking any of it.

The grace's owner is the LoadedModel handle, which already releases it via
a finalizer on an unlocked drop. Record a weakref to the handle on the
CacheRecord (grace_holder) at handle construction; the sweep now clears
only provably orphaned graces — no handle was ever registered (the load
raised between put() and LoadedModel construction) or the handle died
without its finalizer running (lost deferred worker). Live holder = the
loader is coming back; lock() or the finalizer releases the grace.

Also per review: the clear-model-cache button (/empty_model_cache) passes
spare_awaiting_first_use=False through the now-parameterized public
make_room — a user-requested full clear outranks the grace (pre-existing
behavior restored) — and the tautological assertion in the new test file
is replaced by real coverage: grace survival across sibling admissions,
orphan sweeps both ways, finalizer release, and the explicit-clear path.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S4B5exWsbC2z2Uu2tA167A
---
 invokeai/app/api/routers/model_manager.py     |  4 +-
 .../backend/model_manager/load/load_base.py   |  4 +
 .../load/model_cache/cache_record.py          | 37 +++++---
 .../load/model_cache/model_cache.py           | 33 ++++---
 tests/app/routers/test_model_manager.py       |  4 +-
 .../model_cache/test_model_cache_make_room.py | 88 +++++++++++++++++--
 6 files changed, 138 insertions(+), 32 deletions(-)

diff --git a/invokeai/app/api/routers/model_manager.py b/invokeai/app/api/routers/model_manager.py
index 713a26cc92b..3e2abab220e 100644
--- a/invokeai/app/api/routers/model_manager.py
+++ b/invokeai/app/api/routers/model_manager.py
@@ -1524,7 +1524,9 @@ def empty_model_cache(current_admin: AdminUserOrDefault) -> EmptyModelCacheRespo
         if id(cache) in seen_cache_ids:
             continue
         seen_cache_ids.add(id(cache))
-        result = cache.make_room(1000 * 2**30)
+        # spare_awaiting_first_use=False: the user asked for a full clear, which outranks the
+        # admission grace (an in-flight loader survives via the tolerated issue-7513 path).
+        result = cache.make_room(1000 * 2**30, spare_awaiting_first_use=False)
         models_cleared += result.models_cleared
         bytes_freed += result.bytes_freed
     return EmptyModelCacheResponse(models_cleared=models_cleared, bytes_freed=bytes_freed)
diff --git a/invokeai/backend/model_manager/load/load_base.py b/invokeai/backend/model_manager/load/load_base.py
index 98d7681f8c2..c073dcf70cb 100644
--- a/invokeai/backend/model_manager/load/load_base.py
+++ b/invokeai/backend/model_manager/load/load_base.py
@@ -4,6 +4,7 @@
 """
 
 import time
+import weakref
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from logging import Logger
@@ -69,6 +70,9 @@ def __init__(self, cache_record: CacheRecord, cache: ModelCache):
         )
         if self._first_use_finalizer is not None:
             self._first_use_finalizer.atexit = False
+            # This handle owns the grace: put()'s sweep keeps a grace whose holder is alive
+            # (an in-flight multi-model load) and clears one whose holder is gone.
+            cache_record.grace_holder = weakref.ref(self)
 
     def _lock_paced(self, working_mem_bytes: Optional[int]) -> None:
         """Move the model into VRAM in bounded passes, yielding the global load lock between them.
diff --git a/invokeai/backend/model_manager/load/model_cache/cache_record.py b/invokeai/backend/model_manager/load/model_cache/cache_record.py
index a266cdb79e5..5f2340931e2 100644
--- a/invokeai/backend/model_manager/load/model_cache/cache_record.py
+++ b/invokeai/backend/model_manager/load/model_cache/cache_record.py
@@ -1,3 +1,4 @@
+import weakref
 from dataclasses import dataclass
 
 from invokeai.backend.model_manager.load.model_cache.cached_model.cached_model_only_full_load import (
@@ -24,20 +25,32 @@ class CacheRecord:
     is_stale: bool = False
     # Post-admission grace: set by ModelCache.put() (unless the admission is a prefetch of a
     # model nothing will come back for) and cleared on the entry's first lock(). A freshly
-    # admitted model is about to be used — its loader calls get() as soon as put() returns, then
-    # locks it for inference — so the asynchronous eviction paths (shared-budget reconcile,
-    # peer-requested eviction) must not treat it as idle: they would evict the record out from
-    # under the in-flight load, breaking the loader's get() or detaching a live model from the
-    # cache's RAM accounting. The grace deliberately survives get(): get() is synchronized, and
-    # its own lock-release hook may run a pending reconcile before the caller can lock the record
-    # it was just handed. The flag cannot shield a record forever: the cache's local make_room
-    # path ignores it (cold loads are serialized under MODEL_LOAD_LOCK, so make_room can never
-    # see another loader's entry inside the put()->lock() window), and the next admission on the
-    # same cache clears any flag still standing (see the sweep in ModelCache.put()), so a load
-    # that errors out — or a LoadedModel dropped without ever locking — cannot dodge budget
-    # reconciles indefinitely.
+    # admitted model is about to be used — its loader calls get() as soon as put() returns,
+    # constructs a LoadedModel handle, and locks it for inference — so no eviction path
+    # (make_room, shared-budget reconcile, peer-requested eviction) may treat it as idle:
+    # evicting it frees nothing (the handle keeps the model alive) while detaching the record
+    # from the cache's RAM accounting. The window is NOT confined to a single load: multi-model
+    # invocations load their whole set (e.g. text encoder, then tokenizer, then processor)
+    # before locking any of it, so a sibling's cold load legitimately runs make_room — and
+    # put() — while earlier entries sit graced with live handles. The grace deliberately
+    # survives get(): get() is synchronized, and its own lock-release hook may run a pending
+    # reconcile before the caller can lock the record it was just handed.
+    #
+    # The flag cannot shield a record forever. Its owner is the LoadedModel handle
+    # (`grace_holder` below): a dropped handle releases the grace through its finalizer, and
+    # ModelCache.put()'s sweep clears any grace that is provably orphaned — no handle was ever
+    # registered (the load raised between put() and LoadedModel construction), or the handle
+    # died without its finalizer running (deferred worker lost) — so an orphaned record cannot
+    # dodge budget reconciles indefinitely. The keep-alive timeout clear also ignores the grace:
+    # after an idle period it is abandoned by definition.
     awaiting_first_use: bool = False
 
+    # Weak reference to the LoadedModel handle that owns `awaiting_first_use`, registered at
+    # handle construction (see LoadedModelWithoutConfig.__init__). None until then — which is
+    # exactly what put()'s sweep uses to tell an in-flight sibling (live holder: keep the grace)
+    # from an orphaned admission (no holder, or a dead one: clear it).
+    grace_holder: "weakref.ref[object] | None" = None
+
     def lock(self) -> None:
         """Lock this record."""
         self._locks += 1
diff --git a/invokeai/backend/model_manager/load/model_cache/model_cache.py b/invokeai/backend/model_manager/load/model_cache/model_cache.py
index b9119e61b43..79d04361448 100644
--- a/invokeai/backend/model_manager/load/model_cache/model_cache.py
+++ b/invokeai/backend/model_manager/load/model_cache/model_cache.py
@@ -637,16 +637,23 @@ def put(
         # unused cache.
         self._ensure_deferred_worker()
 
-        # Any entry still carrying the post-admission grace belongs to an earlier load: cold loads
-        # are serialized under MODEL_LOAD_LOCK's write lock and each load's only graced put() is
-        # its final one, so a flag that survives to the next admission is stale — its loader
-        # either errored out before retrieving the model or dropped the LoadedModel without ever
-        # locking it. Clear such flags so an orphaned record cannot dodge budget reconciles
-        # indefinitely. (An entry retrieved but not yet locked loses its shield here; if a
-        # reconcile then evicts it, lock() falls back to the tolerated issue-7513 path and
-        # proceeds on the detached record.)
+        # A grace that survives to the next admission is NOT necessarily stale: multi-model
+        # invocations load their whole set (text encoder, then tokenizer, then processor) before
+        # locking any of it, so earlier siblings' graces legitimately outlive later puts — and
+        # sweeping them let a sibling's make_room evict a just-admitted multi-GB model to fit a
+        # tiny one, freeing nothing (the loader's handle keeps it alive) while detaching the
+        # record from all accounting. The grace's owner is the LoadedModel handle: clear only
+        # provably orphaned graces — no handle was ever registered (the load raised between
+        # put() and LoadedModel construction), or the handle died without its finalizer running
+        # (deferred worker lost). A live holder means lock() or the finalizer will release the
+        # grace. (Residual race, tolerated via the issue-7513 path: a sibling's put() landing in
+        # the instructions between a load's MODEL_LOAD_LOCK release and its LoadedModel
+        # construction sees a grace with no holder yet and sweeps it.)
         for stale_entry in self._cached_models.values():
-            stale_entry.awaiting_first_use = False
+            if not stale_entry.awaiting_first_use:
+                continue
+            if stale_entry.grace_holder is None or stale_entry.grace_holder() is None:
+                stale_entry.awaiting_first_use = False
 
         size = calc_model_size_by_data(self._logger, model)
         self._make_room_internal(size)
@@ -1656,14 +1663,18 @@ def _log_cache_state(self, title: str = "Model cache state:", include_entry_deta
         self._logger.debug(log)
 
     @synchronized
-    def make_room(self, bytes_needed: int) -> CacheClearResult:
+    def make_room(self, bytes_needed: int, spare_awaiting_first_use: bool = True) -> CacheClearResult:
         """Make enough room in the cache to accommodate a new model of indicated size.
 
         Note: This function deletes all of the cache's internal references to a model in order to free it. If there are
         external references to the model, there's nothing that the cache can do about it, and those models will not be
         garbage-collected.
+
+        `spare_awaiting_first_use=False` (the explicit clear-cache paths) also evicts entries
+        still inside their admission grace — a user asking for a full clear outranks the grace,
+        and the tolerated issue-7513 path covers an in-flight loader.
         """
-        return self._make_room_internal(bytes_needed)
+        return self._make_room_internal(bytes_needed, spare_awaiting_first_use=spare_awaiting_first_use)
 
     def _make_room_internal(self, bytes_needed: int, spare_awaiting_first_use: bool = True) -> CacheClearResult:
         """Internal implementation of make_room(). Assumes the lock is already held.
diff --git a/tests/app/routers/test_model_manager.py b/tests/app/routers/test_model_manager.py
index c8ffee28015..8165d78d5e3 100644
--- a/tests/app/routers/test_model_manager.py
+++ b/tests/app/routers/test_model_manager.py
@@ -284,7 +284,7 @@ def __init__(self, models_cleared: int, bytes_freed: int) -> None:
             self._result = CacheClearResult(models_cleared=models_cleared, bytes_freed=bytes_freed)
             self.requested: list[int] = []
 
-        def make_room(self, bytes_needed: int) -> CacheClearResult:
+        def make_room(self, bytes_needed: int, spare_awaiting_first_use: bool = True) -> CacheClearResult:
             self.requested.append(bytes_needed)
             return self._result
 
@@ -313,7 +313,7 @@ class _Cache:
         def __init__(self) -> None:
             self.calls = 0
 
-        def make_room(self, bytes_needed: int) -> CacheClearResult:
+        def make_room(self, bytes_needed: int, spare_awaiting_first_use: bool = True) -> CacheClearResult:
             self.calls += 1
             return CacheClearResult(models_cleared=2, bytes_freed=100)
 
diff --git a/tests/backend/model_manager/load/model_cache/test_model_cache_make_room.py b/tests/backend/model_manager/load/model_cache/test_model_cache_make_room.py
index c8547d07805..abb298f5f58 100644
--- a/tests/backend/model_manager/load/model_cache/test_model_cache_make_room.py
+++ b/tests/backend/model_manager/load/model_cache/test_model_cache_make_room.py
@@ -1,10 +1,11 @@
 """Tests for make_room()'s eviction guards."""
 
+import gc
 import logging
+import time
 from unittest.mock import MagicMock
 
-import torch
-
+from invokeai.backend.model_manager.load.load_base import LoadedModelWithoutConfig
 from invokeai.backend.model_manager.load.model_cache.model_cache import ModelCache
 from tests.backend.model_manager.load.model_cache.cached_model.utils import DummyModule
 
@@ -70,7 +71,82 @@ def test_make_room_never_evicts_locked_entries():
         cache.unlock(record)
 
 
-def test_dummy_module_is_cpu():
-    # Guard against the fixture silently needing CUDA: everything above must run CPU-only.
-    assert not any(p.is_cuda for p in DummyModule().parameters())
-    assert torch.cuda.is_available() or True
+def test_grace_survives_sibling_admissions_while_the_handle_lives():
+    """Multi-model invocations load their whole set before locking any of it: an earlier
+    sibling's grace must survive later puts (whose sweep previously cleared every grace) and the
+    make_rooms they trigger, for as long as its LoadedModel handle is alive."""
+    cache = _make_cache()
+    cache.put("first", DummyModule())
+    record = cache.get("first")
+    handle = LoadedModelWithoutConfig(cache_record=record, cache=cache)
+
+    # Two more siblings admitted — each put() runs the stale-grace sweep and a make_room.
+    cache.put("second", DummyModule())
+    cache.put("third", DummyModule())
+    assert record.awaiting_first_use, "a sibling's put() swept a grace whose handle is alive"
+
+    cache.make_room(10**15)
+    assert "first" in cache._cached_models
+
+    # The handle finally locks: grace released, ordinary cache content again.
+    with handle:
+        pass
+    assert not record.awaiting_first_use
+    cache.make_room(10**15)
+    assert "first" not in cache._cached_models
+
+
+def test_sweep_clears_orphaned_graces():
+    """A grace with no registered handle (the load raised between put() and LoadedModel
+    construction) — or whose handle died without its finalizer running (lost deferred worker) —
+    is orphaned, and the next admission's sweep must clear it so the record cannot dodge budget
+    reconciles indefinitely."""
+    cache = _make_cache()
+
+    # No handle ever registered.
+    cache.put("orphan", DummyModule())
+    orphan = cache.get("orphan")
+    assert orphan.grace_holder is None
+    cache.put("sibling", DummyModule())
+    assert not orphan.awaiting_first_use
+
+    # Handle registered, then dies with its finalizer disarmed (simulated lost worker).
+    cache.put("undead", DummyModule())
+    undead = cache.get("undead")
+    handle = LoadedModelWithoutConfig(cache_record=undead, cache=cache)
+    assert handle._first_use_finalizer is not None
+    handle._first_use_finalizer.detach()
+    del handle
+    gc.collect()
+    assert undead.grace_holder is not None and undead.grace_holder() is None
+    cache.put("sibling-2", DummyModule())
+    assert not undead.awaiting_first_use
+
+
+def test_dropped_handle_releases_the_grace_through_its_finalizer():
+    cache = _make_cache()
+    cache.put("dropped", DummyModule())
+    record = cache.get("dropped")
+    handle = LoadedModelWithoutConfig(cache_record=record, cache=cache)
+
+    del handle
+    gc.collect()
+    # The finalizer dispatches the release through the deferred worker; give it a moment.
+    for _ in range(100):
+        if not record.awaiting_first_use:
+            break
+        time.sleep(0.02)
+    assert not record.awaiting_first_use
+
+
+def test_explicit_clear_ignores_the_grace():
+    """The clear-model-cache button path (spare_awaiting_first_use=False) evicts graced
+    entries too — a user-requested full clear outranks the grace."""
+    cache = _make_cache()
+    cache.put("fresh", DummyModule())
+    record = cache.get("fresh")
+    assert record.awaiting_first_use
+
+    result = cache.make_room(10**15, spare_awaiting_first_use=False)
+    assert "fresh" not in cache._cached_models
+    assert result.models_cleared == 1

From e3612c217a1368218334915aca1ace65babc1ee6 Mon Sep 17 00:00:00 2001
From: Lincoln Stein <lincoln.stein@gmail.com>
Date: Sun, 30 Aug 2026 22:58:18 -0400
Subject: [PATCH 3/3] fix(model cache): holder-identity check in the grace
 release, per second review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- The finalizer's release now carries the dying handle's weakref: when the
  record's grace has since been re-registered to a newer, still-live
  handle, the release leaves it alone (single-slot limitation documented —
  no current code path constructs two pre-lock handles for one record).
- The deferred worker unpins the unpacked CacheRecord local before
  blocking on the next queue item (the existing anti-pinning test caught
  the tuple unpacking keeping the record alive).
- The /empty_model_cache route tests now assert the
  spare_awaiting_first_use=False plumbing, not just tolerate the kwarg.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S4B5exWsbC2z2Uu2tA167A
---
 .../backend/model_manager/load/load_base.py   | 19 +++++-----
 .../load/model_cache/model_cache.py           | 38 +++++++++++++++----
 tests/app/routers/test_model_manager.py       |  5 +++
 3 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/invokeai/backend/model_manager/load/load_base.py b/invokeai/backend/model_manager/load/load_base.py
index c073dcf70cb..f8a4a21a39a 100644
--- a/invokeai/backend/model_manager/load/load_base.py
+++ b/invokeai/backend/model_manager/load/load_base.py
@@ -63,16 +63,17 @@ def __init__(self, cache_record: CacheRecord, cache: ModelCache):
         self._cache_record = cache_record
         self._cache = cache
         release_first_use_grace = getattr(cache, "release_first_use_grace", None)
-        self._first_use_finalizer = (
-            finalize(self, release_first_use_grace, cache_record)
-            if cache_record.awaiting_first_use and release_first_use_grace is not None
-            else None
-        )
-        if self._first_use_finalizer is not None:
-            self._first_use_finalizer.atexit = False
+        if cache_record.awaiting_first_use and release_first_use_grace is not None:
             # This handle owns the grace: put()'s sweep keeps a grace whose holder is alive
-            # (an in-flight multi-model load) and clears one whose holder is gone.
-            cache_record.grace_holder = weakref.ref(self)
+            # (an in-flight multi-model load) and clears one whose holder is gone. The finalizer
+            # carries its own ref so the release can tell whether the grace has since been
+            # re-registered to a newer, still-live handle.
+            holder_ref = weakref.ref(self)
+            cache_record.grace_holder = holder_ref
+            self._first_use_finalizer = finalize(self, release_first_use_grace, cache_record, holder_ref)
+            self._first_use_finalizer.atexit = False
+        else:
+            self._first_use_finalizer = None
 
     def _lock_paced(self, working_mem_bytes: Optional[int]) -> None:
         """Move the model into VRAM in bounded passes, yielding the global load lock between them.
diff --git a/invokeai/backend/model_manager/load/model_cache/model_cache.py b/invokeai/backend/model_manager/load/model_cache/model_cache.py
index 79d04361448..1717b6be79e 100644
--- a/invokeai/backend/model_manager/load/model_cache/model_cache.py
+++ b/invokeai/backend/model_manager/load/model_cache/model_cache.py
@@ -72,6 +72,7 @@ def _run_deferred_work(cache_ref: "weakref.ReferenceType[ModelCache]", work_queu
     while True:
         work = work_queue.get()
         cache = None
+        record = None
         try:
             if work is _DEFERRED_STOP:
                 return
@@ -83,6 +84,11 @@ def _run_deferred_work(cache_ref: "weakref.ReferenceType[ModelCache]", work_queu
                 continue
             if work is _DEFERRED_RECONCILE:
                 cache._reconcile_budget_if_pending()
+            elif isinstance(work, tuple):
+                record, holder_ref = work
+                assert isinstance(record, CacheRecord)
+                cache._release_first_use_grace(record, holder_ref)
+                holder_ref = None
             else:
                 assert isinstance(work, CacheRecord)
                 cache._release_first_use_grace(work)
@@ -91,13 +97,14 @@ def _run_deferred_work(cache_ref: "weakref.ReferenceType[ModelCache]", work_queu
                 cache._logger.exception("Error processing deferred model-cache work")
         finally:
             # Drop both references before blocking on the next get(): locals stay bound for as long
-            # as this frame lives. `work` may be a CacheRecord, which transitively holds its model's
+            # as this frame lives. `work` may be (or hold) a CacheRecord, which transitively holds its model's
             # CPU weights — and _release_first_use_grace's release hook can evict that very record,
             # removing it from the cache AND subtracting its bytes from the RamBudget, so holding it
             # would leave the budget under-reporting a model that is still resident. `cache` must go
             # for the same reason this function takes a weakref at all.
             work = None
             cache = None
+            record = None
 
 
 class _ModelLoadReadWriteLock:
@@ -813,7 +820,9 @@ def cached_model_keys(self) -> set[str]:
             if self._ram_budget is not None and self._budget_reconcile_pending.is_set() and not self._lock._is_owned():
                 self._dispatch_deferred(_DEFERRED_RECONCILE)
 
-    def release_first_use_grace(self, cache_entry: CacheRecord) -> None:
+    def release_first_use_grace(
+        self, cache_entry: CacheRecord, holder_ref: "weakref.ref[object] | None" = None
+    ) -> None:
         """Make an abandoned, never-locked record available for budget eviction.
 
         Called from a `weakref.finalize` callback (see LoadedModelWithoutConfig), which runs at an
@@ -839,7 +848,7 @@ def release_first_use_grace(self, cache_entry: CacheRecord) -> None:
         # release. Losing a race here at worst queues work that no-ops under the lock.
         if not cache_entry.awaiting_first_use:
             return
-        self._dispatch_deferred(cache_entry)
+        self._dispatch_deferred((cache_entry, holder_ref))
 
     def _ensure_deferred_worker(self) -> None:
         """Start the background worker if it is not currently running. Caller must hold the lock.
@@ -914,10 +923,25 @@ def _dispatch_deferred(self, work: object) -> None:
         self._deferred_work_queue.put(work)
 
     @synchronized
-    def _release_first_use_grace(self, cache_entry: CacheRecord) -> None:
-        """Clear an abandoned record's grace, then let the release hook reconcile the budget."""
-        if self._cached_models.get(cache_entry.key) is cache_entry and not cache_entry.is_locked:
-            cache_entry.awaiting_first_use = False
+    def _release_first_use_grace(
+        self, cache_entry: CacheRecord, holder_ref: "weakref.ref[object] | None" = None
+    ) -> None:
+        """Clear an abandoned record's grace, then let the release hook reconcile the budget.
+
+        `holder_ref` identifies the handle whose death triggered this release. When the record's
+        current `grace_holder` is a DIFFERENT, still-live handle (a newer handle was constructed
+        for the same graced record and re-registered itself), the grace belongs to that handle
+        now — leave it. (Single-slot limitation, documented: if the NEWER handle dies first while
+        an older one lives, the slot points at the dead ref and the grace clears anyway. No
+        current code path constructs two pre-lock handles for one record — one session worker per
+        device cache — so this stays a contract note, not a reachable bug.)
+        """
+        if self._cached_models.get(cache_entry.key) is not cache_entry or cache_entry.is_locked:
+            return
+        current = cache_entry.grace_holder
+        if holder_ref is not None and current is not None and current is not holder_ref and current() is not None:
+            return
+        cache_entry.awaiting_first_use = False
 
     @synchronized
     def _get_cache_snapshot(self) -> dict[str, CacheEntrySnapshot]:
diff --git a/tests/app/routers/test_model_manager.py b/tests/app/routers/test_model_manager.py
index 8165d78d5e3..78d0958f307 100644
--- a/tests/app/routers/test_model_manager.py
+++ b/tests/app/routers/test_model_manager.py
@@ -283,9 +283,11 @@ class _Cache:
         def __init__(self, models_cleared: int, bytes_freed: int) -> None:
             self._result = CacheClearResult(models_cleared=models_cleared, bytes_freed=bytes_freed)
             self.requested: list[int] = []
+            self.spared_grace: list[bool] = []
 
         def make_room(self, bytes_needed: int, spare_awaiting_first_use: bool = True) -> CacheClearResult:
             self.requested.append(bytes_needed)
+            self.spared_grace.append(spare_awaiting_first_use)
             return self._result
 
     cache_0 = _Cache(models_cleared=2, bytes_freed=100)
@@ -302,6 +304,9 @@ def make_room(self, bytes_needed: int, spare_awaiting_first_use: bool = True) ->
     # Both devices were actually asked to clear, not just the API thread's default cache.
     assert len(cache_0.requested) == 1
     assert len(cache_1.requested) == 1
+    # A user-requested full clear outranks the admission grace.
+    assert cache_0.spared_grace == [False]
+    assert cache_1.spared_grace == [False]
 
 
 def test_empty_model_cache_clears_duplicate_cache_objects_once(monkeypatch: Any, client: TestClient) -> None: