From 79b628c91477c84ba9aa8777f05f7839a8a20177 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:19:46 +0000 Subject: [PATCH 01/12] docs(hashing): add design spec for ITL-522 match_mtime flag Co-Authored-By: Claude Sonnet 4.6 --- ...6-07-13-itl-522-match-mtime-flag-design.md | 306 ++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md diff --git a/superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md b/superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md new file mode 100644 index 00000000..9b09cc47 --- /dev/null +++ b/superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md @@ -0,0 +1,306 @@ +# ITL-522: FileHasher Cache — `match_mtime` Flag for Opt-In mtime Bypass + +## Overview + +Cache hits in `CachedFileHasher` currently require **path + mtime_ns + size** to all +match the stored entry. In environments where mtime is churned by benign operations +(rsync, restore-from-backup, container remounts, `touch` in CI), this causes spurious +misses and unnecessary re-hashing. + +Add an opt-in `match_mtime: bool = True` flag to each hash cacher backend. When set +to `False`, the cache lookup ignores `mtime_ns` and matches on **path + size** only, +returning the entry with the **latest `mtime_ns`** when multiple hits exist. The write +path is unchanged — `mtime_ns` is always stored. + +--- + +## Goals & Success Criteria + +- `InMemoryHashCacher`, `SqliteHashCacher`, and `PostgresHashCacher` each accept + `match_mtime: bool = True` as a keyword-only constructor argument. +- `enable_file_hash_caching()` exposes `match_mtime` and forwards it to the cacher. +- Default (`match_mtime=True`): existing strict behavior — cache hit only when + `path`, `mtime_ns`, and `size` all match. +- `match_mtime=False`: cache hit when `path` and `size` match; `mtime_ns` is ignored + in the lookup. Among multiple matching entries, the one with the **highest + `mtime_ns`** is returned. +- Write path is unchanged in all cachers — `(path, mtime_ns, size, hash)` is always + stored. Switching the flag on later still produces hits against entries written + under `match_mtime=True`. +- Composes cleanly with `read_only` and `min_cache_size_bytes` (ITL-519 guards). +- `__repr__` on each cacher includes `match_mtime`. +- All five test scenarios from the issue spec pass (see Tests section). +- `docs/concepts/file-hash-caching.md` updated with the new knob. + +--- + +## Design + +### 1. Cacher constructor changes + +All three cachers gain `match_mtime` as a new keyword-only argument after the +existing `min_cache_size_bytes`: + +```python +class InMemoryHashCacher: + def __init__( + self, + *, + read_only: bool = False, + min_cache_size_bytes: int | None = None, + match_mtime: bool = True, + ) -> None: + ... + self._match_mtime = match_mtime + +class SqliteHashCacher: + def __init__( + self, + db_path: Path | None = None, + *, + read_only: bool = False, + min_cache_size_bytes: int | None = None, + match_mtime: bool = True, + ) -> None: + ... + self._match_mtime = match_mtime + +class PostgresHashCacher: + def __init__( + self, + conninfo: str, + *, + read_only: bool = False, + min_cache_size_bytes: int | None = None, + match_mtime: bool = True, + ) -> None: + ... + self._match_mtime = match_mtime +``` + +### 2. `get()` changes + +**`InMemoryHashCacher`:** + +```python +def get(self, key: FileHashKey) -> ContentHash | None: + if self._match_mtime: + return self._cache.get(key) + # match_mtime=False: find all (path, size) matches and return + # the one with the highest mtime_ns. + best_key: FileHashKey | None = None + best_value: ContentHash | None = None + for cached_key, value in self._cache.items(): + if cached_key.path == key.path and cached_key.size == key.size: + if best_key is None or cached_key.mtime_ns > best_key.mtime_ns: + best_key = cached_key + best_value = value + return best_value +``` + +O(n) over the dict — acceptable since `InMemoryHashCacher` is intended for +testing and short-lived in-process use. + +**`SqliteHashCacher`:** + +```python +def get(self, key: FileHashKey) -> ContentHash | None: + conn = self._connection() + if self._match_mtime: + cursor = conn.execute( + "SELECT hash FROM file_hash_cache WHERE path=? AND mtime_ns=? AND size=?", + (str(key.path), key.mtime_ns, key.size), + ) + else: + cursor = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=? AND size=? ORDER BY mtime_ns DESC LIMIT 1", + (str(key.path), key.size), + ) + row = cursor.fetchone() + if row is None: + return None + blob: bytes = row[0] + method_bytes, digest = blob.split(b":", 1) + return ContentHash(method=method_bytes.decode("ascii"), digest=digest) +``` + +**`PostgresHashCacher`:** + +```python +def get(self, key: FileHashKey) -> ContentHash | None: + conn = self._connection() + if self._match_mtime: + row = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=%s AND mtime_ns=%s AND size=%s", + (str(key.path), key.mtime_ns, key.size), + ).fetchone() + else: + row = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=%s AND size=%s ORDER BY mtime_ns DESC LIMIT 1", + (str(key.path), key.size), + ).fetchone() + if row is None: + return None + blob: bytes = bytes(row[0]) + method_bytes, digest = blob.split(b":", 1) + return ContentHash(method=method_bytes.decode("ascii"), digest=digest) +``` + +### 3. `put()` — unchanged + +All three cachers continue to store `(path, mtime_ns, size, hash)` on every write. +No schema changes required. + +### 4. `__repr__` updates + +Each cacher's `__repr__` gains `match_mtime=...`: + +```python +# InMemoryHashCacher +f"InMemoryHashCacher(read_only={self._read_only!r}, " \ +f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " \ +f"match_mtime={self._match_mtime!r})" + +# SqliteHashCacher +f"SqliteHashCacher(db_path={str(self.db_path)!r}, " \ +f"read_only={self._read_only!r}, " \ +f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " \ +f"match_mtime={self._match_mtime!r})" + +# PostgresHashCacher +f"PostgresHashCacher(conninfo={_redact_conninfo(self._conninfo)!r}, " \ +f"read_only={self._read_only!r}, " \ +f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " \ +f"match_mtime={self._match_mtime!r})" +``` + +### 5. `enable_file_hash_caching()` update + +```python +def enable_file_hash_caching( + *, + db_path: "Path | None" = None, + conninfo: str | None = None, + read_only: bool = False, + min_cache_size_bytes: int | None = None, + match_mtime: bool = True, # ← new +) -> None: +``` + +`match_mtime` is forwarded to `SqliteHashCacher(...)` or `PostgresHashCacher(...)`. +No other changes to the function body. + +--- + +## Tests + +New file: `tests/test_hashing/test_hash_cacher_match_mtime.py` + +### Core scenarios (run against `InMemoryHashCacher` and `SqliteHashCacher`) + +| # | Scenario | Expected | +|---|---|---| +| T1 | `match_mtime=True` (default), mtime changed, size unchanged | Cache **miss** → recompute | +| T2 | `match_mtime=False`, mtime changed, size unchanged | Cache **hit** → no recompute | +| T3 | `match_mtime=False`, content changed **and** size changed | Cache **miss** (size guard still works) | +| T4 | `match_mtime=False`, content changed but size preserved | Cache **hit** — known trade-off, tested and documented | +| T5 | Unchanged file: `match_mtime=True` vs `False` both return the same hash | + +### Multi-entry tie-breaking (SQLite + in-memory) + +- Populate two entries with the same `(path, size)` but different `mtime_ns` values. +- Verify `get()` with `match_mtime=False` returns the hash from the entry with the + **higher** `mtime_ns`. + +### `CachedFileHasher` integration (real files, in-memory cacher) + +- T2 implemented via a real file: write content, populate cache, advance the file's + mtime via `os.utime()`, then confirm `hash_file()` returns the cached hash without + re-reading the file. +- T3 and T4 implemented via real files with `write_bytes()` and controlled sizes. + +### `__repr__` tests + +- `match_mtime=True` appears in `repr()`. +- `match_mtime=False` appears in `repr()`. + +### `enable_file_hash_caching()` integration + +- `match_mtime=False` → the `SqliteHashCacher` inside the registered + `CachedFileHasher` has `_match_mtime=False`. + +--- + +## Documentation + +File: `docs/concepts/file-hash-caching.md` + +Add a new section **"Ignoring mtime in cache lookups"** after the existing +"Controlling when the cache is written" section: + +--- + +### Ignoring mtime in cache lookups + +By default, a cache hit requires **path, mtime_ns, and size** to all match the stored +entry. In environments where mtime is churned by benign operations — rsync, +restore-from-backup, container mount remounts, `touch` in CI — these changes cause +spurious misses and unnecessary re-hashing even when the file content has not changed. + +Set `match_mtime=False` to drop mtime from the lookup criterion. A cache hit then +requires only **path and size** to match: + +```python +import orcapod as op + +op.enable_file_hash_caching(match_mtime=False) +``` + +When multiple cache entries share the same path and size (recorded at different +mtimes), Orcapod returns the hash from the entry with the most recent `mtime_ns`. + +**The write path is unchanged.** mtime is always recorded when a new entry is +inserted. Switching `match_mtime=False` on a cache that was already populated under +the default `match_mtime=True` still produces hits — no cache rebuild is needed. + +#### Known trade-off + +With `match_mtime=False`, a file modification that preserves the file's byte count +will **not** be detected by the cache. The stored hash from the previous version will +be returned silently. This is rare in practice (most writes change file size), but +you should be aware of the trade-off before enabling this mode. + +Use `match_mtime=False` only in environments where mtime changes are known to be +unreliable. For most local-disk or NFS deployments the default (`match_mtime=True`) +is the right choice. + +--- + +## Scope & Boundaries + +In scope: +- `InMemoryHashCacher`, `SqliteHashCacher`, `PostgresHashCacher`: constructor + + `get()` + `__repr__` +- `enable_file_hash_caching()` signature and body +- New test file `tests/test_hashing/test_hash_cacher_match_mtime.py` +- `docs/concepts/file-hash-caching.md` update + +Out of scope: +- Per-file or per-namespace toggles — global on/off for v1 +- `CachedFileHasher` changes — flag lives entirely in the cacher layer +- `FileHashKey` changes — write path and stored schema are unchanged +- Database schema changes — no migration needed +- Auto-detecting unreliable mtime environments + +--- + +## Dependencies + +- ITL-472 (FileHasher + HashCacher + `SqliteHashCacher`) — complete. +- ITL-519 (`read_only` + `min_cache_size_bytes`) — complete; `match_mtime` follows + the same cacher-level pattern. +- ITL-520 (`PostgresHashCacher`) — complete; `match_mtime` applied identically. +- ITL-511 (activation docs) — complete; new knob added to same doc. From 216eec1f52685be893f146de5f009199946d00d3 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:37:14 +0000 Subject: [PATCH 02/12] docs(hashing): expand unreliable-mtime environment list in ITL-522 spec Co-Authored-By: Claude Sonnet 4.6 --- ...6-07-13-itl-522-match-mtime-flag-design.md | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md b/superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md index 9b09cc47..5e90e381 100644 --- a/superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md +++ b/superpowers/specs/2026-07-13-itl-522-match-mtime-flag-design.md @@ -246,9 +246,24 @@ Add a new section **"Ignoring mtime in cache lookups"** after the existing ### Ignoring mtime in cache lookups By default, a cache hit requires **path, mtime_ns, and size** to all match the stored -entry. In environments where mtime is churned by benign operations — rsync, -restore-from-backup, container mount remounts, `touch` in CI — these changes cause -spurious misses and unnecessary re-hashing even when the file content has not changed. +entry. In several common deployment scenarios, mtime is unreliable — it changes even +when file content has not — causing spurious cache misses and unnecessary re-hashing: + +- **rsync / file transfer tools** — rsync and similar tools do not preserve mtime by + default. Even with `--times`, sub-second precision is often lost on destination + filesystems, producing a different `mtime_ns` for otherwise identical files. +- **Restore from backup** — backup and restore pipelines (tar, restic, Borg, cloud + storage sync) frequently reset mtime to the restore timestamp rather than the + original file timestamp. +- **Container bind mounts and volume remounts** — remounting a volume or restarting a + container can reset or truncate mtime precision depending on the host filesystem and + container runtime (Docker, Podman, Kubernetes). +- **CI `touch` / build system side-effects** — build scripts, test harnesses, and CI + pipelines sometimes call `touch` on input files to force rebuilds, or copy files in + ways that update mtime without changing content. +- **Network filesystems (NFS, CIFS/SMB)** — clock skew between the client and server, + or coarse mtime granularity on older NFS versions, can produce stale or shifted + timestamps that differ from the values recorded in the cache. Set `match_mtime=False` to drop mtime from the lookup criterion. A cache hit then requires only **path and size** to match: From cd7e2dd0d056c17a6452fcf414f2c02fc7f5e442 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:42:46 +0000 Subject: [PATCH 03/12] docs(hashing): add implementation plan for ITL-522 match_mtime flag Co-Authored-By: Claude Sonnet 4.6 --- .../2026-07-13-itl-522-match-mtime-flag.md | 898 ++++++++++++++++++ 1 file changed, 898 insertions(+) create mode 100644 docs/metamorphic/plans/2026-07-13-itl-522-match-mtime-flag.md diff --git a/docs/metamorphic/plans/2026-07-13-itl-522-match-mtime-flag.md b/docs/metamorphic/plans/2026-07-13-itl-522-match-mtime-flag.md new file mode 100644 index 00000000..100b63b4 --- /dev/null +++ b/docs/metamorphic/plans/2026-07-13-itl-522-match-mtime-flag.md @@ -0,0 +1,898 @@ +# ITL-522: `match_mtime` Flag Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `match_mtime: bool = True` to all three hash cacher backends so that cache lookups can optionally ignore `mtime_ns` and match on path + size only, returning the entry with the highest `mtime_ns` on multi-hit. + +**Architecture:** The flag lives on each cacher (`InMemoryHashCacher`, `SqliteHashCacher`, `PostgresHashCacher`) and affects only `get()` — the write path (`put()`) is unchanged. `enable_file_hash_caching()` in `contexts/__init__.py` is updated to accept and forward the flag. All tests use TDD (write test first, run to see it fail, implement, run again to see it pass). + +**Tech Stack:** Python 3.11+, SQLite3 (stdlib), psycopg3, pytest, uv run + +--- + +## File Map + +| Action | File | What changes | +|--------|------|-------------| +| Modify | `src/orcapod/hashing/hash_cachers.py` | `InMemoryHashCacher` and `SqliteHashCacher`: constructor + `get()` + `__repr__` | +| Modify | `src/orcapod/hashing/postgres_hash_cacher.py` | `PostgresHashCacher`: constructor + `get()` + `__repr__` | +| Modify | `src/orcapod/contexts/__init__.py` | `enable_file_hash_caching()`: new `match_mtime` kwarg + forwarding | +| Create | `tests/test_hashing/test_hash_cacher_match_mtime.py` | All `match_mtime` unit + integration tests | +| Modify | `tests/test_hashing/test_hash_cachers.py` | Add one test to `TestEnableFileHashCaching` | +| Modify | `tests/test_hashing/test_postgres_hash_cacher.py` | Add `match_mtime` tests to postgres suite | +| Modify | `docs/concepts/file-hash-caching.md` | New "Ignoring mtime in cache lookups" section | + +--- + +## Task 1: `InMemoryHashCacher` — `match_mtime` flag + +**Files:** +- Create: `tests/test_hashing/test_hash_cacher_match_mtime.py` +- Modify: `src/orcapod/hashing/hash_cachers.py` + +- [ ] **Step 1: Create the test file with `InMemoryHashCacher` tests** + +Create `tests/test_hashing/test_hash_cacher_match_mtime.py` with this content: + +```python +"""Tests for hash cacher match_mtime flag (ITL-522). + +Covers InMemoryHashCacher and SqliteHashCacher for the match_mtime flag, +plus CachedFileHasher integration with real files. +""" +from __future__ import annotations + +import os + +import pytest +from upath import UPath + +from orcapod.hashing.file_hashers import CachedFileHasher, FileHashKey, FileHasher +from orcapod.hashing.hash_cachers import InMemoryHashCacher, SqliteHashCacher +from orcapod.types import ContentHash + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_key( + path: str = "/a/b.txt", mtime_ns: int = 1000, size: int = 100 +) -> FileHashKey: + return FileHashKey(path=UPath(path), mtime_ns=mtime_ns, size=size) + + +def make_hash(digest: bytes = b"\xab" * 32) -> ContentHash: + return ContentHash(method="sha256", digest=digest) + + +# --------------------------------------------------------------------------- +# InMemoryHashCacher — match_mtime +# --------------------------------------------------------------------------- + + +class TestInMemoryHashCacherMatchMtime: + def test_default_true_mtime_change_causes_miss(self): + """Default (match_mtime=True): different mtime → cache miss.""" + cacher = InMemoryHashCacher() + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=100)) is None + + def test_false_mtime_change_is_hit(self): + """match_mtime=False: different mtime, same path+size → cache hit.""" + cacher = InMemoryHashCacher(match_mtime=False) + value = make_hash() + cacher.put(make_key(mtime_ns=1000, size=100), value) + assert cacher.get(make_key(mtime_ns=2000, size=100)) == value + + def test_false_size_change_is_miss(self): + """match_mtime=False: different size → cache miss (size guard still applies).""" + cacher = InMemoryHashCacher(match_mtime=False) + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=200)) is None + + def test_false_different_path_is_miss(self): + """match_mtime=False: different path → cache miss.""" + cacher = InMemoryHashCacher(match_mtime=False) + cacher.put(make_key(path="/a/b.txt", size=100), make_hash()) + assert cacher.get(make_key(path="/a/c.txt", size=100)) is None + + def test_false_returns_latest_mtime_entry(self): + """match_mtime=False: multiple entries for same path+size → returns highest mtime_ns.""" + cacher = InMemoryHashCacher(match_mtime=False) + hash_old = make_hash(b"\xaa" * 32) + hash_new = make_hash(b"\xbb" * 32) + cacher.put(make_key(mtime_ns=1000, size=100), hash_old) + cacher.put(make_key(mtime_ns=2000, size=100), hash_new) + result = cacher.get(make_key(mtime_ns=3000, size=100)) + assert result == hash_new + + def test_false_no_entries_returns_none(self): + """match_mtime=False: no matching path+size → None.""" + cacher = InMemoryHashCacher(match_mtime=False) + assert cacher.get(make_key()) is None + + def test_repr_shows_match_mtime_true(self): + assert "match_mtime=True" in repr(InMemoryHashCacher()) + + def test_repr_shows_match_mtime_false(self): + assert "match_mtime=False" in repr(InMemoryHashCacher(match_mtime=False)) +``` + +- [ ] **Step 2: Run to verify tests fail** + +```bash +uv run pytest tests/test_hashing/test_hash_cacher_match_mtime.py::TestInMemoryHashCacherMatchMtime -v +``` + +Expected: errors like `TypeError: __init__() got an unexpected keyword argument 'match_mtime'` + +- [ ] **Step 3: Add `match_mtime` to `InMemoryHashCacher`** + +In `src/orcapod/hashing/hash_cachers.py`, update `InMemoryHashCacher.__init__`: + +```python +def __init__( + self, + *, + read_only: bool = False, + min_cache_size_bytes: int | None = None, + match_mtime: bool = True, +) -> None: + if min_cache_size_bytes is not None and min_cache_size_bytes < 0: + raise ValueError( + f"min_cache_size_bytes must be None or a non-negative integer, " + f"got {min_cache_size_bytes!r}" + ) + self._cache: dict[FileHashKey, ContentHash] = {} + self._read_only = read_only + self._min_cache_size_bytes = min_cache_size_bytes + self._match_mtime = match_mtime +``` + +Replace `InMemoryHashCacher.get()`: + +```python +def get(self, key: FileHashKey) -> ContentHash | None: + """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. + + When ``match_mtime=True`` (default), all three key fields must match. + When ``match_mtime=False``, only ``path`` and ``size`` are compared; + among all matching entries the one with the highest ``mtime_ns`` is + returned. + + Args: + key: File hash cache key. + + Returns: + Cached ``ContentHash``, or ``None`` if not found. + """ + if self._match_mtime: + return self._cache.get(key) + best_key: FileHashKey | None = None + best_value: ContentHash | None = None + for cached_key, value in self._cache.items(): + if cached_key.path == key.path and cached_key.size == key.size: + if best_key is None or cached_key.mtime_ns > best_key.mtime_ns: + best_key = cached_key + best_value = value + return best_value +``` + +Replace `InMemoryHashCacher.__repr__`: + +```python +def __repr__(self) -> str: + return ( + f"InMemoryHashCacher(" + f"read_only={self._read_only!r}, " + f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " + f"match_mtime={self._match_mtime!r})" + ) +``` + +Also update the class docstring to document the new arg: + +```python +class InMemoryHashCacher: + """Dict-backed file hash cacher for testing and ephemeral in-process use. + + No persistence, no thread-safety guarantees beyond the GIL, no eviction. + Use ``SqliteHashCacher`` for production workloads. + + Args: + read_only: When ``True``, all ``put()`` calls are silent no-ops. + ``get()`` still works normally. Defaults to ``False``. + min_cache_size_bytes: When set to a positive integer, files whose + ``key.size`` is strictly below this threshold are not inserted. + ``None`` and ``0`` disable the threshold (default behaviour). + Negative values raise ``ValueError``. Defaults to ``None``. + match_mtime: When ``True`` (default), cache hits require + ``path``, ``mtime_ns``, and ``size`` to all match. When + ``False``, only ``path`` and ``size`` are compared; among + multiple matching entries the one with the highest ``mtime_ns`` + is returned. The write path is unaffected — ``mtime_ns`` is + always stored. + """ +``` + +- [ ] **Step 4: Run tests and verify they pass** + +```bash +uv run pytest tests/test_hashing/test_hash_cacher_match_mtime.py::TestInMemoryHashCacherMatchMtime -v +``` + +Expected: all 8 tests PASS + +- [ ] **Step 5: Verify no regressions in existing cacher tests** + +```bash +uv run pytest tests/test_hashing/test_hash_cachers.py tests/test_hashing/test_hash_cacher_write_guards.py -v +``` + +Expected: all pass + +- [ ] **Step 6: Commit** + +```bash +git add src/orcapod/hashing/hash_cachers.py tests/test_hashing/test_hash_cacher_match_mtime.py +git commit -m "feat(hashing): add match_mtime flag to InMemoryHashCacher (ITL-522)" +``` + +--- + +## Task 2: `SqliteHashCacher` — `match_mtime` flag + +**Files:** +- Modify: `tests/test_hashing/test_hash_cacher_match_mtime.py` +- Modify: `src/orcapod/hashing/hash_cachers.py` + +- [ ] **Step 1: Add `SqliteHashCacher` tests to the test file** + +Append this class to `tests/test_hashing/test_hash_cacher_match_mtime.py`: + +```python +# --------------------------------------------------------------------------- +# SqliteHashCacher — match_mtime +# --------------------------------------------------------------------------- + + +class TestSqliteHashCacherMatchMtime: + def test_default_true_mtime_change_causes_miss(self, tmp_path): + """Default (match_mtime=True): different mtime → cache miss.""" + cacher = SqliteHashCacher(tmp_path / "cache.db") + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=100)) is None + + def test_false_mtime_change_is_hit(self, tmp_path): + """match_mtime=False: different mtime, same path+size → cache hit.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + value = make_hash() + cacher.put(make_key(mtime_ns=1000, size=100), value) + result = cacher.get(make_key(mtime_ns=2000, size=100)) + assert result is not None + assert result.digest == value.digest + + def test_false_size_change_is_miss(self, tmp_path): + """match_mtime=False: different size → cache miss.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=200)) is None + + def test_false_different_path_is_miss(self, tmp_path): + """match_mtime=False: different path → cache miss.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + cacher.put(make_key(path="/a/b.txt", size=100), make_hash()) + assert cacher.get(make_key(path="/a/c.txt", size=100)) is None + + def test_false_returns_latest_mtime_entry(self, tmp_path): + """match_mtime=False: multiple entries for same path+size → returns highest mtime_ns.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + hash_old = make_hash(b"\xaa" * 32) + hash_new = make_hash(b"\xbb" * 32) + cacher.put(make_key(mtime_ns=1000, size=100), hash_old) + cacher.put(make_key(mtime_ns=2000, size=100), hash_new) + result = cacher.get(make_key(mtime_ns=3000, size=100)) + assert result is not None + assert result.digest == hash_new.digest + + def test_false_no_entries_returns_none(self, tmp_path): + """match_mtime=False: no matching path+size → None.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + assert cacher.get(make_key()) is None + + def test_repr_shows_match_mtime_true(self, tmp_path): + cacher = SqliteHashCacher(tmp_path / "cache.db") + assert "match_mtime=True" in repr(cacher) + + def test_repr_shows_match_mtime_false(self, tmp_path): + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + assert "match_mtime=False" in repr(cacher) +``` + +- [ ] **Step 2: Run to verify tests fail** + +```bash +uv run pytest tests/test_hashing/test_hash_cacher_match_mtime.py::TestSqliteHashCacherMatchMtime -v +``` + +Expected: `TypeError: __init__() got an unexpected keyword argument 'match_mtime'` + +- [ ] **Step 3: Add `match_mtime` to `SqliteHashCacher`** + +In `src/orcapod/hashing/hash_cachers.py`, update `SqliteHashCacher.__init__`: + +```python +def __init__( + self, + db_path: Path | None = None, + *, + read_only: bool = False, + min_cache_size_bytes: int | None = None, + match_mtime: bool = True, +) -> None: + if min_cache_size_bytes is not None and min_cache_size_bytes < 0: + raise ValueError( + f"min_cache_size_bytes must be None or a non-negative integer, " + f"got {min_cache_size_bytes!r}" + ) + self.db_path = Path( + db_path + or os.environ.get("ORCAPOD_HASH_CACHE_DB") + or self.DEFAULT_DB_PATH + ) + self._read_only = read_only + self._min_cache_size_bytes = min_cache_size_bytes + self._match_mtime = match_mtime + self._local = threading.local() + self._ensure_schema() +``` + +Replace `SqliteHashCacher.get()`: + +```python +def get(self, key: FileHashKey) -> ContentHash | None: + """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. + + When ``match_mtime=True`` (default), the query filters on + ``path``, ``mtime_ns``, and ``size``. When ``match_mtime=False``, + only ``path`` and ``size`` are filtered and results are ordered + by ``mtime_ns DESC`` so the most recent entry is returned. + + Args: + key: File hash cache key. + + Returns: + Cached ``ContentHash``, or ``None`` if not found. + """ + conn = self._connection() + if self._match_mtime: + cursor = conn.execute( + "SELECT hash FROM file_hash_cache WHERE path=? AND mtime_ns=? AND size=?", + (str(key.path), key.mtime_ns, key.size), + ) + else: + cursor = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=? AND size=? ORDER BY mtime_ns DESC LIMIT 1", + (str(key.path), key.size), + ) + row = cursor.fetchone() + if row is None: + return None + blob: bytes = row[0] + method_bytes, digest = blob.split(b":", 1) + return ContentHash(method=method_bytes.decode("ascii"), digest=digest) +``` + +Replace `SqliteHashCacher.__repr__`: + +```python +def __repr__(self) -> str: + return ( + f"SqliteHashCacher(" + f"db_path={str(self.db_path)!r}, " + f"read_only={self._read_only!r}, " + f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " + f"match_mtime={self._match_mtime!r})" + ) +``` + +Also add `match_mtime` to the `SqliteHashCacher` class docstring Args section: + +``` + match_mtime: When ``True`` (default), cache hits require + ``path``, ``mtime_ns``, and ``size`` to all match. When + ``False``, only ``path`` and ``size`` are compared; among + multiple matching entries the one with the highest ``mtime_ns`` + is returned. The write path is unaffected — ``mtime_ns`` is + always stored. +``` + +- [ ] **Step 4: Run tests and verify they pass** + +```bash +uv run pytest tests/test_hashing/test_hash_cacher_match_mtime.py::TestSqliteHashCacherMatchMtime -v +``` + +Expected: all 8 tests PASS + +- [ ] **Step 5: Verify no regressions** + +```bash +uv run pytest tests/test_hashing/test_hash_cachers.py tests/test_hashing/test_hash_cacher_write_guards.py tests/test_hashing/test_sqlite_cacher.py -v +``` + +Expected: all pass + +- [ ] **Step 6: Commit** + +```bash +git add src/orcapod/hashing/hash_cachers.py tests/test_hashing/test_hash_cacher_match_mtime.py +git commit -m "feat(hashing): add match_mtime flag to SqliteHashCacher (ITL-522)" +``` + +--- + +## Task 3: `PostgresHashCacher` — `match_mtime` flag + +**Files:** +- Modify: `tests/test_hashing/test_postgres_hash_cacher.py` +- Modify: `src/orcapod/hashing/postgres_hash_cacher.py` + +- [ ] **Step 1: Add `match_mtime` tests to the Postgres test file** + +In `tests/test_hashing/test_postgres_hash_cacher.py`, add a new test class after the existing `TestPostgresHashCacherWriteGuards` class: + +```python +@pytest.mark.postgres +class TestPostgresHashCacherMatchMtime: + def test_default_true_mtime_change_causes_miss(self, pg_conninfo): + """Default (match_mtime=True): different mtime → cache miss.""" + cacher = PostgresHashCacher(pg_conninfo) + cacher.clear() + cacher.put(make_key(mtime_ns=1000, size=100), make_hash(b"\xaa" * 32)) + assert cacher.get(make_key(mtime_ns=2000, size=100)) is None + cacher.close() + + def test_false_mtime_change_is_hit(self, pg_conninfo): + """match_mtime=False: different mtime, same path+size → cache hit.""" + cacher = PostgresHashCacher(pg_conninfo, match_mtime=False) + cacher.clear() + value = make_hash(b"\xbb" * 32) + cacher.put(make_key(mtime_ns=1000, size=100), value) + result = cacher.get(make_key(mtime_ns=2000, size=100)) + assert result is not None + assert result.digest == value.digest + cacher.close() + + def test_false_size_change_is_miss(self, pg_conninfo): + """match_mtime=False: different size → cache miss.""" + cacher = PostgresHashCacher(pg_conninfo, match_mtime=False) + cacher.clear() + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=200)) is None + cacher.close() + + def test_false_returns_latest_mtime_entry(self, pg_conninfo): + """match_mtime=False: multiple entries for same path+size → returns highest mtime_ns.""" + cacher = PostgresHashCacher(pg_conninfo, match_mtime=False) + cacher.clear() + hash_old = make_hash(b"\xaa" * 32) + hash_new = make_hash(b"\xbb" * 32) + cacher.put(make_key(mtime_ns=1000, size=100), hash_old) + cacher.put(make_key(mtime_ns=2000, size=100), hash_new) + result = cacher.get(make_key(mtime_ns=3000, size=100)) + assert result is not None + assert result.digest == hash_new.digest + cacher.close() + + def test_repr_shows_match_mtime_true(self, pg_conninfo): + """__repr__ includes match_mtime=True by default.""" + cacher = PostgresHashCacher(pg_conninfo) + assert "match_mtime=True" in repr(cacher) + cacher.close() + + def test_repr_shows_match_mtime_false(self, pg_conninfo): + """__repr__ includes match_mtime=False.""" + cacher = PostgresHashCacher(pg_conninfo, match_mtime=False) + assert "match_mtime=False" in repr(cacher) + cacher.close() +``` + +- [ ] **Step 2: Add `match_mtime` to `PostgresHashCacher`** + +In `src/orcapod/hashing/postgres_hash_cacher.py`, update `PostgresHashCacher.__init__`: + +```python +def __init__( + self, + conninfo: str, + *, + read_only: bool = False, + min_cache_size_bytes: int | None = None, + match_mtime: bool = True, +) -> None: + if min_cache_size_bytes is not None and min_cache_size_bytes < 0: + raise ValueError( + f"min_cache_size_bytes must be None or a non-negative integer, " + f"got {min_cache_size_bytes!r}" + ) + self._conninfo = conninfo + self._read_only = read_only + self._min_cache_size_bytes = min_cache_size_bytes + self._match_mtime = match_mtime + self._local = threading.local() + self._ensure_schema() +``` + +Replace `PostgresHashCacher.get()`: + +```python +def get(self, key: FileHashKey) -> ContentHash | None: + """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. + + When ``match_mtime=True`` (default), the query filters on + ``path``, ``mtime_ns``, and ``size``. When ``match_mtime=False``, + only ``path`` and ``size`` are filtered and results are ordered + by ``mtime_ns DESC`` so the most recent entry is returned. + + Args: + key: File hash cache key. + + Returns: + Cached ``ContentHash``, or ``None`` if not found. + """ + conn = self._connection() + if self._match_mtime: + row = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=%s AND mtime_ns=%s AND size=%s", + (str(key.path), key.mtime_ns, key.size), + ).fetchone() + else: + row = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=%s AND size=%s ORDER BY mtime_ns DESC LIMIT 1", + (str(key.path), key.size), + ).fetchone() + if row is None: + return None + blob: bytes = bytes(row[0]) + method_bytes, digest = blob.split(b":", 1) + return ContentHash(method=method_bytes.decode("ascii"), digest=digest) +``` + +Replace `PostgresHashCacher.__repr__`: + +```python +def __repr__(self) -> str: + return ( + f"PostgresHashCacher(" + f"conninfo={_redact_conninfo(self._conninfo)!r}, " + f"read_only={self._read_only!r}, " + f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " + f"match_mtime={self._match_mtime!r})" + ) +``` + +Also add `match_mtime` to the `PostgresHashCacher` class docstring Args section: + +``` + match_mtime: When ``True`` (default), cache hits require + ``path``, ``mtime_ns``, and ``size`` to all match. When + ``False``, only ``path`` and ``size`` are compared; among + multiple matching entries the one with the highest ``mtime_ns`` + is returned. The write path is unaffected — ``mtime_ns`` is + always stored. +``` + +- [ ] **Step 3: Run non-postgres tests to verify no regressions** + +```bash +uv run pytest tests/test_hashing/ -v --ignore=tests/test_hashing/test_postgres_hash_cacher.py +``` + +Expected: all pass (the postgres tests require `--postgres` flag and a Docker daemon) + +- [ ] **Step 4: Commit** + +```bash +git add src/orcapod/hashing/postgres_hash_cacher.py tests/test_hashing/test_postgres_hash_cacher.py +git commit -m "feat(hashing): add match_mtime flag to PostgresHashCacher (ITL-522)" +``` + +--- + +## Task 4: `CachedFileHasher` integration + `enable_file_hash_caching()` + +**Files:** +- Modify: `tests/test_hashing/test_hash_cacher_match_mtime.py` +- Modify: `tests/test_hashing/test_hash_cachers.py` +- Modify: `src/orcapod/contexts/__init__.py` + +- [ ] **Step 1: Add `CachedFileHasher` integration tests** + +Append this class to `tests/test_hashing/test_hash_cacher_match_mtime.py`: + +```python +# --------------------------------------------------------------------------- +# CachedFileHasher integration — match_mtime with real files +# --------------------------------------------------------------------------- + + +class TestCachedFileHasherMatchMtime: + def test_t1_default_mtime_change_causes_miss(self, tmp_path): + """T1: Default (match_mtime=True), mtime changed, size unchanged → cache miss.""" + f = tmp_path / "file.bin" + f.write_bytes(b"x" * 50) + + cacher = InMemoryHashCacher(match_mtime=True) + cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) + cached.hash_file(f) + + stat = f.stat() + os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + path = UPath(f).resolve() + new_stat = path.stat() + new_key = FileHashKey(path, new_stat.st_mtime_ns, new_stat.st_size) + assert cacher.get(new_key) is None + + def test_t2_match_mtime_false_mtime_change_is_hit(self, tmp_path): + """T2: match_mtime=False, mtime changed, size unchanged → cache hit.""" + f = tmp_path / "file.bin" + f.write_bytes(b"x" * 50) + + cacher = InMemoryHashCacher(match_mtime=False) + cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) + first_hash = cached.hash_file(f) + + stat = f.stat() + os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + second_hash = cached.hash_file(f) + assert second_hash == first_hash + + def test_t3_match_mtime_false_size_change_is_miss(self, tmp_path): + """T3: match_mtime=False, content and size changed → cache miss.""" + f = tmp_path / "file.bin" + f.write_bytes(b"hello") # 5 bytes + + cacher = InMemoryHashCacher(match_mtime=False) + cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) + first_hash = cached.hash_file(f) + + f.write_bytes(b"hello world") # 11 bytes — different size + + second_hash = cached.hash_file(f) + assert second_hash != first_hash + + def test_t4_match_mtime_false_same_size_content_flip_is_hit(self, tmp_path): + """T4: match_mtime=False, content changed but size preserved → cache hit (known trade-off).""" + f = tmp_path / "file.bin" + f.write_bytes(b"aaaaa") # 5 bytes + + cacher = InMemoryHashCacher(match_mtime=False) + cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) + first_hash = cached.hash_file(f) + + f.write_bytes(b"bbbbb") # 5 bytes — same size, different content + + second_hash = cached.hash_file(f) + assert second_hash == first_hash # stale hit — known trade-off + + def test_t5_unchanged_file_both_modes_agree(self, tmp_path): + """T5: Unchanged file — match_mtime=True and match_mtime=False return the same hash.""" + f = tmp_path / "file.bin" + f.write_bytes(b"stable content") + + inner = FileHasher() + hash_strict = CachedFileHasher( + file_hasher=inner, + cacher=InMemoryHashCacher(match_mtime=True), + ).hash_file(f) + hash_relaxed = CachedFileHasher( + file_hasher=inner, + cacher=InMemoryHashCacher(match_mtime=False), + ).hash_file(f) + assert hash_strict == hash_relaxed +``` + +- [ ] **Step 2: Run integration tests to verify they pass** + +```bash +uv run pytest tests/test_hashing/test_hash_cacher_match_mtime.py::TestCachedFileHasherMatchMtime -v +``` + +Expected: all 5 tests PASS (these exercise already-implemented cacher code) + +- [ ] **Step 3: Add `enable_file_hash_caching` test** + +In `tests/test_hashing/test_hash_cachers.py`, inside the `TestEnableFileHashCaching` class, add this test method after `test_min_cache_size_bytes_kwarg_passes_through_to_cacher`: + +```python + def test_match_mtime_kwarg_passes_through_to_cacher( + self, restore_default_file_handler, tmp_path + ): + from orcapod.contexts import enable_file_hash_caching, get_default_context + from orcapod.extension_types.file_type import File + from orcapod.hashing.hash_cachers import SqliteHashCacher + + enable_file_hash_caching(db_path=tmp_path / "cache.db", match_mtime=False) + + context = get_default_context() + registry = context.semantic_hasher.type_handler_registry + handler = registry.get_handler_for_type(File) + cacher = handler.file_hasher.cacher + + assert isinstance(cacher, SqliteHashCacher) + assert cacher._match_mtime is False +``` + +- [ ] **Step 4: Run to verify test fails** + +```bash +uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching::test_match_mtime_kwarg_passes_through_to_cacher -v +``` + +Expected: `TypeError: enable_file_hash_caching() got an unexpected keyword argument 'match_mtime'` + +- [ ] **Step 5: Add `match_mtime` to `enable_file_hash_caching()`** + +In `src/orcapod/contexts/__init__.py`, update the function signature: + +```python +def enable_file_hash_caching( + *, + db_path: "Path | None" = None, + conninfo: str | None = None, + read_only: bool = False, + min_cache_size_bytes: int | None = None, + match_mtime: bool = True, +) -> None: +``` + +Update the docstring Args section to add: + +``` + match_mtime: When ``False``, the cacher matches cache entries by + ``path`` and ``size`` only, ignoring ``mtime_ns``. The entry + with the highest ``mtime_ns`` is returned on multi-hit. Use in + environments where mtime is unreliable (rsync, container + remounts, restore-from-backup). Defaults to ``True`` (strict + path + mtime_ns + size matching). +``` + +Update the `SqliteHashCacher` construction in the function body: + +```python + cacher = SqliteHashCacher( + db_path, + read_only=read_only, + min_cache_size_bytes=min_cache_size_bytes, + match_mtime=match_mtime, + ) +``` + +Update the `PostgresHashCacher` construction in the function body: + +```python + cacher = PostgresHashCacher( + conninfo, + read_only=read_only, + min_cache_size_bytes=min_cache_size_bytes, + match_mtime=match_mtime, + ) +``` + +- [ ] **Step 6: Run to verify test passes** + +```bash +uv run pytest tests/test_hashing/test_hash_cachers.py::TestEnableFileHashCaching::test_match_mtime_kwarg_passes_through_to_cacher -v +``` + +Expected: PASS + +- [ ] **Step 7: Run full hashing test suite** + +```bash +uv run pytest tests/test_hashing/ -v --ignore=tests/test_hashing/test_postgres_hash_cacher.py +``` + +Expected: all pass + +- [ ] **Step 8: Commit** + +```bash +git add tests/test_hashing/test_hash_cacher_match_mtime.py tests/test_hashing/test_hash_cachers.py src/orcapod/contexts/__init__.py +git commit -m "feat(hashing): add match_mtime to enable_file_hash_caching; integration tests (ITL-522)" +``` + +--- + +## Task 5: Documentation + +**Files:** +- Modify: `docs/concepts/file-hash-caching.md` + +- [ ] **Step 1: Add the new section to the docs** + +In `docs/concepts/file-hash-caching.md`, insert a new section **"Ignoring mtime in cache lookups"** immediately after the closing of the "Controlling when the cache is written" section (after the "Combining both" subsection, before "## Directory hashing (op.Directory)"). + +The section to insert: + +```markdown +## Ignoring mtime in cache lookups + +By default, a cache hit requires **path, mtime_ns, and size** to all match the stored +entry. In several common deployment scenarios, mtime is unreliable — it changes even +when file content has not — causing spurious cache misses and unnecessary re-hashing: + +- **rsync / file transfer tools** — rsync and similar tools do not preserve mtime by + default. Even with `--times`, sub-second precision is often lost on destination + filesystems, producing a different `mtime_ns` for otherwise identical files. +- **Restore from backup** — backup and restore pipelines (tar, restic, Borg, cloud + storage sync) frequently reset mtime to the restore timestamp rather than the + original file timestamp. +- **Container bind mounts and volume remounts** — remounting a volume or restarting a + container can reset or truncate mtime precision depending on the host filesystem and + container runtime (Docker, Podman, Kubernetes). +- **CI `touch` / build system side-effects** — build scripts, test harnesses, and CI + pipelines sometimes call `touch` on input files to force rebuilds, or copy files in + ways that update mtime without changing content. +- **Network filesystems (NFS, CIFS/SMB)** — clock skew between the client and server, + or coarse mtime granularity on older NFS versions, can produce stale or shifted + timestamps that differ from the values recorded in the cache. + +Set `match_mtime=False` to drop mtime from the lookup criterion. A cache hit then +requires only **path and size** to match: + +```python +import orcapod as op + +op.enable_file_hash_caching(match_mtime=False) +``` + +When multiple cache entries share the same path and size (recorded at different +mtimes), Orcapod returns the hash from the entry with the most recent `mtime_ns`. + +**The write path is unchanged.** mtime is always recorded when a new entry is +inserted. Switching `match_mtime=False` on a cache that was already populated under +the default `match_mtime=True` still produces hits — no cache rebuild is needed. + +### Known trade-off + +With `match_mtime=False`, a file modification that preserves the file's byte count +will **not** be detected by the cache. The stored hash from the previous version will +be returned silently. This is rare in practice (most writes change file size), but +you should be aware of the trade-off before enabling this mode. + +Use `match_mtime=False` only in environments where mtime changes are known to be +unreliable. For most local-disk or NFS deployments the default (`match_mtime=True`) +is the right choice. +``` + +- [ ] **Step 2: Verify the docs file renders correctly (spot-check)** + +```bash +grep -n "Ignoring mtime" docs/concepts/file-hash-caching.md +``` + +Expected: line number printed, confirming the section was inserted. + +- [ ] **Step 3: Run full test suite one final time** + +```bash +uv run pytest tests/test_hashing/ -v --ignore=tests/test_hashing/test_postgres_hash_cacher.py +``` + +Expected: all pass + +- [ ] **Step 4: Commit** + +```bash +git add docs/concepts/file-hash-caching.md +git commit -m "docs(hashing): document match_mtime flag in file-hash-caching guide (ITL-522)" +``` From 706b515cf7de3881403a04a338e1772cc70383d2 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:47:13 +0000 Subject: [PATCH 04/12] feat(hashing): add match_mtime flag to InMemoryHashCacher (ITL-522) When match_mtime=False, the get() method ignores mtime_ns and instead scans for entries matching (path, size), returning the one with the highest mtime_ns. The put() method is unaffected and always stores the full key. The default behavior (match_mtime=True) preserves backward compatibility by requiring all three key fields to match. Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/hashing/hash_cachers.py | 27 ++++++- .../test_hash_cacher_match_mtime.py | 80 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 tests/test_hashing/test_hash_cacher_match_mtime.py diff --git a/src/orcapod/hashing/hash_cachers.py b/src/orcapod/hashing/hash_cachers.py index bd8ef5d1..866e6609 100644 --- a/src/orcapod/hashing/hash_cachers.py +++ b/src/orcapod/hashing/hash_cachers.py @@ -32,6 +32,12 @@ class InMemoryHashCacher: ``key.size`` is strictly below this threshold are not inserted. ``None`` and ``0`` disable the threshold (default behaviour). Negative values raise ``ValueError``. Defaults to ``None``. + match_mtime: When ``True`` (default), cache hits require + ``path``, ``mtime_ns``, and ``size`` to all match. When + ``False``, only ``path`` and ``size`` are compared; among + multiple matching entries the one with the highest ``mtime_ns`` + is returned. The write path is unaffected — ``mtime_ns`` is + always stored. """ def __init__( @@ -39,6 +45,7 @@ def __init__( *, read_only: bool = False, min_cache_size_bytes: int | None = None, + match_mtime: bool = True, ) -> None: if min_cache_size_bytes is not None and min_cache_size_bytes < 0: raise ValueError( @@ -48,17 +55,32 @@ def __init__( self._cache: dict[FileHashKey, ContentHash] = {} self._read_only = read_only self._min_cache_size_bytes = min_cache_size_bytes + self._match_mtime = match_mtime def get(self, key: FileHashKey) -> ContentHash | None: """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. + When ``match_mtime=True`` (default), all three key fields must match. + When ``match_mtime=False``, only ``path`` and ``size`` are compared; + among all matching entries the one with the highest ``mtime_ns`` is + returned. + Args: key: File hash cache key. Returns: Cached ``ContentHash``, or ``None`` if not found. """ - return self._cache.get(key) + if self._match_mtime: + return self._cache.get(key) + best_key: FileHashKey | None = None + best_value: ContentHash | None = None + for cached_key, value in self._cache.items(): + if cached_key.path == key.path and cached_key.size == key.size: + if best_key is None or cached_key.mtime_ns > best_key.mtime_ns: + best_key = cached_key + best_value = value + return best_value def put(self, key: FileHashKey, value: ContentHash) -> None: """Store ``value`` under ``key``. @@ -84,7 +106,8 @@ def __repr__(self) -> str: return ( f"InMemoryHashCacher(" f"read_only={self._read_only!r}, " - f"min_cache_size_bytes={self._min_cache_size_bytes!r})" + f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " + f"match_mtime={self._match_mtime!r})" ) diff --git a/tests/test_hashing/test_hash_cacher_match_mtime.py b/tests/test_hashing/test_hash_cacher_match_mtime.py new file mode 100644 index 00000000..6ebb94b6 --- /dev/null +++ b/tests/test_hashing/test_hash_cacher_match_mtime.py @@ -0,0 +1,80 @@ +"""Tests for hash cacher match_mtime flag (ITL-522). + +Covers InMemoryHashCacher and SqliteHashCacher for the match_mtime flag, +plus CachedFileHasher integration with real files. +""" +from __future__ import annotations + +from upath import UPath + +from orcapod.hashing.file_hashers import FileHashKey +from orcapod.hashing.hash_cachers import InMemoryHashCacher +from orcapod.types import ContentHash + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_key( + path: str = "/a/b.txt", mtime_ns: int = 1000, size: int = 100 +) -> FileHashKey: + return FileHashKey(path=UPath(path), mtime_ns=mtime_ns, size=size) + + +def make_hash(digest: bytes = b"\xab" * 32) -> ContentHash: + return ContentHash(method="sha256", digest=digest) + + +# --------------------------------------------------------------------------- +# InMemoryHashCacher — match_mtime +# --------------------------------------------------------------------------- + + +class TestInMemoryHashCacherMatchMtime: + def test_default_true_mtime_change_causes_miss(self): + """Default (match_mtime=True): different mtime → cache miss.""" + cacher = InMemoryHashCacher() + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=100)) is None + + def test_false_mtime_change_is_hit(self): + """match_mtime=False: different mtime, same path+size → cache hit.""" + cacher = InMemoryHashCacher(match_mtime=False) + value = make_hash() + cacher.put(make_key(mtime_ns=1000, size=100), value) + assert cacher.get(make_key(mtime_ns=2000, size=100)) == value + + def test_false_size_change_is_miss(self): + """match_mtime=False: different size → cache miss (size guard still applies).""" + cacher = InMemoryHashCacher(match_mtime=False) + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=200)) is None + + def test_false_different_path_is_miss(self): + """match_mtime=False: different path → cache miss.""" + cacher = InMemoryHashCacher(match_mtime=False) + cacher.put(make_key(path="/a/b.txt", size=100), make_hash()) + assert cacher.get(make_key(path="/a/c.txt", size=100)) is None + + def test_false_returns_latest_mtime_entry(self): + """match_mtime=False: multiple entries for same path+size → returns highest mtime_ns.""" + cacher = InMemoryHashCacher(match_mtime=False) + hash_old = make_hash(b"\xaa" * 32) + hash_new = make_hash(b"\xbb" * 32) + cacher.put(make_key(mtime_ns=1000, size=100), hash_old) + cacher.put(make_key(mtime_ns=2000, size=100), hash_new) + result = cacher.get(make_key(mtime_ns=3000, size=100)) + assert result == hash_new + + def test_false_no_entries_returns_none(self): + """match_mtime=False: no matching path+size → None.""" + cacher = InMemoryHashCacher(match_mtime=False) + assert cacher.get(make_key()) is None + + def test_repr_shows_match_mtime_true(self): + assert "match_mtime=True" in repr(InMemoryHashCacher()) + + def test_repr_shows_match_mtime_false(self): + assert "match_mtime=False" in repr(InMemoryHashCacher(match_mtime=False)) From 372bded96212eb219fac2d23fdaf9a26ab58f9da Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:52:53 +0000 Subject: [PATCH 05/12] feat(hashing): add match_mtime flag to SqliteHashCacher (ITL-522) --- src/orcapod/hashing/hash_cachers.py | 37 +++++++++--- .../test_hash_cacher_match_mtime.py | 60 ++++++++++++++++++- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/src/orcapod/hashing/hash_cachers.py b/src/orcapod/hashing/hash_cachers.py index 866e6609..c8544c5b 100644 --- a/src/orcapod/hashing/hash_cachers.py +++ b/src/orcapod/hashing/hash_cachers.py @@ -10,14 +10,14 @@ import threading from pathlib import Path +from orcapod.hashing.file_hashers import FileHashKey +from orcapod.types import ContentHash + # Current SQLite schema version. Stored in PRAGMA user_version. # V0 (default) = legacy schema without the cached_at column. # V1 = current schema: added cached_at column. _SQLITE_SCHEMA_VERSION = 1 -from orcapod.hashing.file_hashers import FileHashKey -from orcapod.types import ContentHash - class InMemoryHashCacher: """Dict-backed file hash cacher for testing and ephemeral in-process use. @@ -131,6 +131,12 @@ class SqliteHashCacher: ``key.size`` is strictly below this threshold are not inserted. ``None`` and ``0`` disable the threshold (default behaviour). Negative values raise ``ValueError``. Defaults to ``None``. + match_mtime: When ``True`` (default), cache hits require + ``path``, ``mtime_ns``, and ``size`` to all match. When + ``False``, only ``path`` and ``size`` are compared; among + multiple matching entries the one with the highest ``mtime_ns`` + is returned. The write path is unaffected — ``mtime_ns`` is + always stored. Note: Heavy multi-writer scenarios are a known SQLite limitation. A Turso @@ -145,6 +151,7 @@ def __init__( *, read_only: bool = False, min_cache_size_bytes: int | None = None, + match_mtime: bool = True, ) -> None: if min_cache_size_bytes is not None and min_cache_size_bytes < 0: raise ValueError( @@ -158,6 +165,7 @@ def __init__( ) self._read_only = read_only self._min_cache_size_bytes = min_cache_size_bytes + self._match_mtime = match_mtime self._local = threading.local() self._ensure_schema() @@ -237,6 +245,11 @@ def _connection(self) -> sqlite3.Connection: def get(self, key: FileHashKey) -> ContentHash | None: """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. + When ``match_mtime=True`` (default), the query filters on + ``path``, ``mtime_ns``, and ``size``. When ``match_mtime=False``, + only ``path`` and ``size`` are filtered and results are ordered + by ``mtime_ns DESC`` so the most recent entry is returned. + Args: key: File hash cache key. @@ -244,10 +257,17 @@ def get(self, key: FileHashKey) -> ContentHash | None: Cached ``ContentHash``, or ``None`` if not found. """ conn = self._connection() - cursor = conn.execute( - "SELECT hash FROM file_hash_cache WHERE path=? AND mtime_ns=? AND size=?", - (str(key.path), key.mtime_ns, key.size), - ) + if self._match_mtime: + cursor = conn.execute( + "SELECT hash FROM file_hash_cache WHERE path=? AND mtime_ns=? AND size=?", + (str(key.path), key.mtime_ns, key.size), + ) + else: + cursor = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=? AND size=? ORDER BY mtime_ns DESC LIMIT 1", + (str(key.path), key.size), + ) row = cursor.fetchone() if row is None: return None @@ -298,7 +318,8 @@ def __repr__(self) -> str: f"SqliteHashCacher(" f"db_path={str(self.db_path)!r}, " f"read_only={self._read_only!r}, " - f"min_cache_size_bytes={self._min_cache_size_bytes!r})" + f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " + f"match_mtime={self._match_mtime!r})" ) def __enter__(self) -> "SqliteHashCacher": diff --git a/tests/test_hashing/test_hash_cacher_match_mtime.py b/tests/test_hashing/test_hash_cacher_match_mtime.py index 6ebb94b6..236a19d5 100644 --- a/tests/test_hashing/test_hash_cacher_match_mtime.py +++ b/tests/test_hashing/test_hash_cacher_match_mtime.py @@ -8,7 +8,7 @@ from upath import UPath from orcapod.hashing.file_hashers import FileHashKey -from orcapod.hashing.hash_cachers import InMemoryHashCacher +from orcapod.hashing.hash_cachers import InMemoryHashCacher, SqliteHashCacher from orcapod.types import ContentHash @@ -78,3 +78,61 @@ def test_repr_shows_match_mtime_true(self): def test_repr_shows_match_mtime_false(self): assert "match_mtime=False" in repr(InMemoryHashCacher(match_mtime=False)) + + +# --------------------------------------------------------------------------- +# SqliteHashCacher — match_mtime +# --------------------------------------------------------------------------- + + +class TestSqliteHashCacherMatchMtime: + def test_default_true_mtime_change_causes_miss(self, tmp_path): + """Default (match_mtime=True): different mtime → cache miss.""" + cacher = SqliteHashCacher(tmp_path / "cache.db") + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=100)) is None + + def test_false_mtime_change_is_hit(self, tmp_path): + """match_mtime=False: different mtime, same path+size → cache hit.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + value = make_hash() + cacher.put(make_key(mtime_ns=1000, size=100), value) + result = cacher.get(make_key(mtime_ns=2000, size=100)) + assert result is not None + assert result.digest == value.digest + + def test_false_size_change_is_miss(self, tmp_path): + """match_mtime=False: different size → cache miss.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=200)) is None + + def test_false_different_path_is_miss(self, tmp_path): + """match_mtime=False: different path → cache miss.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + cacher.put(make_key(path="/a/b.txt", size=100), make_hash()) + assert cacher.get(make_key(path="/a/c.txt", size=100)) is None + + def test_false_returns_latest_mtime_entry(self, tmp_path): + """match_mtime=False: multiple entries for same path+size → returns highest mtime_ns.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + hash_old = make_hash(b"\xaa" * 32) + hash_new = make_hash(b"\xbb" * 32) + cacher.put(make_key(mtime_ns=1000, size=100), hash_old) + cacher.put(make_key(mtime_ns=2000, size=100), hash_new) + result = cacher.get(make_key(mtime_ns=3000, size=100)) + assert result is not None + assert result.digest == hash_new.digest + + def test_false_no_entries_returns_none(self, tmp_path): + """match_mtime=False: no matching path+size → None.""" + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + assert cacher.get(make_key()) is None + + def test_repr_shows_match_mtime_true(self, tmp_path): + cacher = SqliteHashCacher(tmp_path / "cache.db") + assert "match_mtime=True" in repr(cacher) + + def test_repr_shows_match_mtime_false(self, tmp_path): + cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) + assert "match_mtime=False" in repr(cacher) From e2108fe951343008a040936dac263a9088eb1827 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:01:32 +0000 Subject: [PATCH 06/12] feat(hashing): add match_mtime flag to PostgresHashCacher (ITL-522) --- src/orcapod/hashing/postgres_hash_cacher.py | 33 ++++++++-- .../test_hashing/test_postgres_hash_cacher.py | 60 +++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/src/orcapod/hashing/postgres_hash_cacher.py b/src/orcapod/hashing/postgres_hash_cacher.py index 46fd2220..0a470066 100644 --- a/src/orcapod/hashing/postgres_hash_cacher.py +++ b/src/orcapod/hashing/postgres_hash_cacher.py @@ -77,6 +77,12 @@ class PostgresHashCacher: ``key.size`` is strictly below this threshold are not inserted. ``None`` and ``0`` disable the threshold (default behaviour). Negative values raise ``ValueError``. Defaults to ``None``. + match_mtime: When ``True`` (default), cache hits require + ``path``, ``mtime_ns``, and ``size`` to all match. When + ``False``, only ``path`` and ``size`` are compared; among + multiple matching entries the one with the highest ``mtime_ns`` + is returned. The write path is unaffected — ``mtime_ns`` is + always stored. Raises: ValueError: If ``min_cache_size_bytes`` is negative, or if the @@ -90,6 +96,7 @@ def __init__( *, read_only: bool = False, min_cache_size_bytes: int | None = None, + match_mtime: bool = True, ) -> None: if min_cache_size_bytes is not None and min_cache_size_bytes < 0: raise ValueError( @@ -99,6 +106,7 @@ def __init__( self._conninfo = conninfo self._read_only = read_only self._min_cache_size_bytes = min_cache_size_bytes + self._match_mtime = match_mtime self._local = threading.local() self._ensure_schema() @@ -185,6 +193,11 @@ def _connection(self) -> "psycopg.Connection[tuple[object, ...]]": def get(self, key: FileHashKey) -> ContentHash | None: """Return the cached ``ContentHash`` for ``key``, or ``None`` on miss. + When ``match_mtime=True`` (default), the query filters on + ``path``, ``mtime_ns``, and ``size``. When ``match_mtime=False``, + only ``path`` and ``size`` are filtered and results are ordered + by ``mtime_ns DESC`` so the most recent entry is returned. + Args: key: File hash cache key. @@ -192,11 +205,18 @@ def get(self, key: FileHashKey) -> ContentHash | None: Cached ``ContentHash``, or ``None`` if not found. """ conn = self._connection() - row = conn.execute( - "SELECT hash FROM file_hash_cache " - "WHERE path=%s AND mtime_ns=%s AND size=%s", - (str(key.path), key.mtime_ns, key.size), - ).fetchone() + if self._match_mtime: + row = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=%s AND mtime_ns=%s AND size=%s", + (str(key.path), key.mtime_ns, key.size), + ).fetchone() + else: + row = conn.execute( + "SELECT hash FROM file_hash_cache " + "WHERE path=%s AND size=%s ORDER BY mtime_ns DESC LIMIT 1", + (str(key.path), key.size), + ).fetchone() if row is None: return None blob: bytes = bytes(row[0]) @@ -259,5 +279,6 @@ def __repr__(self) -> str: f"PostgresHashCacher(" f"conninfo={_redact_conninfo(self._conninfo)!r}, " f"read_only={self._read_only!r}, " - f"min_cache_size_bytes={self._min_cache_size_bytes!r})" + f"min_cache_size_bytes={self._min_cache_size_bytes!r}, " + f"match_mtime={self._match_mtime!r})" ) diff --git a/tests/test_hashing/test_postgres_hash_cacher.py b/tests/test_hashing/test_postgres_hash_cacher.py index 53256e95..e40ec32a 100644 --- a/tests/test_hashing/test_postgres_hash_cacher.py +++ b/tests/test_hashing/test_postgres_hash_cacher.py @@ -435,6 +435,66 @@ def test_old_schema_raises_with_migration_hint(self, pg_conninfo): conn.execute("DROP TABLE IF EXISTS file_hash_cache CASCADE") +# --------------------------------------------------------------------------- +# match_mtime flag +# --------------------------------------------------------------------------- + + +@pytest.mark.postgres +class TestPostgresHashCacherMatchMtime: + def test_default_true_mtime_change_causes_miss(self, pg_conninfo): + """Default (match_mtime=True): different mtime → cache miss.""" + cacher = PostgresHashCacher(pg_conninfo) + cacher.clear() + cacher.put(make_key(mtime_ns=1000, size=100), make_hash(b"\xaa" * 32)) + assert cacher.get(make_key(mtime_ns=2000, size=100)) is None + cacher.close() + + def test_false_mtime_change_is_hit(self, pg_conninfo): + """match_mtime=False: different mtime, same path+size → cache hit.""" + cacher = PostgresHashCacher(pg_conninfo, match_mtime=False) + cacher.clear() + value = make_hash(b"\xbb" * 32) + cacher.put(make_key(mtime_ns=1000, size=100), value) + result = cacher.get(make_key(mtime_ns=2000, size=100)) + assert result is not None + assert result.digest == value.digest + cacher.close() + + def test_false_size_change_is_miss(self, pg_conninfo): + """match_mtime=False: different size → cache miss.""" + cacher = PostgresHashCacher(pg_conninfo, match_mtime=False) + cacher.clear() + cacher.put(make_key(mtime_ns=1000, size=100), make_hash()) + assert cacher.get(make_key(mtime_ns=2000, size=200)) is None + cacher.close() + + def test_false_returns_latest_mtime_entry(self, pg_conninfo): + """match_mtime=False: multiple entries for same path+size → returns highest mtime_ns.""" + cacher = PostgresHashCacher(pg_conninfo, match_mtime=False) + cacher.clear() + hash_old = make_hash(b"\xaa" * 32) + hash_new = make_hash(b"\xbb" * 32) + cacher.put(make_key(mtime_ns=1000, size=100), hash_old) + cacher.put(make_key(mtime_ns=2000, size=100), hash_new) + result = cacher.get(make_key(mtime_ns=3000, size=100)) + assert result is not None + assert result.digest == hash_new.digest + cacher.close() + + def test_repr_shows_match_mtime_true(self, pg_conninfo): + """__repr__ includes match_mtime=True by default.""" + cacher = PostgresHashCacher(pg_conninfo) + assert "match_mtime=True" in repr(cacher) + cacher.close() + + def test_repr_shows_match_mtime_false(self, pg_conninfo): + """__repr__ includes match_mtime=False.""" + cacher = PostgresHashCacher(pg_conninfo, match_mtime=False) + assert "match_mtime=False" in repr(cacher) + cacher.close() + + class TestPostgresReprRedaction: def test_repr_redacts_url_password(self): """__repr__ replaces the password in a URL-form conninfo with ***.""" From e57bc6e308108e4d55cc190e6b69fcd77b683a1d Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:11:07 +0000 Subject: [PATCH 07/12] feat(hashing): add match_mtime to enable_file_hash_caching; integration tests (ITL-522) Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/contexts/__init__.py | 11 +++ .../test_hash_cacher_match_mtime.py | 87 ++++++++++++++++++- tests/test_hashing/test_hash_cachers.py | 17 ++++ 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/src/orcapod/contexts/__init__.py b/src/orcapod/contexts/__init__.py index 69d58640..fdd0e462 100644 --- a/src/orcapod/contexts/__init__.py +++ b/src/orcapod/contexts/__init__.py @@ -234,6 +234,7 @@ def enable_file_hash_caching( conninfo: str | None = None, read_only: bool = False, min_cache_size_bytes: int | None = None, + match_mtime: bool = True, ) -> None: """Enable file hash caching on the default Orcapod context. @@ -277,6 +278,14 @@ def enable_file_hash_caching( min_cache_size_bytes: When set, files smaller than this byte count are not inserted into the cache. ``None`` and ``0`` disable the threshold. Defaults to ``None``. + match_mtime: When ``True`` (default), cache lookups require an exact + match on both ``mtime_ns`` and ``size``. When ``False``, only + ``size`` is used for matching — an mtime change alone will not + cause a cache miss. Useful for environments where file timestamps + are unreliable (e.g. network filesystems or build tools that + preserve content but reset mtimes). Note that with + ``match_mtime=False``, a file whose content changes while its + size stays the same will produce a stale cache hit. Raises: ValueError: If both ``conninfo`` and ``db_path`` are provided. @@ -327,6 +336,7 @@ def enable_file_hash_caching( conninfo, read_only=read_only, min_cache_size_bytes=min_cache_size_bytes, + match_mtime=match_mtime, ) else: # SqliteHashCacher import was previously at the top of the function body; @@ -337,6 +347,7 @@ def enable_file_hash_caching( db_path, read_only=read_only, min_cache_size_bytes=min_cache_size_bytes, + match_mtime=match_mtime, ) cached_file_hasher = CachedFileHasher( diff --git a/tests/test_hashing/test_hash_cacher_match_mtime.py b/tests/test_hashing/test_hash_cacher_match_mtime.py index 236a19d5..8e005c2a 100644 --- a/tests/test_hashing/test_hash_cacher_match_mtime.py +++ b/tests/test_hashing/test_hash_cacher_match_mtime.py @@ -5,9 +5,11 @@ """ from __future__ import annotations +import os + from upath import UPath -from orcapod.hashing.file_hashers import FileHashKey +from orcapod.hashing.file_hashers import CachedFileHasher, FileHashKey, FileHasher from orcapod.hashing.hash_cachers import InMemoryHashCacher, SqliteHashCacher from orcapod.types import ContentHash @@ -136,3 +138,86 @@ def test_repr_shows_match_mtime_true(self, tmp_path): def test_repr_shows_match_mtime_false(self, tmp_path): cacher = SqliteHashCacher(tmp_path / "cache.db", match_mtime=False) assert "match_mtime=False" in repr(cacher) + + +# --------------------------------------------------------------------------- +# CachedFileHasher integration — match_mtime with real files +# --------------------------------------------------------------------------- + + +class TestCachedFileHasherMatchMtime: + def test_t1_default_mtime_change_causes_miss(self, tmp_path): + """T1: Default (match_mtime=True), mtime changed, size unchanged → cache miss.""" + f = tmp_path / "file.bin" + f.write_bytes(b"x" * 50) + + cacher = InMemoryHashCacher(match_mtime=True) + cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) + cached.hash_file(f) + + stat = f.stat() + os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + path = UPath(f).resolve() + new_stat = path.stat() + new_key = FileHashKey(path, new_stat.st_mtime_ns, new_stat.st_size) + assert cacher.get(new_key) is None + + def test_t2_match_mtime_false_mtime_change_is_hit(self, tmp_path): + """T2: match_mtime=False, mtime changed, size unchanged → cache hit.""" + f = tmp_path / "file.bin" + f.write_bytes(b"x" * 50) + + cacher = InMemoryHashCacher(match_mtime=False) + cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) + first_hash = cached.hash_file(f) + + stat = f.stat() + os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + second_hash = cached.hash_file(f) + assert second_hash == first_hash + + def test_t3_match_mtime_false_size_change_is_miss(self, tmp_path): + """T3: match_mtime=False, content and size changed → cache miss.""" + f = tmp_path / "file.bin" + f.write_bytes(b"hello") # 5 bytes + + cacher = InMemoryHashCacher(match_mtime=False) + cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) + first_hash = cached.hash_file(f) + + f.write_bytes(b"hello world") # 11 bytes — different size + + second_hash = cached.hash_file(f) + assert second_hash != first_hash + + def test_t4_match_mtime_false_same_size_content_flip_is_hit(self, tmp_path): + """T4: match_mtime=False, content changed but size preserved → cache hit (known trade-off).""" + f = tmp_path / "file.bin" + f.write_bytes(b"aaaaa") # 5 bytes + + cacher = InMemoryHashCacher(match_mtime=False) + cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) + first_hash = cached.hash_file(f) + + f.write_bytes(b"bbbbb") # 5 bytes — same size, different content + + second_hash = cached.hash_file(f) + assert second_hash == first_hash # stale hit — known trade-off + + def test_t5_unchanged_file_both_modes_agree(self, tmp_path): + """T5: Unchanged file — match_mtime=True and match_mtime=False return the same hash.""" + f = tmp_path / "file.bin" + f.write_bytes(b"stable content") + + inner = FileHasher() + hash_strict = CachedFileHasher( + file_hasher=inner, + cacher=InMemoryHashCacher(match_mtime=True), + ).hash_file(f) + hash_relaxed = CachedFileHasher( + file_hasher=inner, + cacher=InMemoryHashCacher(match_mtime=False), + ).hash_file(f) + assert hash_strict == hash_relaxed diff --git a/tests/test_hashing/test_hash_cachers.py b/tests/test_hashing/test_hash_cachers.py index 2742d5f5..42ce7ba1 100644 --- a/tests/test_hashing/test_hash_cachers.py +++ b/tests/test_hashing/test_hash_cachers.py @@ -306,6 +306,23 @@ def test_min_cache_size_bytes_kwarg_passes_through_to_cacher( assert isinstance(cacher, SqliteHashCacher) assert cacher._min_cache_size_bytes == 1024 + def test_match_mtime_kwarg_passes_through_to_cacher( + self, restore_default_file_handler, tmp_path + ): + from orcapod.contexts import enable_file_hash_caching, get_default_context + from orcapod.extension_types.file_type import File + from orcapod.hashing.hash_cachers import SqliteHashCacher + + enable_file_hash_caching(db_path=tmp_path / "cache.db", match_mtime=False) + + context = get_default_context() + registry = context.semantic_hasher.type_handler_registry + handler = registry.get_handler_for_type(File) + cacher = handler.file_hasher.cacher + + assert isinstance(cacher, SqliteHashCacher) + assert cacher._match_mtime is False + class TestEnableFileHashCachingConninfo: def test_conninfo_and_db_path_raises(self, restore_default_file_handler, tmp_path): From 634daafca6fdc4b0c4547c432dda08212fc58804 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:15:35 +0000 Subject: [PATCH 08/12] refactor(hashing): improve T1 integration test; remove stale comment (ITL-522) --- src/orcapod/contexts/__init__.py | 2 -- tests/test_hashing/test_hash_cacher_match_mtime.py | 12 ++++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/orcapod/contexts/__init__.py b/src/orcapod/contexts/__init__.py index fdd0e462..650d99fe 100644 --- a/src/orcapod/contexts/__init__.py +++ b/src/orcapod/contexts/__init__.py @@ -339,8 +339,6 @@ def enable_file_hash_caching( match_mtime=match_mtime, ) else: - # SqliteHashCacher import was previously at the top of the function body; - # it now lives inside this branch only. from orcapod.hashing.hash_cachers import SqliteHashCacher cacher = SqliteHashCacher( diff --git a/tests/test_hashing/test_hash_cacher_match_mtime.py b/tests/test_hashing/test_hash_cacher_match_mtime.py index 8e005c2a..5b275318 100644 --- a/tests/test_hashing/test_hash_cacher_match_mtime.py +++ b/tests/test_hashing/test_hash_cacher_match_mtime.py @@ -147,21 +147,21 @@ def test_repr_shows_match_mtime_false(self, tmp_path): class TestCachedFileHasherMatchMtime: def test_t1_default_mtime_change_causes_miss(self, tmp_path): - """T1: Default (match_mtime=True), mtime changed, size unchanged → cache miss.""" + """T1: Default (match_mtime=True), mtime changed, size unchanged → cache miss → re-hash.""" f = tmp_path / "file.bin" f.write_bytes(b"x" * 50) cacher = InMemoryHashCacher(match_mtime=True) cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) - cached.hash_file(f) + first_hash = cached.hash_file(f) stat = f.stat() os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) - path = UPath(f).resolve() - new_stat = path.stat() - new_key = FileHashKey(path, new_stat.st_mtime_ns, new_stat.st_size) - assert cacher.get(new_key) is None + # match_mtime=True: mtime change causes a cache miss; CachedFileHasher re-hashes + # the file. Content is unchanged so the returned hash must still be correct. + second_hash = cached.hash_file(f) + assert second_hash == first_hash def test_t2_match_mtime_false_mtime_change_is_hit(self, tmp_path): """T2: match_mtime=False, mtime changed, size unchanged → cache hit.""" From 61ad91590f65d154b194f207a2c5da0b5ee1d6be Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:18:14 +0000 Subject: [PATCH 09/12] docs(hashing): document match_mtime flag in file-hash-caching guide (ITL-522) --- docs/concepts/file-hash-caching.md | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/concepts/file-hash-caching.md b/docs/concepts/file-hash-caching.md index 32e53439..1710f812 100644 --- a/docs/concepts/file-hash-caching.md +++ b/docs/concepts/file-hash-caching.md @@ -109,6 +109,55 @@ op.enable_file_hash_caching( ) ``` +## Ignoring mtime in cache lookups + +By default, a cache hit requires **path, mtime_ns, and size** to all match the stored +entry. In several common deployment scenarios, mtime is unreliable — it changes even +when file content has not — causing spurious cache misses and unnecessary re-hashing: + +- **rsync / file transfer tools** — rsync and similar tools do not preserve mtime by + default. Even with `--times`, sub-second precision is often lost on destination + filesystems, producing a different `mtime_ns` for otherwise identical files. +- **Restore from backup** — backup and restore pipelines (tar, restic, Borg, cloud + storage sync) frequently reset mtime to the restore timestamp rather than the + original file timestamp. +- **Container bind mounts and volume remounts** — remounting a volume or restarting a + container can reset or truncate mtime precision depending on the host filesystem and + container runtime (Docker, Podman, Kubernetes). +- **CI `touch` / build system side-effects** — build scripts, test harnesses, and CI + pipelines sometimes call `touch` on input files to force rebuilds, or copy files in + ways that update mtime without changing content. +- **Network filesystems (NFS, CIFS/SMB)** — clock skew between the client and server, + or coarse mtime granularity on older NFS versions, can produce stale or shifted + timestamps that differ from the values recorded in the cache. + +Set `match_mtime=False` to drop mtime from the lookup criterion. A cache hit then +requires only **path and size** to match: + +```python +import orcapod as op + +op.enable_file_hash_caching(match_mtime=False) +``` + +When multiple cache entries share the same path and size (recorded at different +mtimes), Orcapod returns the hash from the entry with the most recent `mtime_ns`. + +**The write path is unchanged.** mtime is always recorded when a new entry is +inserted. Switching `match_mtime=False` on a cache that was already populated under +the default `match_mtime=True` still produces hits — no cache rebuild is needed. + +### Known trade-off + +With `match_mtime=False`, a file modification that preserves the file's byte count +will **not** be detected by the cache. The stored hash from the previous version will +be returned silently. This is rare in practice (most writes change file size), but +you should be aware of the trade-off before enabling this mode. + +Use `match_mtime=False` only in environments where mtime changes are known to be +unreliable. For most local-disk or NFS deployments the default (`match_mtime=True`) +is the right choice. + ## Directory hashing (op.Directory) When Orcapod hashes an `op.Directory`, it traverses the directory tree and From 707754cdfa2f537db85b965fc404e27614fcbfc6 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:23:16 +0000 Subject: [PATCH 10/12] docs(hashing): fix match_mtime docstring to include path in both clauses (ITL-522) --- src/orcapod/contexts/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/orcapod/contexts/__init__.py b/src/orcapod/contexts/__init__.py index 650d99fe..6f256468 100644 --- a/src/orcapod/contexts/__init__.py +++ b/src/orcapod/contexts/__init__.py @@ -279,9 +279,10 @@ def enable_file_hash_caching( are not inserted into the cache. ``None`` and ``0`` disable the threshold. Defaults to ``None``. match_mtime: When ``True`` (default), cache lookups require an exact - match on both ``mtime_ns`` and ``size``. When ``False``, only - ``size`` is used for matching — an mtime change alone will not - cause a cache miss. Useful for environments where file timestamps + match on ``path``, ``mtime_ns``, and ``size``. When ``False``, + only ``path`` and ``size`` are used for matching — an mtime + change alone will not cause a cache miss. Useful for environments + where file timestamps are unreliable (e.g. network filesystems or build tools that preserve content but reset mtimes). Note that with ``match_mtime=False``, a file whose content changes while its From 5340fabca12659cd04eb1579b9c0f1089953f86f Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:37:38 +0000 Subject: [PATCH 11/12] fix(hashing): add spy assertions to T1/T2; add path+size+mtime index (ITL-522) --- src/orcapod/hashing/hash_cachers.py | 10 +++++ src/orcapod/hashing/postgres_hash_cacher.py | 9 ++++ .../test_hash_cacher_match_mtime.py | 42 ++++++++++++------- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/orcapod/hashing/hash_cachers.py b/src/orcapod/hashing/hash_cachers.py index c8544c5b..dd3bb1d7 100644 --- a/src/orcapod/hashing/hash_cachers.py +++ b/src/orcapod/hashing/hash_cachers.py @@ -205,6 +205,16 @@ def _ensure_schema(self) -> None: """ ) + # Supporting index for match_mtime=False lookups: WHERE path=? AND size=? + # ORDER BY mtime_ns DESC. The PK (path, mtime_ns, size) cannot serve this + # efficiently because size is not a leftmost prefix. + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_file_hash_cache_path_size_mtime + ON file_hash_cache (path, size, mtime_ns DESC) + """ + ) + version = conn.execute("PRAGMA user_version").fetchone()[0] # Always validate the cached_at column is present, regardless of diff --git a/src/orcapod/hashing/postgres_hash_cacher.py b/src/orcapod/hashing/postgres_hash_cacher.py index 0a470066..7ba586f0 100644 --- a/src/orcapod/hashing/postgres_hash_cacher.py +++ b/src/orcapod/hashing/postgres_hash_cacher.py @@ -136,6 +136,15 @@ def _ensure_schema(self) -> None: ) """ ) + # Supporting index for match_mtime=False lookups: WHERE path=%s AND size=%s + # ORDER BY mtime_ns DESC. The PK (path, mtime_ns, size) cannot serve this + # efficiently because size is not a leftmost prefix. + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_file_hash_cache_path_size_mtime + ON file_hash_cache (path, size, mtime_ns DESC) + """ + ) # Schema-version metadata table (idempotent). conn.execute( """ diff --git a/tests/test_hashing/test_hash_cacher_match_mtime.py b/tests/test_hashing/test_hash_cacher_match_mtime.py index 5b275318..8834c0c8 100644 --- a/tests/test_hashing/test_hash_cacher_match_mtime.py +++ b/tests/test_hashing/test_hash_cacher_match_mtime.py @@ -6,6 +6,7 @@ from __future__ import annotations import os +from unittest.mock import patch from upath import UPath @@ -151,32 +152,45 @@ def test_t1_default_mtime_change_causes_miss(self, tmp_path): f = tmp_path / "file.bin" f.write_bytes(b"x" * 50) + inner = FileHasher() cacher = InMemoryHashCacher(match_mtime=True) - cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) - first_hash = cached.hash_file(f) + cached = CachedFileHasher(file_hasher=inner, cacher=cacher) - stat = f.stat() - os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + with patch.object(inner, "hash_file", wraps=inner.hash_file) as spy: + first_hash = cached.hash_file(f) + assert spy.call_count == 1 - # match_mtime=True: mtime change causes a cache miss; CachedFileHasher re-hashes - # the file. Content is unchanged so the returned hash must still be correct. - second_hash = cached.hash_file(f) - assert second_hash == first_hash + stat = f.stat() + os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + # match_mtime=True: mtime change causes a cache miss; CachedFileHasher + # re-hashes the file. Content is unchanged so the hash value is equal, + # but spy.call_count == 2 proves the inner hasher was called again. + second_hash = cached.hash_file(f) + assert second_hash == first_hash + assert spy.call_count == 2 def test_t2_match_mtime_false_mtime_change_is_hit(self, tmp_path): """T2: match_mtime=False, mtime changed, size unchanged → cache hit.""" f = tmp_path / "file.bin" f.write_bytes(b"x" * 50) + inner = FileHasher() cacher = InMemoryHashCacher(match_mtime=False) - cached = CachedFileHasher(file_hasher=FileHasher(), cacher=cacher) - first_hash = cached.hash_file(f) + cached = CachedFileHasher(file_hasher=inner, cacher=cacher) - stat = f.stat() - os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + with patch.object(inner, "hash_file", wraps=inner.hash_file) as spy: + first_hash = cached.hash_file(f) + assert spy.call_count == 1 - second_hash = cached.hash_file(f) - assert second_hash == first_hash + stat = f.stat() + os.utime(f, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + # match_mtime=False: mtime change is ignored; CachedFileHasher returns the + # cached hash without calling the inner hasher again. + second_hash = cached.hash_file(f) + assert second_hash == first_hash + assert spy.call_count == 1 # inner hasher NOT called again — confirms the cache hit def test_t3_match_mtime_false_size_change_is_miss(self, tmp_path): """T3: match_mtime=False, content and size changed → cache miss.""" From 3f2f798185be4b19edc61e7852e782e63bc14550 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:42:09 +0000 Subject: [PATCH 12/12] refactor(hashing): use nested dict for InMemoryHashCacher; O(1) get() (ITL-522) --- src/orcapod/hashing/hash_cachers.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/orcapod/hashing/hash_cachers.py b/src/orcapod/hashing/hash_cachers.py index dd3bb1d7..ea9b15ef 100644 --- a/src/orcapod/hashing/hash_cachers.py +++ b/src/orcapod/hashing/hash_cachers.py @@ -52,7 +52,10 @@ def __init__( f"min_cache_size_bytes must be None or a non-negative integer, " f"got {min_cache_size_bytes!r}" ) - self._cache: dict[FileHashKey, ContentHash] = {} + # Nested dict: (path, size) → {mtime_ns: ContentHash}. + # Separating the mtime_ns level from (path, size) makes both lookup + # modes O(1)/O(k) instead of O(n) over all cached entries. + self._cache: dict[tuple, dict[int, ContentHash]] = {} self._read_only = read_only self._min_cache_size_bytes = min_cache_size_bytes self._match_mtime = match_mtime @@ -71,16 +74,16 @@ def get(self, key: FileHashKey) -> ContentHash | None: Returns: Cached ``ContentHash``, or ``None`` if not found. """ + inner = self._cache.get((key.path, key.size)) + if inner is None: + return None if self._match_mtime: - return self._cache.get(key) - best_key: FileHashKey | None = None - best_value: ContentHash | None = None - for cached_key, value in self._cache.items(): - if cached_key.path == key.path and cached_key.size == key.size: - if best_key is None or cached_key.mtime_ns > best_key.mtime_ns: - best_key = cached_key - best_value = value - return best_value + return inner.get(key.mtime_ns) + # match_mtime=False: return the hash from the entry with the + # highest mtime_ns. max() over the inner dict's keys is O(k) + # where k is the number of distinct mtimes for this (path, size) + # pair — almost always 1 in practice. + return inner[max(inner)] def put(self, key: FileHashKey, value: ContentHash) -> None: """Store ``value`` under ``key``. @@ -96,7 +99,7 @@ def put(self, key: FileHashKey, value: ContentHash) -> None: return if self._min_cache_size_bytes is not None and self._min_cache_size_bytes > 0 and key.size < self._min_cache_size_bytes: return - self._cache[key] = value + self._cache.setdefault((key.path, key.size), {})[key.mtime_ns] = value def clear(self) -> None: """Remove all entries from the cache."""