Fix stale media refs after in-place updates

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Fixed
+
+- Re-upload local media when its contents change at the same path, keeping
+  content-derived job names aligned with the media sent to annotators.
+
 ## [0.1.1] - 2026-08-17
 
 ### Fixed

src/humanevals/client.py

@@ -8,6 +8,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import math
 import os
 import time
@@ -111,8 +112,9 @@ def __init__(
             timeout=httpx.Timeout(timeout, read=timeout * 4),
             transport=transport,
         )
-        # Local paths already uploaded this session, keyed by resolved path.
-        self._media_refs: dict[str, str] = {}
+        # Local file versions already uploaded this session. Including the
+        # content hash prevents an in-place overwrite from reusing a stale ref.
+        self._media_refs: dict[tuple[str, str, str], str] = {}
 
     # -- lifecycle -----------------------------------------------------------
 
@@ -202,20 +204,24 @@ def upload_media(self, paths: list[str | Path]) -> list[dict[str, Any]]:
         media: list[dict[str, Any]] = body["media"]
         return media
 
-    def resolve_media(self, media: Media) -> dict[str, str]:
+    def resolve_media(self, media: Media, *, content_sha256: str | None = None) -> dict[str, str]:
         """Turn a :class:`Media` into the ``{"url", "type"}`` item jobs expect.
 
         Remote sources (``https://`` / ``dp://``) pass through unchanged.
-        Local files are uploaded once per client (cached by resolved path)
-        and replaced with their ``dp://`` ref.
+        Each version of a local file is uploaded once per client (cached by
+        resolved path, media type, and content hash) and replaced with its
+        ``dp://`` ref. Scorers pass their precomputed ``content_sha256`` so
+        naming and submission use the same file fingerprint.
         """
         media_type = media.resolved_type()
         if media.is_remote:
             return {"url": str(media.source), "type": media_type}
         path = Path(media.source).expanduser().resolve()
         if not path.is_file():
             raise FileNotFoundError(f"Media file not found: {path}")
-        cache_key = str(path)
+        if content_sha256 is None:
+            content_sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
+        cache_key = (str(path), media_type, content_sha256)
         if cache_key not in self._media_refs:
             uploaded = self.upload_media([path])[0]
             self._media_refs[cache_key] = uploaded["media_ref"]

src/humanevals/scorers.py

@@ -53,6 +53,7 @@
 
 # Shared lazily-created client for scorers constructed without one.
 _default_client: Client | None = None
+_MediaHashes = dict[tuple[str, str], str]
 
 
 def _get_default_client() -> Client:
@@ -167,11 +168,12 @@ def submit(
             if estimate > max_credits:
                 raise BudgetExceededError(estimate, max_credits)
 
-        job_name = name or self._content_name(prepared)
+        media_hashes: _MediaHashes = {}
+        job_name = name or self._content_name(prepared, media_hashes=media_hashes)
         if fresh:
             job_name = f"{job_name}-{uuid.uuid4().hex[:8]}"
 
-        body = self._job_body(prepared)
+        body = self._job_body(prepared, media_hashes=media_hashes)
         body["name"] = job_name
         created = self.client.create_job(body)
         # pricing is null exactly when the API replayed an existing job for
@@ -243,7 +245,9 @@ def _coerce_item(self, item: Any) -> Any:
     def _validate_batch(self, items: list[Any]) -> None:
         """Reject invalid or mixed-mode batches before any network call."""
 
-    def _datapoint(self, item: Any, *, for_naming: bool) -> dict[str, Any]:
+    def _datapoint(
+        self, item: Any, *, for_naming: bool, media_hashes: _MediaHashes
+    ) -> dict[str, Any]:
         """Build one API datapoint.
 
         With ``for_naming=True``, local files are represented by a stable
@@ -267,12 +271,17 @@ def _score_row(self, row: dict[str, Any], item: Any) -> Score:
 
     # -- shared internals ----------------------------------------------------
 
-    def _job_body(self, items: list[Any]) -> dict[str, Any]:
+    def _job_body(
+        self, items: list[Any], *, media_hashes: _MediaHashes | None = None
+    ) -> dict[str, Any]:
+        media_hashes = media_hashes if media_hashes is not None else {}
         body: dict[str, Any] = {
             "instruction": self.instruction,
             "task_type": self._task_type(items),
             "max_responses_per_datapoint": self.responses_per_item,
-            "datapoints": [self._datapoint(i, for_naming=False) for i in items],
+            "datapoints": [
+                self._datapoint(i, for_naming=False, media_hashes=media_hashes) for i in items
+            ],
         }
         options = self._response_options(items)
         if options is not None:
@@ -283,16 +292,19 @@ def _job_body(self, items: list[Any]) -> dict[str, Any]:
             body["serving_environment"] = "sandbox"
         return body
 
-    def _content_name(self, items: list[Any]) -> str:
+    def _content_name(self, items: list[Any], *, media_hashes: _MediaHashes | None = None) -> str:
         """Deterministic job name from the full request content."""
+        media_hashes = media_hashes if media_hashes is not None else {}
         naming_body: dict[str, Any] = {
             "instruction": self.instruction,
             "task_type": self._task_type(items),
             "max_responses_per_datapoint": self.responses_per_item,
             "response_options": self._response_options(items),
             "annotator_filter": self.annotator_filter,
             "sandbox": self.sandbox,
-            "datapoints": [self._datapoint(i, for_naming=True) for i in items],
+            "datapoints": [
+                self._datapoint(i, for_naming=True, media_hashes=media_hashes) for i in items
+            ],
         }
         canonical = json.dumps(naming_body, sort_keys=True, separators=(",", ":"))
         digest = hashlib.sha256(f"humanevals:1:{canonical}".encode()).hexdigest()
@@ -310,15 +322,22 @@ def _check_context_placeholder(self, items: list[Any]) -> None:
                 stacklevel=_user_stacklevel(),
             )
 
-    def _media_item(self, media: Media, *, for_naming: bool) -> dict[str, str]:
+    def _media_item(
+        self, media: Media, *, for_naming: bool, media_hashes: _MediaHashes
+    ) -> dict[str, str]:
         """Resolve a :class:`Media` into an API media item (or naming token)."""
-        if for_naming:
-            if media.is_remote:
-                return {"url": str(media.source), "type": media.resolved_type()}
-            path = Path(media.source).expanduser().resolve()
+        media_type = media.resolved_type()
+        if media.is_remote:
+            return {"url": str(media.source), "type": media_type}
+        path = Path(media.source).expanduser().resolve()
+        cache_key = (str(path), media_type)
+        digest = media_hashes.get(cache_key)
+        if digest is None:
             digest = hashlib.sha256(path.read_bytes()).hexdigest()
+            media_hashes[cache_key] = digest
+        if for_naming:
             return {"content_sha256": digest, "type": media.resolved_type()}
-        return self.client.resolve_media(media)
+        return self.client.resolve_media(media, content_sha256=digest)
 
     def _scores_from_results(
         self,
@@ -466,7 +485,9 @@ def _task_type(self, items: list[Pair]) -> str:
         # media-only); with 2 candidates the math maps back exactly.
         return {"text": "ranking", "comparison": "comparison", "i2v": "i2v_comparison"}[mode]
 
-    def _datapoint(self, item: Pair, *, for_naming: bool) -> dict[str, Any]:
+    def _datapoint(
+        self, item: Pair, *, for_naming: bool, media_hashes: _MediaHashes
+    ) -> dict[str, Any]:
         mode = self._mode(item)
         if mode == "text":
             media: dict[str, Any] = {
@@ -479,12 +500,16 @@ def _datapoint(self, item: Pair, *, for_naming: bool) -> dict[str, Any]:
             assert isinstance(item.a, Media) and isinstance(item.b, Media)
             media = {
                 "candidates": [
-                    self._media_item(item.a, for_naming=for_naming),
-                    self._media_item(item.b, for_naming=for_naming),
+                    self._media_item(item.a, for_naming=for_naming, media_hashes=media_hashes),
+                    self._media_item(item.b, for_naming=for_naming, media_hashes=media_hashes),
                 ]
             }
             if item.reference is not None:
-                media["reference"] = [self._media_item(item.reference, for_naming=for_naming)]
+                media["reference"] = [
+                    self._media_item(
+                        item.reference, for_naming=for_naming, media_hashes=media_hashes
+                    )
+                ]
         datapoint: dict[str, Any] = {"media": media}
         if item.context is not None:
             datapoint["context"] = item.context
@@ -619,12 +644,18 @@ def _response_options(self, items: list[RatingItem]) -> dict[str, Any]:
             options["labels"] = self.labels
         return options
 
-    def _datapoint(self, item: RatingItem, *, for_naming: bool) -> dict[str, Any]:
+    def _datapoint(
+        self, item: RatingItem, *, for_naming: bool, media_hashes: _MediaHashes
+    ) -> dict[str, Any]:
         if isinstance(item.subject, str):
             # Text-only rating: the API requires an explicit empty media dict.
             return {"media": {}, "context": item.subject}
         datapoint: dict[str, Any] = {
-            "media": {"subject": [self._media_item(item.subject, for_naming=for_naming)]}
+            "media": {
+                "subject": [
+                    self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes)
+                ]
+            }
         }
         if item.context is not None:
             datapoint["context"] = item.context
@@ -721,14 +752,18 @@ def _response_options(self, items: list[ChoiceItem]) -> dict[str, Any]:
     def _option_id(index: int) -> str:
         return f"option_{index + 1}"
 
-    def _datapoint(self, item: ChoiceItem, *, for_naming: bool) -> dict[str, Any]:
+    def _datapoint(
+        self, item: ChoiceItem, *, for_naming: bool, media_hashes: _MediaHashes
+    ) -> dict[str, Any]:
         media: dict[str, Any] = {
             "options": [
                 {"id": self._option_id(i), "text": text} for i, text in enumerate(item.options)
             ]
         }
         if item.subject is not None:
-            media["subject"] = [self._media_item(item.subject, for_naming=for_naming)]
+            media["subject"] = [
+                self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes)
+            ]
         return {"media": media, "context": item.question}
 
     def _score_row(self, row: dict[str, Any], item: ChoiceItem | None) -> Score:
@@ -814,15 +849,17 @@ def _candidate_ids(self, item: RankingItem) -> list[str]:
         media_type = candidates[0].resolved_type() if isinstance(candidates[0], Media) else ""
         return [f"{media_type}_{i + 1}" for i in range(len(candidates))]
 
-    def _datapoint(self, item: RankingItem, *, for_naming: bool) -> dict[str, Any]:
+    def _datapoint(
+        self, item: RankingItem, *, for_naming: bool, media_hashes: _MediaHashes
+    ) -> dict[str, Any]:
         candidates = list(item.candidates)
         ids = self._candidate_ids(item)
         payload: list[dict[str, str]]
         if isinstance(candidates[0], str):
             payload = [{"id": ids[i], "text": str(c)} for i, c in enumerate(candidates)]
         else:
             payload = [
-                self._media_item(c, for_naming=for_naming)
+                self._media_item(c, for_naming=for_naming, media_hashes=media_hashes)
                 for c in candidates
                 if isinstance(c, Media)
             ]

tests/test_submit.py

@@ -77,11 +77,39 @@ def test_local_file_naming_uses_content_hash(client: he.Client, api: FakeAPI, tm
     assert name1 == name2
 
     # Changing file *content* changes the name even at the same path.
-    client._media_refs.clear()
     video_a.write_bytes(b"different-content")
     assert scorer.submit(pair).name != name1
 
 
+def test_changed_local_file_reuploads_in_same_client(client: he.Client, api: FakeAPI, tmp_path):
+    video = tmp_path / "0.mp4"
+    video.write_bytes(b"old-video")
+    api.add(
+        "POST",
+        "/media",
+        {"media": [{"media_ref": "dp://old/0.mp4", "media_id": "old", "type": "video"}]},
+    )
+    api.add(
+        "POST",
+        "/media",
+        {"media": [{"media_ref": "dp://new/0.mp4", "media_id": "new", "type": "video"}]},
+    )
+    api.add("POST", "/jobs", create_job_response())
+    scorer = he.HumanRating("Rate it.", client=client)
+    items = [he.Media(video)]
+
+    first = scorer.submit(items)
+    video.write_bytes(b"new-video")
+    second = scorer.submit(items)
+
+    assert first.name != second.name
+    assert api.paths() == ["POST /media", "POST /jobs", "POST /media", "POST /jobs"]
+    first_media = api.body(1)["datapoints"][0]["media"]["subject"][0]
+    second_media = api.body(3)["datapoints"][0]["media"]["subject"][0]
+    assert first_media["url"] == "dp://old/0.mp4"
+    assert second_media["url"] == "dp://new/0.mp4"
+
+
 def test_explicit_name_wins(client: he.Client, api: FakeAPI):
     api.add("POST", "/jobs", create_job_response())
     job = scorer_for(client).submit(PAIRS, name="my-eval-run-7")