From 27a3bcecc9de797cb1fa6c029fe1969bbf073ca2 Mon Sep 17 00:00:00 2001 From: Shayanide Date: Mon, 17 Aug 2026 21:41:09 +0000 Subject: [PATCH] Fix stale media refs after in-place updates --- CHANGELOG.md | 7 ++++ src/humanevals/client.py | 18 ++++++--- src/humanevals/scorers.py | 83 ++++++++++++++++++++++++++++----------- tests/test_submit.py | 30 +++++++++++++- 4 files changed, 108 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1994f3d..aec843d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Re-upload local media when its contents change at the same path, keeping + content-derived job names aligned with the media sent to annotators. + ## [0.1.1] - 2026-08-17 ### Fixed diff --git a/src/humanevals/client.py b/src/humanevals/client.py index 567ce93..6002971 100644 --- a/src/humanevals/client.py +++ b/src/humanevals/client.py @@ -8,6 +8,7 @@ from __future__ import annotations +import hashlib import math import os import time @@ -111,8 +112,9 @@ def __init__( timeout=httpx.Timeout(timeout, read=timeout * 4), transport=transport, ) - # Local paths already uploaded this session, keyed by resolved path. - self._media_refs: dict[str, str] = {} + # Local file versions already uploaded this session. Including the + # content hash prevents an in-place overwrite from reusing a stale ref. + self._media_refs: dict[tuple[str, str, str], str] = {} # -- lifecycle ----------------------------------------------------------- @@ -202,12 +204,14 @@ def upload_media(self, paths: list[str | Path]) -> list[dict[str, Any]]: media: list[dict[str, Any]] = body["media"] return media - def resolve_media(self, media: Media) -> dict[str, str]: + def resolve_media(self, media: Media, *, content_sha256: str | None = None) -> dict[str, str]: """Turn a :class:`Media` into the ``{"url", "type"}`` item jobs expect. Remote sources (``https://`` / ``dp://``) pass through unchanged. - Local files are uploaded once per client (cached by resolved path) - and replaced with their ``dp://`` ref. + Each version of a local file is uploaded once per client (cached by + resolved path, media type, and content hash) and replaced with its + ``dp://`` ref. Scorers pass their precomputed ``content_sha256`` so + naming and submission use the same file fingerprint. """ media_type = media.resolved_type() if media.is_remote: @@ -215,7 +219,9 @@ def resolve_media(self, media: Media) -> dict[str, str]: path = Path(media.source).expanduser().resolve() if not path.is_file(): raise FileNotFoundError(f"Media file not found: {path}") - cache_key = str(path) + if content_sha256 is None: + content_sha256 = hashlib.sha256(path.read_bytes()).hexdigest() + cache_key = (str(path), media_type, content_sha256) if cache_key not in self._media_refs: uploaded = self.upload_media([path])[0] self._media_refs[cache_key] = uploaded["media_ref"] diff --git a/src/humanevals/scorers.py b/src/humanevals/scorers.py index 458683e..e58a85d 100644 --- a/src/humanevals/scorers.py +++ b/src/humanevals/scorers.py @@ -53,6 +53,7 @@ # Shared lazily-created client for scorers constructed without one. _default_client: Client | None = None +_MediaHashes = dict[tuple[str, str], str] def _get_default_client() -> Client: @@ -167,11 +168,12 @@ def submit( if estimate > max_credits: raise BudgetExceededError(estimate, max_credits) - job_name = name or self._content_name(prepared) + media_hashes: _MediaHashes = {} + job_name = name or self._content_name(prepared, media_hashes=media_hashes) if fresh: job_name = f"{job_name}-{uuid.uuid4().hex[:8]}" - body = self._job_body(prepared) + body = self._job_body(prepared, media_hashes=media_hashes) body["name"] = job_name created = self.client.create_job(body) # pricing is null exactly when the API replayed an existing job for @@ -243,7 +245,9 @@ def _coerce_item(self, item: Any) -> Any: def _validate_batch(self, items: list[Any]) -> None: """Reject invalid or mixed-mode batches before any network call.""" - def _datapoint(self, item: Any, *, for_naming: bool) -> dict[str, Any]: + def _datapoint( + self, item: Any, *, for_naming: bool, media_hashes: _MediaHashes + ) -> dict[str, Any]: """Build one API datapoint. With ``for_naming=True``, local files are represented by a stable @@ -267,12 +271,17 @@ def _score_row(self, row: dict[str, Any], item: Any) -> Score: # -- shared internals ---------------------------------------------------- - def _job_body(self, items: list[Any]) -> dict[str, Any]: + def _job_body( + self, items: list[Any], *, media_hashes: _MediaHashes | None = None + ) -> dict[str, Any]: + media_hashes = media_hashes if media_hashes is not None else {} body: dict[str, Any] = { "instruction": self.instruction, "task_type": self._task_type(items), "max_responses_per_datapoint": self.responses_per_item, - "datapoints": [self._datapoint(i, for_naming=False) for i in items], + "datapoints": [ + self._datapoint(i, for_naming=False, media_hashes=media_hashes) for i in items + ], } options = self._response_options(items) if options is not None: @@ -283,8 +292,9 @@ def _job_body(self, items: list[Any]) -> dict[str, Any]: body["serving_environment"] = "sandbox" return body - def _content_name(self, items: list[Any]) -> str: + def _content_name(self, items: list[Any], *, media_hashes: _MediaHashes | None = None) -> str: """Deterministic job name from the full request content.""" + media_hashes = media_hashes if media_hashes is not None else {} naming_body: dict[str, Any] = { "instruction": self.instruction, "task_type": self._task_type(items), @@ -292,7 +302,9 @@ def _content_name(self, items: list[Any]) -> str: "response_options": self._response_options(items), "annotator_filter": self.annotator_filter, "sandbox": self.sandbox, - "datapoints": [self._datapoint(i, for_naming=True) for i in items], + "datapoints": [ + self._datapoint(i, for_naming=True, media_hashes=media_hashes) for i in items + ], } canonical = json.dumps(naming_body, sort_keys=True, separators=(",", ":")) digest = hashlib.sha256(f"humanevals:1:{canonical}".encode()).hexdigest() @@ -310,15 +322,22 @@ def _check_context_placeholder(self, items: list[Any]) -> None: stacklevel=_user_stacklevel(), ) - def _media_item(self, media: Media, *, for_naming: bool) -> dict[str, str]: + def _media_item( + self, media: Media, *, for_naming: bool, media_hashes: _MediaHashes + ) -> dict[str, str]: """Resolve a :class:`Media` into an API media item (or naming token).""" - if for_naming: - if media.is_remote: - return {"url": str(media.source), "type": media.resolved_type()} - path = Path(media.source).expanduser().resolve() + media_type = media.resolved_type() + if media.is_remote: + return {"url": str(media.source), "type": media_type} + path = Path(media.source).expanduser().resolve() + cache_key = (str(path), media_type) + digest = media_hashes.get(cache_key) + if digest is None: digest = hashlib.sha256(path.read_bytes()).hexdigest() + media_hashes[cache_key] = digest + if for_naming: return {"content_sha256": digest, "type": media.resolved_type()} - return self.client.resolve_media(media) + return self.client.resolve_media(media, content_sha256=digest) def _scores_from_results( self, @@ -466,7 +485,9 @@ def _task_type(self, items: list[Pair]) -> str: # media-only); with 2 candidates the math maps back exactly. return {"text": "ranking", "comparison": "comparison", "i2v": "i2v_comparison"}[mode] - def _datapoint(self, item: Pair, *, for_naming: bool) -> dict[str, Any]: + def _datapoint( + self, item: Pair, *, for_naming: bool, media_hashes: _MediaHashes + ) -> dict[str, Any]: mode = self._mode(item) if mode == "text": media: dict[str, Any] = { @@ -479,12 +500,16 @@ def _datapoint(self, item: Pair, *, for_naming: bool) -> dict[str, Any]: assert isinstance(item.a, Media) and isinstance(item.b, Media) media = { "candidates": [ - self._media_item(item.a, for_naming=for_naming), - self._media_item(item.b, for_naming=for_naming), + self._media_item(item.a, for_naming=for_naming, media_hashes=media_hashes), + self._media_item(item.b, for_naming=for_naming, media_hashes=media_hashes), ] } if item.reference is not None: - media["reference"] = [self._media_item(item.reference, for_naming=for_naming)] + media["reference"] = [ + self._media_item( + item.reference, for_naming=for_naming, media_hashes=media_hashes + ) + ] datapoint: dict[str, Any] = {"media": media} if item.context is not None: datapoint["context"] = item.context @@ -619,12 +644,18 @@ def _response_options(self, items: list[RatingItem]) -> dict[str, Any]: options["labels"] = self.labels return options - def _datapoint(self, item: RatingItem, *, for_naming: bool) -> dict[str, Any]: + def _datapoint( + self, item: RatingItem, *, for_naming: bool, media_hashes: _MediaHashes + ) -> dict[str, Any]: if isinstance(item.subject, str): # Text-only rating: the API requires an explicit empty media dict. return {"media": {}, "context": item.subject} datapoint: dict[str, Any] = { - "media": {"subject": [self._media_item(item.subject, for_naming=for_naming)]} + "media": { + "subject": [ + self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes) + ] + } } if item.context is not None: datapoint["context"] = item.context @@ -721,14 +752,18 @@ def _response_options(self, items: list[ChoiceItem]) -> dict[str, Any]: def _option_id(index: int) -> str: return f"option_{index + 1}" - def _datapoint(self, item: ChoiceItem, *, for_naming: bool) -> dict[str, Any]: + def _datapoint( + self, item: ChoiceItem, *, for_naming: bool, media_hashes: _MediaHashes + ) -> dict[str, Any]: media: dict[str, Any] = { "options": [ {"id": self._option_id(i), "text": text} for i, text in enumerate(item.options) ] } if item.subject is not None: - media["subject"] = [self._media_item(item.subject, for_naming=for_naming)] + media["subject"] = [ + self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes) + ] return {"media": media, "context": item.question} def _score_row(self, row: dict[str, Any], item: ChoiceItem | None) -> Score: @@ -814,7 +849,9 @@ def _candidate_ids(self, item: RankingItem) -> list[str]: media_type = candidates[0].resolved_type() if isinstance(candidates[0], Media) else "" return [f"{media_type}_{i + 1}" for i in range(len(candidates))] - def _datapoint(self, item: RankingItem, *, for_naming: bool) -> dict[str, Any]: + def _datapoint( + self, item: RankingItem, *, for_naming: bool, media_hashes: _MediaHashes + ) -> dict[str, Any]: candidates = list(item.candidates) ids = self._candidate_ids(item) payload: list[dict[str, str]] @@ -822,7 +859,7 @@ def _datapoint(self, item: RankingItem, *, for_naming: bool) -> dict[str, Any]: payload = [{"id": ids[i], "text": str(c)} for i, c in enumerate(candidates)] else: payload = [ - self._media_item(c, for_naming=for_naming) + self._media_item(c, for_naming=for_naming, media_hashes=media_hashes) for c in candidates if isinstance(c, Media) ] diff --git a/tests/test_submit.py b/tests/test_submit.py index 826c932..7621b53 100644 --- a/tests/test_submit.py +++ b/tests/test_submit.py @@ -77,11 +77,39 @@ def test_local_file_naming_uses_content_hash(client: he.Client, api: FakeAPI, tm assert name1 == name2 # Changing file *content* changes the name even at the same path. - client._media_refs.clear() video_a.write_bytes(b"different-content") assert scorer.submit(pair).name != name1 +def test_changed_local_file_reuploads_in_same_client(client: he.Client, api: FakeAPI, tmp_path): + video = tmp_path / "0.mp4" + video.write_bytes(b"old-video") + api.add( + "POST", + "/media", + {"media": [{"media_ref": "dp://old/0.mp4", "media_id": "old", "type": "video"}]}, + ) + api.add( + "POST", + "/media", + {"media": [{"media_ref": "dp://new/0.mp4", "media_id": "new", "type": "video"}]}, + ) + api.add("POST", "/jobs", create_job_response()) + scorer = he.HumanRating("Rate it.", client=client) + items = [he.Media(video)] + + first = scorer.submit(items) + video.write_bytes(b"new-video") + second = scorer.submit(items) + + assert first.name != second.name + assert api.paths() == ["POST /media", "POST /jobs", "POST /media", "POST /jobs"] + first_media = api.body(1)["datapoints"][0]["media"]["subject"][0] + second_media = api.body(3)["datapoints"][0]["media"]["subject"][0] + assert first_media["url"] == "dp://old/0.mp4" + assert second_media["url"] == "dp://new/0.mp4" + + def test_explicit_name_wins(client: he.Client, api: FakeAPI): api.add("POST", "/jobs", create_job_response()) job = scorer_for(client).submit(PAIRS, name="my-eval-run-7")