From 70db5af8f5650d4219700d5beb457e12e8a813eb Mon Sep 17 00:00:00 2001 From: jepegit Date: Sat, 26 Sep 2026 15:22:26 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20pluggable=20external=20metadata=20sourc?= =?UTF-8?q?es=20=E2=80=94=20Protocol,=20registry,=20resolver=20hook=20(#78?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cellpy.readers.metadata_sources: MetadataSource / SupportsMetadataPush Protocols, MetaQuery, MetaRecord, ExternalLink, validate_record, error hierarchy; cellpy.metadata_sources entry-point registry (register, names, get_source); null-object fetch_meta (unknown/unreachable => empty layer, auth errors propagate); conformance kit check_metadata_source - MetaResolver.resolve(external=...): records join the journal/db layer below the journal row; Resolution.origins / origin_of / explain() name the winning source - CellpyCell.fetch_meta(source, key=None, kind=, apply=, strict=) and external_links; Data.external_links persisted in v9 meta.json - tests/test_metadata_sources.py (36, 6 essential); docs, HISTORY, design note; CellMeta.uuid filed core-first as cellpy/cellpy-core#151 Closes #784 Co-authored-by: Cursor --- .../03-solved-issues/issue784_original.md | 34 ++ .issueflows/03-solved-issues/issue784_plan.md | 33 ++ .../03-solved-issues/issue784_status.md | 31 ++ .../04-designs-and-guides/metadata-sources.md | 56 +++ .../04-designs-and-guides/test-registry.md | 8 + AGENTS.md | 6 + HISTORY.md | 15 + cellpy/readers/cellpy_file/meta_archive.py | 16 + cellpy/readers/cellpy_file/v9.py | 1 + cellpy/readers/cellreader.py | 108 +++++ cellpy/readers/data_structures.py | 4 + cellpy/readers/meta_resolver.py | 87 +++- cellpy/readers/metadata_sources/__init__.py | 58 +++ cellpy/readers/metadata_sources/contract.py | 262 +++++++++++ cellpy/readers/metadata_sources/registry.py | 233 ++++++++++ cellpy/readers/metadata_sources/testing.py | 123 +++++ docs/agents/index.md | 12 + docs/api/readers.md | 13 + tests/test_metadata_sources.py | 426 ++++++++++++++++++ 19 files changed, 1516 insertions(+), 10 deletions(-) create mode 100644 .issueflows/03-solved-issues/issue784_original.md create mode 100644 .issueflows/03-solved-issues/issue784_plan.md create mode 100644 .issueflows/03-solved-issues/issue784_status.md create mode 100644 .issueflows/04-designs-and-guides/metadata-sources.md create mode 100644 cellpy/readers/metadata_sources/__init__.py create mode 100644 cellpy/readers/metadata_sources/contract.py create mode 100644 cellpy/readers/metadata_sources/registry.py create mode 100644 cellpy/readers/metadata_sources/testing.py create mode 100644 tests/test_metadata_sources.py diff --git a/.issueflows/03-solved-issues/issue784_original.md b/.issueflows/03-solved-issues/issue784_original.md new file mode 100644 index 00000000..1bf6795a --- /dev/null +++ b/.issueflows/03-solved-issues/issue784_original.md @@ -0,0 +1,34 @@ +# Issue #784: Pluggable external metadata sources (query an API for cell/test metadata) + +Source: https://github.com/jepegit/cellpy/issues/784 +Labels: enhancement, v2, cellpy2-stage5 +Captured: 2026-09-26 (Epic M / M1, stage 4 of #783; read scope only) + +--- + +Let cellpy **pull (and optionally push) cell/test metadata from an external source** — starting +with **BatBase** (our in-house Django + PostgreSQL app, HTTP API) — but **source-agnostic**: a +pluggable contract so other labs' stores work too. First transport assumption: an **HTTP API** +(GET to read, POST/PUT to write). + +Realizes the deferred **metadata-plan Step 7** ("DB/API transport — keep the door open"). Full +design: **[cellpy2-metadata-source-integration.md](https://github.com/cellpy/architecture-plan/blob/main/cellpy2-metadata-source-integration.md)**. + +**Approach (mirrors the loader contract):** +- `MetadataSource` **Protocol** + `cellpy.metadata_sources` entry-point registry — third + parties plug in without importing a cellpy base class. +- Feeds the existing **`MetaResolver` JOURNAL/DB layer** (authoritative lab metadata already + outranks raw-file values; provenance extended to name the source). +- **Anti-corruption boundary**: cellpy pins the Protocol + `MetaRecord`; the **BatBase adapter + absorbs API churn** (BatBase API isn't settled yet — adapter can even be a separate plugin + package, keeping the HTTP dep out of cellpy core). +- Adds **`CellMeta.uuid`** + per-source `external_id`/`source_uri` back-link (the identity + parking item, metadata-plan OQ6); vocabulary normalization toward **BattINFO** (OQ4). +- Null-object: an unreachable/unknown source ⇒ empty layer, cell still loads. Push is + side-effecting → **explicit opt-in only**, never automatic. Auth via env-only `SecretStr`. + +**Scope:** read path first (Protocol + `fetch` + resolver + BatBase GET adapter); push a later +opt-in follow-on. **Post-2.2 (2.3+)** — additive, gates nothing in 2.2. + +Adjacent: #243 (dbreader refactor). See design doc for the work breakdown + open questions +(query key, adapter placement, cache/offline, push permission, multi-source precedence). diff --git a/.issueflows/03-solved-issues/issue784_plan.md b/.issueflows/03-solved-issues/issue784_plan.md new file mode 100644 index 00000000..ef87eb59 --- /dev/null +++ b/.issueflows/03-solved-issues/issue784_plan.md @@ -0,0 +1,33 @@ +# Plan: #784 MetadataSource Protocol + resolver hook (M1, read path) + +Non-yolo (public Protocol + resolver precedence). Executed under the +maintainer's "start" after M0 shipped; the PR is the review point and is +**not** auto-merged. + +## Scope (from epic783 M1 spec) + +`MetadataSource` Protocol, `cellpy.metadata_sources` entry-point registry, +`MetaResolver` journal/db-layer hook with provenance naming the source, +null-object on failure, back-link (`external_id` / `source_uri`). No push. + +## Approach + +1. `cellpy/readers/metadata_sources/` — `contract.py` (Protocols, `MetaQuery`, + `MetaRecord`, `ExternalLink`, errors, `validate_record`), `registry.py` + (entry points, `register`, `get_source`, `fetch_meta`), `testing.py` + (`check_metadata_source`, `DictMetadataSource`). +2. `meta_resolver.py`: `external=` on `resolve*`; `Resolution.origins`. +3. `CellpyCell.fetch_meta` + `external_links`; `Data.external_links`; + v9 `meta.json` `"external_links"` round-trip; clone copy. +4. Tests `tests/test_metadata_sources.py` (essential markers on the + precedence / null-object / cell-surface checks). +5. Docs: HISTORY, `docs/api/readers.md`, `docs/agents/index.md`, `AGENTS.md` + agents section, design note `metadata-sources.md`, test registry. + +## Deviations from the issue text + +- `CellMeta.uuid` is a cellpy-core model field → filed as a core-first issue + instead of a cellpy change (see status). Back-link lands on the cellpy + side (`ExternalLink`) and is persisted. +- `batch.from_source`, vocabulary normalisation, cache/TTL: deferred (design + §4 items 4–5; M3+). diff --git a/.issueflows/03-solved-issues/issue784_status.md b/.issueflows/03-solved-issues/issue784_status.md new file mode 100644 index 00000000..dd8f1d6b --- /dev/null +++ b/.issueflows/03-solved-issues/issue784_status.md @@ -0,0 +1,31 @@ +# Issue #784 status (M1 — read path) + +- [x] Done + +## What's done + +- `cellpy/readers/metadata_sources/` package: `MetadataSource` / + `SupportsMetadataPush` Protocols, `MetaQuery`, `MetaRecord`, `ExternalLink`, + `validate_record`, error hierarchy (`MetadataSourceError`, + `MetadataSourceAuthError`, `UnknownMetadataSource`); entry-point registry + (`cellpy.metadata_sources`, `register`, `names`, `get_source`); null-object + `fetch_meta`; conformance kit `testing.check_metadata_source` + + `DictMetadataSource`. +- `MetaResolver.resolve(external=…)` (and `resolve_cell_meta` / + `resolve_test_meta` / `resolve_from_loader_result`): external records join + the journal/db layer below the journal row; `Resolution.origins`, + `origin_of`, `fields_from_origin`, `explain()` name the source. +- `CellpyCell.fetch_meta(...)` + `external_links`; `Data.external_links`; + v9 `meta.json` `"external_links"` round-trip; clone copy in `from_cell`. +- Tests: `tests/test_metadata_sources.py` (36; 6 essential). Full suite + 1757 passed / 187 skipped; `-m essential` 959 passed. +- Docs: HISTORY, `docs/api/readers.md`, `docs/agents/index.md`, `AGENTS.md` + agents section, `metadata-sources.md` design note, test registry. +- `CellMeta.uuid` filed core-first as cellpy/cellpy-core#151 (model lives in + cellpy-core; needs release + re-pin). + +## Remaining work + +None on the cellpy side for the read path. Review the PR (non-yolo). Follow-ups: +cellpy/cellpy-connectors#2 (BatBase adapter, M2), cellpy/cellpy-core#151, +M3 push (2.3), `batch.from_source` / multi-source config (design §4 item 5). diff --git a/.issueflows/04-designs-and-guides/metadata-sources.md b/.issueflows/04-designs-and-guides/metadata-sources.md new file mode 100644 index 00000000..88de6218 --- /dev/null +++ b/.issueflows/04-designs-and-guides/metadata-sources.md @@ -0,0 +1,56 @@ +# External metadata sources (Epic M, read path) + +Issue: #784 (M1; stage 4 of #783). Design origin: +`cellpy-design-and-development/active/cellpy2-metadata-source-integration.md`. +Adapter side: `cellpy-connectors` (#1 `BatBaseClient`, #2 adapter). + +## Context + +cellpy 2 reserved the seam — `MetaResolver` (`kwargs > journal/db > raw file > +config defaults`) with per-field `Resolution` provenance — but nothing outside +the journal fed it, and the resolver is not yet on the raw-load hot path +(`cellreader.from_raw` still fills the legacy meta boxes directly). A lab's +system of record (BatBase first) should contribute mass, area, nominal +capacity, project without cellpy depending on one lab's API. + +## Decisions + +| Topic | Decision | +| --- | --- | +| Package | `cellpy/readers/metadata_sources/` — `contract.py`, `registry.py`, `testing.py`; re-exported from the package | +| Contract | `MetadataSource` Protocol: `name: ClassVar[str]`, `fetch(MetaQuery) -> tuple[MetaRecord, ...]`. Structural, `runtime_checkable`; no base class. `SupportsMetadataPush.register(record) -> str` declared, unused (M3). | +| Shapes | `MetaQuery(key, kind="cell_name", project, extra)`; `MetaRecord(source_name, external_id, source_uri, cell={CellMeta field: value}, test={TestMeta field: value}, fetched_at, raw)`; `ExternalLink` = persisted back-link | +| Validation | `validate_record`: unknown field names, pre-filled provenance (`uuid`, `source_*`, …) and `None` values are errors — a typo in an adapter's field map fails its conformance test rather than losing a mass | +| Registry | `cellpy.metadata_sources` entry-point group; lazy discovery; entry may be a class (instantiated once, no args) or a zero-arg factory; broken plugins skipped with a warning; `register()` for tests/notebooks and pre-configured instances | +| Null object | `fetch_meta(source, query, strict=False)`: unknown/unreachable source or invalid record ⇒ `()` + warning. **`MetadataSourceAuthError` always propagates** — a refused token is the user's to fix, hiding it would look like "no data". `strict=True` raises everything. | +| Resolver | `MetaResolver.resolve(external=…)`: records join the **journal/db layer below the journal row** (kwargs > journal row > external sources (priority order) > raw file > defaults). `Resolution.origins[field]` names the contributor (`"batbase"`, `"journal"`); `origin_of()`, `fields_from_origin()`, `explain()` show it. Old provenance shape unchanged when no externals are given. | +| Cell surface | `CellpyCell.fetch_meta(source, key=None, *, kind, project, apply=True, strict=False, **extra)`; `key` defaults to `cell_name`. Applies the **first** record (warns if several) through `resolve_*_meta(external=record)` → `test_meta.apply_test_meta_to_legacy` so the engine's live legacy boxes are updated; `external_links[source] = ExternalLink(fields=applied)`. Returns all records. | +| Persistence | `Data.external_links: dict[str, ExternalLink]`; v9 `meta.json` key `"external_links"` (omitted when empty); copied by `CellpyCell.from_cell` clone; `apply_meta_document` restores. `raw` payloads are never persisted. | +| Not done here | `CellMeta.uuid` (lives in cellpy-core → core-first issue), `batch.from_source`, multi-source priority config, cache/TTL, push, BattINFO vocabulary map | + +## Precedence rationale + +Journal row above external source: a value the user corrected in their +journal is a deliberate local override; the lab database is authoritative +*relative to the cycler file*, not relative to the user. Both beat the raw +file. Revisit when multi-source config lands (design §6). + +## Alternatives rejected + +- A fifth `Layer.EXTERNAL` between RAW_FILE and JOURNAL — the design says the + source is a *contributor to the journal/db layer*; a separate layer would + change `Layer` ordering that existing provenance tests and docs rely on. + Contributors + `origins` keep the enum stable and still answer "who won". +- Swallowing auth errors like other failures — violates "auth failure + surfaces a clear error, not a silent empty" (design §3.6). +- `fetch(key) -> MetaRecord | None` (epic wording) — the design's tuple form + matches the loader contract and lets a tag match several tests. +- Applying all returned records — ambiguous merge order; first-record + + warning keeps it explicit, adapters narrow the query instead. + +## Conformance kit + +`metadata_sources.testing.check_metadata_source(source, known=, unknown=)`: +capabilities, unknown key ⇒ `()` (never raises), records validate and carry +`source_name == source.name`, determinism. `DictMetadataSource` is the +in-process fake for adapter tests. diff --git a/.issueflows/04-designs-and-guides/test-registry.md b/.issueflows/04-designs-and-guides/test-registry.md index 42a37e20..d0c128e3 100644 --- a/.issueflows/04-designs-and-guides/test-registry.md +++ b/.issueflows/04-designs-and-guides/test-registry.md @@ -256,6 +256,14 @@ current issue**. `/iflow-doctor` may audit the whole suite against this table. | tests/test_cli_connectors_plugin.py::test_help_lists_installed_connectors_without_importing_it | yes | yes | live cellpy-connectors stub | #1060 | skip if dist missing (conda) | | tests/test_cli_connectors_plugin.py::test_connectors_ping_runs_the_installed_plugin | yes | yes | cellpy connectors ping | #1060 | skip if dist missing (conda) | | tests/test_cli_connectors_plugin.py::test_import_cellpy_does_not_import_connectors | yes | yes | import cellpy isolation | #1060 | skip if dist missing (conda) | +| tests/test_metadata_sources.py::test_protocol_is_structural | yes | yes | metadata_sources.contract Protocols | #784 | public contract shape | +| tests/test_metadata_sources.py::test_fetch_meta_returns_validated_records_with_timestamp | yes | | metadata_sources.registry.fetch_meta | #784 | | +| tests/test_metadata_sources.py::test_fetch_meta_unknown_source_is_empty_layer_unless_strict | yes | yes | fetch_meta null object | #784 | cellpy law: empty layer, cell still loads | +| tests/test_metadata_sources.py::test_fetch_meta_auth_error_is_never_swallowed | yes | yes | fetch_meta | #784 | security posture | +| tests/test_metadata_sources.py::test_external_beats_raw_file_and_defaults_but_not_journal_or_kwargs | yes | yes | MetaResolver external= precedence | #784 | precedence + origins provenance | +| tests/test_metadata_sources.py::test_cell_fetch_meta_applies_record_and_links | yes | | CellpyCell.fetch_meta / external_links | #784 | uses `cell` fixture | +| tests/test_metadata_sources.py::test_external_links_survive_save_and_load | no | | v9 meta.json external_links | #784 | save/get round-trip | +| tests/test_metadata_sources.py (other 29) | no | | contract validation, registry discovery, conformance kit, resolver ordering | #784 | offline | **Columns** diff --git a/AGENTS.md b/AGENTS.md index 54f06693..765c9ba0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -379,6 +379,12 @@ Quick facts: the raw file did not change on disk. Also works after `cellpy.get(".cellpy")`. Follow on an interval: `cellpy.utils.live.poll(c, interval=60, on_update=cb)`; batch: `b.refresh()` / `b.poll(interval=, on_update=)`. +- External metadata (lab DB / BatBase): `c.fetch_meta("batbase", key=None, + kind="cell_name")` pulls mass / area / nominal capacity / project onto the + cell like a journal row and records `c.external_links["batbase"]`; + unreachable source ⇒ `()` and no change (`strict=True` raises). Sources: + `cellpy.readers.metadata_sources.names()`; the BatBase adapter is in + `cellpy-connectors`. - Frames: `c.data.raw` / `.steps` / `.summary`; columns via `c.schema.*`. After a raw load, each cycle's raw capacity starts at 0. A forgotten tester reset that 1.x plotted as doubled capacity is rebased on load for every diff --git a/HISTORY.md b/HISTORY.md index 8ffe46d0..2c24a422 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,21 @@ ## [Unreleased] +* Pluggable external metadata sources (read path). New + `cellpy.readers.metadata_sources`: a `MetadataSource` Protocol + (`name`, `fetch(MetaQuery) -> tuple[MetaRecord, ...]`), the + `cellpy.metadata_sources` entry-point registry (`register`, `names`, + `get_source`), and a null-object `fetch_meta` (unreachable or unknown + source ⇒ empty layer; auth errors still raise). `MetaResolver` takes + `external=` records into the journal/db layer *below* the journal row and + `Resolution.origin_of()` / `explain()` name the source that won a field. + `CellpyCell.fetch_meta(source, key=None, kind=, apply=, strict=)` pulls a + record onto the cell and records an `ExternalLink` in + `c.external_links`, persisted in v9 `meta.json` (`"external_links"`). + `metadata_sources.testing.check_metadata_source` is the conformance kit + for adapters. No push. The BatBase adapter ships in `cellpy-connectors`. + (#784) + * Optional `SupportsIncrementalLoad` protocol (`load_since`) with `LoadMarker` and `IncrementalChunk`, hosted in cellpy. Shipped loaders stay full-read until they opt in. (#779) diff --git a/cellpy/readers/cellpy_file/meta_archive.py b/cellpy/readers/cellpy_file/meta_archive.py index 504ff33b..f74ca126 100644 --- a/cellpy/readers/cellpy_file/meta_archive.py +++ b/cellpy/readers/cellpy_file/meta_archive.py @@ -93,6 +93,12 @@ def build_meta_document( } if frames_had_test_id is not None: doc["frames_had_test_id"] = dict(frames_had_test_id) + links = getattr(data, "external_links", None) or {} + if links: + doc["external_links"] = { + name: (link.to_dict() if hasattr(link, "to_dict") else dict(link)) + for name, link in links.items() + } return doc @@ -168,6 +174,16 @@ def apply_meta_document(data: "Data", meta_doc: Mapping[str, Any]) -> None: active_id = int(meta_doc.get("active_test_id", 0)) data._active_test_id = active_id + links_doc = meta_doc.get("external_links") or {} + if isinstance(links_doc, Mapping): + from cellpy.readers.metadata_sources.contract import ExternalLink + + data.external_links = { + str(name): ExternalLink.from_dict(payload) + for name, payload in links_doc.items() + if isinstance(payload, Mapping) + } + tests_doc = meta_doc.get("tests") or {} cell_fallback = meta_doc.get("cell") or {} diff --git a/cellpy/readers/cellpy_file/v9.py b/cellpy/readers/cellpy_file/v9.py index 794c1b1f..63f02ba5 100644 --- a/cellpy/readers/cellpy_file/v9.py +++ b/cellpy/readers/cellpy_file/v9.py @@ -169,6 +169,7 @@ def save( scratch._extra_tests = dict(getattr(data, "_extra_tests", {}) or {}) scratch._active_test_id = data.active_test_id scratch._provenance = dict(getattr(data, "_provenance", {}) or {}) + scratch.external_links = dict(getattr(data, "external_links", {}) or {}) scratch.raw_units = dict(data.raw_units) scratch.raw_limits = dict(data.raw_limits) scratch.raw_data_files = list(data.raw_data_files or []) diff --git a/cellpy/readers/cellreader.py b/cellpy/readers/cellreader.py index e44fb721..05039a27 100644 --- a/cellpy/readers/cellreader.py +++ b/cellpy/readers/cellreader.py @@ -687,6 +687,9 @@ def vacant(cls, cell=None): new_cell.data._extra_tests = dict(cell.data._extra_tests) new_cell.data._active_test_id = cell.data._active_test_id new_cell.data._provenance = dict(cell.data._provenance) + new_cell.data.external_links = dict( + getattr(cell.data, "external_links", None) or {} + ) new_cell.data.raw_data_files = cell.data.raw_data_files new_cell.data.raw_data_files_length = cell.data.raw_data_files_length @@ -2130,6 +2133,111 @@ def _update_full_reload(self, fids, **loader_kwargs) -> bool: # -------------------- incremental refresh end ----------------------- + # -------------------- external metadata sources (#784) --------------- + + @property + def external_links(self) -> dict: + """Back-links to external metadata sources, keyed by source name. + + Each value is an ``ExternalLink`` (``external_id``, ``source_uri``, + ``fetched_at``, ``fields`` supplied). Empty until `fetch_meta` has + applied a record; survives ``save()`` / load of v9 cellpy-files. + """ + return dict(getattr(self.data, "external_links", None) or {}) + + def fetch_meta( + self, + source: str, + key: Optional[str] = None, + *, + kind: str = "cell_name", + project: Optional[str] = None, + apply: bool = True, + strict: bool = False, + **extra, + ) -> tuple: + """Pull cell/test metadata from an external source and apply it. + + Read-only towards the source. The record is merged the way a batch + journal row would be — above what the instrument file wrote, below + anything you set explicitly afterwards — and a back-link is kept in + `external_links` so the fetch is reproducible. + + Args: + source: registered source name (``"batbase"``); see + ``cellpy.readers.metadata_sources.names()``. + key: lookup value; defaults to this cell's ``cell_name``. + kind: what ``key`` is (``"cell_name"``, ``"tag"``, ``"serial"``, + ``"external_id"``, ...). Sources document what they accept. + project: optional project scope. + apply: write the first record's fields onto this cell. With + ``False`` the records are only returned. + strict: raise when the source is unknown or unreachable instead + of returning ``()`` (an auth failure raises either way). + **extra: source-specific filters. + + Returns: + The matching ``MetaRecord`` tuple — possibly empty, in which case + nothing was changed. + + Example: + >>> c = cellpy.get("cell_042.res") + >>> c.fetch_meta("batbase", kind="tag", key="SAL_010") + >>> c.data.meta_common.mass + """ + from cellpy.readers.metadata_sources import MetaQuery, fetch_meta + + if key is None: + key = self.cell_name + query = MetaQuery(key=key, kind=kind, project=project, extra=extra) + records = fetch_meta(source, query, strict=strict) + if not records: + logging.info(f"fetch_meta: {source!r} had nothing for {query.describe()}") + return records + if apply: + if len(records) > 1: + logging.warning( + f"fetch_meta: {source!r} returned {len(records)} records for " + f"{query.describe()}; applying the first " + f"(external_id={records[0].external_id!r})" + ) + self._apply_meta_record(records[0]) + return records + + def _apply_meta_record(self, record) -> None: + """Write one ``MetaRecord`` onto the legacy meta boxes and link it.""" + from cellpycore.metadata.models import CellMeta, TestMeta + + from cellpy.readers.meta_resolver import resolve_cell_meta, resolve_test_meta + + cell_record, cell_res = resolve_cell_meta(CellMeta(), external=record) + test_record, test_res = resolve_test_meta(TestMeta(), external=record) + test_record.cell = cell_record + test_record.test_id = self.data.active_test_id + # ``test_meta`` here is the helper module imported at the top. + test_meta.apply_test_meta_to_legacy( + test_record, self.data.meta_common, self.data.meta_test_dependent + ) + applied = tuple( + sorted( + set(cell_res.fields_from_origin(record.source_name)) + | set(test_res.fields_from_origin(record.source_name)) + ) + ) + if "cell_name" in applied and test_record.cell_name: + self._cell_name = test_record.cell_name + link = record.link() + from dataclasses import replace + + link = replace(link, fields=applied) + if not hasattr(self.data, "external_links") or self.data.external_links is None: + self.data.external_links = {} + self.data.external_links[record.source_name] = link + logging.info( + f"fetch_meta: applied {len(applied)} field(s) from {record.source_name!r} " + f"(external_id={record.external_id!r}): {', '.join(applied) or '-'}" + ) + def merge(self, cells, mode="campaign", renumber_cycles=True, **kwargs): """Merge other cells/datasets into this one. diff --git a/cellpy/readers/data_structures.py b/cellpy/readers/data_structures.py index f1581f57..72668626 100644 --- a/cellpy/readers/data_structures.py +++ b/cellpy/readers/data_structures.py @@ -538,6 +538,10 @@ def __init__(self, **kwargs): # loaded_datetime). Filled by CellpyCell.from_raw; merged into the # derived TestMeta record; NOT persisted in cellpy-file v8 (#510). self._provenance: Dict[str, Any] = {} + # Back-links to external metadata sources (#784): source name -> + # ExternalLink (external id, uri, fetched_at, fields supplied). + # Filled by CellpyCell.fetch_meta; persisted in v9 meta.json. + self.external_links: Dict[str, Any] = {} # Compact per-test grouping key of the active test (0 = single, # unmerged; matches the engine's test_id convention). Note: the legacy # ``meta_test_dependent.test_ID`` is the *tester-assigned* id (e.g. diff --git a/cellpy/readers/meta_resolver.py b/cellpy/readers/meta_resolver.py index d4d6f5f6..d53d0cb9 100644 --- a/cellpy/readers/meta_resolver.py +++ b/cellpy/readers/meta_resolver.py @@ -50,28 +50,69 @@ class Resolution: #: field name -> the layer that supplied the winning value sources: dict[str, Layer] = field(default_factory=dict) + #: field name -> which *contributor* inside the layer won, when the layer + #: has more than one (the journal/db layer: an external metadata source + #: such as ``"batbase"``, or ``"journal"`` for the batch journal row). + origins: dict[str, str] = field(default_factory=dict) def source_of(self, name: str) -> Layer | None: """Which layer supplied ``name``, or None if nothing did.""" return self.sources.get(name) + def origin_of(self, name: str) -> str | None: + """Which contributor supplied ``name`` (e.g. ``"batbase"``), if known.""" + return self.origins.get(name) + def fields_from(self, layer: Layer) -> tuple[str, ...]: """Every field this layer won.""" return tuple( sorted(name for name, won in self.sources.items() if won is layer) ) + def fields_from_origin(self, origin: str) -> tuple[str, ...]: + """Every field a named contributor (external source) won.""" + return tuple( + sorted(name for name, who in self.origins.items() if who == origin) + ) + def explain(self) -> str: """Human-readable per-field provenance, for logs and debugging.""" if not self.sources: return "no metadata resolved" - lines = [ - f" {name}: {layer.label}" - for name, layer in sorted(self.sources.items(), key=lambda kv: kv[0]) - ] + lines = [] + for name, layer in sorted(self.sources.items(), key=lambda kv: kv[0]): + origin = self.origins.get(name) + suffix = f" ({origin})" if origin else "" + lines.append(f" {name}: {layer.label}{suffix}") return "resolved metadata:\n" + "\n".join(lines) +def _iter_external(external: Any) -> Iterable[tuple[str, Mapping[str, Any]]]: + """Yield ``(source_name, field mapping)`` per external record, in order. + + Accepts a `MetaRecord`, an iterable of them, or plain ``(name, mapping)`` + pairs. Cell and test drafts are merged into one mapping — the resolver's + ``target_fields`` filter keeps each ``into`` to its own fields. + """ + if external is None: + return + if _looks_like_record(external): + external = (external,) + for item in external: + if _looks_like_record(item): + yield item.source_name, {**item.cell, **item.test} + elif isinstance(item, tuple) and len(item) == 2: + yield str(item[0]), _as_mapping(item[1]) + else: + raise TypeError( + f"external= expects MetaRecord(s) or (name, mapping) pairs, got {item!r}" + ) + + +def _looks_like_record(obj: Any) -> bool: + return hasattr(obj, "source_name") and hasattr(obj, "cell") and hasattr(obj, "test") + + def _as_mapping(source: Any) -> Mapping[str, Any]: """Read a layer as a plain field→value mapping. @@ -113,6 +154,7 @@ def resolve( *, kwargs: Any = None, journal: Any = None, + external: Any = None, raw_file: Any = None, config_defaults: Any = None, into: Any = None, @@ -122,6 +164,11 @@ def resolve( Args: kwargs: what the user passed explicitly. Wins over everything. journal: batch journal or database row. + external: records from external metadata sources (#784) — one + `MetaRecord`, or an iterable of them in **priority order** + (first wins). They join the journal/db layer *below* the + journal row: a mass the user corrected in the journal beats + what the lab database says, and both beat the raw file. raw_file: the loader's draft — what the instrument file knew. config_defaults: the session's ``ScienceDefaults``-style values. into: the object to populate (mutated and returned). Required. @@ -132,15 +179,23 @@ def resolve( if into is None: raise ValueError("resolve() needs an object to populate (into=)") - layers = ( - (Layer.CONFIG_DEFAULT, _as_mapping(config_defaults)), - (Layer.RAW_FILE, _as_mapping(raw_file)), - (Layer.JOURNAL, _as_mapping(journal)), - (Layer.KWARGS, _as_mapping(kwargs)), + # (layer, contributor label or None, values). Within the journal/db + # layer, later contributors win, so externals go lowest-priority + # first and the journal row last. + contributions: list[tuple[Layer, str | None, Mapping[str, Any]]] = [ + (Layer.CONFIG_DEFAULT, None, _as_mapping(config_defaults)), + (Layer.RAW_FILE, None, _as_mapping(raw_file)), + ] + externals = list(_iter_external(external)) + for label, values in reversed(externals): + contributions.append((Layer.JOURNAL, label, values)) + contributions.append( + (Layer.JOURNAL, "journal" if externals else None, _as_mapping(journal)) ) + contributions.append((Layer.KWARGS, None, _as_mapping(kwargs))) resolution = Resolution() - for layer, values in layers: + for layer, origin, values in contributions: for name in self._target_fields: if name not in values: continue @@ -152,6 +207,10 @@ def resolve( continue setattr(into, name, value) resolution.sources[name] = layer + if origin is not None: + resolution.origins[name] = origin + else: + resolution.origins.pop(name, None) return into, resolution @@ -161,6 +220,7 @@ def resolve_cell_meta( *, kwargs: Mapping[str, Any] | None = None, journal: Any = None, + external: Any = None, draft: Any = None, config_defaults: Any = None, ) -> tuple[Any, Resolution]: @@ -169,6 +229,7 @@ def resolve_cell_meta( return resolver.resolve( kwargs=kwargs, journal=journal, + external=external, raw_file=draft, config_defaults=config_defaults, into=cell_meta, @@ -180,6 +241,7 @@ def resolve_test_meta( *, kwargs: Mapping[str, Any] | None = None, journal: Any = None, + external: Any = None, draft: Any = None, config_defaults: Any = None, ) -> tuple[Any, Resolution]: @@ -188,6 +250,7 @@ def resolve_test_meta( return resolver.resolve( kwargs=kwargs, journal=journal, + external=external, raw_file=draft, config_defaults=config_defaults, into=test_meta, @@ -250,6 +313,7 @@ def resolve_from_loader_result( source_type: str, kwargs: Mapping[str, Any] | None = None, journal: Any = None, + external: Any = None, config_defaults: Any = None, ) -> tuple[Any, Any, Resolution, Resolution]: """Turn a loader's drafts into populated metadata, with provenance. @@ -264,6 +328,7 @@ def resolve_from_loader_result( source_type: the loader/instrument name. kwargs: explicit user values. journal: batch journal or database row. + external: `MetaRecord`(s) from external metadata sources (#784). config_defaults: session ``ScienceDefaults``. Returns: @@ -280,6 +345,7 @@ def resolve_from_loader_result( CellMeta(), kwargs=kwargs, journal=journal, + external=external, draft=cell_draft, config_defaults=config_defaults, ) @@ -288,6 +354,7 @@ def resolve_from_loader_result( type(test_draft)() if test_draft is not None else None, kwargs=kwargs, journal=journal, + external=external, draft=test_draft, config_defaults=None, ) diff --git a/cellpy/readers/metadata_sources/__init__.py b/cellpy/readers/metadata_sources/__init__.py new file mode 100644 index 00000000..4e558aea --- /dev/null +++ b/cellpy/readers/metadata_sources/__init__.py @@ -0,0 +1,58 @@ +"""Pluggable external metadata sources (#784). + +Public surface:: + + from cellpy.readers.metadata_sources import ( + MetadataSource, SupportsMetadataPush, # the Protocols + MetaQuery, MetaRecord, ExternalLink, # the data shapes + fetch_meta, get_source, register, names, + MetadataSourceError, MetadataSourceAuthError, UnknownMetadataSource, + ) + +See `cellpy.readers.metadata_sources.contract` for the promises, +`cellpy.readers.metadata_sources.registry` for discovery and the null-object +`fetch_meta`, and `cellpy.readers.metadata_sources.testing` for the +conformance kit adapters run. +""" + +from cellpy.readers.metadata_sources.contract import ( + PROVENANCE_FIELDS, + ExternalLink, + MetadataSource, + MetadataSourceAuthError, + MetadataSourceError, + MetaQuery, + MetaRecord, + SupportsMetadataPush, + UnknownMetadataSource, + validate_record, +) +from cellpy.readers.metadata_sources.registry import ( + ENTRY_POINT_GROUP, + clear_registry, + fetch_meta, + get_registry, + get_source, + names, + register, +) + +__all__ = [ + "ENTRY_POINT_GROUP", + "PROVENANCE_FIELDS", + "ExternalLink", + "MetaQuery", + "MetaRecord", + "MetadataSource", + "MetadataSourceAuthError", + "MetadataSourceError", + "SupportsMetadataPush", + "UnknownMetadataSource", + "clear_registry", + "fetch_meta", + "get_registry", + "get_source", + "names", + "register", + "validate_record", +] diff --git a/cellpy/readers/metadata_sources/contract.py b/cellpy/readers/metadata_sources/contract.py new file mode 100644 index 00000000..c344a48e --- /dev/null +++ b/cellpy/readers/metadata_sources/contract.py @@ -0,0 +1,262 @@ +"""The external metadata-source contract (#784, metadata plan Step 7). + +A lab's system of record — a database, a LIMS, an HTTP API such as BatBase — +knows things about a cell that the cycler file never will: its mass, loading, +project, who built it. cellpy pulls that in through one small contract, so any +store can plug in the same way and cellpy never depends on one lab's API. + +The contract mirrors the loader contract +(`cellpy.readers.instruments.contract`): a `typing.Protocol` third parties +satisfy structurally (no cellpy base class), a `MetaRecord` that is exactly the +mapping shape `MetaResolver` already ingests, and an entry-point registry +(`cellpy.readers.metadata_sources.registry`) that finds sources without a +registration call. + +Rules a source must keep (`testing.check_metadata_source` enforces them): + +- ``fetch`` is **read-only** and returns ``()`` for "not found". It never + raises on an unknown key. Connectivity and auth problems *may* raise; + `fetch_meta` turns the former into an empty layer (null object) and lets + the latter through, because a wrong token is the user's to fix. +- Records carry only fields the source really knows. ``None`` is not "unset" + here; leave the key out. +- Records never pre-fill cellpy provenance (`uuid`, `source_kind`, ...); that + is the framework's to stamp. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, ClassVar, Mapping, Protocol, runtime_checkable + +from cellpy.exceptions import Error + +#: Provenance a source may never fill; the framework stamps these. +PROVENANCE_FIELDS: frozenset[str] = frozenset( + { + "uuid", + "source_kind", + "source_type", + "source_uri", + "source_uuid", + "raw_file_names", + "loaded_datetime", + } +) + + +class MetadataSourceError(Error): + """A metadata source could not answer (unreachable, malformed reply, ...).""" + + +class MetadataSourceAuthError(MetadataSourceError): + """The source refused the credentials. Not swallowed by the null object.""" + + +class UnknownMetadataSource(MetadataSourceError): + """No registered source has that name.""" + + +@dataclass(frozen=True) +class MetaQuery: + """What to look up. + + Args: + key: the lookup value — a cell name, a BatBase tag, a serial, an + external id — as the source understands it. + kind: what ``key`` is. ``"cell_name"`` (default), ``"tag"``, + ``"serial"``, ``"external_id"``, ``"uuid"``; sources document + which kinds they accept and return ``()`` for kinds they do not. + project: optional project scope. + extra: source-specific filters, passed through untouched. + """ + + key: str | None = None + kind: str = "cell_name" + project: str | None = None + extra: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "extra", dict(self.extra or {})) + + def describe(self) -> str: + parts = [f"{self.kind}={self.key!r}"] + if self.project: + parts.append(f"project={self.project!r}") + parts.extend(f"{k}={v!r}" for k, v in sorted(self.extra.items())) + return ", ".join(parts) + + +@dataclass(frozen=True) +class MetaRecord: + """One answer from a source: draft metadata plus the back-link to it. + + ``cell`` and ``test`` are plain ``{field: value}`` mappings over + ``CellMeta`` / ``TestMeta`` field names — the shape `MetaResolver` + ingests as its journal/db layer. Only fields the source knows appear. + + Args: + source_name: the registered source name (``"batbase"``). + external_id: the source's own id for the matched entity, if any. + source_uri: where the record can be re-fetched or inspected. + cell: ``CellMeta`` draft mapping. + test: ``TestMeta`` draft mapping. + fetched_at: ISO-8601 timestamp; `fetch_meta` fills it when left None. + raw: the source's original payload, for debugging. Not persisted. + """ + + source_name: str + external_id: str | None = None + source_uri: str | None = None + cell: Mapping[str, Any] = field(default_factory=dict) + test: Mapping[str, Any] = field(default_factory=dict) + fetched_at: str | None = None + raw: Any = None + + def __post_init__(self) -> None: + object.__setattr__(self, "cell", dict(self.cell or {})) + object.__setattr__(self, "test", dict(self.test or {})) + + @property + def fields(self) -> tuple[str, ...]: + """Every metadata field this record has a value for.""" + return tuple(sorted(set(self.cell) | set(self.test))) + + def is_empty(self) -> bool: + return not self.cell and not self.test + + def link(self) -> "ExternalLink": + return ExternalLink( + source_name=self.source_name, + external_id=self.external_id, + source_uri=self.source_uri, + fetched_at=self.fetched_at, + fields=self.fields, + ) + + +@dataclass(frozen=True) +class ExternalLink: + """The persisted back-link from a cellpy cell to a source record. + + Stored per source on the cell (``Data.external_links``) and in the v9 + ``meta.json`` under ``"external_links"``, so a re-fetch is deterministic + and "where did this mass come from?" has an answer after reload. + """ + + source_name: str + external_id: str | None = None + source_uri: str | None = None + fetched_at: str | None = None + #: metadata fields this source supplied when it was applied + fields: tuple[str, ...] = () + + def to_dict(self) -> dict[str, Any]: + return { + "source_name": self.source_name, + "external_id": self.external_id, + "source_uri": self.source_uri, + "fetched_at": self.fetched_at, + "fields": list(self.fields), + } + + @classmethod + def from_dict(cls, payload: Mapping[str, Any]) -> "ExternalLink": + return cls( + source_name=str(payload.get("source_name", "")), + external_id=payload.get("external_id"), + source_uri=payload.get("source_uri"), + fetched_at=payload.get("fetched_at"), + fields=tuple(payload.get("fields") or ()), + ) + + +@runtime_checkable +class MetadataSource(Protocol): + """What a metadata source must provide. Structural; no base class. + + Attributes: + name: the registry key (``"batbase"``). Class level. + """ + + name: ClassVar[str] + + def fetch(self, query: MetaQuery) -> tuple[MetaRecord, ...]: + """Look up metadata. Read-only. + + Returns every record matching ``query``; ``()`` when nothing matched. + Never raise for "unknown key". May raise `MetadataSourceError` + (connectivity) or `MetadataSourceAuthError` (credentials). + """ + ... + + +@runtime_checkable +class SupportsMetadataPush(Protocol): + """Optional: a source that can take a record back. Explicit opt-in only. + + Declared here so the shape is pinned; nothing in cellpy calls it yet + (Epic M stage M3). A push is side-effecting and must only ever happen on + an explicit user call, never as part of a load. + """ + + def register(self, record: MetaRecord) -> str: + """Push ``record`` to the source and return the source's id for it.""" + ... + + +def validate_record( + record: Any, + *, + cell_fields: Mapping[str, Any] | frozenset[str] | tuple[str, ...] | None = None, + test_fields: Mapping[str, Any] | frozenset[str] | tuple[str, ...] | None = None, + source: str = "metadata source", +) -> MetaRecord: + """Check that ``record`` keeps the contract, and return it. + + Raises `MetadataSourceError` naming the broken promise. Unknown field + names are an error rather than silently dropped: a typo in an adapter's + field map should fail its conformance test, not lose a lab's mass. + """ + if not isinstance(record, MetaRecord): + raise MetadataSourceError( + f"{source}: fetch() must return MetaRecord instances, got {type(record)!r}" + ) + if not record.source_name: + raise MetadataSourceError(f"{source}: MetaRecord.source_name is empty") + + if cell_fields is None or test_fields is None: + known_cell, known_test = _known_meta_fields() + cell_fields = cell_fields if cell_fields is not None else known_cell + test_fields = test_fields if test_fields is not None else known_test + + unknown_cell = sorted(set(record.cell) - set(cell_fields)) + unknown_test = sorted(set(record.test) - set(test_fields)) + if unknown_cell or unknown_test: + raise MetadataSourceError( + f"{source}: MetaRecord uses field names that are not metadata fields: " + f"cell={unknown_cell} test={unknown_test}" + ) + stamped = sorted((set(record.cell) | set(record.test)) & PROVENANCE_FIELDS) + if stamped: + raise MetadataSourceError( + f"{source}: MetaRecord pre-fills provenance {stamped}; " + "that is the framework's to stamp." + ) + nones = sorted(k for k, v in {**record.cell, **record.test}.items() if v is None) + if nones: + raise MetadataSourceError( + f"{source}: MetaRecord carries None for {nones}; leave unknown fields out." + ) + return record + + +def _known_meta_fields() -> tuple[frozenset[str], frozenset[str]]: + from dataclasses import fields as dc_fields + + from cellpycore.metadata.models import CellMeta, TestMeta + + return ( + frozenset(f.name for f in dc_fields(CellMeta)), + frozenset(f.name for f in dc_fields(TestMeta)) - {"cell"}, + ) diff --git a/cellpy/readers/metadata_sources/registry.py b/cellpy/readers/metadata_sources/registry.py new file mode 100644 index 00000000..d40691ac --- /dev/null +++ b/cellpy/readers/metadata_sources/registry.py @@ -0,0 +1,233 @@ +"""Metadata-source discovery via entry points, and the null-object `fetch_meta`. + +A source package declares itself and cellpy finds it — no import of cellpy at +registration time, no base class:: + + # in the adapter package's pyproject.toml + [project.entry-points."cellpy.metadata_sources"] + batbase = "cellpy_connectors.batbase_source:BatBaseMetadataSource" + +The entry point may name a class (instantiated with no arguments) or a +zero-argument factory returning a source. Discovery is lazy: ``import cellpy`` +never scans or imports third-party code. + +Failure posture mirrors the loader registry: a plugin that cannot be imported +or does not satisfy the contract is skipped with a warning, it does not take +cellpy down. `fetch_meta` adds the cellpy law for external data — an +unreachable or unknown source yields an **empty layer**, never a failed load — +with one deliberate exception: `MetadataSourceAuthError` propagates, because a +refused token is a configuration problem the user needs to see. +""" + +from __future__ import annotations + +import datetime +import logging +from importlib.metadata import EntryPoint, entry_points +from typing import Any, Callable, Iterable + +from cellpy.readers.metadata_sources.contract import ( + MetadataSource, + MetadataSourceAuthError, + MetadataSourceError, + MetaQuery, + MetaRecord, + UnknownMetadataSource, + validate_record, +) + +ENTRY_POINT_GROUP = "cellpy.metadata_sources" + +#: name -> source class or zero-arg factory +_REGISTRY: dict[str, Callable[[], MetadataSource]] | None = None +#: name -> instantiated source (built on first use) +_INSTANCES: dict[str, MetadataSource] = {} + + +def _validate_factory(factory: Any, source: str) -> None: + name = getattr(factory, "name", None) + if not isinstance(name, str) or not name: + raise MetadataSourceError( + f"{source}: {factory!r} must declare a non-empty class-level " + "``name`` so the registry can route on it." + ) + # ``issubclass`` is unavailable for Protocols with data members (``name``), + # so check the one method structurally. + if isinstance(factory, type) and not callable(getattr(factory, "fetch", None)): + raise MetadataSourceError( + f"{source}: {factory!r} does not satisfy the MetadataSource " + "contract; it must provide fetch(query)." + ) + if not isinstance(factory, type) and not callable(factory): + raise MetadataSourceError(f"{source}: {factory!r} is not a class or factory.") + + +def _iter_entry_points() -> Iterable[EntryPoint]: + return entry_points(group=ENTRY_POINT_GROUP) + + +def _discover() -> dict[str, Callable[[], MetadataSource]]: + found: dict[str, Callable[[], MetadataSource]] = {} + for entry_point in _iter_entry_points(): + try: + factory = entry_point.load() + except Exception as exc: + logging.warning( + "could not load metadata source %r from %s: %s", + entry_point.name, + getattr(entry_point, "value", "?"), + exc, + ) + continue + try: + _validate_factory(factory, f"entry point {entry_point.name!r}") + except MetadataSourceError as exc: + logging.warning("%s", exc) + continue + key = factory.name + if key in found: + logging.warning( + "metadata source %r declared more than once; keeping %r", key, found[key] + ) + continue + found[key] = factory + return found + + +def get_registry(*, refresh: bool = False) -> dict[str, Callable[[], MetadataSource]]: + """Registered source factories keyed by ``name``. Discovers on first use.""" + global _REGISTRY + if _REGISTRY is None or refresh: + _REGISTRY = _discover() + _INSTANCES.clear() + return dict(_REGISTRY) + + +def clear_registry() -> None: + """Forget discovery results (tests, and after installing a plugin).""" + global _REGISTRY + _REGISTRY = None + _INSTANCES.clear() + + +def register(source: Any) -> None: + """Register a source class, factory or instance directly. + + For tests and notebook-defined sources. Packaged sources should declare + an entry point instead so they are found without a call. An *instance* + is registered as-is (useful for a pre-configured client). + """ + get_registry() + assert _REGISTRY is not None + if isinstance(source, type) or not isinstance(source, MetadataSource): + _validate_factory(source, "direct registration") + _REGISTRY[source.name] = source + _INSTANCES.pop(source.name, None) + return + name = getattr(source, "name", None) + if not isinstance(name, str) or not name: + raise MetadataSourceError("direct registration: instance has no ``name``") + _REGISTRY[name] = lambda: source + _INSTANCES[name] = source + + +def names() -> tuple[str, ...]: + """Registered source names, sorted.""" + return tuple(sorted(get_registry())) + + +def get_source(name: str) -> MetadataSource: + """The source registered as ``name``, instantiated once and cached. + + Raises: + UnknownMetadataSource: nothing is registered under ``name``. + MetadataSourceError: the factory failed or returned a non-source. + """ + registry = get_registry() + if name not in registry: + known = ", ".join(names()) or "none" + raise UnknownMetadataSource( + f"no metadata source named {name!r} is registered (known: {known}). " + "Install the adapter package or register() one directly." + ) + if name not in _INSTANCES: + try: + instance = registry[name]() + except MetadataSourceError: + raise + except Exception as exc: + raise MetadataSourceError( + f"metadata source {name!r} could not be created: {exc}" + ) from exc + if not isinstance(instance, MetadataSource): + raise MetadataSourceError( + f"metadata source {name!r} factory returned {instance!r}, " + "which does not provide fetch(query)." + ) + _INSTANCES[name] = instance + return _INSTANCES[name] + + +def fetch_meta( + source: str | MetadataSource, + query: MetaQuery | str, + *, + strict: bool = False, +) -> tuple[MetaRecord, ...]: + """Ask one source for metadata; degrade to ``()`` when it cannot answer. + + Args: + source: a registered name or a source object. + query: a `MetaQuery`, or a bare string taken as ``MetaQuery(key)``. + strict: re-raise every failure instead of returning an empty layer. + Auth errors are raised regardless. + + Returns: + Validated records with ``fetched_at`` filled, or ``()``. + """ + if isinstance(query, str): + query = MetaQuery(key=query) + + try: + obj = get_source(source) if isinstance(source, str) else source + raw_records = obj.fetch(query) + label = getattr(obj, "name", repr(obj)) + except MetadataSourceAuthError: + raise + except Exception as exc: + if strict: + if isinstance(exc, MetadataSourceError): + raise + raise MetadataSourceError( + f"metadata source {source!r} failed for {query.describe()}: {exc}" + ) from exc + logging.warning( + "metadata source %r unavailable for %s (%s: %s); continuing without it", + source if isinstance(source, str) else getattr(source, "name", source), + query.describe(), + type(exc).__name__, + exc, + ) + return () + + if raw_records is None: + raw_records = () + now = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds") + records: list[MetaRecord] = [] + for record in raw_records: + try: + validate_record(record, source=f"metadata source {label!r}") + except MetadataSourceError as exc: + if strict: + raise + logging.warning("%s; dropping the record", exc) + continue + if record.fetched_at is None: + from dataclasses import replace + + record = replace(record, fetched_at=now) + records.append(record) + logging.debug( + "metadata source %r answered %s with %d record(s)", label, query.describe(), len(records) + ) + return tuple(records) diff --git a/cellpy/readers/metadata_sources/testing.py b/cellpy/readers/metadata_sources/testing.py new file mode 100644 index 00000000..c604127f --- /dev/null +++ b/cellpy/readers/metadata_sources/testing.py @@ -0,0 +1,123 @@ +"""Conformance kit for metadata sources. + +Ships with cellpy so an adapter can prove it keeps the contract without +reverse-engineering cellpy's internals:: + + from cellpy.readers.metadata_sources.testing import check_metadata_source + + def test_batbase_source_conforms(fake_batbase): + check_metadata_source( + BatBaseMetadataSource(client=fake_batbase), + known=MetaQuery(key="SAL_010", kind="tag"), + unknown=MetaQuery(key="does-not-exist", kind="tag"), + ) + +Every check corresponds to a promise in `contract`; failures raise +`AssertionError` naming the broken promise. Run it against an in-process fake +of the store, not the live one — the kit checks the adapter, not the network. +""" + +from __future__ import annotations + +from typing import Any + +from cellpy.readers.metadata_sources.contract import ( + MetadataSource, + MetadataSourceError, + MetaQuery, + MetaRecord, + validate_record, +) +from cellpy.readers.metadata_sources.registry import _validate_factory + + +def check_capabilities(source: Any) -> None: + """The source declares a ``name`` and provides ``fetch``.""" + try: + _validate_factory(type(source) if not isinstance(source, type) else source, "conformance") + except MetadataSourceError as exc: + raise AssertionError(str(exc)) from exc + assert callable(getattr(source, "fetch", None)), ( + f"{source!r} does not satisfy MetadataSource (needs fetch(query))" + ) + if not isinstance(source, type): + assert isinstance(source, MetadataSource), f"{source!r} is not a MetadataSource" + + +def check_unknown_key_is_empty(source: MetadataSource, unknown: MetaQuery) -> None: + """An unknown key returns ``()`` — it never raises.""" + try: + result = source.fetch(unknown) + except Exception as exc: # noqa: BLE001 - the point is to report it + raise AssertionError( + f"fetch() raised {type(exc).__name__} for an unknown key " + f"({unknown.describe()}); the contract says return ()" + ) from exc + assert isinstance(result, tuple), f"fetch() must return a tuple, got {type(result)!r}" + assert result == (), f"unknown key {unknown.describe()} returned {len(result)} record(s)" + + +def check_known_key_records(source: MetadataSource, known: MetaQuery) -> tuple[MetaRecord, ...]: + """A known key returns ≥1 well-formed records that only use real fields.""" + result = source.fetch(known) + assert isinstance(result, tuple), f"fetch() must return a tuple, got {type(result)!r}" + assert result, f"known key {known.describe()} returned no records" + for record in result: + try: + validate_record(record, source="conformance") + except MetadataSourceError as exc: + raise AssertionError(str(exc)) from exc + assert record.source_name == source.name, ( + f"record.source_name {record.source_name!r} != source.name {source.name!r}" + ) + assert not record.is_empty(), "a returned record carries no metadata at all" + return result + + +def check_deterministic(source: MetadataSource, known: MetaQuery) -> None: + """Two fetches of the same key agree (ignoring fetched_at / raw).""" + + def strip(records: tuple[MetaRecord, ...]) -> list[tuple[Any, ...]]: + return [ + (r.source_name, r.external_id, r.source_uri, sorted(r.cell.items()), sorted(r.test.items())) + for r in records + ] + + first, second = source.fetch(known), source.fetch(known) + assert strip(first) == strip(second), "fetch() is not deterministic for the same query" + + +def check_metadata_source( + source: MetadataSource, + *, + known: MetaQuery, + unknown: MetaQuery, +) -> tuple[MetaRecord, ...]: + """Run every conformance check. Returns the records for ``known``.""" + check_capabilities(source) + check_unknown_key_is_empty(source, unknown) + records = check_known_key_records(source, known) + check_deterministic(source, known) + return records + + +class DictMetadataSource: + """A source backed by a dict — for tests, notebooks and adapter fixtures. + + ``rows`` maps ``(kind, key)`` or just ``key`` to a ``MetaRecord`` (or a + tuple of them). Anything else → ``()``. + """ + + name = "dict" + + def __init__(self, rows: dict[Any, MetaRecord | tuple[MetaRecord, ...]], name: str = "dict") -> None: + self.rows = rows + self.name = name + self.queries: list[MetaQuery] = [] + + def fetch(self, query: MetaQuery) -> tuple[MetaRecord, ...]: + self.queries.append(query) + hit = self.rows.get((query.kind, query.key), self.rows.get(query.key)) + if hit is None: + return () + return tuple(hit) if isinstance(hit, tuple) else (hit,) diff --git a/docs/agents/index.md b/docs/agents/index.md index a7e9fdf3..1b25cfa9 100644 --- a/docs/agents/index.md +++ b/docs/agents/index.md @@ -184,6 +184,18 @@ Useful methods on `CellpyCell` (non-exhaustive): it calls `update()` each tick, runs `on_update(c)` when frames changed, and records the run on `c.poll_status`. Pass `sleep=` to drive it from your own scheduler. `Ctrl-C` stops it cleanly and returns the cell. +- `fetch_meta(source, key=None, kind="cell_name", apply=True, strict=False)` + — pull cell/test metadata (mass, area, nominal capacity, project, …) from + an external source such as a lab database. `source` is a registered name + (`from cellpy.readers.metadata_sources import names`); the BatBase adapter + comes with the `cellpy-connectors` package. Returns the matching + `MetaRecord`s and, with `apply=True`, writes the first one onto the cell the + way a journal row would (above the raw file, below explicit user values); + `c.external_links[source]` then holds the back-link (`external_id`, + `source_uri`, `fields`) and survives `save()`. An unreachable or unknown + source returns `()` and changes nothing (`strict=True` raises); a rejected + credential always raises `MetadataSourceAuthError`. Call + `refresh_after(("mass",))` afterwards if a summary already exists. - `save` / `to_csv` / Excel helpers — persist for the user's workflow Deeper shape docs: [Data structure](../fundamentals/data_structure.md). diff --git a/docs/api/readers.md b/docs/api/readers.md index 75524c8e..60c9abfc 100644 --- a/docs/api/readers.md +++ b/docs/api/readers.md @@ -12,3 +12,16 @@ The cell object itself (`CellpyCell`) has [its own page](cell.md). ::: cellpy.readers.provenance ::: cellpy.readers.journal_layer + +## External metadata sources + +Pluggable lab databases / APIs as a journal-level metadata layer (#784). +Adapters satisfy the `MetadataSource` Protocol and declare a +`cellpy.metadata_sources` entry point; `CellpyCell.fetch_meta` pulls a record +onto a cell. + +::: cellpy.readers.metadata_sources.contract + +::: cellpy.readers.metadata_sources.registry + +::: cellpy.readers.metadata_sources.testing diff --git a/tests/test_metadata_sources.py b/tests/test_metadata_sources.py new file mode 100644 index 00000000..b863226a --- /dev/null +++ b/tests/test_metadata_sources.py @@ -0,0 +1,426 @@ +"""External metadata sources (#784, Epic M / M1): contract, registry, resolver hook. + +The read path in one sentence: a source answers a `MetaQuery` with +`MetaRecord`s, `fetch_meta` degrades failures to an empty layer, the resolver +merges records into the journal/db layer below the journal row, and the cell +keeps an `ExternalLink` so the fetch is reproducible after save/load. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Any + +import pytest +from cellpycore.metadata.models import CellMeta, TestMeta + +from cellpy import log +from cellpy.readers import metadata_sources as ms +from cellpy.readers.cellpy_file import meta_archive +from cellpy.readers.meta_resolver import Layer, resolve_cell_meta, resolve_test_meta +from cellpy.readers.metadata_sources import ( + ExternalLink, + MetadataSource, + MetadataSourceAuthError, + MetadataSourceError, + MetaQuery, + MetaRecord, + SupportsMetadataPush, + UnknownMetadataSource, + fetch_meta, + validate_record, +) +from cellpy.readers.metadata_sources import registry as registry_module +from cellpy.readers.metadata_sources.testing import ( + DictMetadataSource, + check_metadata_source, +) + +log.setup_logging(default_level=logging.DEBUG, testing=True) + +RECORD = MetaRecord( + "labdb", + external_id="42", + source_uri="https://labdb.test/api/cell/42/", + cell={"mass": 1.23, "nom_cap": 3.5, "active_electrode_area": 1.767}, + test={"cell_name": "SAL_010", "test_family": "formation"}, +) + + +@pytest.fixture +def clean_registry(monkeypatch): + monkeypatch.setattr(registry_module, "_iter_entry_points", lambda: ()) + ms.clear_registry() + yield + ms.clear_registry() + + +@pytest.fixture +def labdb(clean_registry) -> DictMetadataSource: + source = DictMetadataSource({("cell_name", "SAL_010"): RECORD, ("tag", "SAL_010"): RECORD}, name="labdb") + ms.register(source) + return source + + +# -- contract ------------------------------------------------------------------ + + +@pytest.mark.essential +def test_protocol_is_structural(): + class Source: + name = "x" + + def fetch(self, query): + return () + + class Pusher(Source): + def register(self, record): + return "id" + + assert isinstance(Source(), MetadataSource) + assert not isinstance(Source(), SupportsMetadataPush) + assert isinstance(Pusher(), SupportsMetadataPush) + assert not isinstance(object(), MetadataSource) + + +def test_record_fields_and_link(): + assert RECORD.fields == ("active_electrode_area", "cell_name", "mass", "nom_cap", "test_family") + link = RECORD.link() + assert link.source_name == "labdb" and link.external_id == "42" + assert ExternalLink.from_dict(link.to_dict()) == link + assert MetaRecord("x").is_empty() + + +def test_query_describe_and_extra_copy(): + extra = {"b": 2, "a": 1} + q = MetaQuery(key="k", kind="tag", project="P", extra=extra) + extra["c"] = 3 + assert "c" not in q.extra + assert q.describe() == "tag='k', project='P', a=1, b=2" + + +@pytest.mark.parametrize( + ("record", "fragment"), + [ + (MetaRecord("x", cell={"mas": 1.0}), "not metadata fields"), + (MetaRecord("x", test={"uuid": "abc"}), "provenance"), + (MetaRecord("x", cell={"mass": None}), "None"), + (MetaRecord("", cell={"mass": 1.0}), "source_name"), + ("not a record", "MetaRecord instances"), + ], +) +def test_validate_record_rejects_broken_promises(record, fragment): + with pytest.raises(MetadataSourceError, match=fragment): + validate_record(record) + + +def test_validate_record_accepts_real_fields(): + assert validate_record(RECORD) is RECORD + + +# -- registry ------------------------------------------------------------------ + + +def test_register_and_get_source(labdb): + assert ms.names() == ("labdb",) + assert ms.get_source("labdb") is labdb + + +def test_unknown_source_names_known_ones(labdb): + with pytest.raises(UnknownMetadataSource, match="known: labdb"): + ms.get_source("nope") + + +def test_register_rejects_class_without_name(clean_registry): + class Nameless: + def fetch(self, query): + return () + + with pytest.raises(MetadataSourceError, match="name"): + ms.register(Nameless) + + +def test_register_rejects_class_without_fetch(clean_registry): + class NoFetch: + name = "nofetch" + + with pytest.raises(MetadataSourceError, match="fetch"): + ms.register(NoFetch) + + +def test_class_registration_instantiates_once(clean_registry): + made = [] + + class Source: + name = "counted" + + def __init__(self): + made.append(self) + + def fetch(self, query): + return () + + ms.register(Source) + assert ms.get_source("counted") is ms.get_source("counted") + assert len(made) == 1 + + +def test_entry_point_discovery_skips_broken_plugins(monkeypatch, caplog): + class Good: + name = "good" + + def fetch(self, query): + return () + + class EP: + def __init__(self, name, value, load): + self.name, self.value, self._load = name, value, load + + def load(self): + return self._load() + + def boom(): + raise ImportError("no such module") + + class BadContract: + name = "bad" + + eps = ( + EP("good", "pkg:Good", lambda: Good), + EP("broken", "pkg:Broken", boom), + EP("bad", "pkg:Bad", lambda: BadContract), + ) + monkeypatch.setattr(registry_module, "_iter_entry_points", lambda: eps) + ms.clear_registry() + try: + with caplog.at_level(logging.WARNING): + assert ms.names() == ("good",) + assert "could not load metadata source 'broken'" in caplog.text + assert "does not satisfy the MetadataSource contract" in caplog.text + finally: + ms.clear_registry() + + +# -- fetch_meta null object ---------------------------------------------------- + + +@pytest.mark.essential +def test_fetch_meta_returns_validated_records_with_timestamp(labdb): + (record,) = fetch_meta("labdb", "SAL_010") + assert record.cell["mass"] == 1.23 + assert record.fetched_at is not None + assert labdb.queries[-1] == MetaQuery(key="SAL_010") + + +def test_fetch_meta_unknown_key_is_empty(labdb): + assert fetch_meta("labdb", MetaQuery(key="nope", kind="tag")) == () + + +@pytest.mark.essential +def test_fetch_meta_unknown_source_is_empty_layer_unless_strict(clean_registry, caplog): + with caplog.at_level(logging.WARNING): + assert fetch_meta("ghost", "SAL_010") == () + assert "unavailable" in caplog.text + with pytest.raises(UnknownMetadataSource): + fetch_meta("ghost", "SAL_010", strict=True) + + +def test_fetch_meta_unreachable_source_is_empty_layer(clean_registry, caplog): + class Offline: + name = "offline" + + def fetch(self, query): + raise ConnectionError("network is down") + + ms.register(Offline) + with caplog.at_level(logging.WARNING): + assert fetch_meta("offline", "SAL_010") == () + assert "network is down" in caplog.text + with pytest.raises(MetadataSourceError, match="network is down"): + fetch_meta("offline", "SAL_010", strict=True) + + +@pytest.mark.essential +def test_fetch_meta_auth_error_is_never_swallowed(clean_registry): + class Locked: + name = "locked" + + def fetch(self, query): + raise MetadataSourceAuthError("token rejected") + + ms.register(Locked) + with pytest.raises(MetadataSourceAuthError, match="token rejected"): + fetch_meta("locked", "SAL_010") + + +def test_fetch_meta_drops_invalid_records_unless_strict(clean_registry, caplog): + bad = MetaRecord("sloppy", cell={"mas": 1.0}) + ms.register(DictMetadataSource({"k": (bad, MetaRecord("sloppy", cell={"mass": 2.0}))}, name="sloppy")) + with caplog.at_level(logging.WARNING): + records = fetch_meta("sloppy", "k") + assert [r.cell for r in records] == [{"mass": 2.0}] + assert "dropping the record" in caplog.text + with pytest.raises(MetadataSourceError, match="not metadata fields"): + fetch_meta("sloppy", "k", strict=True) + + +def test_fetch_meta_accepts_source_object(clean_registry): + source = DictMetadataSource({"k": RECORD}, name="direct") + assert fetch_meta(source, "k")[0].external_id == "42" + + +# -- resolver hook ------------------------------------------------------------- + + +@pytest.mark.essential +def test_external_beats_raw_file_and_defaults_but_not_journal_or_kwargs(): + meta, res = resolve_cell_meta( + CellMeta(), + kwargs={"nom_cap": 9.0}, + journal={"mass": 2.0}, + external=RECORD, # mass 1.23, nom_cap 3.5, area 1.767 + draft=CellMeta(mass=3.0, nom_cap=1.0, active_electrode_area=0.5), + config_defaults={"mass": 4.0}, + ) + assert (meta.mass, meta.nom_cap, meta.active_electrode_area) == (2.0, 9.0, 1.767) + assert res.source_of("mass") is Layer.JOURNAL and res.origin_of("mass") == "journal" + assert res.source_of("nom_cap") is Layer.KWARGS and res.origin_of("nom_cap") is None + assert res.source_of("active_electrode_area") is Layer.JOURNAL + assert res.origin_of("active_electrode_area") == "labdb" + assert res.fields_from_origin("labdb") == ("active_electrode_area",) + assert "active_electrode_area: journal/db (labdb)" in res.explain() + + +def test_external_sources_are_applied_in_priority_order(): + first = MetaRecord("primary", cell={"mass": 1.0}) + second = MetaRecord("secondary", cell={"mass": 2.0, "nom_cap": 5.0}) + meta, res = resolve_cell_meta(CellMeta(), external=[first, second]) + assert meta.mass == 1.0 and meta.nom_cap == 5.0 + assert res.origin_of("mass") == "primary" and res.origin_of("nom_cap") == "secondary" + + +def test_external_test_fields_reach_test_meta_only(): + cell, _ = resolve_cell_meta(CellMeta(), external=RECORD) + test, res = resolve_test_meta(TestMeta(), external=RECORD) + assert test.cell_name == "SAL_010" and test.test_family == "formation" + assert not hasattr(cell, "cell_name") + assert res.origin_of("cell_name") == "labdb" + + +def test_external_accepts_name_mapping_pairs_and_rejects_junk(): + meta, res = resolve_cell_meta(CellMeta(), external=[("sheet", {"mass": 7.0})]) + assert meta.mass == 7.0 and res.origin_of("mass") == "sheet" + with pytest.raises(TypeError): + resolve_cell_meta(CellMeta(), external=[42]) + + +def test_no_external_keeps_old_provenance_shape(): + _, res = resolve_cell_meta(CellMeta(), journal={"mass": 2.0}) + assert res.origins == {} + assert res.explain() == "resolved metadata:\n mass: journal/db" + + +# -- conformance kit ----------------------------------------------------------- + + +def test_conformance_kit_passes_for_dict_source(): + source = DictMetadataSource({("tag", "SAL_010"): RECORD}, name="labdb") + records = check_metadata_source( + source, + known=MetaQuery(key="SAL_010", kind="tag"), + unknown=MetaQuery(key="nope", kind="tag"), + ) + assert records == (RECORD,) + + +def test_conformance_kit_names_the_broken_promise(): + class RaisesOnUnknown: + name = "raiser" + + def fetch(self, query): + if query.key == "nope": + raise KeyError(query.key) + return (MetaRecord("raiser", cell={"mass": 1.0}),) + + with pytest.raises(AssertionError, match="unknown key"): + check_metadata_source(RaisesOnUnknown(), known=MetaQuery(key="k"), unknown=MetaQuery(key="nope")) + + class WrongName: + name = "right" + + def fetch(self, query): + return () if query.key == "nope" else (MetaRecord("wrong", cell={"mass": 1.0}),) + + with pytest.raises(AssertionError, match="source_name"): + check_metadata_source(WrongName(), known=MetaQuery(key="k"), unknown=MetaQuery(key="nope")) + + +# -- CellpyCell surface -------------------------------------------------------- + + +@pytest.mark.essential +def test_cell_fetch_meta_applies_record_and_links(cell, labdb): + cell.cell_name = "SAL_010" + before = cell.data.meta_common.nom_cap + records = cell.fetch_meta("labdb") + assert len(records) == 1 + assert cell.data.meta_common.mass == pytest.approx(1.23) + assert cell.data.meta_common.nom_cap == pytest.approx(3.5) + assert cell.data.meta_common.nom_cap != before or before == 3.5 + assert cell.data.meta_common.active_electrode_area == pytest.approx(1.767) + link = cell.external_links["labdb"] + assert isinstance(link, ExternalLink) + assert link.external_id == "42" and link.source_uri.endswith("/cell/42/") + assert "mass" in link.fields and "cell_name" in link.fields + assert labdb.queries[-1].key == "SAL_010" + + +def test_cell_fetch_meta_no_match_changes_nothing(cell, labdb): + mass = cell.data.meta_common.mass + assert cell.fetch_meta("labdb", "unknown-cell") == () + assert cell.data.meta_common.mass == mass + assert cell.external_links == {} + + +def test_cell_fetch_meta_apply_false_only_returns(cell, labdb): + mass = cell.data.meta_common.mass + records = cell.fetch_meta("labdb", "SAL_010", apply=False) + assert records and cell.data.meta_common.mass == mass + assert cell.external_links == {} + + +def test_cell_fetch_meta_unknown_source_is_soft_unless_strict(cell, clean_registry): + assert cell.fetch_meta("ghost", "SAL_010") == () + with pytest.raises(UnknownMetadataSource): + cell.fetch_meta("ghost", "SAL_010", strict=True) + + +def test_cell_fetch_meta_passes_kind_project_and_extra(cell, labdb): + cell.fetch_meta("labdb", "SAL_010", kind="tag", project="LongLife", include_batch=True) + q = labdb.queries[-1] + assert (q.kind, q.project, q.extra) == ("tag", "LongLife", {"include_batch": True}) + + +def test_external_links_survive_v9_meta_document(cell, labdb): + cell.fetch_meta("labdb", "SAL_010") + doc = meta_archive.build_meta_document(cell.data) + assert doc["external_links"]["labdb"]["external_id"] == "42" + + from cellpy.readers.data_structures import Data + + fresh = Data() + meta_archive.apply_meta_document(fresh, doc) + assert fresh.external_links["labdb"] == cell.external_links["labdb"] + + +def test_external_links_survive_save_and_load(cell, labdb, tmp_path): + cell.fetch_meta("labdb", "SAL_010") + path = tmp_path / "linked.cellpy" + cell.save(path) + + import cellpy + + loaded = cellpy.get(path) + assert loaded.external_links["labdb"].external_id == "42" + assert loaded.data.meta_common.mass == pytest.approx(1.23)