From 003d49194416354532823747d62b2b5b423c71dd Mon Sep 17 00:00:00 2001 From: John Walsh Date: Fri, 10 Apr 2026 09:42:55 -0400 Subject: [PATCH 1/4] add product parsing --- pyproject.toml | 1 + src/ifcbkit/__init__.py | 4 ++ src/ifcbkit/products.py | 143 ++++++++++++++++++++++++++++++++++------ tests/test_products.py | 104 +++++++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 19 deletions(-) create mode 100644 tests/test_products.py diff --git a/pyproject.toml b/pyproject.toml index 5aa6b88..7d29f41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ ] [project.optional-dependencies] +hdf5 = ["h5py"] s3 = ["amplify-storage-utils[s3] @ git+https://github.com/WHOIGit/amplify-storage-utils.git@v1.7.0"] dev = ["pytest", "pytest-asyncio"] diff --git a/src/ifcbkit/__init__.py b/src/ifcbkit/__init__.py index 8eaae19..a679cf4 100644 --- a/src/ifcbkit/__init__.py +++ b/src/ifcbkit/__init__.py @@ -62,11 +62,15 @@ async_blob_path, async_class_scores_path, async_features_path, + read_blobs, + read_class_scores, + read_features, sync_find_product_file, sync_list_product_files, sync_blob_path, sync_class_scores_path, sync_features_path, + ClassScoresRows, ) # Store base classes (always available) diff --git a/src/ifcbkit/products.py b/src/ifcbkit/products.py index 097ecad..05acbbb 100644 --- a/src/ifcbkit/products.py +++ b/src/ifcbkit/products.py @@ -1,22 +1,60 @@ """ -Product file discovery for IFCB derived data (blobs, features, class scores). +Product file discovery and parsing for IFCB derived data. Helpers for locating derived product files alongside raw data: - find_product_file: recursive search - list_product_files: pattern-matched listing - Convenience: blob_path, class_scores_path, features_path +- Readers for blobs, features, and v3 class scores -Both sync and async APIs are provided. +Both sync and async discovery APIs are provided. Parsing helpers are sync-only. """ +from __future__ import annotations + +import csv import os import re +from dataclasses import dataclass +from pathlib import Path +from zipfile import ZipFile import aiofiles.os as aios import aiofiles.ospath as aiopath -# --- Async API --- +@dataclass(slots=True) +class ClassScoresRows: + """Class score rows for one bin.""" + + class_names: list[str] + rows: list[tuple[int, dict[str, float]]] + + +def _parse_scalar(value: str) -> int | float: + try: + return int(value) + except ValueError: + return float(value) + + +def _require_fieldnames(fieldnames: list[str] | None, context: str) -> list[str]: + if not fieldnames: + raise ValueError(f"{context} is missing a header row.") + return fieldnames + + +def _resolve_roi_field(fieldnames: list[str], context: str) -> str: + for candidate in ("roi_number", "roiNumber"): + if candidate in fieldnames: + return candidate + raise ValueError( + f"{context} is missing an ROI number column. Expected one of: " + f"roi_number, roiNumber." + ) + + +# --- Async discovery API --- async def async_find_product_file(directory, filename, exhaustive=False): """Recursively search for a product file by name. @@ -74,29 +112,29 @@ async def async_product_path(directory, filename, exhaustive=False): """Find a product file or raise FileNotFoundError.""" path = await async_find_product_file(directory, filename, exhaustive=exhaustive) if not path: - raise FileNotFoundError(f'Product file {filename} not found in {directory}') + raise FileNotFoundError(f"Product file {filename} not found in {directory}") return path async def async_blob_path(directory, pid, version=4): """Find the blob ZIP file for a given PID.""" - filename = f'{pid}_blobs_v{version}.zip' + filename = f"{pid}_blobs_v{version}.zip" return await async_product_path(directory, filename) -async def async_class_scores_path(directory, pid, version=4): - """Find the class scores CSV file for a given PID.""" - filename = f'{pid}.csv' +async def async_class_scores_path(directory, pid): + """Find the v3 class scores HDF5 file for a given PID.""" + filename = f"{pid}_class.h5" return await async_product_path(directory, filename) async def async_features_path(directory, pid, version=4): - """Find the features ZIP file for a given PID.""" - filename = f'{pid}_features_v{version}.zip' + """Find the features CSV file for a given PID.""" + filename = f"{pid}_fea_v{version}.csv" return await async_product_path(directory, filename) -# --- Sync API --- +# --- Sync discovery API --- def sync_find_product_file(directory, filename, exhaustive=False): """Recursively search for a product file by name (synchronous). @@ -144,7 +182,7 @@ def sync_list_product_files(directory, regex): for name in names: path = os.path.join(directory, name) if os.path.isdir(path): - yield from sync_list_product_files(path, regex) + yield from sync_list_product_files(directory=path, regex=regex) elif re.match(regex, name): yield path @@ -153,23 +191,90 @@ def sync_product_path(directory, filename, exhaustive=False): """Find a product file or raise FileNotFoundError (synchronous).""" path = sync_find_product_file(directory, filename, exhaustive=exhaustive) if not path: - raise FileNotFoundError(f'Product file {filename} not found in {directory}') + raise FileNotFoundError(f"Product file {filename} not found in {directory}") return path def sync_blob_path(directory, pid, version=4): """Find the blob ZIP file for a given PID (synchronous).""" - filename = f'{pid}_blobs_v{version}.zip' + filename = f"{pid}_blobs_v{version}.zip" return sync_product_path(directory, filename) -def sync_class_scores_path(directory, pid, version=4): - """Find the class scores CSV file for a given PID (synchronous).""" - filename = f'{pid}.csv' +def sync_class_scores_path(directory, pid): + """Find the v3 class scores HDF5 file for a given PID (synchronous).""" + filename = f"{pid}_class.h5" return sync_product_path(directory, filename) def sync_features_path(directory, pid, version=4): - """Find the features ZIP file for a given PID (synchronous).""" - filename = f'{pid}_features_v{version}.zip' + """Find the features CSV file for a given PID (synchronous).""" + filename = f"{pid}_fea_v{version}.csv" return sync_product_path(directory, filename) + + +# --- Sync parsing API --- + +def read_blobs(path: str | os.PathLike[str]): + """Yield ``(roi_id, png_bytes)`` for every blob in the archive.""" + with ZipFile(path) as zf: + for name in zf.namelist(): + if not name.lower().endswith(".png"): + continue + roi_id = Path(name).name.removesuffix(".png") + yield roi_id, zf.read(name) + + +def read_features( + path: str | os.PathLike[str], +) -> list[tuple[int, dict[str, int | float]]]: + """Read IFCB features CSV rows as ``(roi_number, feature_dict)`` pairs.""" + rows: list[tuple[int, dict[str, int | float]]] = [] + with open(path, newline="") as handle: + reader = csv.DictReader(handle) + fieldnames = _require_fieldnames(reader.fieldnames, f"Features file {path}") + roi_field = _resolve_roi_field(fieldnames, f"Features file {path}") + + for row in reader: + roi_number_raw = row.get(roi_field) + if roi_number_raw in (None, ""): + raise ValueError(f"Features file {path} contains a row with no ROI number.") + + values: dict[str, int | float] = {} + for key, value in row.items(): + if key == roi_field or value in (None, ""): + continue + values[key] = _parse_scalar(value) + rows.append((int(roi_number_raw), values)) + return rows + + +def read_class_scores( + path: str | os.PathLike[str], +) -> ClassScoresRows: + """Read a v3 IFCB class scores HDF5 file.""" + try: + import h5py + except ImportError as exc: + raise ImportError( + "read_class_scores requires h5py; install with `pip install ifcbkit[hdf5]`" + ) from exc + + with h5py.File(path, "r") as handle: + scores = handle["output_scores"][:] + class_names = [label.decode("ascii") for label in handle["class_labels"][:]] + roi_numbers = handle["roi_numbers"][:] + + rows: list[tuple[int, dict[str, float]]] = [] + for roi_number, score_row in zip(roi_numbers, scores, strict=True): + rows.append( + ( + int(roi_number), + { + class_name: float(score) + for class_name, score in zip(class_names, score_row, strict=True) + }, + ) + ) + + return ClassScoresRows(class_names=class_names, rows=rows) diff --git a/tests/test_products.py b/tests/test_products.py new file mode 100644 index 0000000..61ecb04 --- /dev/null +++ b/tests/test_products.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from zipfile import ZipFile + +import pytest + +from ifcbkit import ( + ClassScoresRows, + read_blobs, + read_class_scores, + read_features, + sync_blob_path, + sync_class_scores_path, + sync_features_path, +) + + +def test_sync_features_path_uses_real_filename(tmp_path): + path = tmp_path / "D20250101T020023_IFCB010_fea_v4.csv" + path.write_text("roi_number,Area\n1,2\n") + + found = sync_features_path(tmp_path, "D20250101T020023_IFCB010") + + assert found == str(path) + + +def test_sync_class_scores_path_uses_real_filename(tmp_path): + path = tmp_path / "D20250101T020023_IFCB010_class.h5" + path.write_bytes(b"") + + found = sync_class_scores_path(tmp_path, "D20250101T020023_IFCB010") + + assert found == str(path) + + +def test_sync_blob_path_uses_real_filename(tmp_path): + path = tmp_path / "D20250101T020023_IFCB010_blobs_v4.zip" + path.write_bytes(b"") + + found = sync_blob_path(tmp_path, "D20250101T020023_IFCB010") + + assert found == str(path) + + +def test_read_features_returns_roi_number_and_scalar_dict(tmp_path): + path = tmp_path / "features.csv" + path.write_text( + "roi_number,Area,Count,MajorAxisLength\n" + "2,1.5,3,4.25\n" + "5,2.0,4,6.5\n" + ) + + rows = read_features(path) + + assert rows == [ + (2, {"Area": 1.5, "Count": 3, "MajorAxisLength": 4.25}), + (5, {"Area": 2.0, "Count": 4, "MajorAxisLength": 6.5}), + ] + + +def test_read_features_accepts_roi_number_camel_case(tmp_path): + path = tmp_path / "features.csv" + path.write_text( + "roiNumber,Area,Count\n" + "7,1.5,3\n" + ) + + rows = read_features(path) + + assert rows == [(7, {"Area": 1.5, "Count": 3})] + + +def test_read_blobs_yields_roi_id_and_png_bytes(tmp_path): + path = tmp_path / "blobs.zip" + with ZipFile(path, "w") as zf: + zf.writestr("D20250101T020023_IFCB010_00002.png", b"png-a") + zf.writestr("nested/D20250101T020023_IFCB010_00003.png", b"png-b") + zf.writestr("ignore.txt", b"nope") + + rows = list(read_blobs(path)) + + assert rows == [ + ("D20250101T020023_IFCB010_00002", b"png-a"), + ("D20250101T020023_IFCB010_00003", b"png-b"), + ] + + +def test_read_class_scores_returns_class_names_and_rows(tmp_path): + h5py = pytest.importorskip("h5py") + + path = tmp_path / "scores.h5" + with h5py.File(path, "w") as handle: + handle.create_dataset("output_scores", data=[[0.1, 0.9], [0.8, 0.2]]) + handle.create_dataset("class_labels", data=[b"detritus", b"diatom"]) + handle.create_dataset("roi_numbers", data=[2, 5]) + + result = read_class_scores(path) + + assert isinstance(result, ClassScoresRows) + assert result.class_names == ["detritus", "diatom"] + assert result.rows == [ + (2, {"detritus": 0.1, "diatom": 0.9}), + (5, {"detritus": 0.8, "diatom": 0.2}), + ] From 804c8c86927cc324ac9d9a023b13bd279ada183b Mon Sep 17 00:00:00 2001 From: John Walsh Date: Fri, 10 Apr 2026 09:45:42 -0400 Subject: [PATCH 2/4] remove async parsing --- src/ifcbkit/__init__.py | 5 -- src/ifcbkit/products.py | 101 ++++------------------------------------ 2 files changed, 9 insertions(+), 97 deletions(-) diff --git a/src/ifcbkit/__init__.py b/src/ifcbkit/__init__.py index a679cf4..9a7de35 100644 --- a/src/ifcbkit/__init__.py +++ b/src/ifcbkit/__init__.py @@ -57,11 +57,6 @@ # Products from .products import ( - async_find_product_file, - async_list_product_files, - async_blob_path, - async_class_scores_path, - async_features_path, read_blobs, read_class_scores, read_features, diff --git a/src/ifcbkit/products.py b/src/ifcbkit/products.py index 05acbbb..1f35c42 100644 --- a/src/ifcbkit/products.py +++ b/src/ifcbkit/products.py @@ -7,7 +7,7 @@ - Convenience: blob_path, class_scores_path, features_path - Readers for blobs, features, and v3 class scores -Both sync and async discovery APIs are provided. Parsing helpers are sync-only. +All APIs in this module are synchronous. """ from __future__ import annotations @@ -19,9 +19,6 @@ from pathlib import Path from zipfile import ZipFile -import aiofiles.os as aios -import aiofiles.ospath as aiopath - @dataclass(slots=True) class ClassScoresRows: @@ -54,90 +51,10 @@ def _resolve_roi_field(fieldnames: list[str], context: str) -> str: ) -# --- Async discovery API --- - -async def async_find_product_file(directory, filename, exhaustive=False): - """Recursively search for a product file by name. - - :param directory: root directory to search - :param filename: the filename to find - :param exhaustive: if False, only recurse into directories whose name - appears in the filename (faster). If True, search all subdirs. - :returns: full path to the file, or None - """ - candidate = os.path.join(directory, filename) - if await aiopath.exists(candidate): - return candidate - - try: - names = await aios.listdir(directory) - except FileNotFoundError: - return None - - for name in names: - path = os.path.join(directory, name) - if await aiopath.isdir(path): - if not exhaustive and name not in filename: - continue - result = await async_find_product_file(path, filename, exhaustive=exhaustive) - if result is not None: - return result - elif name == filename: - return path - - return None - - -async def async_list_product_files(directory, regex): - """Async generator yielding paths to product files matching a regex. - - :param directory: root directory to search - :param regex: regex pattern to match filenames against - """ - try: - names = await aios.listdir(directory) - except FileNotFoundError: - return - - for name in names: - path = os.path.join(directory, name) - if await aiopath.isdir(path): - async for p in async_list_product_files(path, regex): - yield p - elif re.match(regex, name): - yield path - - -async def async_product_path(directory, filename, exhaustive=False): - """Find a product file or raise FileNotFoundError.""" - path = await async_find_product_file(directory, filename, exhaustive=exhaustive) - if not path: - raise FileNotFoundError(f"Product file {filename} not found in {directory}") - return path - - -async def async_blob_path(directory, pid, version=4): - """Find the blob ZIP file for a given PID.""" - filename = f"{pid}_blobs_v{version}.zip" - return await async_product_path(directory, filename) - - -async def async_class_scores_path(directory, pid): - """Find the v3 class scores HDF5 file for a given PID.""" - filename = f"{pid}_class.h5" - return await async_product_path(directory, filename) - - -async def async_features_path(directory, pid, version=4): - """Find the features CSV file for a given PID.""" - filename = f"{pid}_fea_v{version}.csv" - return await async_product_path(directory, filename) - - -# --- Sync discovery API --- +# --- Discovery API --- def sync_find_product_file(directory, filename, exhaustive=False): - """Recursively search for a product file by name (synchronous). + """Recursively search for a product file by name. :param directory: root directory to search :param filename: the filename to find @@ -169,7 +86,7 @@ def sync_find_product_file(directory, filename, exhaustive=False): def sync_list_product_files(directory, regex): - """Generator yielding paths to product files matching a regex (synchronous). + """Generator yielding paths to product files matching a regex. :param directory: root directory to search :param regex: regex pattern to match filenames against @@ -188,7 +105,7 @@ def sync_list_product_files(directory, regex): def sync_product_path(directory, filename, exhaustive=False): - """Find a product file or raise FileNotFoundError (synchronous).""" + """Find a product file or raise FileNotFoundError.""" path = sync_find_product_file(directory, filename, exhaustive=exhaustive) if not path: raise FileNotFoundError(f"Product file {filename} not found in {directory}") @@ -196,24 +113,24 @@ def sync_product_path(directory, filename, exhaustive=False): def sync_blob_path(directory, pid, version=4): - """Find the blob ZIP file for a given PID (synchronous).""" + """Find the blob ZIP file for a given PID.""" filename = f"{pid}_blobs_v{version}.zip" return sync_product_path(directory, filename) def sync_class_scores_path(directory, pid): - """Find the v3 class scores HDF5 file for a given PID (synchronous).""" + """Find the v3 class scores HDF5 file for a given PID.""" filename = f"{pid}_class.h5" return sync_product_path(directory, filename) def sync_features_path(directory, pid, version=4): - """Find the features CSV file for a given PID (synchronous).""" + """Find the features CSV file for a given PID.""" filename = f"{pid}_fea_v{version}.csv" return sync_product_path(directory, filename) -# --- Sync parsing API --- +# --- Parsing API --- def read_blobs(path: str | os.PathLike[str]): """Yield ``(roi_id, png_bytes)`` for every blob in the archive.""" From 08cb448ae79fea0dc13ff5aad166a9ef89849b0f Mon Sep 17 00:00:00 2001 From: John Walsh Date: Fri, 10 Apr 2026 09:48:55 -0400 Subject: [PATCH 3/4] re-add async helpers --- src/ifcbkit/__init__.py | 5 +++ src/ifcbkit/products.py | 87 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/src/ifcbkit/__init__.py b/src/ifcbkit/__init__.py index 9a7de35..a679cf4 100644 --- a/src/ifcbkit/__init__.py +++ b/src/ifcbkit/__init__.py @@ -57,6 +57,11 @@ # Products from .products import ( + async_find_product_file, + async_list_product_files, + async_blob_path, + async_class_scores_path, + async_features_path, read_blobs, read_class_scores, read_features, diff --git a/src/ifcbkit/products.py b/src/ifcbkit/products.py index 1f35c42..35d3db4 100644 --- a/src/ifcbkit/products.py +++ b/src/ifcbkit/products.py @@ -7,7 +7,7 @@ - Convenience: blob_path, class_scores_path, features_path - Readers for blobs, features, and v3 class scores -All APIs in this module are synchronous. +Both sync and async discovery APIs are provided. Readers are sync-only. """ from __future__ import annotations @@ -19,6 +19,9 @@ from pathlib import Path from zipfile import ZipFile +import aiofiles.os as aios +import aiofiles.ospath as aiopath + @dataclass(slots=True) class ClassScoresRows: @@ -51,7 +54,87 @@ def _resolve_roi_field(fieldnames: list[str], context: str) -> str: ) -# --- Discovery API --- +# --- Async discovery API --- + +async def async_find_product_file(directory, filename, exhaustive=False): + """Recursively search for a product file by name. + + :param directory: root directory to search + :param filename: the filename to find + :param exhaustive: if False, only recurse into directories whose name + appears in the filename (faster). If True, search all subdirs. + :returns: full path to the file, or None + """ + candidate = os.path.join(directory, filename) + if await aiopath.exists(candidate): + return candidate + + try: + names = await aios.listdir(directory) + except FileNotFoundError: + return None + + for name in names: + path = os.path.join(directory, name) + if await aiopath.isdir(path): + if not exhaustive and name not in filename: + continue + result = await async_find_product_file(path, filename, exhaustive=exhaustive) + if result is not None: + return result + elif name == filename: + return path + + return None + + +async def async_list_product_files(directory, regex): + """Async generator yielding paths to product files matching a regex. + + :param directory: root directory to search + :param regex: regex pattern to match filenames against + """ + try: + names = await aios.listdir(directory) + except FileNotFoundError: + return + + for name in names: + path = os.path.join(directory, name) + if await aiopath.isdir(path): + async for p in async_list_product_files(path, regex): + yield p + elif re.match(regex, name): + yield path + + +async def async_product_path(directory, filename, exhaustive=False): + """Find a product file or raise FileNotFoundError.""" + path = await async_find_product_file(directory, filename, exhaustive=exhaustive) + if not path: + raise FileNotFoundError(f"Product file {filename} not found in {directory}") + return path + + +async def async_blob_path(directory, pid, version=4): + """Find the blob ZIP file for a given PID.""" + filename = f"{pid}_blobs_v{version}.zip" + return await async_product_path(directory, filename) + + +async def async_class_scores_path(directory, pid): + """Find the v3 class scores HDF5 file for a given PID.""" + filename = f"{pid}_class.h5" + return await async_product_path(directory, filename) + + +async def async_features_path(directory, pid, version=4): + """Find the features CSV file for a given PID.""" + filename = f"{pid}_fea_v{version}.csv" + return await async_product_path(directory, filename) + + +# --- Sync discovery API --- def sync_find_product_file(directory, filename, exhaustive=False): """Recursively search for a product file by name. From f1e3ffdc5cee6671dc23d149d71b9c9f8696f724 Mon Sep 17 00:00:00 2001 From: John Walsh Date: Fri, 10 Apr 2026 10:02:07 -0400 Subject: [PATCH 4/4] remove annotation evaluation deferral --- src/ifcbkit/products.py | 2 -- tests/test_products.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/ifcbkit/products.py b/src/ifcbkit/products.py index 35d3db4..ec8d7fd 100644 --- a/src/ifcbkit/products.py +++ b/src/ifcbkit/products.py @@ -10,8 +10,6 @@ Both sync and async discovery APIs are provided. Readers are sync-only. """ -from __future__ import annotations - import csv import os import re diff --git a/tests/test_products.py b/tests/test_products.py index 61ecb04..f57d432 100644 --- a/tests/test_products.py +++ b/tests/test_products.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from zipfile import ZipFile import pytest