diff --git a/README.md b/README.md index a75fd48..ab57a35 100644 --- a/README.md +++ b/README.md @@ -150,3 +150,9 @@ adc = parse_adc_file(bin_id, '/path/to/bin.adc', extended=True) ``` Low-level stitching functions (`detect_pairs`, `stitch_pair`, `infill_stitched_image`) and raw extraction utilities (`extract_roi_images`, `extract_roi_image`) are available for specialized use cases. + +## Note: corrected ADC files (`adcmod`) + +Some datasets (e.g. MVCO) keep corrected ADC files outside the raw data directory so the raw data stays untouched. These live in an `adcmod` directory that is strictly a sibling of the raw data root, laid out as `adcmod//.adc.mod`, where `` is the name of the directory containing the raw fileset. The `.adc.mod` format is byte-compatible with `.adc`. + +`ifcbkit` resolves these transparently: when listing or fetching a fileset it uses the corrected ADC in place of the raw `.adc` if one exists. Only the ADC file is substituted — `.hdr` and `.roi` always come from the raw data directory — and a raw `.adc` must still be present for the bin to be discovered. diff --git a/src/ifcbkit/fileset.py b/src/ifcbkit/fileset.py index 9222c2f..3b911c6 100644 --- a/src/ifcbkit/fileset.py +++ b/src/ifcbkit/fileset.py @@ -20,6 +20,45 @@ DEFAULT_EXCLUDE = ['skip', 'beads'] DEFAULT_INCLUDE = ['data'] +# Corrected ("modified") ADC files live in a directory named ``adcmod`` that is +# a sibling of the raw data root directory, laid out as +# ``adcmod//.adc.mod`` and byte-compatible with the raw ``.adc``. +# Most datasets have no such sibling. +ADCMOD_DIR = 'adcmod' +ADCMOD_EXT = '.adc.mod' + + +def _adcmod_path(fileset_dir, pid, root_path): + """Return the path a corrected ADC file would have for this fileset. + + The ``adcmod`` directory is strictly a sibling of ``root_path``. The day + subdirectory name is the fileset's own containing directory name. + + :param fileset_dir: directory containing the raw fileset + :param pid: the bin ID + :param root_path: the raw data root directory + """ + day = os.path.basename(os.path.normpath(fileset_dir)) + adcmod_root = os.path.join( + os.path.dirname(os.path.abspath(root_path)), ADCMOD_DIR) + return os.path.join(adcmod_root, day, pid + ADCMOD_EXT) + + +def sync_resolve_adc_path(fileset_dir, pid, root_path): + """Return a corrected ``.adc.mod`` path if present, else the raw ``.adc``.""" + cand = _adcmod_path(fileset_dir, pid, root_path) + if os.path.exists(cand): + return cand + return os.path.join(fileset_dir, pid + '.adc') + + +async def async_resolve_adc_path(fileset_dir, pid, root_path): + """Return a corrected ``.adc.mod`` path if present, else the raw ``.adc``.""" + cand = _adcmod_path(fileset_dir, pid, root_path) + if await aiopath.exists(cand): + return cand + return os.path.join(fileset_dir, pid + '.adc') + def validate_path( filepath, @@ -393,9 +432,12 @@ def paths(self, pid): exists, fs = self._exists(pid) if not exists: raise KeyError(pid) + adc = None + if self.require_adc: + adc = sync_resolve_adc_path(os.path.dirname(fs), os.path.basename(fs), self.root_path) return { 'hdr': fs + '.hdr', - 'adc': fs + '.adc' if self.require_adc else None, + 'adc': adc, 'roi': fs + '.roi' if self.require_roi else None, } @@ -409,7 +451,7 @@ def list(self): yield { 'pid': bn, 'hdr': os.path.join(dp, bn + '.hdr'), - 'adc': os.path.join(dp, bn + '.adc') if self.require_adc else None, + 'adc': sync_resolve_adc_path(dp, bn, self.root_path) if self.require_adc else None, 'roi': os.path.join(dp, bn + '.roi') if self.require_roi else None, } @@ -520,9 +562,12 @@ async def paths(self, pid): exists, fs = await self._exists(pid) if not exists: raise KeyError(pid) + adc = None + if self.require_adc: + adc = await async_resolve_adc_path(os.path.dirname(fs), os.path.basename(fs), self.root_path) return { 'hdr': fs + '.hdr', - 'adc': fs + '.adc' if self.require_adc else None, + 'adc': adc, 'roi': fs + '.roi' if self.require_roi else None, } @@ -533,10 +578,13 @@ async def list(self): exclude=self.exclude, include=self.include, require_adc=self.require_adc, require_roi=self.require_roi, ): + adc = None + if self.require_adc: + adc = await async_resolve_adc_path(dp, bn, self.root_path) yield { 'pid': bn, 'hdr': os.path.join(dp, bn + '.hdr'), - 'adc': os.path.join(dp, bn + '.adc') if self.require_adc else None, + 'adc': adc, 'roi': os.path.join(dp, bn + '.roi') if self.require_roi else None, } diff --git a/src/ifcbkit/stores/filesystem.py b/src/ifcbkit/stores/filesystem.py index 5286042..0739e35 100644 --- a/src/ifcbkit/stores/filesystem.py +++ b/src/ifcbkit/stores/filesystem.py @@ -2,13 +2,14 @@ Filesystem-backed stores for IFCB bin data and ROI images. """ +import os from io import BytesIO import aiofiles from ..identifiers import parse_roi_id from ..fileset import ( - async_find_fileset, + async_find_fileset, async_resolve_adc_path, SyncIfcbDataDirectory, AsyncIfcbDataDirectory, DEFAULT_INCLUDE, DEFAULT_EXCLUDE, ) @@ -41,6 +42,11 @@ async def _find_path(self, bin_id: str, ext: str) -> str | None: ) if basepath is None: return None + if ext == 'adc': + return await async_resolve_adc_path( + os.path.dirname(basepath), os.path.basename(basepath), + self.root_path, + ) return f"{basepath}.{ext}" async def exists(self, key: str) -> bool: diff --git a/tests/test_fileset.py b/tests/test_fileset.py new file mode 100644 index 0000000..851ad22 --- /dev/null +++ b/tests/test_fileset.py @@ -0,0 +1,173 @@ +"""Tests for fileset path resolution, including corrected (adcmod) ADC files.""" + +import asyncio +import os +import shutil + +import pytest + +from ifcbkit.fileset import SyncIfcbDataDirectory, AsyncIfcbDataDirectory + +PID = 'D20170426T164105_IFCB009' +DAY = 'D20170426' + +# Real I-style fixture (the actual adcmod use case). Its containing directory +# name is what the adcmod tree mirrors as the "day" subdirectory. +I_PID = 'IFCB5_2012_028_081515' +I_DAY = 'IFCB5_2012_028' +I_FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'data', I_PID) + + +def _copy_i_fileset(day_dir): + os.makedirs(day_dir, exist_ok=True) + for ext in ('hdr', 'adc', 'roi'): + shutil.copy( + os.path.join(I_FIXTURE_DIR, f'{I_PID}.{ext}'), + os.path.join(day_dir, f'{I_PID}.{ext}'), + ) + + +def _make_fileset(dirpath, pid): + os.makedirs(dirpath, exist_ok=True) + for ext in ('hdr', 'adc', 'roi'): + with open(os.path.join(dirpath, f'{pid}.{ext}'), 'w') as f: + f.write('') + + +def _make_adcmod(adcmod_root, day, pid, content='mod'): + day_dir = os.path.join(adcmod_root, day) + os.makedirs(day_dir, exist_ok=True) + path = os.path.join(day_dir, f'{pid}.adc.mod') + with open(path, 'w') as f: + f.write(content) + return path + + +def test_no_adcmod_uses_raw_adc(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + dd = SyncIfcbDataDirectory(str(root)) + assert dd.paths(PID)['adc'] == str(root / DAY / f'{PID}.adc') + + +def test_adcmod_sibling_of_data_dir(tmp_path): + # /data//.adc and /adcmod//.adc.mod + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + dd = SyncIfcbDataDirectory(str(root)) + assert dd.paths(PID)['adc'] == mod + + +def test_adcmod_with_year_level(tmp_path): + # /data/2017//... and /adcmod//.adc.mod + root = tmp_path / 'data' + _make_fileset(str(root / '2017' / DAY), PID) + mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + dd = SyncIfcbDataDirectory(str(root)) + assert dd.paths(PID)['adc'] == mod + + +def test_adcmod_appears_in_list(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + dd = SyncIfcbDataDirectory(str(root)) + entries = list(dd.list()) + assert len(entries) == 1 + assert entries[0]['adc'] == mod + + +def test_adcmod_inside_root_is_ignored(tmp_path): + # adcmod is strictly a sibling of root_path; one nested inside root (here a + # sibling of the intermediate year directory) must not be used. + root = tmp_path / 'data' + _make_fileset(str(root / '2017' / DAY), PID) + _make_adcmod(str(root / 'adcmod'), DAY, PID) + dd = SyncIfcbDataDirectory(str(root)) + assert dd.paths(PID)['adc'] == str(root / '2017' / DAY / f'{PID}.adc') + + +def test_adcmod_sibling_of_nonroot_ancestor_is_ignored(tmp_path): + # is /raw/data, so only /raw/adcmod counts -- an adcmod + # sibling of /raw's own parent must not be used. + root = tmp_path / 'raw' / 'data' + _make_fileset(str(root / DAY), PID) + _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + dd = SyncIfcbDataDirectory(str(root)) + assert dd.paths(PID)['adc'] == str(root / DAY / f'{PID}.adc') + + +def test_adcmod_sibling_of_root_with_trailing_slash(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + dd = SyncIfcbDataDirectory(str(root) + os.sep) + assert dd.paths(PID)['adc'] == mod + + +def test_async_adcmod_resolution(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + dd = AsyncIfcbDataDirectory(str(root)) + paths = asyncio.run(dd.paths(PID)) + assert paths['adc'] == mod + + +# --- I-style bins (the real adcmod use case) --- + +def test_istyle_adcmod_path_resolves(tmp_path): + root = tmp_path / 'data' + _copy_i_fileset(str(root / I_DAY)) + # corrected ADC is a byte-identical copy of the raw ADC + raw_adc = os.path.join(I_FIXTURE_DIR, f'{I_PID}.adc') + mod = _make_adcmod( + str(tmp_path / 'adcmod'), I_DAY, I_PID, + content=open(raw_adc).read(), + ) + dd = SyncIfcbDataDirectory(str(root)) + assert dd.paths(I_PID)['adc'] == mod + + +def test_istyle_read_images_through_adcmod(tmp_path): + # End-to-end: I-style stitching must work reading the corrected ADC. + root = tmp_path / 'data' + _copy_i_fileset(str(root / I_DAY)) + raw_adc = os.path.join(I_FIXTURE_DIR, f'{I_PID}.adc') + + # baseline: raw ADC (no adcmod sibling) + baseline = SyncIfcbDataDirectory(str(root)).read_images(I_PID) + assert len(baseline) > 0 + + # with a byte-identical corrected ADC present, results must match + _make_adcmod(str(tmp_path / 'adcmod'), I_DAY, I_PID, content=open(raw_adc).read()) + dd = SyncIfcbDataDirectory(str(root)) + assert dd.paths(I_PID)['adc'].endswith('.adc.mod') + corrected = dd.read_images(I_PID) + assert set(corrected.keys()) == set(baseline.keys()) + for t in baseline: + assert corrected[t].size == baseline[t].size + + +def test_istyle_corrected_adc_content_is_used(tmp_path): + # Prove the corrected ADC (not the raw one) drives the output: zero out + # one ROI's width in the .adc.mod so that target drops from the results. + root = tmp_path / 'data' + _copy_i_fileset(str(root / I_DAY)) + raw_adc = os.path.join(I_FIXTURE_DIR, f'{I_PID}.adc') + + baseline = SyncIfcbDataDirectory(str(root)).read_images(I_PID) + dropped = sorted(baseline.keys())[0] + + # I-style: width at column 11, height at 12 (0-based) + lines = open(raw_adc).read().splitlines() + fields = lines[dropped - 1].split(',') + fields[11] = '0' + fields[12] = '0' + lines[dropped - 1] = ','.join(fields) + _make_adcmod(str(tmp_path / 'adcmod'), I_DAY, I_PID, content='\n'.join(lines) + '\n') + + corrected = SyncIfcbDataDirectory(str(root)).read_images(I_PID) + assert dropped in baseline + assert dropped not in corrected diff --git a/tests/test_stores.py b/tests/test_stores.py new file mode 100644 index 0000000..01858e5 --- /dev/null +++ b/tests/test_stores.py @@ -0,0 +1,73 @@ +"""Tests for filesystem-backed stores, including corrected (adcmod) ADC files.""" + +import asyncio +import os + +from ifcbkit.stores.filesystem import AsyncFilesystemBinStore + +PID = 'D20170426T164105_IFCB009' +DAY = 'D20170426' + + +def _make_fileset(dirpath, pid): + os.makedirs(dirpath, exist_ok=True) + for ext in ('hdr', 'adc', 'roi'): + with open(os.path.join(dirpath, f'{pid}.{ext}'), 'w') as f: + f.write('') + + +def _make_adcmod(adcmod_root, day, pid, content='mod'): + day_dir = os.path.join(adcmod_root, day) + os.makedirs(day_dir, exist_ok=True) + path = os.path.join(day_dir, f'{pid}.adc.mod') + with open(path, 'w') as f: + f.write(content) + return path + + +def test_bin_store_no_adcmod_uses_raw_adc(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + store = AsyncFilesystemBinStore(str(root)) + path = asyncio.run(store.get_path(f'{PID}.adc')) + assert path == str(root / DAY / f'{PID}.adc') + + +def test_bin_store_resolves_adcmod(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + store = AsyncFilesystemBinStore(str(root)) + assert asyncio.run(store.get_path(f'{PID}.adc')) == mod + + +def test_bin_store_get_reads_adcmod_content(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID, content='corrected') + store = AsyncFilesystemBinStore(str(root)) + assert asyncio.run(store.get(f'{PID}.adc')) == b'corrected' + + +def test_bin_store_adcmod_does_not_affect_hdr_or_roi(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + store = AsyncFilesystemBinStore(str(root)) + assert asyncio.run(store.get_path(f'{PID}.hdr')) == str(root / DAY / f'{PID}.hdr') + assert asyncio.run(store.get_path(f'{PID}.roi')) == str(root / DAY / f'{PID}.roi') + + +def test_bin_store_adcmod_with_year_level(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / '2017' / DAY), PID) + mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID) + store = AsyncFilesystemBinStore(str(root)) + assert asyncio.run(store.get_path(f'{PID}.adc')) == mod + + +def test_bin_store_exists_unknown_bin(tmp_path): + root = tmp_path / 'data' + _make_fileset(str(root / DAY), PID) + store = AsyncFilesystemBinStore(str(root)) + assert asyncio.run(store.get_path('D20991231T000000_IFCB009.adc')) is None