diff --git a/datamint/__init__.py b/datamint/__init__.py index 9986cdf7..2dd623bc 100644 --- a/datamint/__init__.py +++ b/datamint/__init__.py @@ -16,7 +16,7 @@ __getattr__, __dir__, __all__ = lazy.attach( __name__, - submodules=['dataset', "dataset.dataset"], + submodules=['dataset', "dataset.dataset", "examples"], submod_attrs={ "api.client": ["Api"], # New modular dataset classes diff --git a/datamint/__main__.py b/datamint/__main__.py index 02380d28..90ed54d4 100644 --- a/datamint/__main__.py +++ b/datamint/__main__.py @@ -11,6 +11,7 @@ "init": "datamint.client_cmd_tools.datamint_init", "train": "datamint.client_cmd_tools.datamint_train", "inference": "datamint.client_cmd_tools.datamint_inference", + "example": "datamint.client_cmd_tools.datamint_example", } diff --git a/datamint/client_cmd_tools/datamint_example.py b/datamint/client_cmd_tools/datamint_example.py new file mode 100644 index 00000000..a7952b8c --- /dev/null +++ b/datamint/client_cmd_tools/datamint_example.py @@ -0,0 +1,77 @@ +"""datamint example command-line tool. + +Populates a Datamint project with a small, ready-to-use example dataset - +no need to bring your own data first. See ``datamint example --help`` for +the list of available datasets. +""" +import argparse +import logging +import sys + +from datamint.client_cmd_tools.datamint_upload import handle_api_key +from datamint.exceptions import DatamintException +from datamint.utils.logging_utils import load_cmdline_logging_config + +_LOGGER = logging.getLogger(__name__) +_USER_LOGGER = logging.getLogger('user_logger') + +_DATASETS = ('bccd', 'busi', 'synapse', 'fracatlas') + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description='Populate a Datamint project with an example dataset.', + epilog=""" +Examples: + datamint example bccd # Blood cell detection (BCCD) + datamint example busi --project MyBusiProject # Breast ultrasound segmentation (BUSI) + datamint example synapse # Multi-organ CT segmentation (Synapse) + datamint example fracatlas # Fracture classification (FracAtlas) + +More Documentation: https://sonanceai.github.io/datamint-python-api/command_line_tools.html + """, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument('dataset', choices=_DATASETS, help='Which example dataset to populate.') + parser.add_argument('--project', type=str, default=None, + help='Name of the project to create. Defaults to a dataset-specific name.') + parser.add_argument('--verbose', action='store_true', default=False, help='Print debug messages.') + + return parser.parse_args() + + +def main() -> None: + load_cmdline_logging_config() + + args = _parse_args() + + if args.verbose: + logging.getLogger().handlers[0].setLevel(logging.DEBUG) + logging.getLogger('datamint').setLevel(logging.DEBUG) + _LOGGER.setLevel(logging.DEBUG) + _USER_LOGGER.setLevel(logging.DEBUG) + + try: + api_key = handle_api_key() + if api_key is None: + _USER_LOGGER.error('API key not provided. Aborting.') + sys.exit(1) + import os + + from datamint import configs + os.environ[configs.ENV_VARS[configs.APIKEY_KEY]] = api_key + + from datamint import examples + module = getattr(examples, f'{args.dataset}_dataset') + kwargs = {'project_name': args.project} if args.project else {} + module.create(**kwargs) + except DatamintException as e: + _USER_LOGGER.error(f'❌ {e}') + sys.exit(1) + except KeyboardInterrupt: + _USER_LOGGER.warning('\nCancelled by user.') + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/datamint/client_cmd_tools/datamint_init.py b/datamint/client_cmd_tools/datamint_init.py index 999a6650..204d5cf6 100644 --- a/datamint/client_cmd_tools/datamint_init.py +++ b/datamint/client_cmd_tools/datamint_init.py @@ -1395,6 +1395,29 @@ def _overlay(ax, img_np, anns): """ +# --------------------------------------------------------------------------- +# Example-data one-liner (replaces 01_upload_data.py when the user opts in) +# --------------------------------------------------------------------------- + +_EXAMPLE_SCRIPT_01 = """\ +# Docs: https://sonanceai.github.io/datamint-python-api/command_line_tools.html +# +# Populates this project with the __EXAMPLE_DATASET_LABEL__ example dataset - +# no need to bring your own data. Run this script once. + +from datamint.examples import __EXAMPLE_MODULE__ + +__EXAMPLE_MODULE__.create("__PROJECT_NAME__") +""" + +_EXAMPLE_DATASETS = { + "detection": ("bccd_dataset", "BCCD (blood cell detection)"), + "classification": ("fracatlas_dataset", "FracAtlas (fracture classification)"), + "segmentation_2d": ("busi_dataset", "BUSI (breast ultrasound segmentation)"), + "segmentation_3d": ("synapse_dataset", "Synapse (multi-organ CT segmentation)"), +} + + # --------------------------------------------------------------------------- # Template rendering # --------------------------------------------------------------------------- @@ -1403,8 +1426,15 @@ def _render(template: str, project_name: str) -> str: return template.replace("__PROJECT_NAME__", project_name) -def _generate_detection_files(project_name: str) -> dict[str, str]: - return { +def _render_example_script_01(project_name: str, example_module: str, dataset_label: str) -> str: + return (_EXAMPLE_SCRIPT_01 + .replace("__PROJECT_NAME__", project_name) + .replace("__EXAMPLE_MODULE__", example_module) + .replace("__EXAMPLE_DATASET_LABEL__", dataset_label)) + + +def _generate_detection_files(project_name: str, example_key: str | None = None) -> dict[str, str]: + files = { "README.md": _render(_README, project_name), "01_upload_data.py": _render(_SCRIPT_01, project_name), "02_explore.py": _render(_SCRIPT_02, project_name), @@ -1413,10 +1443,14 @@ def _generate_detection_files(project_name: str) -> dict[str, str]: "05_evaluate.py": _render(_SCRIPT_05, project_name), "06_deploy.py": _render(_SCRIPT_06, project_name), } + if example_key is not None: + module, label = _EXAMPLE_DATASETS[example_key] + files["01_upload_data.py"] = _render_example_script_01(project_name, module, label) + return files -def _generate_classification_files(project_name: str) -> dict[str, str]: - return { +def _generate_classification_files(project_name: str, example_key: str | None = None) -> dict[str, str]: + files = { "README.md": _render(_CLS_README, project_name), "01_upload_data.py": _render(_CLS_SCRIPT_01, project_name), "02_explore.py": _render(_CLS_SCRIPT_02, project_name), @@ -1425,10 +1459,14 @@ def _generate_classification_files(project_name: str) -> dict[str, str]: "05_evaluate.py": _render(_CLS_SCRIPT_05, project_name), "06_deploy.py": _render(_CLS_SCRIPT_06, project_name), } + if example_key is not None: + module, label = _EXAMPLE_DATASETS[example_key] + files["01_upload_data.py"] = _render_example_script_01(project_name, module, label) + return files -def _generate_segmentation_files(project_name: str) -> dict[str, str]: - return { +def _generate_segmentation_files(project_name: str, example_key: str | None = None) -> dict[str, str]: + files = { "README.md": _render(_SEG_README, project_name), "01_upload_data.py": _render(_SEG_SCRIPT_01, project_name), "02_explore.py": _render(_SEG_SCRIPT_02, project_name), @@ -1437,6 +1475,10 @@ def _generate_segmentation_files(project_name: str) -> dict[str, str]: "05_evaluate.py": _render(_SEG_SCRIPT_05, project_name), "06_deploy.py": _render(_SEG_SCRIPT_06, project_name), } + if example_key is not None: + module, label = _EXAMPLE_DATASETS[example_key] + files["01_upload_data.py"] = _render_example_script_01(project_name, module, label) + return files # --------------------------------------------------------------------------- @@ -1486,6 +1528,15 @@ def main() -> None: choices=["detection", "segmentation", "classification"], console=console, ).strip() + + example_key = None + if Confirm.ask(" Populate this project with example data instead of your own?", + default=False, console=console): + if task == "segmentation": + dims = Prompt.ask(" 2D or 3D?", choices=["2d", "3d"], console=console).strip() + example_key = f"segmentation_{dims}" + else: + example_key = task except (KeyboardInterrupt, EOFError): console.print() sys.exit(0) @@ -1504,11 +1555,11 @@ def main() -> None: out_dir.mkdir(exist_ok=True) if task == "classification": - files = _generate_classification_files(project_name) + files = _generate_classification_files(project_name, example_key) elif task == "segmentation": - files = _generate_segmentation_files(project_name) + files = _generate_segmentation_files(project_name, example_key) else: - files = _generate_detection_files(project_name) + files = _generate_detection_files(project_name, example_key) console.print() console.print(f" Generating scripts in [bold]./{project_name}/[/bold] ...") diff --git a/datamint/examples/__init__.py b/datamint/examples/__init__.py index af0da1fb..7d17ae06 100644 --- a/datamint/examples/__init__.py +++ b/datamint/examples/__init__.py @@ -1 +1,2 @@ -from .example_projects import ProjectMR \ No newline at end of file +from .example_projects import ProjectMR +from . import bccd_dataset, busi_dataset, synapse_dataset, fracatlas_dataset \ No newline at end of file diff --git a/datamint/examples/_common.py b/datamint/examples/_common.py new file mode 100644 index 00000000..d1387321 --- /dev/null +++ b/datamint/examples/_common.py @@ -0,0 +1,41 @@ +import logging +from pathlib import Path + +from datamint import Api +from datamint.entities import Project + +_LOGGER = logging.getLogger(__name__) + + +def get_or_create_project(project_name: str, + description: str, + api: Api) -> tuple[Project, bool]: + """Return (project, already_existed). + + If a project with this name already exists, it is returned as-is (not modified). + Otherwise a new, empty project is created for the caller to populate. + """ + existing = api.projects.get_by_name(project_name) + if existing is not None: + return existing, True + + proj = api.projects.create(name=project_name, description=description, exists_ok=True) + return proj, False + + +def print_skip_summary(dataset_name: str, proj: Project) -> None: + print(dataset_name) + print(f" project '{proj.name}' already exists, skipping data population.") + print(f' {proj.url}') + + +def print_summary(dataset_name: str, + n_files: int, + n_annotated: int, + cache_path: Path, + proj: Project) -> None: + pct = (n_annotated / n_files * 100) if n_files else 0.0 + print(dataset_name) + print(f' {n_files} files uploaded, {n_annotated} annotated ({pct:.0f}%)') + print(f' cached at {cache_path}') + print(f' project: {proj.name} ({proj.url})') diff --git a/datamint/examples/_download.py b/datamint/examples/_download.py new file mode 100644 index 00000000..73766257 --- /dev/null +++ b/datamint/examples/_download.py @@ -0,0 +1,55 @@ +import logging +import zipfile +from pathlib import Path + +import requests +from tqdm.auto import tqdm + +from datamint import configs + +_LOGGER = logging.getLogger(__name__) + +_DONE_MARKER = '.datamint_download_complete' + + +def cache_dir(dataset_subdir: str) -> Path: + if configs.DATAMINT_DATA_DIR is None: + raise RuntimeError('Could not determine a local data directory to cache the dataset.') + return Path(configs.DATAMINT_DATA_DIR) / 'examples' / dataset_subdir + + +def is_cached(dataset_subdir: str) -> bool: + return (cache_dir(dataset_subdir) / _DONE_MARKER).exists() + + +def download_and_extract(url: str, dataset_subdir: str) -> Path: + """Download a zip file from `url` and extract it under the examples cache directory. + + Idempotent: if the dataset was already downloaded and extracted, the cached + directory is returned without hitting the network again. + """ + out_dir = cache_dir(dataset_subdir) + marker = out_dir / _DONE_MARKER + if marker.exists(): + _LOGGER.info(f'Using cached dataset at {out_dir}') + return out_dir + + out_dir.mkdir(parents=True, exist_ok=True) + zip_path = out_dir / 'download.zip' + + _LOGGER.info(f'Downloading {url}...') + with requests.get(url, stream=True) as resp: + resp.raise_for_status() + total = int(resp.headers.get('content-length', 0)) + with open(zip_path, 'wb') as f, tqdm(total=total, unit='B', unit_scale=True, desc=dataset_subdir) as pbar: + for chunk in resp.iter_content(chunk_size=1024 * 1024): + f.write(chunk) + pbar.update(len(chunk)) + + _LOGGER.info(f'Extracting {zip_path}...') + with zipfile.ZipFile(zip_path) as zf: + zf.extractall(out_dir) + zip_path.unlink() + marker.touch() + + return out_dir diff --git a/datamint/examples/bccd_dataset.py b/datamint/examples/bccd_dataset.py new file mode 100644 index 00000000..6e90eece --- /dev/null +++ b/datamint/examples/bccd_dataset.py @@ -0,0 +1,102 @@ +import logging +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path + +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +from . import _common, _download + +_LOGGER = logging.getLogger(__name__) + +_BCCD_URL = 'https://github.com/Shenggan/BCCD_Dataset/archive/refs/heads/master.zip' +_DATASET_NAME = 'BCCD Detection Example' +_DESCRIPTION = ('Blood cell detection example dataset (BCCD), ' + 'auto-populated by datamint.examples.bccd_dataset.') + + +@dataclass +class _Box: + label: str + x1: float + y1: float + x2: float + y2: float + + +@dataclass +class _Sample: + image_path: Path + boxes: list[_Box] = field(default_factory=list) + + +def _parse_voc_xml(xml_path: Path) -> list[_Box]: + root = ET.parse(xml_path).getroot() + boxes = [] + for obj in root.findall('object'): + name = obj.findtext('name', default='').strip() + bb = obj.find('bndbox') + if bb is None: + continue + boxes.append(_Box( + label=name, + x1=float(bb.findtext('xmin')), + y1=float(bb.findtext('ymin')), + x2=float(bb.findtext('xmax')), + y2=float(bb.findtext('ymax')), + )) + return boxes + + +def create(project_name: str = _DATASET_NAME, api: Api | None = None) -> Project: + """Download the BCCD blood-cell detection dataset and upload it as a Datamint project. + + Source: https://github.com/Shenggan/BCCD_Dataset (MIT License). + """ + api = api or Api() + + proj, already_existed = _common.get_or_create_project(project_name, _DESCRIPTION, api) + if already_existed: + _LOGGER.warning(f"Project '{project_name}' already exists. Skipping data population.") + _common.print_skip_summary(_DATASET_NAME, proj) + return proj + + data_dir = _download.download_and_extract(_BCCD_URL, 'bccd') + images_dir = data_dir / 'BCCD_Dataset-master' / 'BCCD' / 'JPEGImages' + annots_dir = data_dir / 'BCCD_Dataset-master' / 'BCCD' / 'Annotations' + + image_files = sorted(images_dir.glob('*.jpg')) + samples = [] + for img_path in image_files: + xml_path = annots_dir / img_path.with_suffix('.xml').name + samples.append(_Sample( + image_path=img_path, + boxes=_parse_voc_xml(xml_path) if xml_path.exists() else [], + )) + + resource_ids = api.resources.upload_resources( + [str(s.image_path) for s in samples], + tags=['bccd'], + publish_to=proj, + progress_bar=True, + ) + + n_annotated = 0 + for sample, resource_id in tqdm(zip(samples, resource_ids), total=len(samples), + desc='Uploading annotations'): + if not sample.boxes: + continue + n_annotated += 1 + for box in sample.boxes: + api.annotations.add_box_annotation( + point1=(box.x1, box.y1), + point2=(box.x2, box.y2), + resource=resource_id, + identifier=box.label, + ) + + _common.print_summary(_DATASET_NAME, len(samples), n_annotated, data_dir, proj) + return proj diff --git a/datamint/examples/busi_dataset.py b/datamint/examples/busi_dataset.py new file mode 100644 index 00000000..52006d94 --- /dev/null +++ b/datamint/examples/busi_dataset.py @@ -0,0 +1,67 @@ +import logging + +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +from . import _common, _download + +_LOGGER = logging.getLogger(__name__) + +_BUSI_URL = 'https://www.kaggle.com/api/v1/datasets/download/sabahesaraki/breast-ultrasound-images-dataset' +_DATASET_NAME = 'BUSI Segmentation Example' +_DESCRIPTION = ('Breast ultrasound segmentation example dataset (BUSI), ' + 'auto-populated by datamint.examples.busi_dataset.') +_CLASSES = ('benign', 'malignant', 'normal') + + +def create(project_name: str = _DATASET_NAME, api: Api | None = None) -> Project: + """Download the BUSI breast-ultrasound segmentation dataset and upload it as a Datamint project. + + Source: https://www.kaggle.com/datasets/sabahesaraki/breast-ultrasound-images-dataset. + """ + api = api or Api() + + proj, already_existed = _common.get_or_create_project(project_name, _DESCRIPTION, api) + if already_existed: + _LOGGER.warning(f"Project '{project_name}' already exists. Skipping data population.") + _common.print_skip_summary(_DATASET_NAME, proj) + return proj + + data_dir = _download.download_and_extract(_BUSI_URL, 'busi') + base_dir = data_dir / 'Dataset_BUSI_with_GT' + + image_paths = [] + mask_paths = [] + for cls in _CLASSES: + cls_dir = base_dir / cls + cls_images = sorted(p for p in cls_dir.glob('*.png') if '_mask' not in p.name) + for img_path in cls_images: + mask_path = cls_dir / f'{img_path.stem}_mask.png' + image_paths.append(img_path) + mask_paths.append(mask_path if mask_path.exists() else None) + + resource_ids = api.resources.upload_resources( + [str(p) for p in image_paths], + tags=['busi', 'ultrasound', 'breast'], + publish_to=proj, + progress_bar=True, + ) + + n_annotated = 0 + for img_path, mask_path, resource_id in tqdm(zip(image_paths, mask_paths, resource_ids), + total=len(image_paths), + desc='Uploading annotations'): + if mask_path is None: + continue # normal images have no lesion mask + n_annotated += 1 + api.annotations.upload_segmentations( + resource=resource_id, + file_path=mask_path, + name=img_path.parent.name, # 'benign' or 'malignant' + imported_from='Original GT BUSI Dataset', + ) + + _common.print_summary(_DATASET_NAME, len(image_paths), n_annotated, data_dir, proj) + return proj diff --git a/datamint/examples/fracatlas_dataset.py b/datamint/examples/fracatlas_dataset.py new file mode 100644 index 00000000..592e81e2 --- /dev/null +++ b/datamint/examples/fracatlas_dataset.py @@ -0,0 +1,79 @@ +import logging + +import requests +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +from . import _common, _download + +_LOGGER = logging.getLogger(__name__) + +_FIGSHARE_ARTICLE_URL = 'https://api.figshare.com/v2/articles/22363012' +_DATASET_NAME = 'FracAtlas Classification Example' +_DESCRIPTION = ('Fracture classification example dataset (FracAtlas), ' + 'auto-populated by datamint.examples.fracatlas_dataset.') +_LABEL_IDENTIFIER = 'has_fracture' + + +def _get_download_url() -> str: + resp = requests.get(_FIGSHARE_ARTICLE_URL) + resp.raise_for_status() + return resp.json()['files'][0]['download_url'] + + +def create(project_name: str = _DATASET_NAME, api: Api | None = None) -> Project: + """Download the FracAtlas fracture-classification dataset and upload it as a Datamint project. + + Source: https://doi.org/10.6084/m9.figshare.22363012. + """ + api = api or Api() + + proj, already_existed = _common.get_or_create_project(project_name, _DESCRIPTION, api) + if already_existed: + _LOGGER.warning(f"Project '{project_name}' already exists. Skipping data population.") + _common.print_skip_summary(_DATASET_NAME, proj) + return proj + + if not _download.is_cached('fracatlas'): + print('FracAtlas is ~1.2GB compressed - this download may take a few minutes.') + + download_url = _get_download_url() + data_dir = _download.download_and_extract(download_url, 'fracatlas') + + fractured_dir = next(data_dir.rglob('Fractured')) + non_fractured_dir = next(data_dir.rglob('Non_fractured')) + + fractured_paths = sorted(p for p in fractured_dir.iterdir() if p.is_file()) + non_fractured_paths = sorted(p for p in non_fractured_dir.iterdir() if p.is_file()) + + non_fractured_ids = api.resources.upload_resources( + [str(p) for p in non_fractured_paths], + tags=['fracatlas', 'non-fractured'], + publish_to=proj, + progress_bar=True, + ) + fractured_ids = api.resources.upload_resources( + [str(p) for p in fractured_paths], + tags=['fracatlas', 'fractured'], + publish_to=proj, + progress_bar=True, + ) + + for resource_id in tqdm(non_fractured_ids, desc='Uploading annotations'): + api.annotations.create_image_classification( + resource=resource_id, + identifier=_LABEL_IDENTIFIER, + value='no', + ) + for resource_id in tqdm(fractured_ids, desc='Uploading annotations'): + api.annotations.create_image_classification( + resource=resource_id, + identifier=_LABEL_IDENTIFIER, + value='yes', + ) + + n_files = len(non_fractured_ids) + len(fractured_ids) + _common.print_summary(_DATASET_NAME, n_files, n_files, data_dir, proj) + return proj diff --git a/datamint/examples/synapse_dataset.py b/datamint/examples/synapse_dataset.py new file mode 100644 index 00000000..3b34825a --- /dev/null +++ b/datamint/examples/synapse_dataset.py @@ -0,0 +1,106 @@ +import logging + +import numpy as np +import nibabel as nib +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +from . import _common, _download + +_LOGGER = logging.getLogger(__name__) + +_SYNAPSE_URL = 'https://www.kaggle.com/api/v1/datasets/download/dogcdt/synapse' +_DATASET_NAME = 'Synapse Segmentation Example' +_DESCRIPTION = ('Synapse Multi-Organ CT 3D segmentation example dataset, ' + 'auto-populated by datamint.examples.synapse_dataset.') + +_SYNAPSE_CLASSES = { + 1: 'aorta', + 2: 'gallbladder', + 3: 'spleen', + 4: 'left_kidney', + 5: 'right_kidney', + 6: 'liver', + 7: 'stomach', + 8: 'pancreas', +} + + +def _convert_to_nifti(data_dir): + try: + import h5py + except ImportError as e: + raise ImportError( + "h5py is required to convert the Synapse dataset's HDF5 volumes to NIfTI. " + 'Run: pip install h5py' + ) from e + + h5_dir = data_dir / 'Synapse' / 'test_vol_h5' + h5_files = sorted(h5_dir.glob('*.npy.gz')) or sorted(h5_dir.glob('*.h5')) + + nii_dir = data_dir / 'nifti' / 'images' + label_dir = data_dir / 'nifti' / 'labels' + nii_dir.mkdir(parents=True, exist_ok=True) + label_dir.mkdir(parents=True, exist_ok=True) + + image_paths = [] + label_paths = [] + + for h5_path in h5_files: + case_id = h5_path.stem.split('.')[0] # 'case0001.npy.h5' -> 'case0001' + nii_path = nii_dir / f'{case_id}.nii.gz' + lbl_path = label_dir / f'{case_id}_label.nii.gz' + + if not nii_path.exists() or not lbl_path.exists(): + with h5py.File(h5_path, 'r') as f: + image = f['image'][:] # (H, W, D) float + label = f['label'][:] # (H, W, D) int + + image = image[::2, ::2, :] # downsample in-plane + label = label[::2, ::2, :] + + nib.save(nib.Nifti1Image(image.astype(np.float32), affine=np.eye(4)), nii_path) + nib.save(nib.Nifti1Image(label.astype(np.uint8), affine=np.eye(4)), lbl_path) + + image_paths.append(nii_path) + label_paths.append(lbl_path) + + return image_paths, label_paths + + +def create(project_name: str = _DATASET_NAME, api: Api | None = None) -> Project: + """Download the Synapse Multi-Organ CT dataset and upload it as a Datamint project. + + Source: https://www.kaggle.com/datasets/dogcdt/synapse. + """ + api = api or Api() + + proj, already_existed = _common.get_or_create_project(project_name, _DESCRIPTION, api) + if already_existed: + _LOGGER.warning(f"Project '{project_name}' already exists. Skipping data population.") + _common.print_skip_summary(_DATASET_NAME, proj) + return proj + + data_dir = _download.download_and_extract(_SYNAPSE_URL, 'synapse') + image_paths, label_paths = _convert_to_nifti(data_dir) + + resource_ids = api.resources.upload_resources( + [str(p) for p in image_paths], + tags=['synapse', 'ct', 'abdomen'], + publish_to=proj, + progress_bar=True, + ) + + for lbl_path, resource_id in tqdm(zip(label_paths, resource_ids), total=len(label_paths), + desc='Uploading annotations'): + api.annotations.upload_volume_segmentation( + resource=resource_id, + file_path=str(lbl_path), + name=_SYNAPSE_CLASSES, + imported_from='Synapse Multi-Organ CT', + ) + + _common.print_summary(_DATASET_NAME, len(image_paths), len(label_paths), data_dir, proj) + return proj diff --git a/dev/DAT-991-example-data-plan.md b/dev/DAT-991-example-data-plan.md new file mode 100644 index 00000000..3e6c092f --- /dev/null +++ b/dev/DAT-991-example-data-plan.md @@ -0,0 +1,189 @@ +# DAT-991: One-line population of example project data + +## Context + +Trying Datamint's one-line trainers or other features currently requires the user to +already have their own annotated data uploaded. There's no quick way to spin up a working +project with real data to experiment with. The ask: a one-liner like +`datamint.examples.busi_dataset.create('MyBusiDataset')`, plus (per brainstorm) a CLI form. + +There's a small existing precedent, `datamint/examples/example_projects.py`'s `ProjectMR` +class — it downloads one tiny pydicom test-fixture DICOM + one hardcoded mask PNG, uploads +them, and creates a project (skipping with a warning if the project already exists). It's +narrow (one hardcoded image, not a real dataset) and not wired into `datamint`'s top-level +lazy-loader, but its shape (check-exists → download → upload → create) is the right pattern +to generalize. + +**Key finding from research, discussed with you:** of the three datasets you originally +proposed — BCCD (detection), BUSI (2D segmentation), Synapse (3D segmentation) — BCCD is +genuinely zero-setup (public GitHub zip, MIT license, no auth). BUSI and Synapse are fetched +via `https://www.kaggle.com/api/v1/datasets/download//` in the existing +notebooks. Verified live (2026-07-07): both URLs return the real dataset zip via a bare +`curl -L`/`requests.get`, no `KAGGLE_USERNAME`/`KAGGLE_KEY`, no `~/.kaggle/kaggle.json`, no +cookies — this specific Kaggle download endpoint currently serves public datasets +anonymously, matching exactly what the notebooks already do. So all four datasets are +zero-setup today. Caveat: this is Kaggle's browser-facing download URL, not their documented +authenticated API — it's not a published contract, so it could start requiring auth or get +rate-limited for anonymous/scripted access without notice. Synapse is additionally the most +access-restricted of the three by dataset license (the notebook itself has an "⚠️ Dataset +Access" warning) — it's almost certainly the BTCV/Synapse.org challenge dataset, which +commonly disallows redistribution outside its own registered-access mechanism; that's a +licensing concern independent of the download-auth question. + +We're adding a fourth dataset, **FracAtlas** (binary fracture classification), covering the +`classification` task type — it turns out to be the *best* candidate of all four for +zero-setup: hosted on Figshare with a plain public API (`api.figshare.com`), no auth +required at all, same as BCCD. Only caveat is size: ~1.2GB compressed, so it's a slower +download than the other three, worth surfacing to the user up front. + +**Decisions confirmed with you:** +- All four datasets stay zero-setup: BUSI/Synapse use the same anonymous Kaggle + download-endpoint curl the notebooks already use, no Kaggle credentials required. No + re-hosting, no new licensing exposure beyond what the notebooks already do today. +- CLI: both a standalone `datamint example ` subcommand, and a prompt folded into the + existing `datamint init` wizard. +- Don't refactor the existing notebooks to share code with the new module in this ticket — + they keep their own inline download/upload logic; the new `datamint.examples.*` code is + new, independent code modeled on the same approach. +- All four of `datamint init`'s task types now have a matching example dataset: detection → + BCCD, segmentation (2D) → BUSI, segmentation (3D) → Synapse, classification → FracAtlas. + +## Design + +### Package layout + +``` +datamint/examples/ + __init__.py # existing export (ProjectMR) + new: bccd_dataset, busi_dataset, synapse_dataset, fracatlas_dataset + example_projects.py # existing, untouched + _common.py # NEW: get_or_create_project(project_name, api) -> (Project, already_existed) + # shared by all three new modules (same skip-with-warning behavior as ProjectMR.create) + _download.py # NEW: download_and_extract(url, cache_subdir) -> Path + # one function for all four datasets, including BUSI/Synapse's + # Kaggle URLs — no auth needed (verified live), so no separate + # kaggle_download()/credential handling + bccd_dataset.py # NEW: create(project_name='BCCD Detection Example') -> Project + busi_dataset.py # NEW: create(project_name='BUSI Segmentation Example') -> Project + synapse_dataset.py # NEW: create(project_name='Synapse Segmentation Example') -> Project + fracatlas_dataset.py # NEW: create(project_name='FracAtlas Classification Example') -> Project +``` + +Each `create()` re-implements (not imports from) the relevant notebook's fetch/parse/upload +logic as a proper function: +- `bccd_dataset.create()`: download+extract the BCCD zip via `_download.download_and_extract`, + parse Pascal VOC XML, upload resources + box annotations (mirrors + `notebooks/06_end_to_end/slice_based/03_bccd_detection.ipynb`). +- `busi_dataset.create()`: `_download.download_and_extract('https://www.kaggle.com/api/v1/datasets/download/sabahesaraki/breast-ultrasound-images-dataset', ...)`, + upload resources + PNG mask segmentation annotations (mirrors `02_busi_segmentation.ipynb`). +- `synapse_dataset.create()`: `_download.download_and_extract('https://www.kaggle.com/api/v1/datasets/download/dogcdt/synapse', ...)`, + convert HDF5 → NIfTI per case, upload resources + volume segmentation annotations (mirrors + `01_synapse_unetrpp.ipynb` / `02_synapse_nnunet.ipynb`). +- `fracatlas_dataset.create()`: query `https://api.figshare.com/v2/articles/22363012` for the + download URL, fetch+extract via `_download.download_and_extract` (no auth needed, same as + BCCD), upload the `Fractured`/`Non_fractured` folders with tags, then + `api.annotations.create_image_classification(identifier='has_fracture', value='yes'/'no')` + per resource (mirrors `notebooks/06_end_to_end/slice_based/01_fracatlas_classification.ipynb`). + Print a heads-up before downloading (~1.2GB compressed, a few minutes) since it's + noticeably bigger than the other three. + +Downloaded/extracted data is cached under `configs.DATAMINT_DATA_DIR/examples//` +(i.e. `~/.datamint/examples/bccd/`, etc.) — a new sibling namespace next to the existing +`resources`/`annotations` cache namespaces, so repeated calls don't re-download, and so it's +naturally covered by `datamint config --list-local-data` / `--clean-local-data examples` +(small addition to `datamint_config.py`'s known-namespace list). + +### Console output + +Each `create()` prints a summary after upload finishes — not just the CLI wrapper, since +`datamint.examples.busi_dataset.create(...)` called directly (your literal one-liner) should +give the same feedback as `datamint example busi`: +``` +FracAtlas Classification Example + 717 files uploaded, 717 annotated (100%) + cached at ~/.datamint/examples/fracatlas/ + project: MyProject (https://app.datamint.io/projects/) +``` +- Dataset name: the module's display name (e.g. "FracAtlas Classification Example"). +- File count: total resources uploaded this run. +- Annotated count: resources that got at least one annotation (box/mask/classification + label), as `n (pct%)` — for these four datasets every uploaded file is annotated, so it'll + read 100%, but computing it from the actual upload/annotation calls (not hardcoding "100%") + keeps it honest if a dataset ever has partially-labeled data. +- Cache path: the `configs.DATAMINT_DATA_DIR/examples//` path data was + downloaded/extracted to. +- Project name + link: reuses whatever `_common.get_or_create_project` already returns. + +On the already-exists-skip path, print a shorter variant (name + "already exists, skipping" ++ project link) rather than the full stats block, matching `ProjectMR.create`'s existing +skip-with-warning behavior. + +### Public API wiring + +`datamint/__init__.py` uses `lazy_loader.attach` with an explicit `submodules` list that +currently omits `examples` entirely — meaning `datamint.examples.busi_dataset.create(...)` +(your literal proposed usage) doesn't actually resolve via the top-level package today; only +an explicit `import datamint.examples` does. Add `examples` to that submodules list so the +literal one-liner works. + +### CLI + +**New standalone subcommand** — `datamint/client_cmd_tools/datamint_example.py`: +```bash +datamint example bccd +datamint example busi --project MyBusiProject +datamint example fracatlas +``` +Positional `dataset` (choices: `bccd`, `busi`, `synapse`, `fracatlas`), optional `--project` +(defaults to each module's own default name). Dispatches to the matching +`datamint.examples..create()`. Registered only in `datamint/__main__.py`'s `_COMMANDS` +dict (unified form) — **no new hyphenated `datamint-example` script**, since that legacy +pattern (established in DAT-986) is for backward compatibility with pre-existing commands, +not something to add for a brand-new one. + +**Folded into `datamint init`** (`datamint_init.py`): after the existing task-type prompt +(`detection`/`segmentation`/`classification`), add a prompt: "Populate this project with +example data instead of your own?". If yes: +- `detection` → BCCD. +- `segmentation` → follow-up prompt "2D or 3D?" → BUSI or Synapse (task-type prompt doesn't + currently distinguish these; needs the one extra prompt). +- `classification` → FracAtlas. + +When example data is chosen, `01_upload_data.py` is generated as a short one-liner instead of +the generic "point this at your own data" template, e.g.: +```python +from datamint.examples import busi_dataset +busi_dataset.create("MyProject") +``` + +### Docs + +Add a short section (in `docs/source/command_line_tools.rst` or a new page, whichever reads +better once drafted) documenting `datamint example` and the four datasets. Note FracAtlas's +larger download size (~1.2GB) up front. For `busi`/`synapse`, note that the download relies +on Kaggle's public anonymous download endpoint (no credentials needed today) and that if +Kaggle starts gating it, the fix is to configure Kaggle credentials +(`KAGGLE_USERNAME`/`KAGGLE_KEY` or `~/.kaggle/kaggle.json`) — not a day-one requirement, just +a documented fallback. + +## Tests + +Per repo convention (CLAUDE.md: no real network calls in tests), all downloads are mocked: +- `tests/test_examples_bccd.py`, `test_examples_busi.py`, `test_examples_synapse.py`, + `test_examples_fracatlas.py`: mock `_download.download_and_extract` and the `Api` calls + (`respx`/`httpx.MockTransport`, matching existing test conventions), assert resources + + annotations get uploaded, assert the already-exists-skip-with-warning branch, and assert + the printed summary (capsys) has the right file/annotated counts and cache path. +- `tests/test_datamint_example_cmd.py`: exercises the new CLI module the same way + `tests/test_datamint_config.py` exercises `datamint config` (sys.argv patching + mocked + `create()`). +- Extend `tests/test_datamint_init.py` (if it exists) or add coverage for the new + example-data prompt branch in the init wizard. + +## Verification +- Run `datamint example bccd --project TestBCCD` for real once (small, zero-auth dataset) to + confirm the full download → parse → upload → project-creation path works end-to-end against + a real Datamint server. Do the same for `fracatlas` at least once too (zero-auth, but budget + for the larger ~1.2GB download). +- Run `datamint init` interactively, choose "use example data", confirm the generated + `01_upload_data.py` is the short one-liner and actually runs. +- `pytest tests` — confirm new tests pass and nothing else regresses. diff --git a/docs/source/command_line_tools.rst b/docs/source/command_line_tools.rst index 7fe53d92..815f65b6 100644 --- a/docs/source/command_line_tools.rst +++ b/docs/source/command_line_tools.rst @@ -33,11 +33,12 @@ You should see this in the first line: usage: datamint config [-h] [--api-key API_KEY] [--default-url DEFAULT_URL] [-i] [...] -There are five command-line tools available: +There are six command-line tools available: - ``datamint config``: To configure the Datamint API key and URL. - ``datamint upload``: To upload DICOM, NIfTI, video, image, and segmentation files to the Datamint server. - ``datamint init``: To scaffold a ready-to-run project (upload, train, and deploy scripts). +- ``datamint example``: To populate a project with a ready-made example dataset, no data of your own required. - ``datamint train``: To train a model on a Datamint project using a built-in one-line trainer. - ``datamint inference``: To run local inference with a registered Datamint model against a local file. @@ -275,6 +276,42 @@ See all available options by running ``datamint upload --help``: --version show program's version number and exit --verbose Print debug messages +Populating a project with example data +--------------------------------------- + +If you don't have your own annotated data yet, ``datamint example`` downloads a small +public dataset and uploads it into a new Datamint project so you can try the one-line +trainers immediately: + +.. code-block:: bash + + datamint example bccd + datamint example busi --project MyBusiProject + +Four datasets are available, one per supported task type: + +- ``bccd``: Blood cell object detection (`BCCD `_). +- ``busi``: Breast ultrasound 2D segmentation (`BUSI `_). +- ``synapse``: Multi-organ CT 3D segmentation (`Synapse `_). +- ``fracatlas``: Fracture image classification (`FracAtlas `_). + +Downloaded data is cached under ``~/.datamint/examples//`` so re-running the same +command doesn't re-download. It shows up alongside your other local data in +``datamint config --list-local-data`` / ``--clean-local-data``. + +You can also populate a project this way without the CLI: + +.. code-block:: python + + from datamint.examples import busi_dataset + + busi_dataset.create("MyBusiProject") + +``datamint init`` offers this as an option too. Answer "yes" when asked whether to +populate the scaffolded project with example data instead of your own. + +See all available options by running ``datamint example --help``. + Training a model ----------------- diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index eb866b28..80dccd60 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -45,6 +45,10 @@ It asks for a project name and task type, then writes six scripts into a new dir (upload data, explore, build a dataset, train, evaluate, and deploy), so you can follow them in order without writing boilerplate. +Don't have your own data yet? ``datamint init`` can populate the project with a small +public example dataset instead — see :ref:`command_line_tools` for details, or run +``datamint example --help`` directly. + Your first API call =================== diff --git a/pyproject.toml b/pyproject.toml index e4dd3dea..b4b86936 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ respx = { version = ">=0.22.0", optional = true } nnunetv2 = { version = ">=2.4,<3.0", optional = true } filelock = { version = ">=3.0", optional = true } yolox-datamint = { version = ">=0.3.1", optional = true } +h5py = { version = ">=3.0", optional = true } [tool.poetry.group.dev.dependencies] # for `poetry install` @@ -86,6 +87,7 @@ docs = ["sphinx", "sphinx_rtd_theme", "sphinx-tabs", "setuptools"] dev = ["pytest", "pytest-cov", "responses", "aioresponses", "respx"] nnunet = ["nnunetv2", "filelock"] detection = ["yolox-datamint"] +examples = ["h5py"] [build-system] requires = ["poetry-core>=1.0.0"]