diff --git a/datamint/__init__.py b/datamint/__init__.py index c6c45e84..00578c3c 100644 --- a/datamint/__init__.py +++ b/datamint/__init__.py @@ -15,6 +15,9 @@ from .dataset.image_dataset import ImageDataset as ImageDataset from .dataset.volume_dataset import VolumeDataset as VolumeDataset from .default_project import select_project as select_project + from .importers.coco import COCOImporter as COCOImporter + from .importers.pascal_voc import PascalVOCImporter as PascalVOCImporter + from .importers.yolo import YOLOImporter as YOLOImporter from .mlflow.flavors.validation import ( ModelValidationError as ModelValidationError, ValidationIssue as ValidationIssue, @@ -27,7 +30,7 @@ __getattr__, __dir__, __all__ = lazy.attach( __name__, - submodules=['dataset', "examples"], + submodules=['dataset', "examples", "importers"], submod_attrs={ "api.client": ["Api"], # New modular dataset classes @@ -36,6 +39,9 @@ "mlflow.flavors.validation": ["validate_model", "ValidationReport", "ValidationIssue", "ModelValidationError"], "default_project": ["select_project"], + "importers.coco": ["COCOImporter"], + "importers.pascal_voc": ["PascalVOCImporter"], + "importers.yolo": ["YOLOImporter"], }, ) diff --git a/datamint/importers/__init__.py b/datamint/importers/__init__.py new file mode 100644 index 00000000..11adee3d --- /dev/null +++ b/datamint/importers/__init__.py @@ -0,0 +1,9 @@ +from .coco import COCOBox, COCOImporter, COCOImportResult, COCOParseResult, COCOSample +from .pascal_voc import PascalVOCBox, PascalVOCImporter, PascalVOCImportResult, PascalVOCParseResult, PascalVOCSample +from .yolo import YOLOBox, YOLOImporter, YOLOImportResult, YOLOParseResult, YOLOSample + +__all__ = [ + 'COCOImporter', 'COCOParseResult', 'COCOImportResult', 'COCOSample', 'COCOBox', + 'PascalVOCImporter', 'PascalVOCParseResult', 'PascalVOCImportResult', 'PascalVOCSample', 'PascalVOCBox', + 'YOLOImporter', 'YOLOParseResult', 'YOLOImportResult', 'YOLOSample', 'YOLOBox', +] diff --git a/datamint/importers/_common.py b/datamint/importers/_common.py new file mode 100644 index 00000000..11296b14 --- /dev/null +++ b/datamint/importers/_common.py @@ -0,0 +1,86 @@ +import logging +from typing import Callable, Literal, Sequence + +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +_LOGGER = logging.getLogger(__name__) + + +def import_boxes_to_project(result, + project: Project | str, + api: Api | None, + *, + box_points: Callable[[object], tuple[tuple[float, float], tuple[float, float]]], + source_label: str, + tags: Sequence[str] | None, + imported_from: str, + on_error: Literal['raise', 'skip'], + progress_bar: bool, + result_cls: type): + """Shared upload loop behind every ``*Importer.import_to_project()``. + + ``result`` is a parse result with ``samples``, ``missing_images``, and + ``unsupported_annotations`` attributes. ``box_points`` maps a format's box + dataclass to a ``(point1, point2)`` pair for ``add_box_annotation``. + + """ + api = api or Api() + + if result.missing_images: + _LOGGER.warning(f'{len(result.missing_images)} image(s) referenced in {source_label} ' + f'were not found on disk and will be skipped.') + if result.unsupported_annotations: + _LOGGER.warning(f'{result.unsupported_annotations} annotation(s) in {source_label} use an ' + f'unsupported annotation type (e.g. polygon/segmentation) and were not ' + f'imported; only bounding boxes are supported.') + + uploaded = api.resources.upload_resources( + [str(s.image_path) for s in result.samples], + tags=tags, + publish_to=project, + on_error=on_error, + progress_bar=progress_bar, + ) + + resource_ids: list[str] = [] + errors: list[tuple[str, Exception]] = [] + n_boxes_uploaded = 0 + + iterator = zip(result.samples, uploaded) + if progress_bar: + iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations') + + for sample, resource_id in iterator: + if isinstance(resource_id, Exception): + errors.append((sample.file_name, resource_id)) + continue + resource_ids.append(resource_id) + + for box in sample.boxes: + point1, point2 = box_points(box) + try: + api.annotations.add_box_annotation( + point1=point1, + point2=point2, + resource=resource_id, + identifier=box.label, + imported_from=imported_from, + ) + n_boxes_uploaded += 1 + except (KeyboardInterrupt, SystemExit): + raise + except Exception as e: + if on_error == 'raise': + raise + errors.append((sample.file_name, e)) + + return result_cls( + project=project, + resource_ids=resource_ids, + n_images_uploaded=len(resource_ids), + n_boxes_uploaded=n_boxes_uploaded, + errors=errors, + ) diff --git a/datamint/importers/coco.py b/datamint/importers/coco.py new file mode 100644 index 00000000..1fa8d77c --- /dev/null +++ b/datamint/importers/coco.py @@ -0,0 +1,150 @@ +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Sequence + +from datamint import Api +from datamint.entities import Project + +from . import _common + + +@dataclass +class COCOBox: + label: str + x: float + y: float + width: float + height: float + + +@dataclass +class COCOSample: + image_path: Path + file_name: str + boxes: list[COCOBox] = field(default_factory=list) + + +@dataclass +class COCOParseResult: + samples: list[COCOSample] + class_names: list[str] + missing_images: list[str] + unsupported_annotations: int + + @property + def num_images(self) -> int: + return len(self.samples) + + @property + def num_boxes(self) -> int: + return sum(len(s.boxes) for s in self.samples) + + +@dataclass +class COCOImportResult: + project: Project | str + resource_ids: list[str] + n_images_uploaded: int + n_boxes_uploaded: int + errors: list[tuple[str, Exception]] = field(default_factory=list) + + +class COCOImporter: + """Parse a COCO-format annotations file and upload it to a Datamint project. """ + + def __init__(self, annotations_file: str | Path, images_dir: str | Path | None = None): + self.annotations_file = Path(annotations_file) + self.images_dir = Path(images_dir) if images_dir is not None else self.annotations_file.parent + self._result: COCOParseResult | None = None + + def parse(self, force: bool = False) -> COCOParseResult: + """Read and validate the COCO JSON file. + + Cached after the first call; pass ``force=True`` to reparse. + + Raises: + ValueError: If the file is structurally invalid (missing required + keys, or an annotation references an unknown category/image id). + """ + if self._result is not None and not force: + return self._result + + with open(self.annotations_file) as f: + data = json.load(f) + + for key in ('images', 'annotations', 'categories'): + if key not in data: + raise ValueError(f"Invalid COCO file: missing required key '{key}'.") + + categories = {cat['id']: cat['name'] for cat in data['categories']} + + images_by_id: dict[int, tuple[str, Path]] = {} + missing_images: list[str] = [] + samples_by_id: dict[int, COCOSample] = {} + for img in data['images']: + file_name = img['file_name'] + image_path = self.images_dir / file_name + images_by_id[img['id']] = (file_name, image_path) + if not image_path.exists(): + missing_images.append(file_name) + continue + samples_by_id[img['id']] = COCOSample(image_path=image_path, file_name=file_name) + + used_class_names: set[str] = set() + unsupported_annotations = 0 + for ann in data['annotations']: + image_id = ann['image_id'] + if image_id not in images_by_id: + raise ValueError(f"Annotation {ann.get('id')} references unknown image_id {image_id}.") + if image_id not in samples_by_id: + continue # image file missing on disk, already recorded above + + category_id = ann['category_id'] + if category_id not in categories: + raise ValueError(f"Annotation {ann.get('id')} references unknown category_id {category_id}.") + + bbox = ann.get('bbox') + if bbox is None: + # COCO allows annotations without bounding boxes (e.g., segmentation-only annotations) + if ann.get('segmentation'): + unsupported_annotations += 1 + continue + + x, y, width, height = bbox + label = categories[category_id] + samples_by_id[image_id].boxes.append(COCOBox(label=label, x=x, y=y, width=width, height=height)) + used_class_names.add(label) + + self._result = COCOParseResult( + samples=list(samples_by_id.values()), + class_names=sorted(used_class_names), + missing_images=missing_images, + unsupported_annotations=unsupported_annotations, + ) + return self._result + + def import_to_project(self, + project: Project | str, + api: Api | None = None, + *, + tags: Sequence[str] | None = None, + imported_from: str = 'coco-import', + on_error: Literal['raise', 'skip'] = 'raise', + progress_bar: bool = True) -> COCOImportResult: + """Upload the parsed images and box annotations to a Datamint project. + + Calls :meth:`parse` first (reusing the cached result if already called). + ``api`` defaults to a new :class:`~datamint.api.client.Api` instance if not given. + """ + result = self.parse() + return _common.import_boxes_to_project( + result, project, api, + box_points=lambda b: ((b.x, b.y), (b.x + b.width, b.y + b.height)), + source_label=str(self.annotations_file), + tags=tags, + imported_from=imported_from, + on_error=on_error, + progress_bar=progress_bar, + result_cls=COCOImportResult, + ) diff --git a/datamint/importers/pascal_voc.py b/datamint/importers/pascal_voc.py new file mode 100644 index 00000000..846e6c33 --- /dev/null +++ b/datamint/importers/pascal_voc.py @@ -0,0 +1,153 @@ +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Sequence + +from datamint import Api +from datamint.entities import Project + +from . import _common + + +@dataclass +class PascalVOCBox: + label: str + x1: float + y1: float + x2: float + y2: float + difficult: bool = False + + +@dataclass +class PascalVOCSample: + image_path: Path + file_name: str + boxes: list[PascalVOCBox] = field(default_factory=list) + + +@dataclass +class PascalVOCParseResult: + samples: list[PascalVOCSample] + class_names: list[str] + missing_images: list[str] + unsupported_annotations: int + + @property + def num_images(self) -> int: + return len(self.samples) + + @property + def num_boxes(self) -> int: + return sum(len(s.boxes) for s in self.samples) + + +@dataclass +class PascalVOCImportResult: + project: Project | str + resource_ids: list[str] + n_images_uploaded: int + n_boxes_uploaded: int + errors: list[tuple[str, Exception]] = field(default_factory=list) + + +class PascalVOCImporter: + """Parse a Pascal VOC-format annotations directory and upload it to a Datamint project. + + Only bounding-box annotations (the ``bndbox`` field) are imported. + """ + + def __init__(self, annotations_dir: str | Path, images_dir: str | Path): + self.annotations_dir = Path(annotations_dir) + self.images_dir = Path(images_dir) + self._result: PascalVOCParseResult | None = None + + def parse(self, force: bool = False) -> PascalVOCParseResult: + """Read and validate the Pascal VOC XML annotation files. + + Cached after the first call; pass ``force=True`` to reparse. + + Raises: + ValueError: If ``annotations_dir`` doesn't exist, or an XML file is + missing its required ``filename`` element. + """ + if self._result is not None and not force: + return self._result + + if not self.annotations_dir.is_dir(): + raise ValueError(f"Invalid Pascal VOC annotations directory: '{self.annotations_dir}' does not exist.") + + samples: list[PascalVOCSample] = [] + missing_images: list[str] = [] + used_class_names: set[str] = set() + unsupported_annotations = 0 + + for xml_path in sorted(self.annotations_dir.glob('*.xml')): + root = ET.parse(xml_path).getroot() + + file_name = root.findtext('filename', default='').strip() + if not file_name: + raise ValueError(f"Invalid Pascal VOC annotation '{xml_path}': missing required element 'filename'.") + + image_path = self.images_dir / file_name + if not image_path.exists(): + missing_images.append(file_name) + continue + + sample = PascalVOCSample(image_path=image_path, file_name=file_name) + for obj in root.findall('object'): + label = obj.findtext('name', default='').strip() + bb = obj.find('bndbox') + if bb is None: + # e.g. a segmentation object instead of -- not supported + unsupported_annotations += 1 + continue + if not label: + # incomplete object entry skip it + continue + + difficult = obj.findtext('difficult', default='0').strip() == '1' + sample.boxes.append(PascalVOCBox( + label=label, + x1=float(bb.findtext('xmin')), + y1=float(bb.findtext('ymin')), + x2=float(bb.findtext('xmax')), + y2=float(bb.findtext('ymax')), + difficult=difficult, + )) + used_class_names.add(label) + + samples.append(sample) + + self._result = PascalVOCParseResult( + samples=samples, + class_names=sorted(used_class_names), + missing_images=missing_images, + unsupported_annotations=unsupported_annotations, + ) + return self._result + + def import_to_project(self, + project: Project | str, + api: Api | None = None, + *, + tags: Sequence[str] | None = None, + imported_from: str = 'pascal-voc-import', + on_error: Literal['raise', 'skip'] = 'raise', + progress_bar: bool = True) -> PascalVOCImportResult: + """Upload the parsed images and box annotations to a Datamint project. + + Calls :meth:`parse` first (reusing the cached result if already called). + ``api`` defaults to a new :class:`~datamint.api.client.Api` instance if not given. + """ + result = self.parse() + return _common.import_boxes_to_project( + result, project, api, + box_points=lambda b: ((b.x1, b.y1), (b.x2, b.y2)), + source_label=str(self.annotations_dir), + tags=tags, + imported_from=imported_from, + on_error=on_error, + progress_bar=progress_bar, + result_cls=PascalVOCImportResult, + ) diff --git a/datamint/importers/yolo.py b/datamint/importers/yolo.py new file mode 100644 index 00000000..7de757dc --- /dev/null +++ b/datamint/importers/yolo.py @@ -0,0 +1,204 @@ +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Sequence + +import yaml +from PIL import Image + +from datamint import Api +from datamint.entities import Project + +from . import _common + +_DEFAULT_IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.bmp', '.tif', '.tiff', '.webp') + + +@dataclass +class YOLOBox: + label: str + x1: float + y1: float + x2: float + y2: float + + +@dataclass +class YOLOSample: + image_path: Path + file_name: str + boxes: list[YOLOBox] = field(default_factory=list) + + +@dataclass +class YOLOParseResult: + samples: list[YOLOSample] + class_names: list[str] + missing_images: list[str] + unsupported_annotations: int + + @property + def num_images(self) -> int: + return len(self.samples) + + @property + def num_boxes(self) -> int: + return sum(len(s.boxes) for s in self.samples) + + +@dataclass +class YOLOImportResult: + project: Project | str + resource_ids: list[str] + n_images_uploaded: int + n_boxes_uploaded: int + errors: list[tuple[str, Exception]] = field(default_factory=list) + + +def _load_names_from_yaml(path: Path) -> dict[int, str]: + with open(path) as f: + data = yaml.safe_load(f) + + names = data.get('names') + if names is None: + raise ValueError(f"Invalid YOLO data.yaml '{path}': missing required key 'names'.") + + if isinstance(names, dict): + return {int(idx): str(name) for idx, name in names.items()} + return {idx: str(name) for idx, name in enumerate(names)} + + +class YOLOImporter: + """Parse a YOLO-format (images + normalized-bbox label .txt files) dataset and + upload it to a Datamint project. + + """ + + def __init__(self, + images_dir: str | Path, + labels_dir: str | Path, + *, + class_names: Sequence[str] | None = None, + data_yaml: str | Path | None = None, + image_extensions: Sequence[str] = _DEFAULT_IMAGE_EXTENSIONS): + self.images_dir = Path(images_dir) + self.labels_dir = Path(labels_dir) + self.class_names_override = list(class_names) if class_names is not None else None + self.data_yaml = Path(data_yaml) if data_yaml is not None else None + self.image_extensions = image_extensions + self._result: YOLOParseResult | None = None + + def _resolve_class_names(self) -> dict[int, str]: + if self.class_names_override is not None: + return dict(enumerate(self.class_names_override)) + + if self.data_yaml is not None: + return _load_names_from_yaml(self.data_yaml) + + raise ValueError('No class names available: pass class_names or data_yaml explicitly.') + + def _find_image(self, stem: str) -> Path | None: + for ext in self.image_extensions: + candidate = self.images_dir / f'{stem}{ext}' + if candidate.exists(): + return candidate + return None + + def parse(self, force: bool = False) -> YOLOParseResult: + """Read and validate the YOLO label files. + + Cached after the first call; pass ``force=True`` to reparse. + + Raises: + ValueError: If class names can't be resolved, a label line + references an unknown class index, or a normalized coordinate + is outside the expected ``[0, 1]`` range. + """ + if self._result is not None and not force: + return self._result + + class_names_by_id = self._resolve_class_names() + + samples: list[YOLOSample] = [] + missing_images: list[str] = [] + used_class_names: set[str] = set() + unsupported_annotations = 0 + + for txt_path in sorted(self.labels_dir.glob('*.txt')): + if txt_path.name in ('classes.txt',): + continue + + image_path = self._find_image(txt_path.stem) + if image_path is None: + missing_images.append(txt_path.name) + continue + + img_w, img_h = Image.open(image_path).size + + sample = YOLOSample(image_path=image_path, file_name=image_path.name) + with open(txt_path) as f: + for line in f: + parts = line.split() + if not parts: + continue + if len(parts) != 5: + # segmentation/OBB/keypoint variants have a different field count -- not supported + unsupported_annotations += 1 + continue + + class_id = int(parts[0]) + if class_id not in class_names_by_id: + raise ValueError(f"Label '{txt_path}' references unknown class_id {class_id}.") + + cx, cy, w, h = (float(v) for v in parts[1:]) + for coord_name, value in (('x_center', cx), ('y_center', cy), ('width', w), ('height', h)): + if not (0.0 <= value <= 1.0): + raise ValueError(f"Label '{txt_path}' has out-of-range normalized " + f"{coord_name}={value!r} (expected 0<=v<=1).") + + cx, w = cx * img_w, w * img_w + cy, h = cy * img_h, h * img_h + + label = class_names_by_id[class_id] + sample.boxes.append(YOLOBox( + label=label, + x1=cx - w / 2, + y1=cy - h / 2, + x2=cx + w / 2, + y2=cy + h / 2, + )) + used_class_names.add(label) + + samples.append(sample) + + self._result = YOLOParseResult( + samples=samples, + class_names=sorted(used_class_names), + missing_images=missing_images, + unsupported_annotations=unsupported_annotations, + ) + return self._result + + def import_to_project(self, + project: Project | str, + api: Api | None = None, + *, + tags: Sequence[str] | None = None, + imported_from: str = 'yolo-import', + on_error: Literal['raise', 'skip'] = 'raise', + progress_bar: bool = True) -> YOLOImportResult: + """Upload the parsed images and box annotations to a Datamint project. + + Calls :meth:`parse` first (reusing the cached result if already called). + ``api`` defaults to a new :class:`~datamint.api.client.Api` instance if not given. + """ + result = self.parse() + return _common.import_boxes_to_project( + result, project, api, + box_points=lambda b: ((b.x1, b.y1), (b.x2, b.y2)), + source_label=str(self.labels_dir), + tags=tags, + imported_from=imported_from, + on_error=on_error, + progress_bar=progress_bar, + result_cls=YOLOImportResult, + ) diff --git a/docs/source/client_api_content.rst b/docs/source/client_api_content.rst index 872adc04..495fb151 100644 --- a/docs/source/client_api_content.rst +++ b/docs/source/client_api_content.rst @@ -459,6 +459,60 @@ Organize resources with channels See also the tutorial notebooks: `upload_data.ipynb `_ +Importing External Dataset Formats +----------------------------------- + +If you already have a dataset labeled in a common format, ``datamint.importers`` +(see :doc:`datamint.importers` for the full reference) saves you from +hand-rolling the upload-images-then-loop-over-annotations glue code: each +importer parses the on-disk format and uploads images plus box annotations to +a project in one call. + +.. list-table:: + :header-rows: 1 + + * - Importer + - Format + - Constructor + * - :py:class:`~datamint.importers.coco.COCOImporter` + - COCO JSON (``images``/``annotations``/``categories``) + - ``COCOImporter(annotations_file, images_dir=None)`` + * - :py:class:`~datamint.importers.pascal_voc.PascalVOCImporter` + - Pascal VOC XML (one ``.xml`` per image, ```` elements) + - ``PascalVOCImporter(annotations_dir, images_dir)`` + * - :py:class:`~datamint.importers.yolo.YOLOImporter` + - YOLO ``.txt`` labels (normalized ``class x_center y_center width height``) + - ``YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None)`` + +Only bounding boxes are imported. If a dataset contains polygon/segmentation +annotations (or, for YOLO, OBB/keypoint label lines), ``parse()`` counts them +and ``import_to_project()`` logs a warning naming how many were skipped, +rather than failing or silently dropping them. + +Every importer follows the same two-step shape: :py:meth:`~datamint.importers.coco.COCOImporter.parse` +reads and validates the dataset with no network calls (useful to preview +image/box counts and class names before uploading anything), and +:py:meth:`~datamint.importers.coco.COCOImporter.import_to_project` reuses that +parsed result to upload the images and their box annotations. ``api`` is +optional and defaults to a new :py:class:`~datamint.api.client.Api` instance +if you don't already have one: + +.. code-block:: python + + from datamint import COCOImporter + + importer = COCOImporter("dataset/train/_annotations.coco.json") + + # No network calls yet -- inspect what would be uploaded + preview = importer.parse() + print(preview.num_images, preview.num_boxes, preview.class_names) + + # Uploads images + box annotations, reusing the parsed result above + result = importer.import_to_project("My Project", tags=["coco-import"]) + print(result.n_images_uploaded, result.n_boxes_uploaded, result.errors) + +See also the tutorial notebook: `05_import_dataset.ipynb `_ + Working with Models -------------------- diff --git a/docs/source/datamint.importers.rst b/docs/source/datamint.importers.rst new file mode 100644 index 00000000..c8eb12c0 --- /dev/null +++ b/docs/source/datamint.importers.rst @@ -0,0 +1,31 @@ +datamint.importers +=================== + +Format-specific importers that parse an externally-labeled dataset and upload +it to a Datamint project. See :ref:`client_python_api` for a narrative +walkthrough and the `05_import_dataset.ipynb `_ +tutorial notebook. + +COCO +---- + +.. automodule:: datamint.importers.coco + :members: + :undoc-members: + :show-inheritance: + +Pascal VOC +---------- + +.. automodule:: datamint.importers.pascal_voc + :members: + :undoc-members: + :show-inheritance: + +YOLO +---- + +.. automodule:: datamint.importers.yolo + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/index.rst b/docs/source/index.rst index 7f1eace4..a65ccd8e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -318,6 +318,7 @@ Community & Support datamint.apihandler datamint.api.base_classes datamint.dataset + datamint.importers datamint.entities datamint.lightning_api datamint.mlflow_api diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index 1e84a033..6601a282 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -23,6 +23,7 @@ Datasets * `02_patient_wise_splits.ipynb `_: Split datasets by patient to avoid data leakage between train and test sets. * `03_build_dataset.ipynb `_: Build and configure a PyTorch dataset from a Datamint project. * `04_volume_dataset.ipynb `_: Work with 3D volume datasets (NIfTI, DICOM series). +* `05_import_dataset.ipynb `_: Import an already-labeled dataset (COCO, Pascal VOC, or YOLO format) into a project in one call. Experiment Tracking ------------------- diff --git a/notebooks/03_datasets/05_import_dataset.ipynb b/notebooks/03_datasets/05_import_dataset.ipynb new file mode 100644 index 00000000..47b9deaa --- /dev/null +++ b/notebooks/03_datasets/05_import_dataset.ipynb @@ -0,0 +1,366 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Importing External Dataset Formats\n", + "\n", + "This tutorial shows how to bring an already-labeled dataset into a Datamint project using `datamint.importers`, instead of hand-rolling `upload_resources()` plus a loop of `add_box_annotation()` calls.\n", + "\n", + "Three formats are supported:\n", + "\n", + "| Importer | Format |\n", + "|---|---|\n", + "| `COCOImporter` | COCO JSON (`images`/`annotations`/`categories`) |\n", + "| `PascalVOCImporter` | Pascal VOC XML (one `.xml` file per image) |\n", + "| `YOLOImporter` | YOLO `.txt` labels (normalized `class x_center y_center width height`) |\n", + "\n", + "All three share the same two-step shape:\n", + "- `.parse()` reads and validates the dataset on disk. Use it to preview image/box counts and class names before uploading anything. Unsupported annotations (e.g. polygons) are counted, not silently dropped.\n", + "- `.import_to_project(project)` reuses the parsed result to upload the images and their box annotations. \n", + "\n", + "This notebook builds tiny synthetic datasets in each format (a couple of generated images) so it runs end-to-end without needing an external download." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Make sure you've run `datamint config` in a terminal (or set the `DATAMINT_API_KEY` environment variable) before running this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "from datamint import Api\n", + "\n", + "api = Api()\n", + "\n", + "workdir = Path(tempfile.mkdtemp(prefix=\"datamint_import_tutorial_\"))\n", + "print(f\"Working directory: {workdir}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The three sections below each build their own tiny 2-image dataset from this shared layout, so they can be run independently." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from PIL import Image, ImageDraw\n", + "\n", + "IMAGE_SIZE = (128, 128)\n", + "\n", + "# file_name -> (label, (x, y, width, height))\n", + "DEMO_BOXES = {\n", + " \"image_0.png\": (\"cat\", (10, 10, 40, 30)),\n", + " \"image_1.png\": (\"dog\", (20, 15, 35, 25)),\n", + "}\n", + "\n", + "\n", + "def new_demo_image(bbox):\n", + " img = Image.new(\"RGB\", IMAGE_SIZE, color=\"white\")\n", + " x, y, w, h = bbox\n", + " ImageDraw.Draw(img).rectangle([x, y, x + w, y + h], outline=\"black\")\n", + " return img" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll import each format into its own project, so the results are easy to tell apart in the web app." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_coco = api.projects.create(\n", + " \"Import Tutorial - COCO\", description=\"datamint.importers tutorial\", exists_ok=True\n", + ")\n", + "project_voc = api.projects.create(\n", + " \"Import Tutorial - Pascal VOC\", description=\"datamint.importers tutorial\", exists_ok=True\n", + ")\n", + "project_yolo = api.projects.create(\n", + " \"Import Tutorial - YOLO\", description=\"datamint.importers tutorial\", exists_ok=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Import a COCO dataset\n", + "\n", + "`COCOImporter(annotations_file, images_dir=None)` reads a single COCO JSON file. `images_dir` defaults to the annotations file's parent directory.\n", + "\n", + "First, generate a tiny COCO dataset: two images, one box each.\n", + "\n", + "*(`build_coco_dataset` below is demo-data setup only, not part of the importer API -- feel free to skip to the `COCOImporter` cell.)*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "\n", + "def build_coco_dataset(workdir: Path) -> Path:\n", + " coco_dir = workdir / \"coco_dataset\"\n", + " coco_dir.mkdir()\n", + "\n", + " images, annotations = [], []\n", + " for i, (file_name, (label, bbox)) in enumerate(DEMO_BOXES.items()):\n", + " new_demo_image(bbox).save(coco_dir / file_name)\n", + "\n", + " images.append({\"id\": i, \"file_name\": file_name, \"width\": IMAGE_SIZE[0], \"height\": IMAGE_SIZE[1]})\n", + " annotations.append({\n", + " \"id\": i,\n", + " \"image_id\": i,\n", + " \"category_id\": 0 if label == \"cat\" else 1,\n", + " \"bbox\": list(bbox),\n", + " })\n", + "\n", + " coco_json = {\n", + " \"images\": images,\n", + " \"annotations\": annotations,\n", + " \"categories\": [{\"id\": 0, \"name\": \"cat\"}, {\"id\": 1, \"name\": \"dog\"}],\n", + " }\n", + " annotations_file = coco_dir / \"_annotations.coco.json\"\n", + " annotations_file.write_text(json.dumps(coco_json))\n", + " return annotations_file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coco_annotations_file = build_coco_dataset(workdir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Parse it first to preview what would be uploaded:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datamint import COCOImporter\n", + "\n", + "coco_importer = COCOImporter(coco_annotations_file)\n", + "\n", + "preview = coco_importer.parse()\n", + "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", + "print(f\"missing images: {preview.missing_images}, unsupported annotations: {preview.unsupported_annotations}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then upload the images and their box annotations. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = coco_importer.import_to_project(project_coco, tags=[\"coco-import-tutorial\"])\n", + "\n", + "print(f\"Uploaded images: {result.n_images_uploaded}\")\n", + "print(f\"Uploaded boxes: {result.n_boxes_uploaded}\")\n", + "print(f\"Errors: {result.errors}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Import a Pascal VOC dataset\n", + "\n", + "`PascalVOCImporter(annotations_dir, images_dir)` -- unlike COCO, both directories are required explicitly.\n", + "\n", + "*(`build_voc_dataset` below is demo-data setup only, not part of the importer API -- feel free to skip to the `PascalVOCImporter` cell.)*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xml.etree.ElementTree as ET\n", + "\n", + "\n", + "def build_voc_dataset(workdir: Path) -> tuple[Path, Path]:\n", + " voc_dir = workdir / \"voc_dataset\"\n", + " images_dir = voc_dir / \"JPEGImages\"\n", + " annotations_dir = voc_dir / \"Annotations\"\n", + " images_dir.mkdir(parents=True)\n", + " annotations_dir.mkdir(parents=True)\n", + "\n", + " for file_name, (label, bbox) in DEMO_BOXES.items():\n", + " new_demo_image(bbox).save(images_dir / file_name)\n", + "\n", + " x, y, w, h = bbox\n", + " annotation = ET.Element(\"annotation\")\n", + " ET.SubElement(annotation, \"filename\").text = file_name\n", + " obj = ET.SubElement(annotation, \"object\")\n", + " ET.SubElement(obj, \"name\").text = label\n", + " bndbox = ET.SubElement(obj, \"bndbox\")\n", + " ET.SubElement(bndbox, \"xmin\").text = str(x)\n", + " ET.SubElement(bndbox, \"ymin\").text = str(y)\n", + " ET.SubElement(bndbox, \"xmax\").text = str(x + w)\n", + " ET.SubElement(bndbox, \"ymax\").text = str(y + h)\n", + " ET.ElementTree(annotation).write(annotations_dir / f\"{Path(file_name).stem}.xml\")\n", + "\n", + " return annotations_dir, images_dir" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "voc_annotations_dir, voc_images_dir = build_voc_dataset(workdir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datamint import PascalVOCImporter\n", + "\n", + "voc_importer = PascalVOCImporter(voc_annotations_dir, voc_images_dir)\n", + "\n", + "preview = voc_importer.parse()\n", + "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", + "\n", + "result = voc_importer.import_to_project(project_voc, tags=[\"voc-import-tutorial\"])\n", + "print(f\"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Import a YOLO dataset\n", + "\n", + "`YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None)` -- YOLO label files are pure numbers with no class name or image filename embedded, so class names must be resolved from one of: an explicit `class_names` list, an auto-detected `data.yaml`/`data.yml` (`names:` key), or an auto-detected legacy `classes.txt`. Coordinates are normalized (`0..1` of image width/height): `class x_center y_center width height`.\n", + "\n", + "*(`build_yolo_dataset` below is demo-data setup only, not part of the importer API -- feel free to skip to the `YOLOImporter` cell.)*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def build_yolo_dataset(workdir: Path) -> tuple[Path, Path, list[str]]:\n", + " yolo_dir = workdir / \"yolo_dataset\"\n", + " images_dir = yolo_dir / \"images\"\n", + " labels_dir = yolo_dir / \"labels\"\n", + " images_dir.mkdir(parents=True)\n", + " labels_dir.mkdir(parents=True)\n", + "\n", + " class_names = [\"cat\", \"dog\"]\n", + " img_w, img_h = IMAGE_SIZE\n", + "\n", + " for file_name, (label, bbox) in DEMO_BOXES.items():\n", + " new_demo_image(bbox).save(images_dir / file_name)\n", + "\n", + " x, y, w, h = bbox\n", + " cx, cy = (x + w / 2) / img_w, (y + h / 2) / img_h\n", + " nw, nh = w / img_w, h / img_h\n", + "\n", + " class_id = class_names.index(label)\n", + " label_path = labels_dir / f\"{Path(file_name).stem}.txt\"\n", + " label_path.write_text(f\"{class_id} {cx} {cy} {nw} {nh}\\n\")\n", + "\n", + " return images_dir, labels_dir, class_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "yolo_images_dir, yolo_labels_dir, yolo_class_names = build_yolo_dataset(workdir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datamint import YOLOImporter\n", + "\n", + "yolo_importer = YOLOImporter(yolo_images_dir, yolo_labels_dir, class_names=yolo_class_names)\n", + "\n", + "preview = yolo_importer.parse()\n", + "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", + "\n", + "result = yolo_importer.import_to_project(project_yolo, tags=[\"yolo-import-tutorial\"])\n", + "print(f\"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/03_datasets/README.md b/notebooks/03_datasets/README.md index eecc1ce6..ed048109 100644 --- a/notebooks/03_datasets/README.md +++ b/notebooks/03_datasets/README.md @@ -8,3 +8,4 @@ PyTorch dataset classes, data splits, and volume loading. | [02_patient_wise_splits](02_patient_wise_splits.ipynb) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Patient-level splitting to prevent data leakage in multi-scan datasets | | [03_build_dataset](03_build_dataset.ipynb) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Use `build_dataset` to auto-detect project type and get the right dataset class | | [04_volume_dataset](04_volume_dataset.ipynb) | ![Advanced](https://img.shields.io/badge/level-advanced-red) | Load 3D volumes, slice along anatomical axes, and apply albumentations transforms | +| [05_import_dataset](05_import_dataset.ipynb) | ![Intermediate](https://img.shields.io/badge/level-beginner-brightgreen) | Import an already-labeled dataset (COCO, Pascal VOC, or YOLO) into a project with `datamint.importers` | diff --git a/notebooks/README.md b/notebooks/README.md index e826aff3..0cbc66bb 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -15,7 +15,7 @@ Folders are numbered in the recommended learning order. |---|---|---| | [01_getting_started](01_getting_started/) | ![Beginner](https://img.shields.io/badge/level-beginner-brightgreen) | Upload data and explore a project | | [02_annotations](02_annotations/) | ![Beginner](https://img.shields.io/badge/level-beginner-brightgreen) | Upload and work with annotations | -| [03_datasets](03_datasets/) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Build PyTorch datasets, splits, and volume loading | +| [03_datasets](03_datasets/) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Build PyTorch datasets, splits, volume loading, and importing external dataset formats | | [04_experiment_tracking](04_experiment_tracking/) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Log metrics and artifacts, and manage the model registry, with MLflow | | [05_deployment](05_deployment/) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Deploy registered and external models | | [06_end_to_end](06_end_to_end/) | ![Advanced](https://img.shields.io/badge/level-advanced-red) | Full pipelines from data to deployed model | @@ -35,6 +35,7 @@ Folders are numbered in the recommended learning order. 2. [`02_patient_wise_splits`](03_datasets/02_patient_wise_splits.ipynb) ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) — Avoid data leakage with patient-level splitting 3. [`03_build_dataset`](03_datasets/03_build_dataset.ipynb) ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) — Auto-detect dataset type with `build_dataset` 4. [`04_volume_dataset`](03_datasets/04_volume_dataset.ipynb) ![Advanced](https://img.shields.io/badge/level-advanced-red) — Load 3D volumes, slice into 2D, apply albumentations +5. [`05_import_dataset`](03_datasets/05_import_dataset.ipynb) ![Intermediate](https://img.shields.io/badge/level-beginner-brightgreen) — Import an already-labeled dataset (COCO, Pascal VOC, or YOLO) into a project ### 04 — Experiment Tracking 1. [`01_mlflow_manual_logging`](04_experiment_tracking/01_mlflow_manual_logging.ipynb) ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) — Log metrics, parameters, and models manually with MLflow