Make dataset formats like YOLO/COCO/other-common-formats easy to import on Datamint (DAT-1053)

datamint/__init__.py

@@ -15,6 +15,9 @@
     from .dataset.image_dataset import ImageDataset as ImageDataset
     from .dataset.volume_dataset import VolumeDataset as VolumeDataset
     from .default_project import select_project as select_project
+    from .importers.coco import COCOImporter as COCOImporter
+    from .importers.pascal_voc import PascalVOCImporter as PascalVOCImporter
+    from .importers.yolo import YOLOImporter as YOLOImporter
     from .mlflow.flavors.validation import (
         ModelValidationError as ModelValidationError,
         ValidationIssue as ValidationIssue,
@@ -27,7 +30,7 @@
 
     __getattr__, __dir__, __all__ = lazy.attach(
         __name__,
-        submodules=['dataset', "examples"],
+        submodules=['dataset', "examples", "importers"],
         submod_attrs={
             "api.client": ["Api"],
             # New modular dataset classes
@@ -36,6 +39,9 @@
             "mlflow.flavors.validation": ["validate_model", "ValidationReport",
                                           "ValidationIssue", "ModelValidationError"],
             "default_project": ["select_project"],
+            "importers.coco": ["COCOImporter"],
+            "importers.pascal_voc": ["PascalVOCImporter"],
+            "importers.yolo": ["YOLOImporter"],
         },
     )
 

datamint/importers/__init__.py

@@ -0,0 +1,9 @@
+from .coco import COCOBox, COCOImporter, COCOImportResult, COCOParseResult, COCOSample
+from .pascal_voc import PascalVOCBox, PascalVOCImporter, PascalVOCImportResult, PascalVOCParseResult, PascalVOCSample
+from .yolo import YOLOBox, YOLOImporter, YOLOImportResult, YOLOParseResult, YOLOSample
+
+__all__ = [
+    'COCOImporter', 'COCOParseResult', 'COCOImportResult', 'COCOSample', 'COCOBox',
+    'PascalVOCImporter', 'PascalVOCParseResult', 'PascalVOCImportResult', 'PascalVOCSample', 'PascalVOCBox',
+    'YOLOImporter', 'YOLOParseResult', 'YOLOImportResult', 'YOLOSample', 'YOLOBox',
+]

datamint/importers/_common.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Callable, Literal, Sequence
+
+from tqdm.auto import tqdm
+
+from datamint import Api
+from datamint.entities import Project
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def import_boxes_to_project(result,
+                            project: Project | str,
+                            api: Api | None,
+                            *,
+                            box_points: Callable[[object], tuple[tuple[float, float], tuple[float, float]]],
+                            source_label: str,
+                            tags: Sequence[str] | None,
+                            imported_from: str,
+                            on_error: Literal['raise', 'skip'],
+                            progress_bar: bool,
+                            result_cls: type):
+    """Shared upload loop behind every ``*Importer.import_to_project()``.
+
+    ``result`` is a parse result with ``samples``, ``missing_images``, and
+    ``unsupported_annotations`` attributes. ``box_points`` maps a format's box
+    dataclass to a ``(point1, point2)`` pair for ``add_box_annotation``.
+
+    """
+    api = api or Api()
+
+    if result.missing_images:
+        _LOGGER.warning(f'{len(result.missing_images)} image(s) referenced in {source_label} '
+                        f'were not found on disk and will be skipped.')
+    if result.unsupported_annotations:
+        _LOGGER.warning(f'{result.unsupported_annotations} annotation(s) in {source_label} use an '
+                        f'unsupported annotation type (e.g. polygon/segmentation) and were not '
+                        f'imported; only bounding boxes are supported.')
+
+    uploaded = api.resources.upload_resources(
+        [str(s.image_path) for s in result.samples],
+        tags=tags,
+        publish_to=project,
+        on_error=on_error,
+        progress_bar=progress_bar,
+    )
+
+    resource_ids: list[str] = []
+    errors: list[tuple[str, Exception]] = []
+    n_boxes_uploaded = 0
+
+    iterator = zip(result.samples, uploaded)
+    if progress_bar:
+        iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations')
+
+    for sample, resource_id in iterator:
+        if isinstance(resource_id, Exception):
+            errors.append((sample.file_name, resource_id))
+            continue
+        resource_ids.append(resource_id)
+
+        for box in sample.boxes:
+            point1, point2 = box_points(box)
+            try:
+                api.annotations.add_box_annotation(
+                    point1=point1,
+                    point2=point2,
+                    resource=resource_id,
+                    identifier=box.label,
+                    imported_from=imported_from,
+                )
+                n_boxes_uploaded += 1
+            except (KeyboardInterrupt, SystemExit):
+                raise
+            except Exception as e:
+                if on_error == 'raise':
+                    raise
+                errors.append((sample.file_name, e))
+
+    return result_cls(
+        project=project,
+        resource_ids=resource_ids,
+        n_images_uploaded=len(resource_ids),
+        n_boxes_uploaded=n_boxes_uploaded,
+        errors=errors,
+    )

datamint/importers/coco.py

@@ -0,0 +1,150 @@
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal, Sequence
+
+from datamint import Api
+from datamint.entities import Project
+
+from . import _common
+
+
+@dataclass
+class COCOBox:
+    label: str
+    x: float
+    y: float
+    width: float
+    height: float
+
+
+@dataclass
+class COCOSample:
+    image_path: Path
+    file_name: str
+    boxes: list[COCOBox] = field(default_factory=list)
+
+
+@dataclass
+class COCOParseResult:
+    samples: list[COCOSample]
+    class_names: list[str]
+    missing_images: list[str]
+    unsupported_annotations: int
+
+    @property
+    def num_images(self) -> int:
+        return len(self.samples)
+
+    @property
+    def num_boxes(self) -> int:
+        return sum(len(s.boxes) for s in self.samples)
+
+
+@dataclass
+class COCOImportResult:
+    project: Project | str
+    resource_ids: list[str]
+    n_images_uploaded: int
+    n_boxes_uploaded: int
+    errors: list[tuple[str, Exception]] = field(default_factory=list)
+
+
+class COCOImporter:
+    """Parse a COCO-format annotations file and upload it to a Datamint project. """
+
+    def __init__(self, annotations_file: str | Path, images_dir: str | Path | None = None):
+        self.annotations_file = Path(annotations_file)
+        self.images_dir = Path(images_dir) if images_dir is not None else self.annotations_file.parent
+        self._result: COCOParseResult | None = None
+
+    def parse(self, force: bool = False) -> COCOParseResult:
+        """Read and validate the COCO JSON file.
+
+        Cached after the first call; pass ``force=True`` to reparse.
+
+        Raises:
+            ValueError: If the file is structurally invalid (missing required
+                keys, or an annotation references an unknown category/image id).
+        """
+        if self._result is not None and not force:
+            return self._result
+
+        with open(self.annotations_file) as f:
+            data = json.load(f)
+
+        for key in ('images', 'annotations', 'categories'):
+            if key not in data:
+                raise ValueError(f"Invalid COCO file: missing required key '{key}'.")
+
+        categories = {cat['id']: cat['name'] for cat in data['categories']}
+
+        images_by_id: dict[int, tuple[str, Path]] = {}
+        missing_images: list[str] = []
+        samples_by_id: dict[int, COCOSample] = {}
+        for img in data['images']:
+            file_name = img['file_name']
+            image_path = self.images_dir / file_name
+            images_by_id[img['id']] = (file_name, image_path)
+            if not image_path.exists():
+                missing_images.append(file_name)
+                continue
+            samples_by_id[img['id']] = COCOSample(image_path=image_path, file_name=file_name)
+
+        used_class_names: set[str] = set()
+        unsupported_annotations = 0
+        for ann in data['annotations']:
+            image_id = ann['image_id']
+            if image_id not in images_by_id:
+                raise ValueError(f"Annotation {ann.get('id')} references unknown image_id {image_id}.")
+            if image_id not in samples_by_id:
+                continue  # image file missing on disk, already recorded above
+
+            category_id = ann['category_id']
+            if category_id not in categories:
+                raise ValueError(f"Annotation {ann.get('id')} references unknown category_id {category_id}.")
+
+            bbox = ann.get('bbox')
+            if bbox is None:
+                # COCO allows annotations without bounding boxes (e.g., segmentation-only annotations)
+                if ann.get('segmentation'):
+                    unsupported_annotations += 1
+                continue
+
+            x, y, width, height = bbox
+            label = categories[category_id]
+            samples_by_id[image_id].boxes.append(COCOBox(label=label, x=x, y=y, width=width, height=height))
+            used_class_names.add(label)
+
+        self._result = COCOParseResult(
+            samples=list(samples_by_id.values()),
+            class_names=sorted(used_class_names),
+            missing_images=missing_images,
+            unsupported_annotations=unsupported_annotations,
+        )
+        return self._result
+
+    def import_to_project(self,
+                          project: Project | str,
+                          api: Api | None = None,
+                          *,
+                          tags: Sequence[str] | None = None,
+                          imported_from: str = 'coco-import',
+                          on_error: Literal['raise', 'skip'] = 'raise',
+                          progress_bar: bool = True) -> COCOImportResult:
+        """Upload the parsed images and box annotations to a Datamint project.
+
+        Calls :meth:`parse` first (reusing the cached result if already called).
+        ``api`` defaults to a new :class:`~datamint.api.client.Api` instance if not given.
+        """
+        result = self.parse()
+        return _common.import_boxes_to_project(
+            result, project, api,
+            box_points=lambda b: ((b.x, b.y), (b.x + b.width, b.y + b.height)),
+            source_label=str(self.annotations_file),
+            tags=tags,
+            imported_from=imported_from,
+            on_error=on_error,
+            progress_bar=progress_bar,
+            result_cls=COCOImportResult,
+        )