From 7af7ab3baab6552bbba6a29d01c316e457821c4d Mon Sep 17 00:00:00 2001 From: luandalmazo Date: Thu, 13 Aug 2026 11:46:09 -0300 Subject: [PATCH 1/3] add one-line importers --- datamint/__init__.py | 8 +- datamint/importers/__init__.py | 9 + datamint/importers/coco.py | 189 +++++++++++ datamint/importers/pascal_voc.py | 186 +++++++++++ datamint/importers/yolo.py | 233 +++++++++++++ docs/source/client_api_content.rst | 53 +++ docs/source/datamint.importers.rst | 31 ++ docs/source/index.rst | 1 + docs/source/tutorials.rst | 1 + notebooks/03_datasets/05_import_dataset.ipynb | 308 ++++++++++++++++++ notebooks/03_datasets/README.md | 1 + notebooks/README.md | 3 +- 12 files changed, 1021 insertions(+), 2 deletions(-) create mode 100644 datamint/importers/__init__.py create mode 100644 datamint/importers/coco.py create mode 100644 datamint/importers/pascal_voc.py create mode 100644 datamint/importers/yolo.py create mode 100644 docs/source/datamint.importers.rst create mode 100644 notebooks/03_datasets/05_import_dataset.ipynb diff --git a/datamint/__init__.py b/datamint/__init__.py index c6c45e84..00578c3c 100644 --- a/datamint/__init__.py +++ b/datamint/__init__.py @@ -15,6 +15,9 @@ from .dataset.image_dataset import ImageDataset as ImageDataset from .dataset.volume_dataset import VolumeDataset as VolumeDataset from .default_project import select_project as select_project + from .importers.coco import COCOImporter as COCOImporter + from .importers.pascal_voc import PascalVOCImporter as PascalVOCImporter + from .importers.yolo import YOLOImporter as YOLOImporter from .mlflow.flavors.validation import ( ModelValidationError as ModelValidationError, ValidationIssue as ValidationIssue, @@ -27,7 +30,7 @@ __getattr__, __dir__, __all__ = lazy.attach( __name__, - submodules=['dataset', "examples"], + submodules=['dataset', "examples", "importers"], submod_attrs={ "api.client": ["Api"], # New modular dataset classes @@ -36,6 +39,9 @@ "mlflow.flavors.validation": ["validate_model", "ValidationReport", "ValidationIssue", "ModelValidationError"], "default_project": ["select_project"], + "importers.coco": ["COCOImporter"], + "importers.pascal_voc": ["PascalVOCImporter"], + "importers.yolo": ["YOLOImporter"], }, ) diff --git a/datamint/importers/__init__.py b/datamint/importers/__init__.py new file mode 100644 index 00000000..11adee3d --- /dev/null +++ b/datamint/importers/__init__.py @@ -0,0 +1,9 @@ +from .coco import COCOBox, COCOImporter, COCOImportResult, COCOParseResult, COCOSample +from .pascal_voc import PascalVOCBox, PascalVOCImporter, PascalVOCImportResult, PascalVOCParseResult, PascalVOCSample +from .yolo import YOLOBox, YOLOImporter, YOLOImportResult, YOLOParseResult, YOLOSample + +__all__ = [ + 'COCOImporter', 'COCOParseResult', 'COCOImportResult', 'COCOSample', 'COCOBox', + 'PascalVOCImporter', 'PascalVOCParseResult', 'PascalVOCImportResult', 'PascalVOCSample', 'PascalVOCBox', + 'YOLOImporter', 'YOLOParseResult', 'YOLOImportResult', 'YOLOSample', 'YOLOBox', +] diff --git a/datamint/importers/coco.py b/datamint/importers/coco.py new file mode 100644 index 00000000..b71b4a89 --- /dev/null +++ b/datamint/importers/coco.py @@ -0,0 +1,189 @@ +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Sequence + +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +_LOGGER = logging.getLogger(__name__) + + +@dataclass +class COCOBox: + label: str + x: float + y: float + width: float + height: float + + +@dataclass +class COCOSample: + image_path: Path + file_name: str + boxes: list[COCOBox] = field(default_factory=list) + + +@dataclass +class COCOParseResult: + samples: list[COCOSample] + class_names: list[str] + missing_images: list[str] + + @property + def num_images(self) -> int: + return len(self.samples) + + @property + def num_boxes(self) -> int: + return sum(len(s.boxes) for s in self.samples) + + +@dataclass +class COCOImportResult: + project: Project | str + resource_ids: list[str] + n_images_uploaded: int + n_boxes_uploaded: int + errors: list[tuple[str, Exception]] = field(default_factory=list) + + +class COCOImporter: + """Parse a COCO-format annotations file and upload it to a Datamint project. + + Only bounding-box annotations (the ``bbox`` field) are imported; polygon + ``segmentation`` fields are ignored. + """ + + def __init__(self, annotations_file: str | Path, images_dir: str | Path | None = None): + self.annotations_file = Path(annotations_file) + self.images_dir = Path(images_dir) if images_dir is not None else self.annotations_file.parent + self._result: COCOParseResult | None = None + + def parse(self, force: bool = False) -> COCOParseResult: + """Read and validate the COCO JSON file. + + Cached after the first call; pass ``force=True`` to reparse. + + Raises: + ValueError: If the file is structurally invalid (missing required + keys, or an annotation references an unknown category/image id). + """ + if self._result is not None and not force: + return self._result + + with open(self.annotations_file) as f: + data = json.load(f) + + for key in ('images', 'annotations', 'categories'): + if key not in data: + raise ValueError(f"Invalid COCO file: missing required key '{key}'.") + + categories = {cat['id']: cat['name'] for cat in data['categories']} + + images_by_id: dict[int, tuple[str, Path]] = {} + missing_images: list[str] = [] + samples_by_id: dict[int, COCOSample] = {} + for img in data['images']: + file_name = img['file_name'] + image_path = self.images_dir / file_name + images_by_id[img['id']] = (file_name, image_path) + if not image_path.exists(): + missing_images.append(file_name) + continue + samples_by_id[img['id']] = COCOSample(image_path=image_path, file_name=file_name) + + used_class_names: set[str] = set() + for ann in data['annotations']: + image_id = ann['image_id'] + if image_id not in images_by_id: + raise ValueError(f"Annotation {ann.get('id')} references unknown image_id {image_id}.") + if image_id not in samples_by_id: + continue # image file missing on disk, already recorded above + + category_id = ann['category_id'] + if category_id not in categories: + raise ValueError(f"Annotation {ann.get('id')} references unknown category_id {category_id}.") + + bbox = ann.get('bbox') + if bbox is None: + # COCO allows annotations without bounding boxes (e.g., segmentation-only annotations) + continue + + x, y, width, height = bbox + label = categories[category_id] + samples_by_id[image_id].boxes.append(COCOBox(label=label, x=x, y=y, width=width, height=height)) + used_class_names.add(label) + + self._result = COCOParseResult( + samples=list(samples_by_id.values()), + class_names=sorted(used_class_names), + missing_images=missing_images, + ) + return self._result + + def import_to_project(self, + api: Api, + project: Project | str, + *, + tags: Sequence[str] | None = None, + imported_from: str = 'coco-import', + on_error: Literal['raise', 'skip'] = 'raise', + progress_bar: bool = True) -> COCOImportResult: + """Upload the parsed images and box annotations to a Datamint project. + + Calls :meth:`parse` first (reusing the cached result if already called). + """ + result = self.parse() + if result.missing_images: + _LOGGER.warning(f'{len(result.missing_images)} image(s) referenced in ' + f'{self.annotations_file} were not found under {self.images_dir} and will be skipped.') + + uploaded = api.resources.upload_resources( + [str(s.image_path) for s in result.samples], + tags=tags, + publish_to=project, + on_error=on_error, + progress_bar=progress_bar, + ) + + resource_ids: list[str] = [] + errors: list[tuple[str, Exception]] = [] + n_boxes_uploaded = 0 + + iterator = zip(result.samples, uploaded) + if progress_bar: + iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations') + + for sample, resource_id in iterator: + if isinstance(resource_id, Exception): + errors.append((sample.file_name, resource_id)) + continue + resource_ids.append(resource_id) + + for box in sample.boxes: + try: + api.annotations.add_box_annotation( + point1=(box.x, box.y), + point2=(box.x + box.width, box.y + box.height), + resource=resource_id, + identifier=box.label, + imported_from=imported_from, + ) + n_boxes_uploaded += 1 + except Exception as e: + if on_error == 'raise': + raise + errors.append((sample.file_name, e)) + + return COCOImportResult( + project=project, + resource_ids=resource_ids, + n_images_uploaded=len(resource_ids), + n_boxes_uploaded=n_boxes_uploaded, + errors=errors, + ) diff --git a/datamint/importers/pascal_voc.py b/datamint/importers/pascal_voc.py new file mode 100644 index 00000000..e0c60686 --- /dev/null +++ b/datamint/importers/pascal_voc.py @@ -0,0 +1,186 @@ +import logging +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Sequence + +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +_LOGGER = logging.getLogger(__name__) + + +@dataclass +class PascalVOCBox: + label: str + x1: float + y1: float + x2: float + y2: float + difficult: bool = False + + +@dataclass +class PascalVOCSample: + image_path: Path + file_name: str + boxes: list[PascalVOCBox] = field(default_factory=list) + + +@dataclass +class PascalVOCParseResult: + samples: list[PascalVOCSample] + class_names: list[str] + missing_images: list[str] + + @property + def num_images(self) -> int: + return len(self.samples) + + @property + def num_boxes(self) -> int: + return sum(len(s.boxes) for s in self.samples) + + +@dataclass +class PascalVOCImportResult: + project: Project | str + resource_ids: list[str] + n_images_uploaded: int + n_boxes_uploaded: int + errors: list[tuple[str, Exception]] = field(default_factory=list) + + +class PascalVOCImporter: + """Parse a Pascal VOC-format annotations directory and upload it to a Datamint project. + + Only bounding-box annotations (the ``bndbox`` field) are imported. + """ + + def __init__(self, annotations_dir: str | Path, images_dir: str | Path): + self.annotations_dir = Path(annotations_dir) + self.images_dir = Path(images_dir) + self._result: PascalVOCParseResult | None = None + + def parse(self, force: bool = False) -> PascalVOCParseResult: + """Read and validate the Pascal VOC XML annotation files. + + Cached after the first call; pass ``force=True`` to reparse. + + Raises: + ValueError: If ``annotations_dir`` doesn't exist, or an XML file is + missing its required ``filename`` element. + """ + if self._result is not None and not force: + return self._result + + if not self.annotations_dir.is_dir(): + raise ValueError(f"Invalid Pascal VOC annotations directory: '{self.annotations_dir}' does not exist.") + + samples: list[PascalVOCSample] = [] + missing_images: list[str] = [] + used_class_names: set[str] = set() + + for xml_path in sorted(self.annotations_dir.glob('*.xml')): + root = ET.parse(xml_path).getroot() + + file_name = root.findtext('filename', default='').strip() + if not file_name: + raise ValueError(f"Invalid Pascal VOC annotation '{xml_path}': missing required element 'filename'.") + + image_path = self.images_dir / file_name + if not image_path.exists(): + missing_images.append(file_name) + continue + + sample = PascalVOCSample(image_path=image_path, file_name=file_name) + for obj in root.findall('object'): + label = obj.findtext('name', default='').strip() + bb = obj.find('bndbox') + if not label or bb is None: + # incomplete object entry skip it + continue + + difficult = obj.findtext('difficult', default='0').strip() == '1' + sample.boxes.append(PascalVOCBox( + label=label, + x1=float(bb.findtext('xmin')), + y1=float(bb.findtext('ymin')), + x2=float(bb.findtext('xmax')), + y2=float(bb.findtext('ymax')), + difficult=difficult, + )) + used_class_names.add(label) + + samples.append(sample) + + self._result = PascalVOCParseResult( + samples=samples, + class_names=sorted(used_class_names), + missing_images=missing_images, + ) + return self._result + + def import_to_project(self, + api: Api, + project: Project | str, + *, + tags: Sequence[str] | None = None, + imported_from: str = 'pascal-voc-import', + on_error: Literal['raise', 'skip'] = 'raise', + progress_bar: bool = True) -> PascalVOCImportResult: + """Upload the parsed images and box annotations to a Datamint project. + + Calls :meth:`parse` first (reusing the cached result if already called). + """ + result = self.parse() + if result.missing_images: + _LOGGER.warning(f'{len(result.missing_images)} image(s) referenced in ' + f'{self.annotations_dir} were not found under {self.images_dir} and will be skipped.') + + uploaded = api.resources.upload_resources( + [str(s.image_path) for s in result.samples], + tags=tags, + publish_to=project, + on_error=on_error, + progress_bar=progress_bar, + ) + + resource_ids: list[str] = [] + errors: list[tuple[str, Exception]] = [] + n_boxes_uploaded = 0 + + iterator = zip(result.samples, uploaded) + if progress_bar: + iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations') + + for sample, resource_id in iterator: + if isinstance(resource_id, Exception): + errors.append((sample.file_name, resource_id)) + continue + resource_ids.append(resource_id) + + for box in sample.boxes: + try: + api.annotations.add_box_annotation( + point1=(box.x1, box.y1), + point2=(box.x2, box.y2), + resource=resource_id, + identifier=box.label, + imported_from=imported_from, + ) + n_boxes_uploaded += 1 + except Exception as e: + if on_error == 'raise': + raise + errors.append((sample.file_name, e)) + + return PascalVOCImportResult( + project=project, + resource_ids=resource_ids, + n_images_uploaded=len(resource_ids), + n_boxes_uploaded=n_boxes_uploaded, + errors=errors, + ) diff --git a/datamint/importers/yolo.py b/datamint/importers/yolo.py new file mode 100644 index 00000000..a1dabece --- /dev/null +++ b/datamint/importers/yolo.py @@ -0,0 +1,233 @@ +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Sequence + +import yaml +from PIL import Image +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +_LOGGER = logging.getLogger(__name__) + +_DEFAULT_IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.bmp', '.tif', '.tiff', '.webp') + + +@dataclass +class YOLOBox: + label: str + x1: float + y1: float + x2: float + y2: float + + +@dataclass +class YOLOSample: + image_path: Path + file_name: str + boxes: list[YOLOBox] = field(default_factory=list) + + +@dataclass +class YOLOParseResult: + samples: list[YOLOSample] + class_names: list[str] + missing_images: list[str] + + @property + def num_images(self) -> int: + return len(self.samples) + + @property + def num_boxes(self) -> int: + return sum(len(s.boxes) for s in self.samples) + + +@dataclass +class YOLOImportResult: + project: Project | str + resource_ids: list[str] + n_images_uploaded: int + n_boxes_uploaded: int + errors: list[tuple[str, Exception]] = field(default_factory=list) + + +def _load_names_from_yaml(path: Path) -> dict[int, str]: + with open(path) as f: + data = yaml.safe_load(f) + + names = data.get('names') + if names is None: + raise ValueError(f"Invalid YOLO data.yaml '{path}': missing required key 'names'.") + + if isinstance(names, dict): + return {int(idx): str(name) for idx, name in names.items()} + return {idx: str(name) for idx, name in enumerate(names)} + + +class YOLOImporter: + """Parse a YOLO-format (images + normalized-bbox label .txt files) dataset and + upload it to a Datamint project. + + """ + + def __init__(self, + images_dir: str | Path, + labels_dir: str | Path, + *, + class_names: Sequence[str] | None = None, + data_yaml: str | Path | None = None, + image_extensions: Sequence[str] = _DEFAULT_IMAGE_EXTENSIONS): + self.images_dir = Path(images_dir) + self.labels_dir = Path(labels_dir) + self.class_names_override = list(class_names) if class_names is not None else None + self.data_yaml = Path(data_yaml) if data_yaml is not None else None + self.image_extensions = image_extensions + self._result: YOLOParseResult | None = None + + def _resolve_class_names(self) -> dict[int, str]: + if self.class_names_override is not None: + return dict(enumerate(self.class_names_override)) + + if self.data_yaml is not None: + return _load_names_from_yaml(self.data_yaml) + + raise ValueError('No class names available: pass class_names or data_yaml explicitly.') + + def _find_image(self, stem: str) -> Path | None: + for ext in self.image_extensions: + candidate = self.images_dir / f'{stem}{ext}' + if candidate.exists(): + return candidate + return None + + def parse(self, force: bool = False) -> YOLOParseResult: + """Read and validate the YOLO label files. + + Cached after the first call; pass ``force=True`` to reparse. + + Raises: + ValueError: If class names can't be resolved, or a label line + references an unknown class index. + """ + if self._result is not None and not force: + return self._result + + class_names_by_id = self._resolve_class_names() + + samples: list[YOLOSample] = [] + missing_images: list[str] = [] + used_class_names: set[str] = set() + + for txt_path in sorted(self.labels_dir.glob('*.txt')): + if txt_path.name in ('classes.txt',): + continue + + image_path = self._find_image(txt_path.stem) + if image_path is None: + missing_images.append(txt_path.name) + continue + + img_w, img_h = Image.open(image_path).size + + sample = YOLOSample(image_path=image_path, file_name=image_path.name) + with open(txt_path) as f: + for line in f: + parts = line.split() + if not parts: + continue + if len(parts) != 5: + # unsupported line shape skip + continue + + class_id = int(parts[0]) + if class_id not in class_names_by_id: + raise ValueError(f"Label '{txt_path}' references unknown class_id {class_id}.") + + cx, cy, w, h = (float(v) for v in parts[1:]) + cx, w = cx * img_w, w * img_w + cy, h = cy * img_h, h * img_h + + label = class_names_by_id[class_id] + sample.boxes.append(YOLOBox( + label=label, + x1=cx - w / 2, + y1=cy - h / 2, + x2=cx + w / 2, + y2=cy + h / 2, + )) + used_class_names.add(label) + + samples.append(sample) + + self._result = YOLOParseResult( + samples=samples, + class_names=sorted(used_class_names), + missing_images=missing_images, + ) + return self._result + + def import_to_project(self, + api: Api, + project: Project | str, + *, + tags: Sequence[str] | None = None, + imported_from: str = 'yolo-import', + on_error: Literal['raise', 'skip'] = 'raise', + progress_bar: bool = True) -> YOLOImportResult: + """Upload the parsed images and box annotations to a Datamint project. + + Calls :meth:`parse` first (reusing the cached result if already called). + """ + result = self.parse() + if result.missing_images: + _LOGGER.warning(f'{len(result.missing_images)} label file(s) in {self.labels_dir} had no matching ' + f'image under {self.images_dir} and will be skipped.') + + uploaded = api.resources.upload_resources( + [str(s.image_path) for s in result.samples], + tags=tags, + publish_to=project, + on_error=on_error, + progress_bar=progress_bar, + ) + + resource_ids: list[str] = [] + errors: list[tuple[str, Exception]] = [] + n_boxes_uploaded = 0 + + iterator = zip(result.samples, uploaded) + if progress_bar: + iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations') + + for sample, resource_id in iterator: + if isinstance(resource_id, Exception): + errors.append((sample.file_name, resource_id)) + continue + resource_ids.append(resource_id) + + for box in sample.boxes: + try: + api.annotations.add_box_annotation( + point1=(box.x1, box.y1), + point2=(box.x2, box.y2), + resource=resource_id, + identifier=box.label, + imported_from=imported_from, + ) + n_boxes_uploaded += 1 + except Exception as e: + if on_error == 'raise': + raise + errors.append((sample.file_name, e)) + + return YOLOImportResult( + project=project, + resource_ids=resource_ids, + n_images_uploaded=len(resource_ids), + n_boxes_uploaded=n_boxes_uploaded, + errors=errors, + ) diff --git a/docs/source/client_api_content.rst b/docs/source/client_api_content.rst index 872adc04..1493132f 100644 --- a/docs/source/client_api_content.rst +++ b/docs/source/client_api_content.rst @@ -459,6 +459,59 @@ Organize resources with channels See also the tutorial notebooks: `upload_data.ipynb `_ +Importing External Dataset Formats +----------------------------------- + +If you already have a dataset labeled in a common format, ``datamint.importers`` +(see :doc:`datamint.importers` for the full reference) saves you from +hand-rolling the upload-images-then-loop-over-annotations glue code: each +importer parses the on-disk format and uploads images plus box annotations to +a project in one call. + +.. list-table:: + :header-rows: 1 + + * - Importer + - Format + - Constructor + * - :py:class:`~datamint.importers.coco.COCOImporter` + - COCO JSON (``images``/``annotations``/``categories``) + - ``COCOImporter(annotations_file, images_dir=None)`` + * - :py:class:`~datamint.importers.pascal_voc.PascalVOCImporter` + - Pascal VOC XML (one ``.xml`` per image, ```` elements) + - ``PascalVOCImporter(annotations_dir, images_dir)`` + * - :py:class:`~datamint.importers.yolo.YOLOImporter` + - YOLO ``.txt`` labels (normalized ``class x_center y_center width height``) + - ``YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None)` + +Only bounding boxes are imported; polygon/segmentation annotations in these +formats are not supported yet. + +Every importer follows the same two-step shape: :py:meth:`~datamint.importers.coco.COCOImporter.parse` +reads and validates the dataset with no network calls (useful to preview +image/box counts and class names before uploading anything), and +:py:meth:`~datamint.importers.coco.COCOImporter.import_to_project` reuses that +parsed result to upload the images and their box annotations: + +.. code-block:: python + + from datamint import Api, COCOImporter + + api = Api() + project = api.projects.get_by_name("My Project") + + importer = COCOImporter("dataset/train/_annotations.coco.json") + + # No network calls yet -- inspect what would be uploaded + preview = importer.parse() + print(preview.num_images, preview.num_boxes, preview.class_names) + + # Uploads images + box annotations, reusing the parsed result above + result = importer.import_to_project(api, project, tags=["coco-import"]) + print(result.n_images_uploaded, result.n_boxes_uploaded, result.errors) + +See also the tutorial notebook: `05_import_dataset.ipynb `_ + Working with Models -------------------- diff --git a/docs/source/datamint.importers.rst b/docs/source/datamint.importers.rst new file mode 100644 index 00000000..c8eb12c0 --- /dev/null +++ b/docs/source/datamint.importers.rst @@ -0,0 +1,31 @@ +datamint.importers +=================== + +Format-specific importers that parse an externally-labeled dataset and upload +it to a Datamint project. See :ref:`client_python_api` for a narrative +walkthrough and the `05_import_dataset.ipynb `_ +tutorial notebook. + +COCO +---- + +.. automodule:: datamint.importers.coco + :members: + :undoc-members: + :show-inheritance: + +Pascal VOC +---------- + +.. automodule:: datamint.importers.pascal_voc + :members: + :undoc-members: + :show-inheritance: + +YOLO +---- + +.. automodule:: datamint.importers.yolo + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/index.rst b/docs/source/index.rst index 7f1eace4..a65ccd8e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -318,6 +318,7 @@ Community & Support datamint.apihandler datamint.api.base_classes datamint.dataset + datamint.importers datamint.entities datamint.lightning_api datamint.mlflow_api diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index 1e84a033..6601a282 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -23,6 +23,7 @@ Datasets * `02_patient_wise_splits.ipynb `_: Split datasets by patient to avoid data leakage between train and test sets. * `03_build_dataset.ipynb `_: Build and configure a PyTorch dataset from a Datamint project. * `04_volume_dataset.ipynb `_: Work with 3D volume datasets (NIfTI, DICOM series). +* `05_import_dataset.ipynb `_: Import an already-labeled dataset (COCO, Pascal VOC, or YOLO format) into a project in one call. Experiment Tracking ------------------- diff --git a/notebooks/03_datasets/05_import_dataset.ipynb b/notebooks/03_datasets/05_import_dataset.ipynb new file mode 100644 index 00000000..aa79e853 --- /dev/null +++ b/notebooks/03_datasets/05_import_dataset.ipynb @@ -0,0 +1,308 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Importing External Dataset Formats\n", + "\n", + "This tutorial shows how to bring an already-labeled dataset into a Datamint project using `datamint.importers`, instead of hand-rolling `upload_resources()` plus a loop of `add_box_annotation()` calls.\n", + "\n", + "Three formats are supported:\n", + "\n", + "| Importer | Format |\n", + "|---|---|\n", + "| `COCOImporter` | COCO JSON (`images`/`annotations`/`categories`) |\n", + "| `PascalVOCImporter` | Pascal VOC XML (one `.xml` file per image) |\n", + "| `YOLOImporter` | YOLO `.txt` labels (normalized `class x_center y_center width height`) |\n", + "\n", + "All three share the same two-step shape:\n", + "- `.parse()` reads and validates the dataset on disk. Use it to preview image/box counts and class names before uploading anything.\n", + "- `.import_to_project(api, project)` reuses the parsed result to upload the images and their box annotations.\n", + "\n", + "This notebook builds tiny synthetic datasets in each format (a couple of generated images) so it runs end-to-end without needing an external download." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Make sure you've run `datamint config` in a terminal (or set the `DATAMINT_API_KEY` environment variable) before running this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "from datamint import Api\n", + "\n", + "api = Api()\n", + "\n", + "workdir = Path(tempfile.mkdtemp(prefix=\"datamint_import_tutorial_\"))\n", + "print(f\"Working directory: {workdir}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll import each format into its own project, so the results are easy to tell apart in the web app." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_coco = api.projects.create(\n", + " \"Import Tutorial - COCO\", description=\"datamint.importers tutorial\", exists_ok=True\n", + ")\n", + "project_voc = api.projects.create(\n", + " \"Import Tutorial - Pascal VOC\", description=\"datamint.importers tutorial\", exists_ok=True\n", + ")\n", + "project_yolo = api.projects.create(\n", + " \"Import Tutorial - YOLO\", description=\"datamint.importers tutorial\", exists_ok=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Import a COCO dataset\n", + "\n", + "`COCOImporter(annotations_file, images_dir=None)` reads a single COCO JSON file. `images_dir` defaults to the annotations file's parent directory\n", + "First, generate a tiny COCO dataset: two images, one box each." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "from PIL import Image, ImageDraw\n", + "\n", + "coco_dir = workdir / \"coco_dataset\"\n", + "coco_dir.mkdir()\n", + "\n", + "# file_name -> (label, (x, y, width, height))\n", + "boxes_per_image = {\n", + " \"image_0.png\": (\"cat\", (10, 10, 40, 30)),\n", + " \"image_1.png\": (\"dog\", (20, 15, 35, 25)),\n", + "}\n", + "\n", + "images, annotations = [], []\n", + "for i, (file_name, (label, bbox)) in enumerate(boxes_per_image.items()):\n", + " img = Image.new(\"RGB\", (128, 128), color=\"white\")\n", + " x, y, w, h = bbox\n", + " ImageDraw.Draw(img).rectangle([x, y, x + w, y + h], outline=\"black\")\n", + " img.save(coco_dir / file_name)\n", + "\n", + " images.append({\"id\": i, \"file_name\": file_name, \"width\": 128, \"height\": 128})\n", + " annotations.append({\n", + " \"id\": i,\n", + " \"image_id\": i,\n", + " \"category_id\": 0 if label == \"cat\" else 1,\n", + " \"bbox\": list(bbox),\n", + " })\n", + "\n", + "coco_json = {\n", + " \"images\": images,\n", + " \"annotations\": annotations,\n", + " \"categories\": [{\"id\": 0, \"name\": \"cat\"}, {\"id\": 1, \"name\": \"dog\"}],\n", + "}\n", + "\n", + "coco_annotations_file = coco_dir / \"_annotations.coco.json\"\n", + "coco_annotations_file.write_text(json.dumps(coco_json))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Parse it first to preview what would be uploaded:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datamint import COCOImporter\n", + "\n", + "coco_importer = COCOImporter(coco_annotations_file) \n", + "\n", + "preview = coco_importer.parse()\n", + "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", + "print(f\"missing images: {preview.missing_images}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then upload the images and their box annotations:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = coco_importer.import_to_project(api, project_coco, tags=[\"coco-import-tutorial\"])\n", + "\n", + "print(f\"Uploaded images: {result.n_images_uploaded}\")\n", + "print(f\"Uploaded boxes: {result.n_boxes_uploaded}\")\n", + "print(f\"Errors: {result.errors}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Import a Pascal VOC dataset\n", + "\n", + "`PascalVOCImporter(annotations_dir, images_dir)` -- unlike COCO, both directories are required explicitly. \n", + "\n", + "We'll reuse the same two images, this time with one `.xml` file per image." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import xml.etree.ElementTree as ET\n", + "\n", + "voc_dir = workdir / \"voc_dataset\"\n", + "voc_images_dir = voc_dir / \"JPEGImages\"\n", + "voc_annotations_dir = voc_dir / \"Annotations\"\n", + "voc_images_dir.mkdir(parents=True)\n", + "voc_annotations_dir.mkdir(parents=True)\n", + "\n", + "for file_name, (label, bbox) in boxes_per_image.items():\n", + " Image.open(coco_dir / file_name).save(voc_images_dir / file_name)\n", + "\n", + " x, y, w, h = bbox\n", + " annotation = ET.Element(\"annotation\")\n", + " ET.SubElement(annotation, \"filename\").text = file_name\n", + " obj = ET.SubElement(annotation, \"object\")\n", + " ET.SubElement(obj, \"name\").text = label\n", + " bndbox = ET.SubElement(obj, \"bndbox\")\n", + " ET.SubElement(bndbox, \"xmin\").text = str(x)\n", + " ET.SubElement(bndbox, \"ymin\").text = str(y)\n", + " ET.SubElement(bndbox, \"xmax\").text = str(x + w)\n", + " ET.SubElement(bndbox, \"ymax\").text = str(y + h)\n", + "\n", + " xml_path = voc_annotations_dir / f\"{Path(file_name).stem}.xml\"\n", + " ET.ElementTree(annotation).write(xml_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datamint import PascalVOCImporter\n", + "\n", + "voc_importer = PascalVOCImporter(voc_annotations_dir, voc_images_dir)\n", + "\n", + "preview = voc_importer.parse()\n", + "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", + "\n", + "result = voc_importer.import_to_project(api, project_voc, tags=[\"voc-import-tutorial\"])\n", + "print(f\"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Import a YOLO dataset\n", + "\n", + "`YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None)` -- YOLO label files are pure numbers with no class name or image filename embedded, so class names must be resolved from one of: an explicit `class_names` list, an auto-detected `data.yaml`/`data.yml` (`names:` key), or an auto-detected legacy `classes.txt`. Coordinates are normalized (`0..1` of image width/height): `class x_center y_center width height`." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "yolo_dir = workdir / \"yolo_dataset\"\n", + "yolo_images_dir = yolo_dir / \"images\"\n", + "yolo_labels_dir = yolo_dir / \"labels\"\n", + "yolo_images_dir.mkdir(parents=True)\n", + "yolo_labels_dir.mkdir(parents=True)\n", + "\n", + "class_names = [\"cat\", \"dog\"]\n", + "\n", + "for file_name, (label, bbox) in boxes_per_image.items():\n", + " img = Image.open(coco_dir / file_name)\n", + " img.save(yolo_images_dir / file_name)\n", + "\n", + " img_w, img_h = img.size\n", + " x, y, w, h = bbox\n", + " cx, cy = (x + w / 2) / img_w, (y + h / 2) / img_h\n", + " nw, nh = w / img_w, h / img_h\n", + "\n", + " class_id = class_names.index(label)\n", + " label_path = yolo_labels_dir / f\"{Path(file_name).stem}.txt\"\n", + " label_path.write_text(f\"{class_id} {cx} {cy} {nw} {nh}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datamint import YOLOImporter\n", + "\n", + "yolo_importer = YOLOImporter(yolo_images_dir, yolo_labels_dir, class_names=class_names)\n", + "\n", + "preview = yolo_importer.parse()\n", + "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", + "\n", + "result = yolo_importer.import_to_project(api, project_yolo, tags=[\"yolo-import-tutorial\"])\n", + "print(f\"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/03_datasets/README.md b/notebooks/03_datasets/README.md index eecc1ce6..ed048109 100644 --- a/notebooks/03_datasets/README.md +++ b/notebooks/03_datasets/README.md @@ -8,3 +8,4 @@ PyTorch dataset classes, data splits, and volume loading. | [02_patient_wise_splits](02_patient_wise_splits.ipynb) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Patient-level splitting to prevent data leakage in multi-scan datasets | | [03_build_dataset](03_build_dataset.ipynb) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Use `build_dataset` to auto-detect project type and get the right dataset class | | [04_volume_dataset](04_volume_dataset.ipynb) | ![Advanced](https://img.shields.io/badge/level-advanced-red) | Load 3D volumes, slice along anatomical axes, and apply albumentations transforms | +| [05_import_dataset](05_import_dataset.ipynb) | ![Intermediate](https://img.shields.io/badge/level-beginner-brightgreen) | Import an already-labeled dataset (COCO, Pascal VOC, or YOLO) into a project with `datamint.importers` | diff --git a/notebooks/README.md b/notebooks/README.md index e826aff3..0cbc66bb 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -15,7 +15,7 @@ Folders are numbered in the recommended learning order. |---|---|---| | [01_getting_started](01_getting_started/) | ![Beginner](https://img.shields.io/badge/level-beginner-brightgreen) | Upload data and explore a project | | [02_annotations](02_annotations/) | ![Beginner](https://img.shields.io/badge/level-beginner-brightgreen) | Upload and work with annotations | -| [03_datasets](03_datasets/) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Build PyTorch datasets, splits, and volume loading | +| [03_datasets](03_datasets/) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Build PyTorch datasets, splits, volume loading, and importing external dataset formats | | [04_experiment_tracking](04_experiment_tracking/) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Log metrics and artifacts, and manage the model registry, with MLflow | | [05_deployment](05_deployment/) | ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) | Deploy registered and external models | | [06_end_to_end](06_end_to_end/) | ![Advanced](https://img.shields.io/badge/level-advanced-red) | Full pipelines from data to deployed model | @@ -35,6 +35,7 @@ Folders are numbered in the recommended learning order. 2. [`02_patient_wise_splits`](03_datasets/02_patient_wise_splits.ipynb) ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) — Avoid data leakage with patient-level splitting 3. [`03_build_dataset`](03_datasets/03_build_dataset.ipynb) ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) — Auto-detect dataset type with `build_dataset` 4. [`04_volume_dataset`](03_datasets/04_volume_dataset.ipynb) ![Advanced](https://img.shields.io/badge/level-advanced-red) — Load 3D volumes, slice into 2D, apply albumentations +5. [`05_import_dataset`](03_datasets/05_import_dataset.ipynb) ![Intermediate](https://img.shields.io/badge/level-beginner-brightgreen) — Import an already-labeled dataset (COCO, Pascal VOC, or YOLO) into a project ### 04 — Experiment Tracking 1. [`01_mlflow_manual_logging`](04_experiment_tracking/01_mlflow_manual_logging.ipynb) ![Intermediate](https://img.shields.io/badge/level-intermediate-yellow) — Log metrics, parameters, and models manually with MLflow From a8d1fb1160e6b5962d37ce99a508449261625c34 Mon Sep 17 00:00:00 2001 From: luandalmazo Date: Thu, 13 Aug 2026 15:07:46 -0300 Subject: [PATCH 2/3] update code to address review --- datamint/importers/_common.py | 86 +++++++ datamint/importers/coco.py | 71 ++---- datamint/importers/pascal_voc.py | 69 ++---- datamint/importers/yolo.py | 75 ++---- docs/source/client_api_content.rst | 21 +- notebooks/03_datasets/05_import_dataset.ipynb | 234 +++++++++++------- 6 files changed, 300 insertions(+), 256 deletions(-) create mode 100644 datamint/importers/_common.py diff --git a/datamint/importers/_common.py b/datamint/importers/_common.py new file mode 100644 index 00000000..11296b14 --- /dev/null +++ b/datamint/importers/_common.py @@ -0,0 +1,86 @@ +import logging +from typing import Callable, Literal, Sequence + +from tqdm.auto import tqdm + +from datamint import Api +from datamint.entities import Project + +_LOGGER = logging.getLogger(__name__) + + +def import_boxes_to_project(result, + project: Project | str, + api: Api | None, + *, + box_points: Callable[[object], tuple[tuple[float, float], tuple[float, float]]], + source_label: str, + tags: Sequence[str] | None, + imported_from: str, + on_error: Literal['raise', 'skip'], + progress_bar: bool, + result_cls: type): + """Shared upload loop behind every ``*Importer.import_to_project()``. + + ``result`` is a parse result with ``samples``, ``missing_images``, and + ``unsupported_annotations`` attributes. ``box_points`` maps a format's box + dataclass to a ``(point1, point2)`` pair for ``add_box_annotation``. + + """ + api = api or Api() + + if result.missing_images: + _LOGGER.warning(f'{len(result.missing_images)} image(s) referenced in {source_label} ' + f'were not found on disk and will be skipped.') + if result.unsupported_annotations: + _LOGGER.warning(f'{result.unsupported_annotations} annotation(s) in {source_label} use an ' + f'unsupported annotation type (e.g. polygon/segmentation) and were not ' + f'imported; only bounding boxes are supported.') + + uploaded = api.resources.upload_resources( + [str(s.image_path) for s in result.samples], + tags=tags, + publish_to=project, + on_error=on_error, + progress_bar=progress_bar, + ) + + resource_ids: list[str] = [] + errors: list[tuple[str, Exception]] = [] + n_boxes_uploaded = 0 + + iterator = zip(result.samples, uploaded) + if progress_bar: + iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations') + + for sample, resource_id in iterator: + if isinstance(resource_id, Exception): + errors.append((sample.file_name, resource_id)) + continue + resource_ids.append(resource_id) + + for box in sample.boxes: + point1, point2 = box_points(box) + try: + api.annotations.add_box_annotation( + point1=point1, + point2=point2, + resource=resource_id, + identifier=box.label, + imported_from=imported_from, + ) + n_boxes_uploaded += 1 + except (KeyboardInterrupt, SystemExit): + raise + except Exception as e: + if on_error == 'raise': + raise + errors.append((sample.file_name, e)) + + return result_cls( + project=project, + resource_ids=resource_ids, + n_images_uploaded=len(resource_ids), + n_boxes_uploaded=n_boxes_uploaded, + errors=errors, + ) diff --git a/datamint/importers/coco.py b/datamint/importers/coco.py index b71b4a89..1fa8d77c 100644 --- a/datamint/importers/coco.py +++ b/datamint/importers/coco.py @@ -1,15 +1,12 @@ import json -import logging from dataclasses import dataclass, field from pathlib import Path from typing import Literal, Sequence -from tqdm.auto import tqdm - from datamint import Api from datamint.entities import Project -_LOGGER = logging.getLogger(__name__) +from . import _common @dataclass @@ -33,6 +30,7 @@ class COCOParseResult: samples: list[COCOSample] class_names: list[str] missing_images: list[str] + unsupported_annotations: int @property def num_images(self) -> int: @@ -53,11 +51,7 @@ class COCOImportResult: class COCOImporter: - """Parse a COCO-format annotations file and upload it to a Datamint project. - - Only bounding-box annotations (the ``bbox`` field) are imported; polygon - ``segmentation`` fields are ignored. - """ + """Parse a COCO-format annotations file and upload it to a Datamint project. """ def __init__(self, annotations_file: str | Path, images_dir: str | Path | None = None): self.annotations_file = Path(annotations_file) @@ -98,6 +92,7 @@ def parse(self, force: bool = False) -> COCOParseResult: samples_by_id[img['id']] = COCOSample(image_path=image_path, file_name=file_name) used_class_names: set[str] = set() + unsupported_annotations = 0 for ann in data['annotations']: image_id = ann['image_id'] if image_id not in images_by_id: @@ -112,7 +107,9 @@ def parse(self, force: bool = False) -> COCOParseResult: bbox = ann.get('bbox') if bbox is None: # COCO allows annotations without bounding boxes (e.g., segmentation-only annotations) - continue + if ann.get('segmentation'): + unsupported_annotations += 1 + continue x, y, width, height = bbox label = categories[category_id] @@ -123,12 +120,13 @@ def parse(self, force: bool = False) -> COCOParseResult: samples=list(samples_by_id.values()), class_names=sorted(used_class_names), missing_images=missing_images, + unsupported_annotations=unsupported_annotations, ) return self._result def import_to_project(self, - api: Api, project: Project | str, + api: Api | None = None, *, tags: Sequence[str] | None = None, imported_from: str = 'coco-import', @@ -137,53 +135,16 @@ def import_to_project(self, """Upload the parsed images and box annotations to a Datamint project. Calls :meth:`parse` first (reusing the cached result if already called). + ``api`` defaults to a new :class:`~datamint.api.client.Api` instance if not given. """ result = self.parse() - if result.missing_images: - _LOGGER.warning(f'{len(result.missing_images)} image(s) referenced in ' - f'{self.annotations_file} were not found under {self.images_dir} and will be skipped.') - - uploaded = api.resources.upload_resources( - [str(s.image_path) for s in result.samples], + return _common.import_boxes_to_project( + result, project, api, + box_points=lambda b: ((b.x, b.y), (b.x + b.width, b.y + b.height)), + source_label=str(self.annotations_file), tags=tags, - publish_to=project, + imported_from=imported_from, on_error=on_error, progress_bar=progress_bar, - ) - - resource_ids: list[str] = [] - errors: list[tuple[str, Exception]] = [] - n_boxes_uploaded = 0 - - iterator = zip(result.samples, uploaded) - if progress_bar: - iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations') - - for sample, resource_id in iterator: - if isinstance(resource_id, Exception): - errors.append((sample.file_name, resource_id)) - continue - resource_ids.append(resource_id) - - for box in sample.boxes: - try: - api.annotations.add_box_annotation( - point1=(box.x, box.y), - point2=(box.x + box.width, box.y + box.height), - resource=resource_id, - identifier=box.label, - imported_from=imported_from, - ) - n_boxes_uploaded += 1 - except Exception as e: - if on_error == 'raise': - raise - errors.append((sample.file_name, e)) - - return COCOImportResult( - project=project, - resource_ids=resource_ids, - n_images_uploaded=len(resource_ids), - n_boxes_uploaded=n_boxes_uploaded, - errors=errors, + result_cls=COCOImportResult, ) diff --git a/datamint/importers/pascal_voc.py b/datamint/importers/pascal_voc.py index e0c60686..846e6c33 100644 --- a/datamint/importers/pascal_voc.py +++ b/datamint/importers/pascal_voc.py @@ -1,15 +1,12 @@ -import logging import xml.etree.ElementTree as ET from dataclasses import dataclass, field from pathlib import Path from typing import Literal, Sequence -from tqdm.auto import tqdm - from datamint import Api from datamint.entities import Project -_LOGGER = logging.getLogger(__name__) +from . import _common @dataclass @@ -34,6 +31,7 @@ class PascalVOCParseResult: samples: list[PascalVOCSample] class_names: list[str] missing_images: list[str] + unsupported_annotations: int @property def num_images(self) -> int: @@ -65,7 +63,7 @@ def __init__(self, annotations_dir: str | Path, images_dir: str | Path): self._result: PascalVOCParseResult | None = None def parse(self, force: bool = False) -> PascalVOCParseResult: - """Read and validate the Pascal VOC XML annotation files. + """Read and validate the Pascal VOC XML annotation files. Cached after the first call; pass ``force=True`` to reparse. @@ -82,6 +80,7 @@ def parse(self, force: bool = False) -> PascalVOCParseResult: samples: list[PascalVOCSample] = [] missing_images: list[str] = [] used_class_names: set[str] = set() + unsupported_annotations = 0 for xml_path in sorted(self.annotations_dir.glob('*.xml')): root = ET.parse(xml_path).getroot() @@ -99,7 +98,11 @@ def parse(self, force: bool = False) -> PascalVOCParseResult: for obj in root.findall('object'): label = obj.findtext('name', default='').strip() bb = obj.find('bndbox') - if not label or bb is None: + if bb is None: + # e.g. a segmentation object instead of -- not supported + unsupported_annotations += 1 + continue + if not label: # incomplete object entry skip it continue @@ -120,12 +123,13 @@ def parse(self, force: bool = False) -> PascalVOCParseResult: samples=samples, class_names=sorted(used_class_names), missing_images=missing_images, + unsupported_annotations=unsupported_annotations, ) return self._result def import_to_project(self, - api: Api, project: Project | str, + api: Api | None = None, *, tags: Sequence[str] | None = None, imported_from: str = 'pascal-voc-import', @@ -134,53 +138,16 @@ def import_to_project(self, """Upload the parsed images and box annotations to a Datamint project. Calls :meth:`parse` first (reusing the cached result if already called). + ``api`` defaults to a new :class:`~datamint.api.client.Api` instance if not given. """ result = self.parse() - if result.missing_images: - _LOGGER.warning(f'{len(result.missing_images)} image(s) referenced in ' - f'{self.annotations_dir} were not found under {self.images_dir} and will be skipped.') - - uploaded = api.resources.upload_resources( - [str(s.image_path) for s in result.samples], + return _common.import_boxes_to_project( + result, project, api, + box_points=lambda b: ((b.x1, b.y1), (b.x2, b.y2)), + source_label=str(self.annotations_dir), tags=tags, - publish_to=project, + imported_from=imported_from, on_error=on_error, progress_bar=progress_bar, - ) - - resource_ids: list[str] = [] - errors: list[tuple[str, Exception]] = [] - n_boxes_uploaded = 0 - - iterator = zip(result.samples, uploaded) - if progress_bar: - iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations') - - for sample, resource_id in iterator: - if isinstance(resource_id, Exception): - errors.append((sample.file_name, resource_id)) - continue - resource_ids.append(resource_id) - - for box in sample.boxes: - try: - api.annotations.add_box_annotation( - point1=(box.x1, box.y1), - point2=(box.x2, box.y2), - resource=resource_id, - identifier=box.label, - imported_from=imported_from, - ) - n_boxes_uploaded += 1 - except Exception as e: - if on_error == 'raise': - raise - errors.append((sample.file_name, e)) - - return PascalVOCImportResult( - project=project, - resource_ids=resource_ids, - n_images_uploaded=len(resource_ids), - n_boxes_uploaded=n_boxes_uploaded, - errors=errors, + result_cls=PascalVOCImportResult, ) diff --git a/datamint/importers/yolo.py b/datamint/importers/yolo.py index a1dabece..7de757dc 100644 --- a/datamint/importers/yolo.py +++ b/datamint/importers/yolo.py @@ -1,16 +1,14 @@ -import logging from dataclasses import dataclass, field from pathlib import Path from typing import Literal, Sequence import yaml from PIL import Image -from tqdm.auto import tqdm from datamint import Api from datamint.entities import Project -_LOGGER = logging.getLogger(__name__) +from . import _common _DEFAULT_IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.bmp', '.tif', '.tiff', '.webp') @@ -36,6 +34,7 @@ class YOLOParseResult: samples: list[YOLOSample] class_names: list[str] missing_images: list[str] + unsupported_annotations: int @property def num_images(self) -> int: @@ -71,7 +70,7 @@ def _load_names_from_yaml(path: Path) -> dict[int, str]: class YOLOImporter: """Parse a YOLO-format (images + normalized-bbox label .txt files) dataset and upload it to a Datamint project. - + """ def __init__(self, @@ -110,8 +109,9 @@ def parse(self, force: bool = False) -> YOLOParseResult: Cached after the first call; pass ``force=True`` to reparse. Raises: - ValueError: If class names can't be resolved, or a label line - references an unknown class index. + ValueError: If class names can't be resolved, a label line + references an unknown class index, or a normalized coordinate + is outside the expected ``[0, 1]`` range. """ if self._result is not None and not force: return self._result @@ -121,6 +121,7 @@ def parse(self, force: bool = False) -> YOLOParseResult: samples: list[YOLOSample] = [] missing_images: list[str] = [] used_class_names: set[str] = set() + unsupported_annotations = 0 for txt_path in sorted(self.labels_dir.glob('*.txt')): if txt_path.name in ('classes.txt',): @@ -140,7 +141,8 @@ def parse(self, force: bool = False) -> YOLOParseResult: if not parts: continue if len(parts) != 5: - # unsupported line shape skip + # segmentation/OBB/keypoint variants have a different field count -- not supported + unsupported_annotations += 1 continue class_id = int(parts[0]) @@ -148,6 +150,11 @@ def parse(self, force: bool = False) -> YOLOParseResult: raise ValueError(f"Label '{txt_path}' references unknown class_id {class_id}.") cx, cy, w, h = (float(v) for v in parts[1:]) + for coord_name, value in (('x_center', cx), ('y_center', cy), ('width', w), ('height', h)): + if not (0.0 <= value <= 1.0): + raise ValueError(f"Label '{txt_path}' has out-of-range normalized " + f"{coord_name}={value!r} (expected 0<=v<=1).") + cx, w = cx * img_w, w * img_w cy, h = cy * img_h, h * img_h @@ -167,12 +174,13 @@ def parse(self, force: bool = False) -> YOLOParseResult: samples=samples, class_names=sorted(used_class_names), missing_images=missing_images, + unsupported_annotations=unsupported_annotations, ) return self._result def import_to_project(self, - api: Api, project: Project | str, + api: Api | None = None, *, tags: Sequence[str] | None = None, imported_from: str = 'yolo-import', @@ -181,53 +189,16 @@ def import_to_project(self, """Upload the parsed images and box annotations to a Datamint project. Calls :meth:`parse` first (reusing the cached result if already called). + ``api`` defaults to a new :class:`~datamint.api.client.Api` instance if not given. """ result = self.parse() - if result.missing_images: - _LOGGER.warning(f'{len(result.missing_images)} label file(s) in {self.labels_dir} had no matching ' - f'image under {self.images_dir} and will be skipped.') - - uploaded = api.resources.upload_resources( - [str(s.image_path) for s in result.samples], + return _common.import_boxes_to_project( + result, project, api, + box_points=lambda b: ((b.x1, b.y1), (b.x2, b.y2)), + source_label=str(self.labels_dir), tags=tags, - publish_to=project, + imported_from=imported_from, on_error=on_error, progress_bar=progress_bar, - ) - - resource_ids: list[str] = [] - errors: list[tuple[str, Exception]] = [] - n_boxes_uploaded = 0 - - iterator = zip(result.samples, uploaded) - if progress_bar: - iterator = tqdm(iterator, total=len(result.samples), desc='Uploading annotations') - - for sample, resource_id in iterator: - if isinstance(resource_id, Exception): - errors.append((sample.file_name, resource_id)) - continue - resource_ids.append(resource_id) - - for box in sample.boxes: - try: - api.annotations.add_box_annotation( - point1=(box.x1, box.y1), - point2=(box.x2, box.y2), - resource=resource_id, - identifier=box.label, - imported_from=imported_from, - ) - n_boxes_uploaded += 1 - except Exception as e: - if on_error == 'raise': - raise - errors.append((sample.file_name, e)) - - return YOLOImportResult( - project=project, - resource_ids=resource_ids, - n_images_uploaded=len(resource_ids), - n_boxes_uploaded=n_boxes_uploaded, - errors=errors, + result_cls=YOLOImportResult, ) diff --git a/docs/source/client_api_content.rst b/docs/source/client_api_content.rst index 1493132f..495fb151 100644 --- a/docs/source/client_api_content.rst +++ b/docs/source/client_api_content.rst @@ -479,26 +479,27 @@ a project in one call. - ``COCOImporter(annotations_file, images_dir=None)`` * - :py:class:`~datamint.importers.pascal_voc.PascalVOCImporter` - Pascal VOC XML (one ``.xml`` per image, ```` elements) - - ``PascalVOCImporter(annotations_dir, images_dir)`` + - ``PascalVOCImporter(annotations_dir, images_dir)`` * - :py:class:`~datamint.importers.yolo.YOLOImporter` - YOLO ``.txt`` labels (normalized ``class x_center y_center width height``) - - ``YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None)` + - ``YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None)`` -Only bounding boxes are imported; polygon/segmentation annotations in these -formats are not supported yet. +Only bounding boxes are imported. If a dataset contains polygon/segmentation +annotations (or, for YOLO, OBB/keypoint label lines), ``parse()`` counts them +and ``import_to_project()`` logs a warning naming how many were skipped, +rather than failing or silently dropping them. Every importer follows the same two-step shape: :py:meth:`~datamint.importers.coco.COCOImporter.parse` reads and validates the dataset with no network calls (useful to preview image/box counts and class names before uploading anything), and :py:meth:`~datamint.importers.coco.COCOImporter.import_to_project` reuses that -parsed result to upload the images and their box annotations: +parsed result to upload the images and their box annotations. ``api`` is +optional and defaults to a new :py:class:`~datamint.api.client.Api` instance +if you don't already have one: .. code-block:: python - from datamint import Api, COCOImporter - - api = Api() - project = api.projects.get_by_name("My Project") + from datamint import COCOImporter importer = COCOImporter("dataset/train/_annotations.coco.json") @@ -507,7 +508,7 @@ parsed result to upload the images and their box annotations: print(preview.num_images, preview.num_boxes, preview.class_names) # Uploads images + box annotations, reusing the parsed result above - result = importer.import_to_project(api, project, tags=["coco-import"]) + result = importer.import_to_project("My Project", tags=["coco-import"]) print(result.n_images_uploaded, result.n_boxes_uploaded, result.errors) See also the tutorial notebook: `05_import_dataset.ipynb `_ diff --git a/notebooks/03_datasets/05_import_dataset.ipynb b/notebooks/03_datasets/05_import_dataset.ipynb index aa79e853..d939ad80 100644 --- a/notebooks/03_datasets/05_import_dataset.ipynb +++ b/notebooks/03_datasets/05_import_dataset.ipynb @@ -17,8 +17,8 @@ "| `YOLOImporter` | YOLO `.txt` labels (normalized `class x_center y_center width height`) |\n", "\n", "All three share the same two-step shape:\n", - "- `.parse()` reads and validates the dataset on disk. Use it to preview image/box counts and class names before uploading anything.\n", - "- `.import_to_project(api, project)` reuses the parsed result to upload the images and their box annotations.\n", + "- `.parse()` reads and validates the dataset on disk. Use it to preview image/box counts and class names before uploading anything. Unsupported annotations (e.g. polygons) are counted, not silently dropped.\n", + "- `.import_to_project(project, api=None)` reuses the parsed result to upload the images and their box annotations. `api` is optional and defaults to a new `Api()` if omitted.\n", "\n", "This notebook builds tiny synthetic datasets in each format (a couple of generated images) so it runs end-to-end without needing an external download." ] @@ -49,6 +49,37 @@ "print(f\"Working directory: {workdir}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The three sections below each build their own tiny 2-image dataset from this shared layout, so they can be run independently." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from PIL import Image, ImageDraw\n", + "\n", + "IMAGE_SIZE = (128, 128)\n", + "\n", + "# file_name -> (label, (x, y, width, height))\n", + "DEMO_BOXES = {\n", + " \"image_0.png\": (\"cat\", (10, 10, 40, 30)),\n", + " \"image_1.png\": (\"dog\", (20, 15, 35, 25)),\n", + "}\n", + "\n", + "\n", + "def new_demo_image(bbox):\n", + " img = Image.new(\"RGB\", IMAGE_SIZE, color=\"white\")\n", + " x, y, w, h = bbox\n", + " ImageDraw.Draw(img).rectangle([x, y, x + w, y + h], outline=\"black\")\n", + " return img" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -79,8 +110,11 @@ "source": [ "## 1. Import a COCO dataset\n", "\n", - "`COCOImporter(annotations_file, images_dir=None)` reads a single COCO JSON file. `images_dir` defaults to the annotations file's parent directory\n", - "First, generate a tiny COCO dataset: two images, one box each." + "`COCOImporter(annotations_file, images_dir=None)` reads a single COCO JSON file. `images_dir` defaults to the annotations file's parent directory.\n", + "\n", + "First, generate a tiny COCO dataset: two images, one box each.\n", + "\n", + "*(`build_coco_dataset` below is demo-data setup only, not part of the importer API -- feel free to skip to the `COCOImporter` cell.)*" ] }, { @@ -91,40 +125,40 @@ "source": [ "import json\n", "\n", - "from PIL import Image, ImageDraw\n", - "\n", - "coco_dir = workdir / \"coco_dataset\"\n", - "coco_dir.mkdir()\n", - "\n", - "# file_name -> (label, (x, y, width, height))\n", - "boxes_per_image = {\n", - " \"image_0.png\": (\"cat\", (10, 10, 40, 30)),\n", - " \"image_1.png\": (\"dog\", (20, 15, 35, 25)),\n", - "}\n", "\n", - "images, annotations = [], []\n", - "for i, (file_name, (label, bbox)) in enumerate(boxes_per_image.items()):\n", - " img = Image.new(\"RGB\", (128, 128), color=\"white\")\n", - " x, y, w, h = bbox\n", - " ImageDraw.Draw(img).rectangle([x, y, x + w, y + h], outline=\"black\")\n", - " img.save(coco_dir / file_name)\n", - "\n", - " images.append({\"id\": i, \"file_name\": file_name, \"width\": 128, \"height\": 128})\n", - " annotations.append({\n", - " \"id\": i,\n", - " \"image_id\": i,\n", - " \"category_id\": 0 if label == \"cat\" else 1,\n", - " \"bbox\": list(bbox),\n", - " })\n", - "\n", - "coco_json = {\n", - " \"images\": images,\n", - " \"annotations\": annotations,\n", - " \"categories\": [{\"id\": 0, \"name\": \"cat\"}, {\"id\": 1, \"name\": \"dog\"}],\n", - "}\n", - "\n", - "coco_annotations_file = coco_dir / \"_annotations.coco.json\"\n", - "coco_annotations_file.write_text(json.dumps(coco_json))" + "def build_coco_dataset(workdir: Path) -> Path:\n", + " coco_dir = workdir / \"coco_dataset\"\n", + " coco_dir.mkdir()\n", + "\n", + " images, annotations = [], []\n", + " for i, (file_name, (label, bbox)) in enumerate(DEMO_BOXES.items()):\n", + " new_demo_image(bbox).save(coco_dir / file_name)\n", + "\n", + " images.append({\"id\": i, \"file_name\": file_name, \"width\": IMAGE_SIZE[0], \"height\": IMAGE_SIZE[1]})\n", + " annotations.append({\n", + " \"id\": i,\n", + " \"image_id\": i,\n", + " \"category_id\": 0 if label == \"cat\" else 1,\n", + " \"bbox\": list(bbox),\n", + " })\n", + "\n", + " coco_json = {\n", + " \"images\": images,\n", + " \"annotations\": annotations,\n", + " \"categories\": [{\"id\": 0, \"name\": \"cat\"}, {\"id\": 1, \"name\": \"dog\"}],\n", + " }\n", + " annotations_file = coco_dir / \"_annotations.coco.json\"\n", + " annotations_file.write_text(json.dumps(coco_json))\n", + " return annotations_file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coco_annotations_file = build_coco_dataset(workdir)" ] }, { @@ -142,18 +176,18 @@ "source": [ "from datamint import COCOImporter\n", "\n", - "coco_importer = COCOImporter(coco_annotations_file) \n", + "coco_importer = COCOImporter(coco_annotations_file)\n", "\n", "preview = coco_importer.parse()\n", "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", - "print(f\"missing images: {preview.missing_images}\")" + "print(f\"missing images: {preview.missing_images}, unsupported annotations: {preview.unsupported_annotations}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Then upload the images and their box annotations:" + "Then upload the images and their box annotations. `api` is optional here -- it defaults to a new `Api()` if omitted, but we reuse the one from Setup:" ] }, { @@ -162,7 +196,7 @@ "metadata": {}, "outputs": [], "source": [ - "result = coco_importer.import_to_project(api, project_coco, tags=[\"coco-import-tutorial\"])\n", + "result = coco_importer.import_to_project(project_coco, api=api, tags=[\"coco-import-tutorial\"])\n", "\n", "print(f\"Uploaded images: {result.n_images_uploaded}\")\n", "print(f\"Uploaded boxes: {result.n_boxes_uploaded}\")\n", @@ -175,41 +209,52 @@ "source": [ "## 2. Import a Pascal VOC dataset\n", "\n", - "`PascalVOCImporter(annotations_dir, images_dir)` -- unlike COCO, both directories are required explicitly. \n", + "`PascalVOCImporter(annotations_dir, images_dir)` -- unlike COCO, both directories are required explicitly.\n", "\n", - "We'll reuse the same two images, this time with one `.xml` file per image." + "*(`build_voc_dataset` below is demo-data setup only, not part of the importer API -- feel free to skip to the `PascalVOCImporter` cell.)*" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import xml.etree.ElementTree as ET\n", "\n", - "voc_dir = workdir / \"voc_dataset\"\n", - "voc_images_dir = voc_dir / \"JPEGImages\"\n", - "voc_annotations_dir = voc_dir / \"Annotations\"\n", - "voc_images_dir.mkdir(parents=True)\n", - "voc_annotations_dir.mkdir(parents=True)\n", - "\n", - "for file_name, (label, bbox) in boxes_per_image.items():\n", - " Image.open(coco_dir / file_name).save(voc_images_dir / file_name)\n", "\n", - " x, y, w, h = bbox\n", - " annotation = ET.Element(\"annotation\")\n", - " ET.SubElement(annotation, \"filename\").text = file_name\n", - " obj = ET.SubElement(annotation, \"object\")\n", - " ET.SubElement(obj, \"name\").text = label\n", - " bndbox = ET.SubElement(obj, \"bndbox\")\n", - " ET.SubElement(bndbox, \"xmin\").text = str(x)\n", - " ET.SubElement(bndbox, \"ymin\").text = str(y)\n", - " ET.SubElement(bndbox, \"xmax\").text = str(x + w)\n", - " ET.SubElement(bndbox, \"ymax\").text = str(y + h)\n", - "\n", - " xml_path = voc_annotations_dir / f\"{Path(file_name).stem}.xml\"\n", - " ET.ElementTree(annotation).write(xml_path)" + "def build_voc_dataset(workdir: Path) -> tuple[Path, Path]:\n", + " voc_dir = workdir / \"voc_dataset\"\n", + " images_dir = voc_dir / \"JPEGImages\"\n", + " annotations_dir = voc_dir / \"Annotations\"\n", + " images_dir.mkdir(parents=True)\n", + " annotations_dir.mkdir(parents=True)\n", + "\n", + " for file_name, (label, bbox) in DEMO_BOXES.items():\n", + " new_demo_image(bbox).save(images_dir / file_name)\n", + "\n", + " x, y, w, h = bbox\n", + " annotation = ET.Element(\"annotation\")\n", + " ET.SubElement(annotation, \"filename\").text = file_name\n", + " obj = ET.SubElement(annotation, \"object\")\n", + " ET.SubElement(obj, \"name\").text = label\n", + " bndbox = ET.SubElement(obj, \"bndbox\")\n", + " ET.SubElement(bndbox, \"xmin\").text = str(x)\n", + " ET.SubElement(bndbox, \"ymin\").text = str(y)\n", + " ET.SubElement(bndbox, \"xmax\").text = str(x + w)\n", + " ET.SubElement(bndbox, \"ymax\").text = str(y + h)\n", + " ET.ElementTree(annotation).write(annotations_dir / f\"{Path(file_name).stem}.xml\")\n", + "\n", + " return annotations_dir, images_dir" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "voc_annotations_dir, voc_images_dir = build_voc_dataset(workdir)" ] }, { @@ -225,7 +270,7 @@ "preview = voc_importer.parse()\n", "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", "\n", - "result = voc_importer.import_to_project(api, project_voc, tags=[\"voc-import-tutorial\"])\n", + "result = voc_importer.import_to_project(project_voc, api=api, tags=[\"voc-import-tutorial\"])\n", "print(f\"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}\")" ] }, @@ -235,35 +280,48 @@ "source": [ "## 3. Import a YOLO dataset\n", "\n", - "`YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None)` -- YOLO label files are pure numbers with no class name or image filename embedded, so class names must be resolved from one of: an explicit `class_names` list, an auto-detected `data.yaml`/`data.yml` (`names:` key), or an auto-detected legacy `classes.txt`. Coordinates are normalized (`0..1` of image width/height): `class x_center y_center width height`." + "`YOLOImporter(images_dir, labels_dir, class_names=None, data_yaml=None)` -- YOLO label files are pure numbers with no class name or image filename embedded, so class names must be resolved from one of: an explicit `class_names` list, an auto-detected `data.yaml`/`data.yml` (`names:` key), or an auto-detected legacy `classes.txt`. Coordinates are normalized (`0..1` of image width/height): `class x_center y_center width height`.\n", + "\n", + "*(`build_yolo_dataset` below is demo-data setup only, not part of the importer API -- feel free to skip to the `YOLOImporter` cell.)*" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "yolo_dir = workdir / \"yolo_dataset\"\n", - "yolo_images_dir = yolo_dir / \"images\"\n", - "yolo_labels_dir = yolo_dir / \"labels\"\n", - "yolo_images_dir.mkdir(parents=True)\n", - "yolo_labels_dir.mkdir(parents=True)\n", + "def build_yolo_dataset(workdir: Path) -> tuple[Path, Path, list[str]]:\n", + " yolo_dir = workdir / \"yolo_dataset\"\n", + " images_dir = yolo_dir / \"images\"\n", + " labels_dir = yolo_dir / \"labels\"\n", + " images_dir.mkdir(parents=True)\n", + " labels_dir.mkdir(parents=True)\n", "\n", - "class_names = [\"cat\", \"dog\"]\n", + " class_names = [\"cat\", \"dog\"]\n", + " img_w, img_h = IMAGE_SIZE\n", "\n", - "for file_name, (label, bbox) in boxes_per_image.items():\n", - " img = Image.open(coco_dir / file_name)\n", - " img.save(yolo_images_dir / file_name)\n", + " for file_name, (label, bbox) in DEMO_BOXES.items():\n", + " new_demo_image(bbox).save(images_dir / file_name)\n", "\n", - " img_w, img_h = img.size\n", - " x, y, w, h = bbox\n", - " cx, cy = (x + w / 2) / img_w, (y + h / 2) / img_h\n", - " nw, nh = w / img_w, h / img_h\n", + " x, y, w, h = bbox\n", + " cx, cy = (x + w / 2) / img_w, (y + h / 2) / img_h\n", + " nw, nh = w / img_w, h / img_h\n", + "\n", + " class_id = class_names.index(label)\n", + " label_path = labels_dir / f\"{Path(file_name).stem}.txt\"\n", + " label_path.write_text(f\"{class_id} {cx} {cy} {nw} {nh}\\n\")\n", "\n", - " class_id = class_names.index(label)\n", - " label_path = yolo_labels_dir / f\"{Path(file_name).stem}.txt\"\n", - " label_path.write_text(f\"{class_id} {cx} {cy} {nw} {nh}\\n\")" + " return images_dir, labels_dir, class_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "yolo_images_dir, yolo_labels_dir, yolo_class_names = build_yolo_dataset(workdir)" ] }, { @@ -274,19 +332,19 @@ "source": [ "from datamint import YOLOImporter\n", "\n", - "yolo_importer = YOLOImporter(yolo_images_dir, yolo_labels_dir, class_names=class_names)\n", + "yolo_importer = YOLOImporter(yolo_images_dir, yolo_labels_dir, class_names=yolo_class_names)\n", "\n", "preview = yolo_importer.parse()\n", "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", "\n", - "result = yolo_importer.import_to_project(api, project_yolo, tags=[\"yolo-import-tutorial\"])\n", + "result = yolo_importer.import_to_project(project_yolo, api=api, tags=[\"yolo-import-tutorial\"])\n", "print(f\"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}\")" ] } ], "metadata": { "kernelspec": { - "display_name": "env", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -300,7 +358,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.12" } }, "nbformat": 4, From de39a23f01acd3a6124bd1bedceab8df1984d513 Mon Sep 17 00:00:00 2001 From: luandalmazo Date: Fri, 14 Aug 2026 09:03:54 -0300 Subject: [PATCH 3/3] update notebook to remove api param --- notebooks/03_datasets/05_import_dataset.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/notebooks/03_datasets/05_import_dataset.ipynb b/notebooks/03_datasets/05_import_dataset.ipynb index d939ad80..47b9deaa 100644 --- a/notebooks/03_datasets/05_import_dataset.ipynb +++ b/notebooks/03_datasets/05_import_dataset.ipynb @@ -18,7 +18,7 @@ "\n", "All three share the same two-step shape:\n", "- `.parse()` reads and validates the dataset on disk. Use it to preview image/box counts and class names before uploading anything. Unsupported annotations (e.g. polygons) are counted, not silently dropped.\n", - "- `.import_to_project(project, api=None)` reuses the parsed result to upload the images and their box annotations. `api` is optional and defaults to a new `Api()` if omitted.\n", + "- `.import_to_project(project)` reuses the parsed result to upload the images and their box annotations. \n", "\n", "This notebook builds tiny synthetic datasets in each format (a couple of generated images) so it runs end-to-end without needing an external download." ] @@ -187,7 +187,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Then upload the images and their box annotations. `api` is optional here -- it defaults to a new `Api()` if omitted, but we reuse the one from Setup:" + "Then upload the images and their box annotations. " ] }, { @@ -196,7 +196,7 @@ "metadata": {}, "outputs": [], "source": [ - "result = coco_importer.import_to_project(project_coco, api=api, tags=[\"coco-import-tutorial\"])\n", + "result = coco_importer.import_to_project(project_coco, tags=[\"coco-import-tutorial\"])\n", "\n", "print(f\"Uploaded images: {result.n_images_uploaded}\")\n", "print(f\"Uploaded boxes: {result.n_boxes_uploaded}\")\n", @@ -270,7 +270,7 @@ "preview = voc_importer.parse()\n", "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", "\n", - "result = voc_importer.import_to_project(project_voc, api=api, tags=[\"voc-import-tutorial\"])\n", + "result = voc_importer.import_to_project(project_voc, tags=[\"voc-import-tutorial\"])\n", "print(f\"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}\")" ] }, @@ -337,7 +337,7 @@ "preview = yolo_importer.parse()\n", "print(f\"{preview.num_images} images, {preview.num_boxes} boxes, classes={preview.class_names}\")\n", "\n", - "result = yolo_importer.import_to_project(project_yolo, api=api, tags=[\"yolo-import-tutorial\"])\n", + "result = yolo_importer.import_to_project(project_yolo, tags=[\"yolo-import-tutorial\"])\n", "print(f\"Uploaded images: {result.n_images_uploaded}, boxes: {result.n_boxes_uploaded}, errors: {result.errors}\")" ] }