From 1b9d9c2a64c3846717b1fe12757026452517b5f5 Mon Sep 17 00:00:00 2001 From: luandalmazo Date: Fri, 31 Jul 2026 10:13:20 -0300 Subject: [PATCH 1/2] completely remove deprecated code --- .github/workflows/run_test.yaml | 6 - datamint/__init__.py | 2 +- datamint/api/base_api.py | 4 +- datamint/api/endpoints/annotations_api.py | 56 +- datamint/api/endpoints/annotationsets_api.py | 264 +--- datamint/api/endpoints/datasetsinfo_api.py | 12 +- datamint/api/endpoints/deploy_model_api.py | 11 +- datamint/api/endpoints/inference_api.py | 21 +- datamint/api/endpoints/projects_api.py | 71 +- datamint/api/endpoints/resources_api.py | 10 - datamint/api/endpoints/users_api.py | 21 +- datamint/client_cmd_tools/datamint_config.py | 11 +- .../client_cmd_tools/datamint_inference.py | 7 - datamint/client_cmd_tools/datamint_init.py | 7 - datamint/client_cmd_tools/datamint_train.py | 7 - datamint/client_cmd_tools/datamint_upload.py | 12 - datamint/dataset/annotation.py | 221 --- datamint/dataset/base.py | 73 +- datamint/dataset/base_dataset.py | 1220 ----------------- datamint/dataset/dataset.py | 577 -------- datamint/dataset/factory.py | 9 - datamint/entities/base_entity.py | 2 +- datamint/lightning/datamodule.py | 9 +- datamint/utils/env.py | 7 - docs/source/client_api_content.rst | 4 +- docs/source/command_line_tools.rst | 7 - docs/source/datamint.dataset.rst | 30 +- docs/source/datamint_vs_raw_pytorch.rst | 4 +- pyproject.toml | 6 - tests/test_annotations_api.py | 58 +- tests/test_annotationsets_api.py | 68 +- tests/test_datamint_config.py | 18 - tests/test_datamodule_collate.py | 1 - tests/test_dataset_factory.py | 14 - tests/test_dataset_patient_split.py | 6 - tests/test_datasetsinfo_api.py | 6 +- tests/test_deploy_model_api.py | 6 +- tests/test_imports.py | 8 - tests/test_inference_api.py | 8 +- tests/test_projects_api.py | 35 +- tests/test_resources_api.py | 23 +- tests/test_users_api.py | 33 +- 42 files changed, 69 insertions(+), 2906 deletions(-) delete mode 100644 datamint/dataset/annotation.py delete mode 100644 datamint/dataset/base_dataset.py delete mode 100644 datamint/dataset/dataset.py diff --git a/.github/workflows/run_test.yaml b/.github/workflows/run_test.yaml index 4e821f21..03034090 100644 --- a/.github/workflows/run_test.yaml +++ b/.github/workflows/run_test.yaml @@ -80,12 +80,6 @@ jobs: PYTHONIOENCODING: utf-8 if: runner.os == 'Windows' - - name: Test datamint-config CLI (deprecated alias) - run: datamint-config --api-key testapikey - timeout-minutes: 1 - env: - PYTHONIOENCODING: utf-8 - - name: Test datamint config CLI (unified) run: datamint config --api-key testapikey timeout-minutes: 1 diff --git a/datamint/__init__.py b/datamint/__init__.py index 2ca26517..c88b952c 100644 --- a/datamint/__init__.py +++ b/datamint/__init__.py @@ -17,7 +17,7 @@ __getattr__, __dir__, __all__ = lazy.attach( __name__, - submodules=['dataset', "dataset.dataset", "examples"], + submodules=['dataset', "examples"], submod_attrs={ "api.client": ["Api"], # New modular dataset classes diff --git a/datamint/api/base_api.py b/datamint/api/base_api.py index dfbdb066..636f6666 100644 --- a/datamint/api/base_api.py +++ b/datamint/api/base_api.py @@ -685,7 +685,9 @@ def convert_format(bytes_array: bytes, Args: bytes_array: Raw file content bytes mimetype: Optional MIME type of the content - file_path: deprecated + file_path: Path to the source file. Required when mimetype is a video type + (used to open the file with ``cv2.VideoCapture``) or a NIfTI type + (used to load the file with ``nibabel``). Returns: Converted content in appropriate format (pydicom.Dataset, PIL Image, cv2.VideoCapture, ...) diff --git a/datamint/api/endpoints/annotations_api.py b/datamint/api/endpoints/annotations_api.py index 5b0f352e..f0789b36 100644 --- a/datamint/api/endpoints/annotations_api.py +++ b/datamint/api/endpoints/annotations_api.py @@ -7,7 +7,6 @@ import json import logging import os -import warnings import aiohttp import httpx @@ -117,9 +116,6 @@ def get_list( # type: ignore[override] load_ai_segmentations: bool | None = None, limit: int | None = None, group_by_resource: bool = False, - *, - date_from: date | None = None, - date_to: date | None = None, **kwargs: Any, ) -> Sequence[Annotation] | Sequence[Sequence[Annotation]]: """ @@ -139,8 +135,6 @@ def get_list( # type: ignore[override] limit: Maximum number of annotations to return. group_by_resource: If True, return results grouped by resource. For instance, the first index of the returned list will contain all annotations for the first resource. - date_from: (DEPRECATED) Use ``from_date`` instead. - date_to: (DEPRECATED) Use ``to_date`` instead. Returns: Sequence[Annotation] | Sequence[Sequence[Annotation]]: List of annotations, or list of lists if grouped by resource. @@ -175,17 +169,6 @@ def group_annotations_by_resource(annotations: Sequence[Annotation], resource_annotations_map[ann.resource_id].append(ann) return [resource_annotations_map[rid] for rid in resource_ids] - if date_from is not None: - warnings.warn("The 'date_from' parameter is deprecated. " - "Please use 'from_date' instead", DeprecationWarning) - if from_date is None: - from_date = date_from - if date_to is not None: - warnings.warn("The 'date_to' parameter is deprecated. " - "Please use 'to_date' instead", DeprecationWarning) - if to_date is None: - to_date = date_to - # Build search payload according to POST /annotations/search schema payload = { 'annotation_type': annotation_type, @@ -605,8 +588,6 @@ def upload_volume_segmentation(self, model_name: str | None = None, transpose_segmentation: bool = False, source: str | None = 'imported', - *, - ai_model_name: str | None = None, ) -> list[str]: """ Upload a 3D volume segmentation to a resource. @@ -623,7 +604,6 @@ def upload_volume_segmentation(self, worklist_id: The annotation worklist unique id. model_name: The AI model name. transpose_segmentation: Whether to transpose the segmentation before uploading. - ai_model_name: (DEPRECATED) Use ``model_name`` instead. source: Annotation source tag. Defaults to 'imported' since this is a direct API entry point; :meth:`upload_predictions` overrides it with 'model_pipeline'/'model_deploy'. @@ -653,12 +633,6 @@ def upload_volume_segmentation(self, """ import nest_asyncio - if ai_model_name is not None: - warnings.warn("The 'ai_model_name' parameter is deprecated. " - "Please use 'model_name' instead", DeprecationWarning) - if model_name is None: - model_name = ai_model_name - if isinstance(file_path, Path): file_path = str(file_path) @@ -694,8 +668,6 @@ def upload_segmentations(self, transpose_segmentation: bool = False, model_name: str | None = None, source: str | None = 'imported', - *, - ai_model_name: str | None = None, ) -> list[str]: """ Upload frame-by-frame segmentations to a resource. @@ -727,7 +699,6 @@ def upload_segmentations(self, model_name: Optional AI model name to associate with the segmentation. source: Annotation source tag. Defaults to 'imported' since this is a direct API entry point; :meth:`upload_predictions` overrides it with 'model_pipeline'/'model_deploy'. - ai_model_name: (DEPRECATED) Use ``model_name`` instead. Returns: List of segmentation unique ids. @@ -756,12 +727,6 @@ def upload_segmentations(self, """ import nest_asyncio - if ai_model_name is not None: - warnings.warn("The 'ai_model_name' parameter is deprecated. " - "Please use 'model_name' instead", DeprecationWarning) - if model_name is None: - model_name = ai_model_name - if isinstance(file_path, Path): file_path = str(file_path) @@ -1321,7 +1286,6 @@ def add_line_annotation(self, frame_index: int | None = None, slice_plane: ViewPlane | None = None, metadata: pydicom.Dataset | Nifti1Image | None = None, - dicom_metadata: pydicom.Dataset | None = None, coords_system: CoordinateSystem = 'pixel', worklist_id: str | None = None, imported_from: str | None = None, @@ -1341,8 +1305,6 @@ def add_line_annotation(self, resource: The resource unique id or Resource instance. identifier: The annotation identifier, also as known as the annotation's label. frame_index: The frame index of the annotation. - dicom_metadata: (DEPRECATED) The DICOM metadata of the image. If provided, the coordinates will be converted to the - correct coordinates automatically using the DICOM metadata. coords_system: The coordinate system of the points. Can be 'pixel', or 'patient'. If 'pixel', the points are in pixel coordinates. If 'patient', the points are in patient coordinates (see DICOM patient coordinates). project: The project unique id or name. @@ -1373,13 +1335,6 @@ def add_line_annotation(self, metadata, ) - if dicom_metadata is not None: - import warnings - warnings.warn("The 'dicom_metadata' parameter is deprecated. " - "Please use 'metadata' parameter instead", DeprecationWarning) - if resolved_metadata is None: - resolved_metadata = dicom_metadata - annotation = LineAnnotation.from_points( point1, point2, @@ -1625,9 +1580,7 @@ def bulk_download_file(self, def patch(self, annotation: str | Annotation, identifier: str | None = None, - project: 'str | Project | None' = None, - *, - project_id: str | None = None) -> None: + project: 'str | Project | None' = None) -> None: """ Partially update an annotation's metadata. @@ -1635,19 +1588,12 @@ def patch(self, annotation: The annotation unique id or Annotation instance. identifier: Optional new identifier/label for the annotation. project: Optional project ID or Project instance to associate with the annotation. - project_id: (DEPRECATED) Use ``project`` instead. Raises: ServerError: If the update fails. """ annotation_id = self._entid(annotation) - if project_id is not None: - warnings.warn("The 'project_id' parameter is deprecated. " - "Please use 'project' instead", DeprecationWarning) - if project is None: - project = project_id - payload = {'identifier': identifier, 'project_id': self._entid(project) if project is not None else None} # remove None values diff --git a/datamint/api/endpoints/annotationsets_api.py b/datamint/api/endpoints/annotationsets_api.py index 47899edc..e12f78e9 100644 --- a/datamint/api/endpoints/annotationsets_api.py +++ b/datamint/api/endpoints/annotationsets_api.py @@ -3,7 +3,6 @@ from datamint.entities.annotation_worklist import AnnotationWorklist from typing_extensions import override import logging -import warnings if TYPE_CHECKING: from datamint.entities import Project @@ -33,7 +32,6 @@ def create(self, project: 'str | Project | None' = None, return_url: str | None = None, *, - project_id: str | None = None, return_entity: Literal[True] = True, exists_ok: bool = False ) -> AnnotationWorklist: ... @@ -53,7 +51,6 @@ def create(self, project: 'str | Project | None' = None, return_url: str | None = None, *, - project_id: str | None = None, return_entity: Literal[False], exists_ok: bool = False ) -> str: ... @@ -73,7 +70,6 @@ def create(self, project: 'str | Project | None' = None, return_url: str | None = None, *, - project_id: str | None = None, return_entity: bool = True, exists_ok: bool = False, ) -> str | AnnotationWorklist: @@ -101,17 +97,10 @@ def create(self, editable_ai_annotations: Optional list of AI annotation identifiers to allow editing. project: Optional project ID or Project instance to associate with this worklist. return_url: Optional URL to redirect after annotation. - project_id: (DEPRECATED) Use ``project`` instead. Returns: The ID of the created annotation set. """ - if project_id is not None: - warnings.warn("The 'project_id' parameter is deprecated. " - "Please use 'project' instead", DeprecationWarning) - if project is None: - project = project_id - payload: dict = {'name': name, 'resource_ids': resource_ids} if description is not None: payload['description'] = description @@ -139,9 +128,7 @@ def update_segmentation_group(self, worklist_id: 'str | AnnotationWorklist | None' = None, definitions: list[dict] | None = None, segmentation_value_type: str = 'single_label', - renames: list[str] | None = None, - *, - annotation_worklist: 'str | AnnotationWorklist | None' = None) -> None: + renames: list[str] | None = None) -> None: """Replace the segmentation-group definitions for an annotation worklist. Args: @@ -151,13 +138,7 @@ def update_segmentation_group(self, segmentation_value_type: ``'single_label'`` (default) or ``'multi_label'``. renames: Optional rename pairs (old→new identifier strings). - annotation_worklist: (DEPRECATED) Use ``worklist_id`` instead. """ - if annotation_worklist is not None: - warnings.warn("The 'annotation_worklist' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_worklist if worklist_id is None: raise TypeError("update_segmentation_group() missing required argument: 'worklist_id'") if definitions is None: @@ -176,20 +157,12 @@ def update_segmentation_group(self, json=payload) def get_segmentation_group(self, - worklist_id: str | None = None, - *, - annotation_set: str | None = None) -> dict: + worklist_id: str | None = None) -> dict: """Get the segmentation group for a given worklist ID. Args: worklist_id: The annotation worklist ID. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set if worklist_id is None: raise TypeError("get_segmentation_group() missing required argument: 'worklist_id'") @@ -226,21 +199,13 @@ def delete_segmentation_group( self, worklist_id: str | None = None, identifier: str | None = None, - *, - annotation_set: str | None = None, ) -> None: """Delete a specific segmentation group from a worklist. Args: worklist_id: The annotation worklist ID. identifier: The segmentation group identifier to delete. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set if worklist_id is None: raise TypeError("delete_segmentation_group() missing required argument: 'worklist_id'") if identifier is None: @@ -253,31 +218,16 @@ def get_annotator_status( self, worklist_id: str | None = None, annotator_email: str | None = None, - *, - annotation_set: str | None = None, - email: str | None = None, ) -> dict: """Get a specific annotator's progress status within a worklist. Args: worklist_id: The annotation worklist ID. annotator_email: The annotator's email address. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - email: (DEPRECATED) Use ``annotator_email`` instead. Returns: Dict with annotator status information. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if email is not None: - warnings.warn("The 'email' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = email if worklist_id is None: raise TypeError("get_annotator_status() missing required argument: 'worklist_id'") if annotator_email is None: @@ -292,10 +242,6 @@ def get_segmentations( resource: 'str | Resource | None' = None, all: bool = True, annotator_email: str | None = None, - *, - annotation_set: str | None = None, - resource_id: str | None = None, - annotator: str | None = None, ) -> list[dict]: """Get all segmentations for a resource within a worklist. @@ -304,28 +250,10 @@ def get_segmentations( resource: The resource unique id or a Resource instance. all: Whether to get all segmentations. annotator_email: Optional annotator email filter. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - resource_id: (DEPRECATED) Use ``resource`` instead. - annotator: (DEPRECATED) Use ``annotator_email`` instead. Returns: List of segmentation objects. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id - if annotator is not None: - warnings.warn("The 'annotator' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = annotator if worklist_id is None: raise TypeError("get_segmentations() missing required argument: 'worklist_id'") if resource is None: @@ -345,10 +273,6 @@ def get_annotations( resource: 'str | Resource | None' = None, all: bool = True, annotator_email: str | None = None, - *, - annotation_set: str | None = None, - resource_id: str | None = None, - annotator: str | None = None, ) -> list[dict]: """Get all annotations (non-segmentation types) for a resource within a worklist. @@ -357,28 +281,10 @@ def get_annotations( resource: The resource unique id or a Resource instance. all: Whether to get all annotations. annotator_email: Optional annotator email filter. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - resource_id: (DEPRECATED) Use ``resource`` instead. - annotator: (DEPRECATED) Use ``annotator_email`` instead. Returns: List of annotation objects. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id - if annotator is not None: - warnings.warn("The 'annotator' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = annotator if worklist_id is None: raise TypeError("get_annotations() missing required argument: 'worklist_id'") if resource is None: @@ -396,31 +302,16 @@ def get_ai_segmentations( self, worklist_id: str | None = None, resource: 'str | Resource | None' = None, - *, - annotation_set: str | None = None, - resource_id: str | None = None, ) -> list[dict]: """Get AI-generated segmentations for a resource within a worklist. Args: worklist_id: The annotation worklist ID. resource: The resource unique id or a Resource instance. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - resource_id: (DEPRECATED) Use ``resource`` instead. Returns: List of AI segmentation objects. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id if worklist_id is None: raise TypeError("get_ai_segmentations() missing required argument: 'worklist_id'") if resource is None: @@ -436,9 +327,6 @@ def upload_annotations( resource: 'str | Resource | None' = None, payload: str | None = None, images: list[Any] | None = None, - *, - annotation_set: str | None = None, - resource_id: str | None = None, ) -> dict: """Upload one or more resource segmentations. @@ -460,22 +348,10 @@ def upload_annotations( resource: The resource unique id or a Resource instance. payload: A JSON string containing an array of segmentation items. images: Optional list of file-like objects or file paths to upload. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - resource_id: (DEPRECATED) Use ``resource`` instead. Returns: Response dict with created annotation info. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id if worklist_id is None: raise TypeError("upload_annotations() missing required argument: 'worklist_id'") if resource is None: @@ -515,10 +391,6 @@ def update_annotation_status( annotator_email: str | None = None, message: str | None = None, path: str | None = None, - *, - annotation_set: str | None = None, - resource_id: str | None = None, - annotator: str | None = None, ) -> list[dict]: """Update the annotation status for a resource within a worklist. @@ -529,28 +401,10 @@ def update_annotation_status( annotator_email: Optional annotator email. message: Optional message. path: Optional path. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - resource_id: (DEPRECATED) Use ``resource`` instead. - annotator: (DEPRECATED) Use ``annotator_email`` instead. Returns: List of objects with updated status information. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id - if annotator is not None: - warnings.warn("The 'annotator' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = annotator if worklist_id is None: raise TypeError("update_annotation_status() missing required argument: 'worklist_id'") if resource is None: @@ -575,8 +429,6 @@ def set_annotator( status: Literal['active', 'frozen'] | None = None, expertise_level: Literal['learner', 'trained', 'expert'] | None = None, return_url: str | None = None, - *, - annotation_set: str | None = None, ) -> dict: """Set or update an annotator's status and expertise level in a worklist. @@ -586,16 +438,10 @@ def set_annotator( status: Annotator status (active or frozen). expertise_level: Expertise level (learner, trained, or expert). return_url: Optional return URL. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. Returns: Response dict with created annotator info. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set if worklist_id is None: raise TypeError("set_annotator() missing required argument: 'worklist_id'") if user_id is None: @@ -619,21 +465,13 @@ def remove_annotator( self, worklist_id: str | None = None, user_id: str | None = None, - *, - annotation_set: str | None = None, ) -> None: """Remove an annotator from a worklist. Args: worklist_id: The annotation worklist ID. user_id: The user's UUID. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set if worklist_id is None: raise TypeError("remove_annotator() missing required argument: 'worklist_id'") if user_id is None: @@ -647,10 +485,6 @@ def update_resources( worklist_id: str | None = None, resource_ids_to_add: list[str] | None = None, resource_ids_to_delete: list[str] | None = None, - *, - annotation_set: str | None = None, - resources_to_add: list[str] | None = None, - resources_to_delete: list[str] | None = None, ) -> dict: """Add or remove resources from a worklist. @@ -658,28 +492,10 @@ def update_resources( worklist_id: The annotation worklist ID. resource_ids_to_add: Optional list of resource IDs to add. resource_ids_to_delete: Optional list of resource IDs to delete. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - resources_to_add: (DEPRECATED) Use ``resource_ids_to_add`` instead. - resources_to_delete: (DEPRECATED) Use ``resource_ids_to_delete`` instead. Returns: Response dict with updated worklist info. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if resources_to_add is not None: - warnings.warn("The 'resources_to_add' parameter is deprecated. " - "Please use 'resource_ids_to_add' instead", DeprecationWarning) - if resource_ids_to_add is None: - resource_ids_to_add = resources_to_add - if resources_to_delete is not None: - warnings.warn("The 'resources_to_delete' parameter is deprecated. " - "Please use 'resource_ids_to_delete' instead", DeprecationWarning) - if resource_ids_to_delete is None: - resource_ids_to_delete = resources_to_delete if worklist_id is None: raise TypeError("update_resources() missing required argument: 'worklist_id'") @@ -697,9 +513,6 @@ def get_annotation_statuses( status: str | None = None, user_id: str | None = None, resource: 'str | Resource | None' = None, - *, - annotation_set: str | None = None, - resource_id: str | None = None, ) -> list[dict]: """Get annotation statuses for a worklist. @@ -709,22 +522,10 @@ def get_annotation_statuses( ``closed``, ``approved``, ``revision_request``. user_id: Optional user ID filter. resource: Optional resource unique id, or Resource instance, filter. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - resource_id: (DEPRECATED) Use ``resource`` instead. Returns: List of annotation status dicts. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id if worklist_id is None: raise TypeError("get_annotation_statuses() missing required argument: 'worklist_id'") @@ -744,10 +545,6 @@ def reset_annotator_status( worklist_id: str | None = None, resource: 'str | Resource | None' = None, annotator_email: str | None = None, - *, - annotation_set: str | None = None, - resource_id: str | None = None, - annotator: str | None = None, ) -> None: """Reset one annotator's resource status and delete related annotations. @@ -755,25 +552,7 @@ def reset_annotator_status( worklist_id: The annotation worklist ID. resource: The resource unique id or a Resource instance. annotator_email: The annotator's email address. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - resource_id: (DEPRECATED) Use ``resource`` instead. - annotator: (DEPRECATED) Use ``annotator_email`` instead. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id - if annotator is not None: - warnings.warn("The 'annotator' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = annotator if worklist_id is None: raise TypeError("reset_annotator_status() missing required argument: 'worklist_id'") if resource is None: @@ -789,31 +568,16 @@ def get_annotators_statistics( self, worklist_id: str | None = None, annotator_email: str | None = None, - *, - annotation_set: str | None = None, - email: str | None = None, ) -> list[dict]: """Get annotator statistics for a worklist. Args: worklist_id: The annotation worklist ID. annotator_email: Optional annotator email filter. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. - email: (DEPRECATED) Use ``annotator_email`` instead. Returns: List of per-annotator stat dicts. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set - if email is not None: - warnings.warn("The 'email' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = email if worklist_id is None: raise TypeError("get_annotators_statistics() missing required argument: 'worklist_id'") @@ -827,23 +591,15 @@ def get_annotators_statistics( def get_annotations_statistics( self, worklist_id: str | None = None, - *, - annotation_set: str | None = None, ) -> list[dict]: """Get annotation statistics for a worklist. Args: worklist_id: The annotation worklist ID. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. Returns: List of annotation stat objects. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set if worklist_id is None: raise TypeError("get_annotations_statistics() missing required argument: 'worklist_id'") @@ -858,8 +614,6 @@ def download_annotations( annotators: list[str] | None = None, annotations: list[str] | None = None, format: str = 'csv', - *, - annotation_set: str | None = None, ) -> bytes: """Download annotations as a streamed file. @@ -872,16 +626,10 @@ def download_annotations( annotations: Optional list of annotation identifiers. Accepts either a repeated/array input or a comma-separated string. format: Export format. Allowed values: ``csv`` (default), ``excel``. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. Returns: The raw bytes of the downloaded file. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set if worklist_id is None: raise TypeError("download_annotations() missing required argument: 'worklist_id'") @@ -904,8 +652,6 @@ def upload_segmentation_group( worklist_id: str | None = None, file: Any = None, replace_existing: bool = True, - *, - annotation_set: str | None = None, ) -> dict: """Upload a segmentation group definition file to an annotation set. @@ -914,16 +660,10 @@ def upload_segmentation_group( file: The file to upload. Must have one of these extensions: ``.yaml``, ``.yml``, ``.csv``, ``.json``. Maximum size: 10 MB. replace_existing: Whether to replace existing segmentation data. - annotation_set: (DEPRECATED) Use ``worklist_id`` instead. Returns: Response dict with created segmentation group info. """ - if annotation_set is not None: - warnings.warn("The 'annotation_set' parameter is deprecated. " - "Please use 'worklist_id' instead", DeprecationWarning) - if worklist_id is None: - worklist_id = annotation_set if worklist_id is None: raise TypeError("upload_segmentation_group() missing required argument: 'worklist_id'") if file is None: diff --git a/datamint/api/endpoints/datasetsinfo_api.py b/datamint/api/endpoints/datasetsinfo_api.py index a4a4f67c..eb501d40 100644 --- a/datamint/api/endpoints/datasetsinfo_api.py +++ b/datamint/api/endpoints/datasetsinfo_api.py @@ -5,7 +5,6 @@ from datamint.entities.datasetinfo import DatasetInfo import httpx from tqdm.auto import tqdm -import warnings if TYPE_CHECKING: from datamint.entities import Project @@ -46,9 +45,7 @@ def update_resources(self, dataset: str | DatasetInfo, resource_ids_to_add: list[str] | None = None, resource_ids_to_delete: list[str] | None = None, - project: 'str | Project | None' = None, - *, - project_id: str | None = None) -> None: + project: 'str | Project | None' = None) -> None: """Add or remove resources from a dataset. Args: @@ -56,14 +53,7 @@ def update_resources(self, resource_ids_to_add: List of resource IDs to add. resource_ids_to_delete: List of resource IDs to remove. project: Optional project ID or Project instance context. - project_id: (DEPRECATED) Use ``project`` instead. """ - if project_id is not None: - warnings.warn("The 'project_id' parameter is deprecated. " - "Please use 'project' instead", DeprecationWarning) - if project is None: - project = project_id - payload: dict = {'all_files_selected': False} if resource_ids_to_add is not None: payload['resource_ids_to_add'] = resource_ids_to_add diff --git a/datamint/api/endpoints/deploy_model_api.py b/datamint/api/endpoints/deploy_model_api.py index 145d1bd3..ab7f32db 100644 --- a/datamint/api/endpoints/deploy_model_api.py +++ b/datamint/api/endpoints/deploy_model_api.py @@ -6,7 +6,6 @@ import time import httpx -import warnings from datamint.exceptions import ResourceNotFoundError, JobTimeoutError from ..entity_base_api import EntityBaseApi, ApiConfig @@ -39,9 +38,7 @@ def get_by_id(self, entity_id: str) -> DeployJob: raise def stream_status(self, - job: str | DeployJob | None = None, - *, - job_id: str | None = None) -> Generator[dict[str, Any], None, None]: + job: str | DeployJob | None = None) -> Generator[dict[str, Any], None, None]: """Stream status updates for a deployment job via Server-Sent Events. Yields dictionaries parsed from SSE ``data:`` lines until the @@ -49,16 +46,10 @@ def stream_status(self, Args: job: The job ID string or ``DeployJob`` instance. - job_id: (DEPRECATED) Use ``job`` instead. Yields: Parsed JSON dictionaries for each SSE event. """ - if job_id is not None: - warnings.warn("The 'job_id' parameter is deprecated. " - "Please use 'job' instead", DeprecationWarning) - if job is None: - job = job_id if job is None: raise TypeError("stream_status() missing required argument: 'job'") job_id_str = self._entid(job) diff --git a/datamint/api/endpoints/inference_api.py b/datamint/api/endpoints/inference_api.py index 977a16a5..e5b343ca 100644 --- a/datamint/api/endpoints/inference_api.py +++ b/datamint/api/endpoints/inference_api.py @@ -6,7 +6,6 @@ import time import httpx -import warnings from ..entity_base_api import EntityBaseApi, ApiConfig from datamint.entities.inferencejob import InferenceJob @@ -163,23 +162,15 @@ def submit( # ------------------------------------------------------------------ def get_status(self, - job: str | InferenceJob | None = None, - *, - job_id: str | None = None) -> InferenceJob: + job: str | InferenceJob | None = None) -> InferenceJob: """Get the current status of an inference job. Args: job: The job ID string or ``InferenceJob`` instance. - job_id: (DEPRECATED) Use ``job`` instead. Returns: An ``InferenceJob`` populated with the latest status. """ - if job_id is not None: - warnings.warn("The 'job_id' parameter is deprecated. " - "Please use 'job' instead", DeprecationWarning) - if job is None: - job = job_id if job is None: raise TypeError("get_status() missing required argument: 'job'") job_id_str = self._entid(job) @@ -192,9 +183,7 @@ def get_by_id(self, entity_id: str) -> InferenceJob: return self.get_status(entity_id) def stream_status(self, - job: str | InferenceJob | None = None, - *, - job_id: str | None = None) -> Generator[dict[str, Any], None, None]: + job: str | InferenceJob | None = None) -> Generator[dict[str, Any], None, None]: """Stream status updates for an inference job via Server-Sent Events. Yields dictionaries parsed from SSE ``data:`` lines until the @@ -202,16 +191,10 @@ def stream_status(self, Args: job: The job ID string or ``InferenceJob`` instance. - job_id: (DEPRECATED) Use ``job`` instead. Yields: Parsed JSON dictionaries for each SSE event. """ - if job_id is not None: - warnings.warn("The 'job_id' parameter is deprecated. " - "Please use 'job' instead", DeprecationWarning) - if job is None: - job = job_id if job is None: raise TypeError("stream_status() missing required argument: 'job'") job_id_str = self._entid(job) diff --git a/datamint/api/endpoints/projects_api.py b/datamint/api/endpoints/projects_api.py index ccb86bd3..30819193 100644 --- a/datamint/api/endpoints/projects_api.py +++ b/datamint/api/endpoints/projects_api.py @@ -1,7 +1,6 @@ from typing import Literal, TYPE_CHECKING, overload from collections.abc import Sequence from pathlib import Path -import warnings from ..entity_base_api import ApiConfig, CRUDEntityApi from datamint.entities.project import Project @@ -59,7 +58,6 @@ def create(self, two_up_display: bool = False, segmentation_spec: Literal['single_label', 'multi_label'] = 'single_label', *, - resources_ids: list[str] | None = None, return_entity: Literal[True] = True, exists_ok: bool = False ) -> Project: ... @@ -73,7 +71,6 @@ def create(self, two_up_display: bool = False, segmentation_spec: Literal['single_label', 'multi_label'] = 'single_label', *, - resources_ids: list[str] | None = None, return_entity: Literal[False], exists_ok: bool = False ) -> str: ... @@ -86,7 +83,6 @@ def create(self, two_up_display: bool = False, segmentation_spec: Literal['single_label', 'multi_label'] = 'single_label', *, - resources_ids: list[str] | None = None, return_entity: bool = True, exists_ok: bool = False ) -> str | Project: @@ -102,17 +98,10 @@ def create(self, exists_ok: If ``True``, do not raise an error when a project with the same name already exists. Instead, the existing project is returned when possible. - resources_ids: (DEPRECATED) Use ``resource_ids`` instead. Returns: The id of the created project. """ - if resources_ids is not None: - warnings.warn("The 'resources_ids' parameter is deprecated. " - "Please use 'resource_ids' instead", DeprecationWarning) - if resource_ids is None: - resource_ids = resources_ids - proj = self.get_by_name(name, include_archived=True) if proj is not None: if exists_ok: @@ -384,9 +373,7 @@ def get_annotation_statuses(self, project: str | Project | None = None, status: str | None = None, user_id: str | None = None, - resource: str | Resource | None = None, - *, - resource_id: str | None = None) -> list[dict]: + resource: str | Resource | None = None) -> list[dict]: """Get per-resource annotation statuses for a project. Args: @@ -395,17 +382,10 @@ def get_annotation_statuses(self, status: Optional status filter. user_id: Optional user ID filter. resource: Optional resource unique id, or Resource instance, filter. - resource_id: (DEPRECATED) Use ``resource`` instead. Returns: List of annotation status dicts. """ - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id - project = self._resolve_project_or_default(project) params = {k: v for k, v in {'status': status, 'user_id': user_id, 'resource_id': self._entid(resource) if resource is not None else None @@ -505,9 +485,7 @@ def get_resource_split( def reset_annotator_status(self, resource: str | Resource, annotator_email: str | None = None, - project: str | Project | None = None, - *, - annotator: str | None = None) -> None: + project: str | Project | None = None) -> None: """Reset annotation status for a specific annotator on a resource. Args: @@ -515,13 +493,7 @@ def reset_annotator_status(self, annotator_email: The annotator's email address. project: The project ID or Project instance. Falls back to the session's default project (see `datamint.select_project()`) when omitted. - annotator: (DEPRECATED) Use ``annotator_email`` instead. """ - if annotator is not None: - warnings.warn("The 'annotator' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = annotator if annotator_email is None: raise TypeError("reset_annotator_status() missing required argument: 'annotator_email'") @@ -588,26 +560,17 @@ def download_annotations(self, def get_annotators_stats(self, project: str | Project | None = None, - annotator_email: str | None = None, - *, - email: str | None = None) -> list[dict]: + annotator_email: str | None = None) -> list[dict]: """Get per-annotator completion statistics for a project. Args: project: The project ID or Project instance. Falls back to the session's default project (see `datamint.select_project()`) when omitted. annotator_email: Optional annotator email to filter results. - email: (DEPRECATED) Use ``annotator_email`` instead. Returns: List of per-annotator stat dicts. """ - if email is not None: - warnings.warn("The 'email' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = email - project = self._resolve_project_or_default(project) params = {'email': annotator_email} if annotator_email is not None else None response = self._make_entity_request('GET', project, add_path='annotators-statistic', @@ -644,25 +607,17 @@ def get_files_matrix_stats(self, project: str | Project | None = None) -> dict: def get_annotator_status(self, annotator_email: str | None = None, - project: str | Project | None = None, - *, - email: str | None = None) -> dict: + project: str | Project | None = None) -> dict: """Get a specific annotator's progress status in a project. Args: annotator_email: The annotator's email address. project: The project ID or Project instance. Falls back to the session's default project (see `datamint.select_project()`) when omitted. - email: (DEPRECATED) Use ``annotator_email`` instead. Returns: Annotator status dict. """ - if email is not None: - warnings.warn("The 'email' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = email if annotator_email is None: raise TypeError("get_annotator_status() missing required argument: 'annotator_email'") @@ -679,10 +634,7 @@ def get_review_messages(self, project: str | Project | None = None, annotator_email: str | None = None, resource: str | Resource | None = None, - statuses: list[str] | None = None, - *, - annotator: str | None = None, - resource_id: str | None = None) -> list[dict]: + statuses: list[str] | None = None) -> list[dict]: """Get review feedback messages for a project. Args: @@ -691,23 +643,10 @@ def get_review_messages(self, annotator_email: Optional annotator email filter. resource: Optional resource unique id, or Resource instance, filter. statuses: Optional list of status strings to filter by. - annotator: (DEPRECATED) Use ``annotator_email`` instead. - resource_id: (DEPRECATED) Use ``resource`` instead. Returns: List of review message dicts. """ - if annotator is not None: - warnings.warn("The 'annotator' parameter is deprecated. " - "Please use 'annotator_email' instead", DeprecationWarning) - if annotator_email is None: - annotator_email = annotator - if resource_id is not None: - warnings.warn("The 'resource_id' parameter is deprecated. " - "Please use 'resource' instead", DeprecationWarning) - if resource is None: - resource = resource_id - project = self._resolve_project_or_default(project) params: dict = {} if annotator_email is not None: diff --git a/datamint/api/endpoints/resources_api.py b/datamint/api/endpoints/resources_api.py index 64f8ba38..6f55111b 100644 --- a/datamint/api/endpoints/resources_api.py +++ b/datamint/api/endpoints/resources_api.py @@ -22,7 +22,6 @@ from tqdm.auto import tqdm import asyncio import aiohttp -import warnings from pathlib import Path from PIL import Image import io @@ -702,8 +701,6 @@ def upload_resources(self, metadata: Sequence[str | dict | None] | None = None, discard_dicom_reports: bool = True, progress_bar: bool = False, - *, - ai_model: str | None = None, ) -> Sequence[str | Exception]: """ Upload multiple resources. @@ -733,7 +730,6 @@ def upload_resources(self, model_name (Optional[str]): The name of the AI model to associate with uploaded segmentations. Must match an existing deployed model name on the server. modality (Optional[str]): The modality of the resources. - ai_model (Optional[str]): (DEPRECATED) Use ``model_name`` instead. assemble_dicoms (bool): Whether to assemble the dicom files or not based on the SeriesInstanceUID and InstanceNumber attributes. metadata (Optional[list[str | dict | None]]): JSON metadata to include with each resource. Must have the same length as `files_path`. @@ -747,12 +743,6 @@ def upload_resources(self, list[str | Exception]: A list of resource IDs or errors. """ - if ai_model is not None: - warnings.warn("The 'ai_model' parameter is deprecated. " - "Please use 'model_name' instead", DeprecationWarning) - if model_name is None: - model_name = ai_model - self._validate_upload_params(on_error, files_path) proj = self._resolve_project(publish_to) diff --git a/datamint/api/endpoints/users_api.py b/datamint/api/endpoints/users_api.py index e31a37af..ad081fa5 100644 --- a/datamint/api/endpoints/users_api.py +++ b/datamint/api/endpoints/users_api.py @@ -3,7 +3,6 @@ from ..entity_base_api import CreatableEntityApi, ApiConfig from datamint.entities import User import httpx -import warnings if TYPE_CHECKING: from datamint.entities import Project @@ -99,8 +98,6 @@ def invite(self, project: 'str | Project | None' = None, project_roles: list[str] | None = None, annotation_worklist_id: str | None = None, - *, - project_id: str | None = None, ) -> dict: """Send an invitation email to a new user. @@ -112,17 +109,10 @@ def invite(self, project: Optional project ID or Project instance to add the invitee to. project_roles: Roles to assign in the given project. annotation_worklist_id: Optional annotation worklist to associate. - project_id: (DEPRECATED) Use ``project`` instead. Returns: The server response as a dict. """ - if project_id is not None: - warnings.warn("The 'project_id' parameter is deprecated. " - "Please use 'project' instead", DeprecationWarning) - if project is None: - project = project_id - payload: dict = {'email': email} if firstname is not None: payload['firstname'] = firstname @@ -170,24 +160,15 @@ def delete_user(self, email: str) -> None: self._make_entity_request('DELETE', email) def get_invitations(self, - project: 'str | Project | None' = None, - *, - project_id: str | None = None) -> list[dict]: + project: 'str | Project | None' = None) -> list[dict]: """List pending user invitations. Args: project: Optional project ID or Project instance to filter invitations. - project_id: (DEPRECATED) Use ``project`` instead. Returns: List of invitation dicts. """ - if project_id is not None: - warnings.warn("The 'project_id' parameter is deprecated. " - "Please use 'project' instead", DeprecationWarning) - if project is None: - project = project_id - params: dict = {} if project is not None: params['project_id'] = self._entid(project) diff --git a/datamint/client_cmd_tools/datamint_config.py b/datamint/client_cmd_tools/datamint_config.py index f55894bc..dad22227 100644 --- a/datamint/client_cmd_tools/datamint_config.py +++ b/datamint/client_cmd_tools/datamint_config.py @@ -11,7 +11,6 @@ from typing_extensions import NotRequired from datamint import configs -from datamint.utils.env import is_legacy_cli_invocation from datamint.utils.logging_utils import ConsoleWrapperHandler, load_cmdline_logging_config _LOGGER = logging.getLogger(__name__) @@ -817,9 +816,7 @@ def _build_parser(subparsers: argparse._SubParsersAction | None = None) -> argpa return parser -_COMPLETION_EXECUTABLES = [ - 'datamint', 'datamint-upload', 'datamint-config', 'datamint-train', 'datamint-inference', -] +_COMPLETION_EXECUTABLES = ['datamint'] def _detect_shell() -> str | None: @@ -889,12 +886,6 @@ def main(): if console_handlers: console = console_handlers[0].console - if is_legacy_cli_invocation('config'): - console.print( - "[warning]'datamint-config' is deprecated and will be removed in a future " - "release. Use 'datamint config' instead.[/warning]" - ) - parser = _build_parser() import argcomplete argcomplete.autocomplete(parser) diff --git a/datamint/client_cmd_tools/datamint_inference.py b/datamint/client_cmd_tools/datamint_inference.py index fed56f4b..6b74fcfe 100644 --- a/datamint/client_cmd_tools/datamint_inference.py +++ b/datamint/client_cmd_tools/datamint_inference.py @@ -16,7 +16,6 @@ from datamint.client_cmd_tools.datamint_upload import _is_valid_path_argparse, handle_api_key from datamint.exceptions import DatamintException, ItemNotFoundError -from datamint.utils.env import is_legacy_cli_invocation from datamint.utils.logging_utils import ConsoleWrapperHandler, load_cmdline_logging_config _LOGGER = logging.getLogger(__name__) @@ -215,12 +214,6 @@ def main() -> None: load_cmdline_logging_config() CONSOLE = [h for h in _USER_LOGGER.handlers if isinstance(h, ConsoleWrapperHandler)][0].console - if is_legacy_cli_invocation('inference'): - CONSOLE.print( - "[warning]'datamint-inference' is deprecated and will be removed in a future " - "release. Use 'datamint inference' instead.[/warning]" - ) - args = _parse_args() if args.verbose: diff --git a/datamint/client_cmd_tools/datamint_init.py b/datamint/client_cmd_tools/datamint_init.py index 204d5cf6..8fcd1ce4 100644 --- a/datamint/client_cmd_tools/datamint_init.py +++ b/datamint/client_cmd_tools/datamint_init.py @@ -5,7 +5,6 @@ from rich.prompt import Prompt, Confirm from rich.rule import Rule -from datamint.utils.env import is_legacy_cli_invocation console = Console() @@ -1509,12 +1508,6 @@ def _print_header() -> None: def main() -> None: - if is_legacy_cli_invocation('init'): - console.print( - "[yellow]'datamint-init' is deprecated and will be removed in a future " - "release. Use 'datamint init' instead.[/yellow]" - ) - _print_header() try: diff --git a/datamint/client_cmd_tools/datamint_train.py b/datamint/client_cmd_tools/datamint_train.py index 01d59190..7d3f82f9 100644 --- a/datamint/client_cmd_tools/datamint_train.py +++ b/datamint/client_cmd_tools/datamint_train.py @@ -23,7 +23,6 @@ from datamint import Api, configs from datamint.client_cmd_tools.datamint_upload import handle_api_key from datamint.exceptions import DatamintException -from datamint.utils.env import is_legacy_cli_invocation from datamint.utils.logging_utils import ConsoleWrapperHandler, load_cmdline_logging_config if TYPE_CHECKING: @@ -416,12 +415,6 @@ def main() -> None: load_cmdline_logging_config() CONSOLE = [h for h in _USER_LOGGER.handlers if isinstance(h, ConsoleWrapperHandler)][0].console - if is_legacy_cli_invocation('train'): - CONSOLE.print( - "[warning]'datamint-train' is deprecated and will be removed in a future " - "release. Use 'datamint train' instead.[/warning]" - ) - args = _parse_args() if args.verbose: diff --git a/datamint/client_cmd_tools/datamint_upload.py b/datamint/client_cmd_tools/datamint_upload.py index 653220dd..04e2f24d 100644 --- a/datamint/client_cmd_tools/datamint_upload.py +++ b/datamint/client_cmd_tools/datamint_upload.py @@ -14,7 +14,6 @@ from datamint import __version__ as datamint_version from datamint import configs from datamint.utils.logging_utils import load_cmdline_logging_config, ConsoleWrapperHandler -from datamint.utils.env import is_legacy_cli_invocation from rich.console import Console import yaml from collections.abc import Iterable @@ -555,7 +554,6 @@ def _build_parser(subparsers: argparse._SubParsersAction | None = None) -> argpa default=[], help='Retain the value of a single attribute code specified as hexidecimal integers. \ Example: (0x0008, 0x0050) or just (0008, 0050)') - parser.add_argument('-l', '--label', type=str, action='append', help='Deprecated. Use --tag instead.') parser.add_argument('--tag', type=str, action='append', help='A tag name to be applied to all files') parser.add_argument('--publish', action='store_true', help='Publish the uploaded resources, giving them the status "published" instead of "inbox"') @@ -690,10 +688,6 @@ def _parse_args() -> tuple[Any, list[str], list[dict] | None, list[str] | None]: sys.exit(1) os.environ[configs.ENV_VARS[configs.APIKEY_KEY]] = api_key - if args.tag is not None and args.label is not None: - raise ValueError("Cannot use both --tag and --label. Use --tag instead. --label is deprecated.") - args.tag = args.tag if args.tag is not None else args.label - return args, file_path, segmentation_files, metadata_files except Exception as e: @@ -793,12 +787,6 @@ def main(): load_cmdline_logging_config() CONSOLE = [h for h in _USER_LOGGER.handlers if isinstance(h, ConsoleWrapperHandler)][0].console - if is_legacy_cli_invocation('upload'): - CONSOLE.print( - "[warning]'datamint-upload' is deprecated and will be removed in a future " - "release. Use 'datamint upload' instead.[/warning]" - ) - try: args, files_path, segfiles, metadata_files = _parse_args() except Exception as e: diff --git a/datamint/dataset/annotation.py b/datamint/dataset/annotation.py deleted file mode 100644 index 87fbde7b..00000000 --- a/datamint/dataset/annotation.py +++ /dev/null @@ -1,221 +0,0 @@ -from __future__ import annotations -from dataclasses import dataclass, field, asdict -from typing import Optional, Any, TYPE_CHECKING -from datetime import datetime -from pathlib import Path -import logging -import numpy as np -from PIL import Image -import json - -# if TYPE_CHECKING: -# from datamint.apihandler.annotation_api_handler import AnnotationAPIHandler - -_LOGGER = logging.getLogger(__name__) - - -# Map API field names to class attributes -_FIELD_MAPPING = { - 'type':'annotation_type', - 'name': 'identifier', - 'added_by': 'created_by', - 'index': 'frame_index', -} - -@dataclass -class Annotation: - """ - Class representing an annotation from the Datamint API. - - This class stores annotation data and provides methods for loading - and saving annotations through the API handler. - - Args: - id: Unique identifier for the annotation - identifier: The annotation identifier/label name - scope: Whether annotation applies to 'frame' or 'image' - annotation_type: Type of annotation ('segmentation', 'label', 'category', etc.) - resource_id: ID of the resource this annotation belongs to - annotation_worklist_id: ID of the annotation worklist - created_by: Email of the user who created the annotation - status: Status of the annotation ('published', 'new', etc.) - frame_index: Frame index for frame-scoped annotations - text_value: Text value for category annotations - numeric_value: Numeric value for numeric annotations - units: Units for numeric annotations - geometry: Geometry data for geometric annotations - created_at: When the annotation was created - approved_at: When the annotation was approved - approved_by: Who approved the annotation - associated_file: Path to associated file (for segmentations) - deleted: Whether the annotation is deleted - deleted_at: When the annotation was deleted - deleted_by: Who deleted the annotation - created_by_model: Model ID if created by AI - old_geometry: Previous geometry data - set_name: Set name for grouped annotations - resource_filename: Filename of the associated resource - resource_modality: Modality of the associated resource - annotation_worklist_name: Name of the annotation worklist - user_info: Information about the user who created the annotation - values: Additional values - """ - - id: str - identifier: str - scope: str - annotation_type: str - resource_id: str - created_by: str - annotation_worklist_id: Optional[str] = None - status: Optional[str] = None - frame_index: Optional[int] = None - text_value: Optional[str] = None - numeric_value: Optional[float] = None - units: Optional[str] = None - geometry: list[Any] = field(default_factory=list) - created_at: Optional[str] = None - approved_at: Optional[str] = None - approved_by: Optional[str] = None - associated_file: Optional[str] = None - file: Optional[str] = None - deleted: bool = False - deleted_at: Optional[str] = None - deleted_by: Optional[str] = None - created_by_model: Optional[str] = None - old_geometry: Optional[Any] = None - set_name: Optional[str] = None - resource_filename: Optional[str] = None - resource_modality: Optional[str] = None - annotation_worklist_name: Optional[str] = None - user_info: Optional[dict[str, str]] = None - values: Optional[Any] = None - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> Annotation: - """ - Create an Annotation instance from a dictionary. - - Args: - data: Dictionary containing annotation data from API - - Returns: - Annotation instance - """ - - - # Convert field names and filter valid fields - converted_data = {} - for key, value in data.items(): - # Map field names if needed - mapped_key = _FIELD_MAPPING.get(key, key) - converted_data[mapped_key] = value - - if 'scope' not in converted_data: - converted_data['scope'] = 'image' if converted_data.get('frame_index') is None else 'frame' - - if converted_data['annotation_type'] in ['segmentation']: - if converted_data.get('file') is None: - raise ValueError(f"Segmentation annotations must have an associated file. {data}") - - # Create instance with only valid fields - valid_fields = {f.name for f in cls.__dataclass_fields__.values()} - filtered_data = {k: v for k, v in converted_data.items() if k in valid_fields} - - return cls(**filtered_data) - - def to_dict(self) -> dict[str, Any]: - """ - Convert the annotation to a dictionary format. - - Returns: - Dictionary representation of the annotation - """ - result = {} - for key, value in self.__dict__.items(): - # Handle special serialization cases - if isinstance(value, (np.ndarray, np.generic)): - value = value.tolist() - elif isinstance(value, datetime): - value = value.isoformat() - elif isinstance(value, Path): - value = str(value) - - result[key] = value - if self.annotation_type == 'segmentation' and 'file' not in result: - raise ValueError(f"Segmentation annotations must have an associated file. {self}") - return result - - @property - def name(self) -> str: - """Get the annotation name (alias for identifier).""" - return self.identifier - - @property - def type(self) -> str: - """Get the annotation type.""" - return self.annotation_type - - @property - def value(self) -> Optional[str]: - """Get the annotation value (for category annotations).""" - return self.text_value - - @property - def index(self) -> Optional[int]: - """Get the frame index (alias for frame_index).""" - return self.frame_index - - @property - def added_by(self) -> str: - """Get the creator email (alias for created_by).""" - return self.created_by - - # @property - # def file(self) -> Optional[str]: - # """Get the associated file path.""" - # return self.associated_file - - # @file.setter - # def file(self, value: Optional[str]) -> None: - # """Set the associated file path.""" - # self.associated_file = value - - def is_segmentation(self) -> bool: - """Check if this is a segmentation annotation.""" - return self.annotation_type == 'segmentation' - - def is_label(self) -> bool: - """Check if this is a label annotation.""" - return self.annotation_type == 'label' - - def is_category(self) -> bool: - """Check if this is a category annotation.""" - return self.annotation_type == 'category' - - def is_frame_scoped(self) -> bool: - """Check if this annotation is frame-scoped.""" - return self.scope == 'frame' - - def is_image_scoped(self) -> bool: - """Check if this annotation is image-scoped.""" - return self.scope == 'image' - - def get_created_datetime(self) -> Optional[datetime]: - """ - Get the creation datetime as a datetime object. - - Returns: - datetime object or None if created_at is not set - """ - if self.created_at: - try: - return datetime.fromisoformat(self.created_at.replace('Z', '+00:00')) - except ValueError: - _LOGGER.warning(f"Could not parse created_at datetime: {self.created_at}") - return None - - def __repr__(self) -> str: - """String representation of the annotation.""" - return (f"Annotation(id='{self.id}', identifier='{self.identifier}', " - f"type='{self.annotation_type}', scope='{self.scope}', resource_id='{self.resource_id}')") diff --git a/datamint/dataset/base.py b/datamint/dataset/base.py index 3a9fa1e6..31bf7985 100644 --- a/datamint/dataset/base.py +++ b/datamint/dataset/base.py @@ -1271,14 +1271,13 @@ def split( self, *, seed: int | None = None, - use_server_splits: bool | None = None, use_project_splits: bool | None = None, as_of_timestamp: str | None = None, by_patient: bool = False, none_patient_id_strategy: Literal['individual', 'group', 'skip', 'error'] = 'individual', **splits: float, ) -> 'SplitResult': - + """Split the dataset into multiple named subsets. The mode is selected automatically when no explicit split mode is @@ -1288,15 +1287,14 @@ def split( is used. - If no ratio kwargs are provided and the dataset was loaded from a project, project-scoped split assignments are used. - - Otherwise, server-side ``split:*`` tags on resources are used. Examples:: # Local split parts = dataset.split(train=0.7, val=0.15, test=0.15, seed=42) - train_ds = parts['train'] - - # Patient-wise split + train_ds = parts['train'] + + # Patient-wise split parts = dataset.split(train = 0.8, test = 0.2, by_patient=True, seed=42) # Project-scoped split — inferred for project-backed datasets @@ -1310,8 +1308,8 @@ def split( by_patient: If ``True``, shuffle and assign whole patients to splits rather than individual resources, preventing cross-patient data leakage. Requires ratio kwards; mutually exclusive with - ``use_project_splits`` and ``use_server_splits``. - none_patient_id_strategy: Strategy for handling resources without patient IDs + ``use_project_splits``. + none_patient_id_strategy: Strategy for handling resources without patient IDs when ``by_patient=True``. See :meth:`group_by_patient` for details. use_project_splits: If ``True``, read split assignments from the project splits API. If ``None`` (default), project-backed @@ -1320,10 +1318,9 @@ def split( splits against. When omitted for project-scoped splits, the current UTC timestamp is captured and stored on the resolved split datasets for later reuse. - use_server_splits: (DEPRECATED in favor of ``use_project_splits``) **splits: Named split ratios (e.g. ``train=0.7, test=0.3``). Must sum to 1.0 (±0.01 tolerance). Must be empty when - *use_server_splits* or *use_project_splits* is ``True``. + *use_project_splits* is ``True``. Returns: Dictionary mapping split names to new dataset instances. @@ -1333,11 +1330,11 @@ def split( """ from .split_result import SplitResult - + if by_patient: - if use_project_splits or use_server_splits: + if use_project_splits: raise ValueError( - "by_patient=True cannot be combined with use_project_splits or use_server_splits." + "by_patient=True cannot be combined with use_project_splits." ) if not splits: @@ -1347,7 +1344,7 @@ def split( return SplitResult(self._split_locally_by_patient(dict(splits), seed, none_patient_id_strategy)) _auto_project = False - if use_project_splits is None and use_server_splits is None and not splits: + if use_project_splits is None and not splits: if getattr(self, 'project', None) is not None or as_of_timestamp is not None: use_project_splits = True _auto_project = True @@ -1375,14 +1372,12 @@ def split( 'Set use_project_splits=True or use a project-backed dataset with no ratio kwargs.' ) - if use_server_splits is None: - use_server_splits = not splits # True when no ratios given - - if use_server_splits: - import warnings - warnings.warn("use_server_splits and splitting by resource tags are deprecated in favor of use_project_splits. " - "Please migrate to project-scoped splits for better reproducibility and management.", DeprecationWarning) - return SplitResult(self._split_by_server_tags(splits)) + if not splits: + raise ValueError( + 'No ratio kwargs provided and no project-scoped splits available. ' + 'Provide ratio kwargs (e.g. train=0.7, test=0.3), or use a project-backed ' + 'dataset with use_project_splits=True.' + ) return SplitResult(self._split_locally(splits, seed)) @@ -1430,40 +1425,6 @@ def _split_by_project_api( ds.split_as_of_timestamp = resolved_as_of_timestamp return result - def _split_by_server_tags( - self, - splits: dict[str, float], - ) -> dict[str, 'DatamintBaseDataset']: - """Group resources by ``split:`` tags.""" - if splits: - raise ValueError( - "Ratio kwargs (e.g. train=0.7) must not be provided when " - "use_server_splits=True." - ) - - from collections import defaultdict - split_indices: dict[str, list[int]] = defaultdict(list) - - for idx, resource in enumerate(self.resources): - tags = resource.tags or [] - for tag in tags: - if tag.startswith("split:"): - split_name = tag[len("split:"):] - split_indices[split_name].append(idx) - - if not split_indices: - raise ValueError( - "No resources have 'split:*' tags. Tag resources on the " - "server first or use local splitting (use_server_splits=False)." - ) - - result = {name: self.subset(indices) for name, indices in split_indices.items()} - for name, ds in result.items(): - ds.split_name = name - ds.split_source = 'server_tags' - ds.split_as_of_timestamp = None - return result - def _split_locally( self, splits: dict[str, float], diff --git a/datamint/dataset/base_dataset.py b/datamint/dataset/base_dataset.py deleted file mode 100644 index 7dd8cbd7..00000000 --- a/datamint/dataset/base_dataset.py +++ /dev/null @@ -1,1220 +0,0 @@ -import warnings -import os -import requests -from typing import Optional, Callable, Any, Literal, Sequence -import logging -import shutil -import json -import pydicom -from pydicom.dataset import FileDataset -import numpy as np -from torch.utils.data import DataLoader -import torch -from torch import Tensor -from datamint.exceptions import DatamintException -from medimgkit.readers import read_array_normalized -from medimgkit.format_detection import guess_typez -from medimgkit.nifti_utils import NIFTI_MIMES, get_nifti_shape -from datetime import datetime -from pathlib import Path -from datamint.entities import Annotation, DatasetInfo -from datamint.entities.annotations import annotation_from_dict -import cv2 -from datamint.entities import Resource -import datamint.configs -from deprecated import deprecated - -_LOGGER = logging.getLogger(__name__) - - -class DatamintDatasetException(DatamintException): - pass - - -@deprecated(reason="DatamintBaseDataset is deprecated and may be removed in future versions. " - "Please use 'from datamint.dataset import ImageDataset, VolumeDataset' instead.") -class DatamintBaseDataset: - """Class to download and load datasets from the Datamint API. - - Args: - project_name: Name of the project to download. - root: Root directory of dataset where data already exists or will be downloaded. - auto_update: If True, the dataset will be checked for updates and downloaded if necessary. - api_key: API key to access the Datamint API. If not provided, it will look for the - environment variable 'DATAMINT_API_KEY'. Not necessary if - you don't want to download/update the dataset. - return_dicom: If True, the DICOM object will be returned, if the image is a DICOM file. - return_metainfo: If True, the metainfo of the image will be returned. - return_annotations: If True, the annotations of the image will be returned. - return_frame_by_frame: If True, each frame of a video/DICOM/3d-image will be returned separately. - include_unannotated: If True, images without annotations will be included. - all_annotations: If True, all annotations will be downloaded, including the ones that are not set as closed/done. - server_url: URL of the Datamint server. If not provided, it will use the default server. - include_annotators: List of annotators to include. If None, all annotators will be included. - exclude_annotators: List of annotators to exclude. If None, no annotators will be excluded. - include_segmentation_names: List of segmentation names to include. If None, all segmentations will be included. - exclude_segmentation_names: List of segmentation names to exclude. If None, no segmentations will be excluded. - include_image_label_names: List of image label names to include. If None, all image labels will be included. - exclude_image_label_names: List of image label names to exclude. If None, no image labels will be excluded. - include_frame_label_names: List of frame label names to include. If None, all frame labels will be included. - exclude_frame_label_names: List of frame label names to exclude. If None, no frame labels will be excluded. - """ - - DATAMINT_DATASETS_DIR = "datasets" - - def __init__( - self, - project_name: str, - root: str | None = None, - auto_update: bool = True, - api_key: str | None = None, - server_url: str | None = None, - return_dicom: bool = False, - return_metainfo: bool = True, - return_annotations: bool = True, - return_frame_by_frame: bool = False, - include_unannotated: bool = True, - all_annotations: bool = False, - # Filtering parameters - include_annotators: list[str] | None = None, - exclude_annotators: list[str] | None = None, - include_segmentation_names: list[str] | None = None, - exclude_segmentation_names: list[str] | None = None, - include_image_label_names: list[str] | None = None, - exclude_image_label_names: list[str] | None = None, - include_frame_label_names: list[str] | None = None, - exclude_frame_label_names: list[str] | None = None, - ): - warnings.warn( - "DatamintBaseDataset is deprecated and may be removed in future versions. " - "Please use 'from datamint.dataset import ImageDataset, VolumeDataset' instead.", - DeprecationWarning, - stacklevel=2 - ) - _LOGGER.warning( - "DatamintBaseDataset is a legacy class and may be removed in future versions. " - "Please use 'from datamint.dataset import ImageDataset, VolumeDataset' instead." - ) - self._validate_inputs(project_name, include_annotators, exclude_annotators, - include_segmentation_names, exclude_segmentation_names, - include_image_label_names, exclude_image_label_names, - include_frame_label_names, exclude_frame_label_names) - - self._initialize_config( - project_name, auto_update, all_annotations, return_dicom, - return_metainfo, return_annotations, return_frame_by_frame, - include_unannotated, include_annotators, exclude_annotators, - include_segmentation_names, exclude_segmentation_names, - include_image_label_names, exclude_image_label_names, - include_frame_label_names, exclude_frame_label_names - ) - - self._setup_api_handler(server_url, api_key, auto_update) - self._setup_directories(root) - self._setup_dataset() - self._post_process_data() - - def _validate_inputs( - self, - project_name: str, - include_annotators: Sequence[str] | None, - exclude_annotators: Sequence[str] | None, - include_segmentation_names: Sequence[str] | None, - exclude_segmentation_names: Sequence[str] | None, - include_image_label_names: Sequence[str] | None, - exclude_image_label_names: Sequence[str] | None, - include_frame_label_names: Sequence[str] | None, - exclude_frame_label_names: Sequence[str] | None, - ) -> None: - """Validate input parameters.""" - if project_name is None: - raise ValueError("project_name is required.") - - # Validate mutually exclusive filtering parameters - filter_pairs = [ - (include_annotators, exclude_annotators, "annotators"), - (include_segmentation_names, exclude_segmentation_names, "segmentation_names"), - (include_image_label_names, exclude_image_label_names, "image_label_names"), - (include_frame_label_names, exclude_frame_label_names, "frame_label_names"), - ] - - for include_param, exclude_param, param_name in filter_pairs: - if include_param is not None and exclude_param is not None: - raise ValueError(f"Cannot set both include_{param_name} and exclude_{param_name} at the same time") - - def _initialize_config( - self, - project_name: str, - auto_update: bool, - all_annotations: bool, - return_dicom: bool, - return_metainfo: bool, - return_annotations: bool, - return_frame_by_frame: bool, - include_unannotated: bool, - include_annotators: Optional[list[str]], - exclude_annotators: Optional[list[str]], - include_segmentation_names: Optional[list[str]], - exclude_segmentation_names: Optional[list[str]], - include_image_label_names: Optional[list[str]], - exclude_image_label_names: Optional[list[str]], - include_frame_label_names: Optional[list[str]], - exclude_frame_label_names: Optional[list[str]], - ) -> None: - """Initialize configuration parameters.""" - self.project_name = project_name - self.all_annotations = all_annotations - self.return_dicom = return_dicom - self.return_metainfo = return_metainfo - self.return_frame_by_frame = return_frame_by_frame - self.return_annotations = return_annotations - self.include_unannotated = include_unannotated - self.discard_without_annotations = not include_unannotated - - # Filtering parameters - self.include_annotators = include_annotators - self.exclude_annotators = exclude_annotators - self.include_segmentation_names = include_segmentation_names - self.exclude_segmentation_names = exclude_segmentation_names - self.include_image_label_names = include_image_label_names - self.exclude_image_label_names = exclude_image_label_names - self.include_frame_label_names = include_frame_label_names - self.exclude_frame_label_names = exclude_frame_label_names - - # Internal state - self.__logged_uint16_conversion = False - self.auto_update = auto_update - - def _setup_api_handler(self, server_url: Optional[str], api_key: Optional[str], auto_update: bool) -> None: - """Setup API handler and validate connection.""" - from datamint import Api - self.api = Api( - server_url=server_url, - api_key=api_key, - check_connection=self.auto_update - ) - - def _setup_directories(self, root: str | None) -> None: - """Setup root and dataset directories.""" - if root is None: - root = os.path.join( - datamint.configs.DATAMINT_DATA_DIR, - self.DATAMINT_DATASETS_DIR - ) - os.makedirs(root, exist_ok=True) - else: - root = os.path.expanduser(root) - if not os.path.isdir(root): - raise NotADirectoryError(f"Root directory not found: {root}") - - self.root = root - self.dataset_dir = os.path.join(root, self.project_name) - self.dataset_zippath = os.path.join(root, f'{self.project_name}.zip') - - if not os.path.exists(self.dataset_dir): - os.makedirs(self.dataset_dir, exist_ok=True) - os.makedirs(os.path.join(self.dataset_dir, 'images'), exist_ok=True) - os.makedirs(os.path.join(self.dataset_dir, 'masks'), exist_ok=True) - - def _setup_dataset(self) -> None: - """Setup dataset by downloading or loading existing data.""" - self._server_dataset_info = None - local_load_success = self._load_metadata() - self._handle_dataset_download_or_update(local_load_success) - self._apply_annotation_filters() - - def _handle_dataset_download_or_update(self, local_load_success: bool) -> None: - """Handle dataset download or update logic.""" - - if local_load_success: - _LOGGER.debug(f"Dataset directory already exists: {self.dataset_dir}") - # Check for updates if auto_update is enabled and we have API access - if self.auto_update: - _LOGGER.info("Checking for updates...") - self._check_version() - else: - self._check_version() - - def _init_metainfo(self) -> None: - # get the server info - self.project_info = self.get_info() - self.metainfo = self._get_datasetinfo().asdict().copy() - self.metainfo['updated_at'] = None - self.metainfo['resources'] = [] - self.metainfo['all_annotations'] = self.all_annotations - self.images_metainfo = self.metainfo['resources'] - - def _load_metadata(self) -> bool: - """Load and process dataset metadata.""" - if hasattr(self, 'metainfo'): - _LOGGER.warning("Metadata already loaded.") - metadata_path = os.path.join(self.dataset_dir, 'dataset.json') - if not os.path.isfile(metadata_path): - self._init_metainfo() - return False - else: - with open(metadata_path, 'r') as file: - self.metainfo = json.load(file) - self.images_metainfo = self.metainfo['resources'] - # Convert annotations from dict to Annotation objects - try: - self._convert_metainfo_to_clsobj() - except Exception as e: - _LOGGER.warning(f"Failed to convert annotations. Redownloading dataset. {type(e)}") - self._init_metainfo() - return False - return True - - def _convert_metainfo_to_clsobj(self): - for imginfo in self.images_metainfo: - if 'annotations' in imginfo: - for ann in imginfo['annotations']: - if 'resource_id' not in ann: - ann['resource_id'] = imginfo['id'] - if 'id' not in ann: - ann['id'] = None - imginfo['annotations'] = [annotation_from_dict(ann) if isinstance(ann, dict) else ann - for ann in imginfo['annotations']] - - def _apply_annotation_filters(self) -> None: - """Apply annotation filters and remove unannotated images if needed.""" - # Filter annotations for each image - for imginfo in self.images_metainfo: - imginfo['annotations'] = self._filter_annotations(imginfo['annotations']) - - # Filter out images with no annotations if needed - if self.discard_without_annotations: - original_count = len(self.images_metainfo) - self.images_metainfo = self._filter_items(self.images_metainfo) - _LOGGER.info(f"Discarded {original_count - len(self.images_metainfo)} images without annotations.") - - def _post_process_data(self) -> None: - """Post-process data after loading metadata.""" - self._check_integrity() - self._calculate_dataset_length() - if self.return_frame_by_frame: - self._precompute_frame_data() - self.subset_indices = list(range(self.dataset_length)) - self._setup_labels() - - if self.discard_without_annotations: - self._filter_unannotated() - - def _calculate_dataset_length(self) -> None: - """Calculate the total dataset length based on frame-by-frame setting.""" - if self.return_frame_by_frame: - self.dataset_length = sum( - self.read_number_of_frames(os.path.join(self.dataset_dir, imginfo['file'])) - for imginfo in self.images_metainfo - ) - else: - self.dataset_length = len(self.images_metainfo) - - def _precompute_frame_data(self) -> None: - """Precompute frame-related data for efficient indexing.""" - num_frames_per_resource = self.__compute_num_frames_per_resource() - self._cumulative_frames = np.cumsum([0] + num_frames_per_resource) - - def _setup_labels(self) -> None: - """Setup label sets and mappings.""" - self.frame_lsets, self.frame_lcodes = self._get_labels_set(framed=True) - self.image_lsets, self.image_lcodes = self._get_labels_set(framed=False) - worklist_id = self.get_info()['worklist_id'] - groups: dict[str, dict] = self.api.annotationsets.get_segmentation_group(worklist_id)['groups'] - if not groups: - self.seglabel_list = [] - self.seglabel2code = {} - return - # order by 'index' key - max_index = max([g['index'] for g in groups.values()]) - self.seglabel_list: list[str] = ['UNKNOWN'] * max_index # 1-based - for segname, g in groups.items(): - self.seglabel_list[g['index'] - 1] = segname - - self.seglabel2code = {label: idx + 1 for idx, label in enumerate(self.seglabel_list)} - - def _filter_items(self, images_metainfo: list[dict]) -> list[dict]: - """Filter items that have annotations.""" - return [img for img in images_metainfo if len(img.get('annotations', []))] - - def _filter_unannotated(self) -> None: - """Filter out frames that don't have any segmentations.""" - filtered_indices = [] - for i in range(len(self.subset_indices)): - item_meta = self._get_image_metainfo(i) - annotations = item_meta.get('annotations', []) - - # Check if there are any segmentation annotations - has_segmentations = any(ann.type == 'segmentation' for ann in annotations) - - if has_segmentations: - filtered_indices.append(self.subset_indices[i]) - - self.subset_indices = filtered_indices - _LOGGER.debug(f"Filtered dataset: {len(self.subset_indices)} frames with segmentations") - - def __compute_num_frames_per_resource(self) -> list[int]: - """Compute number of frames for each resource.""" - return [ - self.read_number_of_frames(os.path.join(self.dataset_dir, imginfo['file'])) - for imginfo in self.images_metainfo - ] - - @property - def frame_labels_set(self) -> list[str]: - """Returns the set of independent labels in the dataset (multi-label tasks).""" - return self.frame_lsets['multilabel'] - - @property - def frame_categories_set(self) -> list[tuple[str, str]]: - """Returns the set of categories in the dataset (multi-class tasks).""" - return self.frame_lsets['multiclass'] - - @property - def image_labels_set(self) -> list[str]: - """Returns the set of independent labels in the dataset (multi-label tasks).""" - return self.image_lsets['multilabel'] - - @property - def image_categories_set(self) -> list[tuple[str, str]]: - """Returns the set of categories in the dataset (multi-class tasks).""" - return self.image_lsets['multiclass'] - - @property - def segmentation_labels_set(self) -> list[str]: - """Returns the set of segmentation labels in the dataset.""" - return self.seglabel_list - - def _get_annotations_internal( - self, - annotations: Sequence[Annotation], - type: Literal['label', 'category', 'segmentation', 'all'] = 'all', - scope: Literal['frame', 'image', 'all'] = 'all' - ) -> list[Annotation]: - """Internal method to filter annotations by type and scope.""" - if type not in ['label', 'category', 'segmentation', 'all']: - raise ValueError(f"Invalid value for 'type': {type}") - if scope not in ['frame', 'image', 'all']: - raise ValueError(f"Invalid value for 'scope': {scope}") - - filtered_annotations = [] - for ann in annotations: - ann_scope = 'image' if ann.index is None else 'frame' - - type_matches = type == 'all' or ann.type == type - scope_matches = scope == 'all' or scope == ann_scope - - if type_matches and scope_matches: - filtered_annotations.append(ann) - - return filtered_annotations - - def get_annotations( - self, - index: int, - type: Literal['label', 'category', 'segmentation', 'all'] = 'all', - scope: Literal['frame', 'image', 'all'] = 'all' - ) -> list[Annotation]: - """Returns the annotations of the image at the given index. - - Args: - index: Index of the image. - type: The type of the annotations. Can be 'label', 'category', 'segmentation' or 'all'. - scope: The scope of the annotations. Can be 'frame', 'image' or 'all'. - - Returns: - The annotations of the image. - """ - if index >= len(self): - raise IndexError(f"Index {index} out of bounds for dataset of length {len(self)}") - - imginfo = self._get_image_metainfo(index) - return self._get_annotations_internal(imginfo['annotations'], type=type, scope=scope) - - @staticmethod - def read_number_of_frames(filepath: str) -> int: - """Read the number of frames in a file.""" - - mimetypes, ext = guess_typez(filepath) - mimetype = mimetypes[0] - if mimetype is None: - raise ValueError(f"Could not determine MIME type for file: {filepath}") - - if mimetype == 'application/dicom': - ds = pydicom.dcmread(filepath) - return getattr(ds, 'NumberOfFrames', 1) - elif mimetype.startswith('video/'): - cap = cv2.VideoCapture(filepath) - try: - return int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - finally: - cap.release() - elif mimetype in ('image/png', 'image/jpeg', 'image/jpg', 'image/bmp', 'image/tiff'): - return 1 - elif mimetype in NIFTI_MIMES: - shape = get_nifti_shape(filepath) - if len(shape) == 3: - return shape[-1] - elif len(shape) > 3: - return shape[3] - else: - return 1 - else: - raise ValueError(f"Unsupported file type '{mimetype}' for file {filepath}") - - def get_resources_ids(self) -> list[str]: - """Get list of resource IDs.""" - return [self._get_image_metainfo(i, bypass_subset_indices=True)['metainfo']['id'] - for i in self.subset_indices] - - def _get_labels_set(self, framed: bool) -> tuple[dict, dict[str, dict[str, int]]]: - """Returns the set of labels and mappings to integers. - - Args: - framed: If True, get frame-level labels, otherwise image-level labels. - - Returns: - Tuple containing label sets and label-to-code mappings. - """ - scope = 'frame' if framed else 'image' - - multilabel_set = set() - segmentation_labels = set() - multiclass_set = set() - - for i in range(len(self)): - # Collect labels by type - label_anns = self.get_annotations(i, type='label', scope=scope) - multilabel_set.update(ann.name for ann in label_anns) - - # seg_anns = self.get_annotations(i, type='segmentation', scope=scope) - # segmentation_labels.update(ann.name for ann in seg_anns) - - cat_anns = self.get_annotations(i, type='category', scope=scope) - multiclass_set.update((ann.name, ann.value) for ann in cat_anns) - - # Sort and create mappings - multilabel_list = sorted(multilabel_set) - multiclass_list = sorted(multiclass_set) - # segmentation_list = sorted(segmentation_labels) - - sets = { - 'multilabel': multilabel_list, - # 'segmentation': segmentation_list, - 'multiclass': multiclass_list - } - - codes_map = { - 'multilabel': {label: idx for idx, label in enumerate(multilabel_list)}, - # 'segmentation': {label: idx + 1 for idx, label in enumerate(segmentation_list)}, - 'multiclass': {label: idx for idx, label in enumerate(multiclass_list)} - } - - return sets, codes_map - - def get_framelabel_distribution(self, normalize: bool = False) -> dict[str, float]: - """Returns the distribution of frame labels in the dataset.""" - return self._get_label_distribution('label', 'frame', normalize) - - def get_segmentationlabel_distribution(self, normalize: bool = False) -> dict[str, float]: - """Returns the distribution of segmentation labels in the dataset.""" - return self._get_label_distribution('segmentation', 'all', normalize) - - def _get_label_distribution(self, ann_type: str, scope: str, normalize: bool) -> dict[str, float]: - """Helper method to calculate label distributions.""" - if ann_type == 'label' and scope == 'frame': - labels = self.frame_labels_set - elif ann_type == 'segmentation': - labels = self.segmentation_labels_set - else: - raise ValueError(f"Unsupported combination: type={ann_type}, scope={scope}") - - distribution = {label: 0 for label in labels} - - for imginfo in self.images_metainfo: - for ann in imginfo.get('annotations', []): - condition_met = ( - ann.type == ann_type and - (scope == 'all' or - (scope == 'frame' and ann.index is not None) or - (scope == 'image' and ann.index is None)) - ) - if condition_met and ann.name in distribution: - distribution[ann.name] += 1 - - if normalize: - total = sum(distribution.values()) - if total > 0: - distribution = {k: v / total for k, v in distribution.items()} - - return distribution - - def _check_integrity(self) -> None: - """Check if all image files exist and are not empty. Empty files will be redownloaded.""" - missing_files = [] - empty_resources = [] - for imginfo in self.images_metainfo: - filepath = os.path.join(self.dataset_dir, imginfo['file']) - if not os.path.isfile(filepath): - missing_files.append(imginfo['file']) - elif os.path.getsize(filepath) == 0: - # File exists but is empty, consider it as missing and attempt to redownload - # delete - os.remove(filepath) - empty_resources.append(imginfo) - - if missing_files: - raise DatamintDatasetException(f"Image files not found: {missing_files}") - - if empty_resources: - _LOGGER.warning( - f"Found {len(empty_resources)} empty image file(s). Attempting to redownload..." - ) - self._redownload_resources(empty_resources) - - def _redownload_resources(self, resources: list[dict]) -> None: - """Attempt to redownload the given resources by ID.""" - resource_ids = [r['id'] for r in resources] - resource_paths = [Path(self.dataset_dir) / r['file'] for r in resources] - try: - new_res_paths = self.api.resources.download_multiple_resources( - resource_ids, - save_path=resource_paths, - add_extension=True - ) - for new_rpath, r in zip(new_res_paths, resources): - r['file'] = str(Path(new_rpath).relative_to(self.dataset_dir)) - _LOGGER.info(f"Successfully redownloaded {len(resources)} resource(s).") - except Exception as e: - empty_file_names = [r['file'] for r in resources] - raise DatamintDatasetException( - f"Empty image files found and could not be redownloaded: {empty_file_names}" - ) from e - - def _get_datasetinfo(self) -> DatasetInfo: - """Get dataset information from API.""" - if self._server_dataset_info is not None: - return self._server_dataset_info - all_datasets = self.api._datasetsinfo.get_all() - - for dataset in all_datasets: - if dataset.id == self.dataset_id: - self._server_dataset_info = dataset - return dataset - - available_datasets = [(d.name, d.id) for d in all_datasets] - raise DatamintDatasetException( - f"Dataset with id '{self.dataset_id}' not found. " - f"Available datasets: {available_datasets}" - ) - - def get_info(self) -> dict: - """Get project information from API.""" - if hasattr(self, 'project_info') and self.project_info is not None: - return self.project_info - project = self.api.projects.get_by_name(self.project_name) - if project is None: - raise DatamintDatasetException( - f"Project with name '{self.project_name}' not found." - ) - project = project.asdict() - self.project_info = project - self.dataset_id = project['dataset_id'] - return project - - def _run_request(self, session, request_args) -> requests.Response: - response = session.request(**request_args) - if response.status_code == 400: - _LOGGER.error(f"Bad request: {response.text}") - response.raise_for_status() - return response - - def __repr__(self) -> str: - """String representation of the dataset.""" - head = f"Dataset {self.project_name}" - body = [f"Number of datapoints: {self.__len__()}"] - - if self.root is not None: - body.append(f"Location: {self.dataset_dir}") - - # Add filter information - filter_info = [ - (self.include_annotators, "Including only annotators"), - (self.exclude_annotators, "Excluding annotators"), - (self.include_segmentation_names, "Including only segmentations"), - (self.exclude_segmentation_names, "Excluding segmentations"), - (self.include_image_label_names, "Including only image labels"), - (self.exclude_image_label_names, "Excluding image labels"), - (self.include_frame_label_names, "Including only frame labels"), - (self.exclude_frame_label_names, "Excluding frame labels"), - ] - - for filter_value, description in filter_info: - if filter_value is not None: - body.append(f"{description}: {filter_value}") - - lines = [head] + [" " * 4 + line for line in body] - return "\n".join(lines) - - def _get_dataset_id(self) -> str: - if self.dataset_id is None: - dataset_info = self._get_datasetinfo() - self.dataset_id = dataset_info.id - return self.dataset_id - - def _extract_and_update_metadata(self) -> None: - """Extract downloaded archive and update metadata.""" - from torchvision.datasets.utils import extract_archive - - if os.path.exists(self.dataset_dir): - _LOGGER.info(f"Deleting existing dataset directory: {self.dataset_dir}") - shutil.rmtree(self.dataset_dir) - - extract_archive(self.dataset_zippath, self.dataset_dir, remove_finished=True) - - # Load and update metadata - datasetjson_path = os.path.join(self.dataset_dir, 'dataset.json') - with open(datasetjson_path, 'r') as file: - self.metainfo = json.load(file) - - self._update_metadata_timestamps() - - # Save updated metadata - with open(datasetjson_path, 'w') as file: - json.dump(self.metainfo, file, default=lambda o: o.asdict() if hasattr(o, 'asdict') else o) - - self.images_metainfo = self.metainfo['resources'] - # self._convert_metainfo_to_clsobj() - - def _update_metadata_timestamps(self) -> None: - """Update metadata with correct timestamps.""" - if 'updated_at' not in self.metainfo: - self.metainfo['updated_at'] = self.last_updated_at - else: - try: - local_time = datetime.fromisoformat(self.metainfo['updated_at']) - server_time = datetime.fromisoformat(self.last_updated_at) - - if local_time < server_time: - _LOGGER.warning( - f"Inconsistent updated_at dates detected " - f"({self.metainfo['updated_at']} < {self.last_updated_at}). " - f"Fixing it to {self.last_updated_at}" - ) - self.metainfo['updated_at'] = self.last_updated_at - except Exception as e: - _LOGGER.warning(f"Failed to parse updated_at date: {e}") - - self.metainfo['all_annotations'] = self.all_annotations - - def _load_image(self, filepath: str, index: int | None = None, mimetype: str | None = None) -> tuple[Tensor, FileDataset | None]: - """Load image from file with optional frame index.""" - if os.path.isdir(filepath): - raise NotImplementedError("Loading an image from a directory is not supported yet.") - - # check if file is empty - if os.path.getsize(filepath) == 0: - raise DatamintDatasetException(f"File is empty: {filepath}") - - if mimetype == 'application/octet-stream': - mimetype = None - - if self.return_frame_by_frame: - img, ds = read_array_normalized(filepath, return_metainfo=True, index=index, mime_type=mimetype) - else: - img, ds = read_array_normalized(filepath, return_metainfo=True, mime_type=mimetype) - - img = self._process_image_array(img) - return img, ds - - def _process_image_array(self, img: np.ndarray) -> Tensor: - """Process numpy array to tensor with proper normalization.""" - if img.dtype == np.uint16: - if not self.__logged_uint16_conversion: - _LOGGER.info("Original image is uint16, converting to uint8") - self.__logged_uint16_conversion = True - - # Min-max normalization - img = img.astype(np.float32) - min_val = img.min() - img = (img - min_val) / (img.max() - min_val) * 255 - img = img.astype(np.uint8) - - if not img.flags.writeable: - img = img.copy() - - img_tensor = torch.from_numpy(img).contiguous() - - if isinstance(img_tensor, torch.ByteTensor): - img_tensor = img_tensor.to(dtype=torch.get_default_dtype()).div(255) - - return img_tensor - - def _get_image_metainfo(self, index: int, bypass_subset_indices: bool = False) -> dict[str, Any]: - """Get metadata for image at given index.""" - if not bypass_subset_indices: - index = self.subset_indices[index] - - if self.return_frame_by_frame: - resource_id, frame_index = self.__find_index(index) - img_metainfo = dict(self.images_metainfo[resource_id]) # Copy - img_metainfo['frame_index'] = frame_index - img_metainfo['annotations'] = [ - ann for ann in img_metainfo['annotations'] - if ann.index is None or ann.index == frame_index - ] - else: - img_metainfo = self.images_metainfo[index] - - return img_metainfo - - def __find_index(self, index: int) -> tuple[int, int]: - """Find the resource index and frame index for a given global frame index.""" - resource_index = np.searchsorted(self._cumulative_frames[1:], index, side='right') - frame_index = index - self._cumulative_frames[resource_index] - return resource_index, frame_index - - def __getitem_internal( - self, - index: int, - only_load_metainfo: bool = False - ) -> dict[str, Tensor | FileDataset | dict | list]: - """Internal method to get item at index.""" - if self.return_frame_by_frame: - resource_index, frame_idx = self.__find_index(index) - else: - resource_index = index - frame_idx = None - - img_metainfo = self._get_image_metainfo(index, bypass_subset_indices=True) - - if only_load_metainfo: - return {'metainfo': img_metainfo} - - filepath = os.path.join(self.dataset_dir, img_metainfo['file']) - img, ds = self._load_image(filepath, frame_idx, mimetype=img_metainfo.get('mimetype')) - - return self._build_item_dict(img, ds, img_metainfo) - - def _build_item_dict( - self, - img: Tensor, - ds: FileDataset | None, - img_metainfo: dict - ) -> dict[str, Any]: - """Build the return dictionary for __getitem__.""" - ret = {'image': img} - - if self.return_dicom: - ret['dicom'] = ds - if self.return_metainfo: - ret['metainfo'] = {k: v for k, v in img_metainfo.items() if k != 'annotations'} - if self.return_annotations: - ret['annotations'] = img_metainfo['annotations'] - - return ret - - def _filter_annotations(self, annotations: list[Annotation]) -> list[Annotation]: - """Filter annotations based on the filtering settings.""" - if annotations is None: - return [] - - filtered_annotations = [] - for ann in annotations: - if not self._should_include_annotation(ann): - continue - filtered_annotations.append(ann) - - return filtered_annotations - - def _should_include_annotation(self, ann: Annotation) -> bool: - """Check if an annotation should be included based on all filters.""" - if not self._should_include_annotator(ann.created_by): - return False - - if ann.type == 'segmentation': - return self._should_include_segmentation(ann.name) - elif ann.type == 'label': - if ann.index is None: - return self._should_include_image_label(ann.name) - else: - return self._should_include_frame_label(ann.name) - - return True - - def __getitem__(self, index: int) -> dict[str, Tensor | FileDataset | dict | list]: - """Get item at index. - - Args: - index: Index - - Returns: - A dictionary containing 'image', 'metainfo' and 'annotations' keys. - """ - if index >= len(self): - raise IndexError(f"Index {index} out of bounds for dataset of length {len(self)}") - - return self.__getitem_internal(self.subset_indices[index]) - - def __iter__(self): - """Iterate over dataset items.""" - for index in self.subset_indices: - yield self.__getitem__(index) - # do not use __getitem_internal__ here, so subclass only need to implement __getitem__ - - def __len__(self) -> int: - """Return dataset length.""" - return len(self.subset_indices) - - def _check_version(self) -> None: - """Check if local dataset version is up to date.""" - # metainfo_path = os.path.join(self.dataset_dir, 'dataset.json') - # if not os.path.exists(metainfo_path): - # self.download_project() - # return - - if not hasattr(self, 'project_info'): - self.project_info = self.get_info() - self.dataset_id = self.project_info['dataset_id'] - - local_updated_at = self.metainfo.get('updated_at', None) - local_all_annotations = self.metainfo.get('all_annotations', None) - - try: - external_metadata_info = self._get_datasetinfo() - server_updated_at = external_metadata_info.updated_at - except Exception as e: - _LOGGER.warning(f"Failed to check for updates in {self.project_name}: {e}") - return - - _LOGGER.debug(f"Local updated at: {local_updated_at}, Server updated at: {server_updated_at}") - - annotations_changed = local_all_annotations != self.all_annotations - version_outdated = local_updated_at is None or local_updated_at < server_updated_at - - if annotations_changed: - _LOGGER.info( - f"The 'all_annotations' parameter has changed. " - f"Previous: {local_all_annotations}, Current: {self.all_annotations}." - ) - # self.download_project() - self._incremental_update() - elif version_outdated: - _LOGGER.info( - f"A newer version of the dataset is available. " - f"Your version: {local_updated_at}. Last version: {server_updated_at}." - ) - self._incremental_update() - else: - _LOGGER.info('Local version is up to date with the latest version.') - - def _fetch_new_resources(self, - all_uptodate_resources: list[Resource]) -> list[dict]: - local_resources = self.images_metainfo - local_resources_ids = [res['id'] for res in local_resources] - new_resources = [] - for resource in all_uptodate_resources: - resource = resource.asdict() - if resource['id'] not in local_resources_ids: - resource['file'] = str(self._get_resource_file_path(resource)) - resource['annotations'] = [] - new_resources.append(resource) - return new_resources - - def _fetch_deleted_resources(self, all_uptodate_resources: list[Resource]) -> list[dict]: - local_resources = self.images_metainfo - all_uptodate_resources_ids = [res.id for res in all_uptodate_resources] - deleted_resources = [] - for resource in local_resources: - try: - res_idx = all_uptodate_resources_ids.index(resource['id']) - if resource.get('deleted_at', None): # was deleted on server - if local_resources[res_idx].get('deleted_at_local', None) is None: - deleted_resources.append(resource) - except ValueError: - deleted_resources.append(resource) - - return deleted_resources - - def _incremental_update(self) -> None: - # local_updated_at = self.metainfo.get('updated_at', None) - # external_metadata_info = self._get_datasetinfo() - # server_updated_at = external_metadata_info['updated_at'] - - ### RESOURCES ### - all_uptodate_resources = self.api.projects.get_project_resources(self.get_info()['id']) - new_resources = self._fetch_new_resources(all_uptodate_resources) - deleted_resources = self._fetch_deleted_resources(all_uptodate_resources) - - if new_resources: - for r in new_resources: - self._new_resource_created(r) - new_resources_path = [Path(self.dataset_dir) / r['file'] for r in new_resources] - new_resources_ids = [r['id'] for r in new_resources] - _LOGGER.info(f"Downloading {len(new_resources)} new resources...") - new_res_paths = self.api.resources.download_multiple_resources(new_resources_ids, - save_path=new_resources_path, - add_extension=True) - for new_rpath, r in zip(new_res_paths, new_resources): - r['file'] = str(Path(new_rpath).relative_to(self.dataset_dir)) - _LOGGER.info(f"Downloaded {len(new_resources)} new resources.") - - for r in deleted_resources: - self._resource_deleted(r) - ################ - - ### ANNOTATIONS ### - _LOGGER.info("Fetching new annotations...") - all_annotations = self.api.annotations.get_list(worklist_id=self.project_info['worklist_id'], - status=None if self.all_annotations else 'published', - load_ai_segmentations=self.all_annotations) - - # group annotations by resource ID - annotations_by_resource: dict[str, list[Annotation]] = {} - for ann in all_annotations: - # add the local filepath - filepath = self._get_annotation_file_path(ann) - if filepath is not None: - ann.file = str(filepath) - resource_id = ann.resource_id - if resource_id not in annotations_by_resource: - annotations_by_resource[resource_id] = [] - annotations_by_resource[resource_id].append(ann) - - # Collect all segmentation annotations that need to be downloaded - segmentations_to_download = [] - segmentation_paths = [] - segmentation_resource_map = {} # Maps annotation ID to resource ID for cleanup - - # update annotations in resources - for resource in self.images_metainfo: - resource_id = resource['id'] - new_resource_annotations = annotations_by_resource.get(resource_id, []) - old_resource_annotations = resource.get('annotations', []) - - # check if segmentation annotations need to be downloaded - # Also check if annotations need to be deleted - old_ann_ids = set([ann.id for ann in old_resource_annotations if hasattr(ann, 'id')]) - new_ann_ids = set([ann.id for ann in new_resource_annotations]) - - # Find annotations to add, update, or remove - annotations_to_add = [ann for ann in new_resource_annotations - if ann.id not in old_ann_ids] - annotations_to_remove = [ann for ann in old_resource_annotations - if getattr(ann, 'id', 'NA') not in new_ann_ids] - - for ann in annotations_to_add: - filepath = self._get_annotation_file_path(ann) - if filepath is not None: # None means it is not a segmentation - # Collect for batch download - filepath = Path(self.dataset_dir) / filepath - filepath.parent.mkdir(parents=True, exist_ok=True) - segmentations_to_download.append(ann) - segmentation_paths.append(filepath) - segmentation_resource_map[ann.id] = resource_id - - # Process annotation changes - for ann in annotations_to_remove: - filepath = getattr(ann, 'file', None) if hasattr(ann, 'file') else ann.get('file', None) - if filepath is None: - # Not a segmentation annotation - continue - - try: - filepath = Path(self.dataset_dir) / filepath - # delete the local annotation file if it exists - if filepath.exists(): - os.remove(filepath) - except Exception as e: - _LOGGER.error(f"Error deleting annotation file {filepath}: {e}") - - # Update resource annotations list - convert to Annotation objects - resource['annotations'] = new_resource_annotations - - # Batch download all segmentation files - if segmentations_to_download: - _LOGGER.info(f"Downloading {len(segmentations_to_download)} segmentation files...") - download_results = self.api.annotations.download_multiple_files( - segmentations_to_download, segmentation_paths - ) - - # Process failed downloads - failed_annotations = [result['annotation_id'] for result in download_results if not result['success']] - if failed_annotations: - _LOGGER.warning( - f"Failed to download {len(failed_annotations)} annotations, removing them from metadata") - - # Remove failed annotations from each resource's annotation list - for resource in self.images_metainfo: - resource['annotations'] = [ - ann for ann in resource['annotations'] - if ann.id not in failed_annotations - ] - - _LOGGER.info( - f"Successfully downloaded {len(segmentations_to_download) - len(failed_annotations)} segmentation files.") - - ################### - # update metadata - self.metainfo['updated_at'] = self._get_datasetinfo().updated_at - self.metainfo['all_annotations'] = self.all_annotations - # save updated metadata - datasetjson_path = os.path.join(self.dataset_dir, 'dataset.json') - with open(datasetjson_path, 'w') as file: - json.dump(self.metainfo, file, default=lambda o: o.asdict() if hasattr(o, 'asdict') else o) - - def _get_resource_file_path(self, resource: dict) -> Path: - """Get the local file path for a resource.""" - if 'file' in resource and resource['file'] is not None: - return Path(resource['file']) - else: - # ext = guess_extension(resource['mimetype']) - # if ext is None: - # _LOGGER.warning(f"Could not guess extension for resource {resource['id']}.") - # ext = '' - # return Path('images', f"{resource['id']}{ext}") - return Path('images', resource['id']) - - def _get_annotation_file_path(self, annotation: dict | Annotation) -> Path | None: - """Get the local file path for an annotation.""" - if isinstance(annotation, Annotation): - if annotation.file: - return Path(annotation.file) - elif annotation.type == 'segmentation': - return Path('masks', - annotation.created_by, - annotation.resource_id, - annotation.id) - else: - # Handle dict format for backwards compatibility - if 'file' in annotation: - return Path(annotation['file']) - elif annotation.get('annotation_type', annotation.get('type')) == 'segmentation': - return Path('masks', - annotation['created_by'], - annotation['resource_id'], - annotation['id']) - return None - - def _new_resource_created(self, resource: dict) -> None: - """Handle a new resource created in the dataset.""" - if 'annotations' not in resource: - resource['annotations'] = [] # Initialize as empty list for Annotation objects - self.images_metainfo.append(resource) - - if hasattr(self, 'num_frames_per_resource'): - raise NotImplementedError('Cannot handle new resources after dataset initialization') - - def _resource_deleted(self, resource: dict) -> None: - """Handle a resource deleted from the dataset.""" - - # remove from metadata - for i, imginfo in enumerate(self.images_metainfo): - if imginfo['id'] == resource['id']: - deleted_metainfo = self.images_metainfo.pop(i) - break - else: - _LOGGER.warning(f"Resource {resource['id']} not found in dataset metadata.") - return - - # delete from system file - if os.path.exists(deleted_metainfo['file']): - os.remove(os.path.join(self.dataset_dir, deleted_metainfo['file'])) - - # delete associated annotations - for ann in deleted_metainfo.get('annotations', []): - ann_file = getattr(ann, 'file', None) if hasattr(ann, 'file') else ann.get('file', None) - if ann_file is not None and os.path.exists(os.path.join(self.dataset_dir, ann_file)): - os.remove(os.path.join(self.dataset_dir, ann_file)) - - def __add__(self, other): - """Concatenate datasets.""" - from torch.utils.data import ConcatDataset - return ConcatDataset([self, other]) - - def get_dataloader(self, *args, **kwargs) -> DataLoader: - """Returns a DataLoader for the dataset with proper collate function. - - Args: - *args: Positional arguments for the DataLoader. - **kwargs: Keyword arguments for the DataLoader. - - Returns: - DataLoader instance with custom collate function. - """ - return DataLoader(self, *args, collate_fn=self.get_collate_fn(), **kwargs) - - def get_collate_fn(self) -> Callable: - """Get collate function for DataLoader.""" - def collate_fn(batch: list[dict]) -> dict: - if not batch: - return {} - - keys = batch[0].keys() - collated_batch = {} - - for key in keys: - values = [item[key] for item in batch] - - if isinstance(values[0], torch.Tensor): - shapes = [tensor.shape for tensor in values] - if all(shape == shapes[0] for shape in shapes): - collated_batch[key] = torch.stack(values) - else: - _LOGGER.warning(f"Collating {key} tensors with different shapes: {shapes}") - collated_batch[key] = values - elif isinstance(values[0], np.ndarray): - collated_batch[key] = np.stack(values) - else: - collated_batch[key] = values - - return collated_batch - - return collate_fn - - def subset(self, indices: list[int]) -> 'DatamintBaseDataset': - """Create a subset of the dataset. - - Args: - indices: List of indices to include in the subset. - - Returns: - Self with updated subset indices. - """ - if max(indices, default=-1) >= self.dataset_length: - raise ValueError(f"Subset indices must be less than the dataset length: {self.dataset_length}") - - self.subset_indices = indices - return self - - def _should_include_annotator(self, annotator_id: str) -> bool: - """Check if an annotator should be included based on filtering settings.""" - if self.include_annotators is not None: - return annotator_id in self.include_annotators - if self.exclude_annotators is not None: - return annotator_id not in self.exclude_annotators - return True - - def _should_include_segmentation(self, segmentation_name: str) -> bool: - """Check if a segmentation should be included based on filtering settings.""" - if self.include_segmentation_names is not None: - return segmentation_name in self.include_segmentation_names - if self.exclude_segmentation_names is not None: - return segmentation_name not in self.exclude_segmentation_names - return True - - def _should_include_image_label(self, label_name: str) -> bool: - """Check if an image label should be included based on filtering settings.""" - if self.include_image_label_names is not None: - return label_name in self.include_image_label_names - if self.exclude_image_label_names is not None: - return label_name not in self.exclude_image_label_names - return True - - def _should_include_frame_label(self, label_name: str) -> bool: - """Check if a frame label should be included based on filtering settings.""" - if self.include_frame_label_names is not None: - return label_name in self.include_frame_label_names - if self.exclude_frame_label_names is not None: - return label_name not in self.exclude_frame_label_names - return True diff --git a/datamint/dataset/dataset.py b/datamint/dataset/dataset.py deleted file mode 100644 index 3570fc21..00000000 --- a/datamint/dataset/dataset.py +++ /dev/null @@ -1,577 +0,0 @@ -from .base_dataset import DatamintBaseDataset -from typing import Optional, Callable, Any, Literal, Sequence, TYPE_CHECKING -import torch -from torch import Tensor -import os -import numpy as np -import logging -from PIL import Image -import albumentations -from datamint.entities.annotations.annotation import Annotation -from medimgkit.readers import read_array_normalized - -_LOGGER = logging.getLogger(__name__) -if not TYPE_CHECKING: - _LOGGER.warning( - "DatamintDataset is a legacy class and may be removed in future versions. " - "Please use `from datamint.dataset import ImageDataset, VolumeDataset` instead." - ) - -class DatamintDataset(DatamintBaseDataset): - """ - This Dataset class extends the `DatamintBaseDataset` class to be easily used with PyTorch. - In addition to that, it has functionality to better process annotations and segmentations. - - .. note:: - Import using ``from datamint import Dataset``. - - Args: - root: Root directory of dataset where data already exists or will be downloaded. - project_name: Name of the project to download. - auto_update: If True, the dataset will be checked for updates and downloaded if necessary. - api_key: API key to access the Datamint API. If not provided, it will look for the - environment variable 'DATAMINT_API_KEY'. Not necessary if - you don't want to download/update the dataset. - return_dicom: If True, the DICOM object will be returned, if the image is a DICOM file. - return_metainfo: If True, the metainfo of the image will be returned. - return_annotations: If True, the annotations of the image will be returned. - return_frame_by_frame: If True, each frame of a video/DICOM/3d-image will be returned separately. - include_unannotated: If True, images without annotations will be included. If False, images without annotations will be discarded. - all_annotations: If True, all annotations will be downloaded, including the ones that are not set as closed/done. - server_url: URL of the Datamint server. If not provided, it will use the default server. - return_segmentations: If True (default), the segmentations of the image will be returned in the 'segmentations' key. - return_as_semantic_segmentation: If True, the segmentations will be returned as semantic segmentation. - image_transform: A function to transform the image. - mask_transform: A function to transform the mask. - semantic_seg_merge_strategy: If not None, the segmentations will be merged using this strategy. - Possible values are 'union', 'intersection', 'mode'. - include_annotators: List of annotators to include. If None, all annotators will be included. See parameter ``exclude_annotators``. - exclude_annotators: List of annotators to exclude. If None, no annotators will be excluded. See parameter ``include_annotators``. - include_segmentation_names: List of segmentation names to include. If None, all segmentations will be included. - exclude_segmentation_names: List of segmentation names to exclude. If None, no segmentations will be excluded. - include_image_label_names: List of image label names to include. If None, all image labels will be included. - exclude_image_label_names: List of image label names to exclude. If None, no image labels will be excluded. - include_frame_label_names: List of frame label names to include. If None, all frame labels will be included. - exclude_frame_label_names: List of frame label names to exclude. If None, no frame labels will be excluded. - all_annotations: If True, all annotations will be downloaded, including the ones that are not set as closed/done. - """ - - def __init__(self, - project_name: str, - root: str | None = None, - auto_update: bool = True, - api_key: Optional[str] = None, - server_url: Optional[str] = None, - return_dicom: bool = False, - return_metainfo: bool = True, - return_frame_by_frame: bool = False, - return_annotations: bool = True, - # new parameters - return_segmentations: bool = True, - return_as_semantic_segmentation: bool = False, - image_transform: Callable[[torch.Tensor], Any] | None = None, - mask_transform: Callable[[torch.Tensor], Any] | None = None, - alb_transform: albumentations.BasicTransform | None = None, - semantic_seg_merge_strategy: Optional[Literal['union', 'intersection', 'mode']] = None, - include_unannotated: bool = True, - # filtering parameters - include_annotators: Optional[list[str]] = None, - exclude_annotators: Optional[list[str]] = None, - include_segmentation_names: Optional[list[str]] = None, - exclude_segmentation_names: Optional[list[str]] = None, - include_image_label_names: Optional[list[str]] = None, - exclude_image_label_names: Optional[list[str]] = None, - include_frame_label_names: Optional[list[str]] = None, - exclude_frame_label_names: Optional[list[str]] = None, - all_annotations: bool = False, - ): - super().__init__(root=root, - project_name=project_name, - auto_update=auto_update, - api_key=api_key, - server_url=server_url, - return_dicom=return_dicom, - return_metainfo=return_metainfo, - return_frame_by_frame=return_frame_by_frame, - return_annotations=return_annotations, - include_unannotated=include_unannotated, - all_annotations=all_annotations, - include_annotators=include_annotators, - exclude_annotators=exclude_annotators, - include_segmentation_names=include_segmentation_names, - exclude_segmentation_names=exclude_segmentation_names, - include_image_label_names=include_image_label_names, - exclude_image_label_names=exclude_image_label_names, - include_frame_label_names=include_frame_label_names, - exclude_frame_label_names=exclude_frame_label_names - ) - self.return_segmentations = return_segmentations - self.return_as_semantic_segmentation = return_as_semantic_segmentation - self.image_transform = image_transform - self.mask_transform = mask_transform - self.alb_transform = alb_transform - if alb_transform is not None and return_frame_by_frame == False: - # not supported yet - raise NotImplementedError( - "albumentations transform is not supported yet when return_frame_by_frame is False") - self.semantic_seg_merge_strategy = semantic_seg_merge_strategy - - if return_segmentations == False and return_as_semantic_segmentation == True: - raise ValueError("return_as_semantic_segmentation can only be True if return_segmentations is True") - - if semantic_seg_merge_strategy is not None and not return_as_semantic_segmentation: - raise ValueError("semantic_seg_merge_strategy can only be used if return_as_semantic_segmentation is True") - - def _load_segmentations(self, - annotations: Sequence[Annotation], - img_shape) -> tuple[dict[str, list], dict[str, list], dict[str, Any]]: - """ - Load segmentations from annotations. - - Args: - annotations: list of Annotation objects - img_shape: shape of the image (#frames, C, H, W) - - Returns: - tuple[dict[str, list], dict[str, list], dict[str, Any]]: a tuple of two dictionaries and additional metadata. - The first dictionary is author -> list of #frames tensors, each tensor has shape (#instances_i, H, W). - The second dictionary is author -> list of #frames segmentation labels (tensors). - """ - segmentations = {} - seg_labels = {} - seg_metainfos = {} - - if self.return_frame_by_frame: - assert len(img_shape) == 3, f"img_shape must have 3 dimensions, got {img_shape}" - _, h, w = img_shape - nframes = 1 - else: - assert len(img_shape) == 4, f"img_shape must have 4 dimensions, got {img_shape}" - nframes, _, h, w = img_shape - - # Load segmentation annotations - for ann in annotations: - if ann.type != 'segmentation': - continue - if ann.file is None: - _LOGGER.warning(f"Segmentation annotation without file in annotations {ann}") - continue - author = ann.created_by - - segfilepath = ann.file # png file - segfilepath = os.path.join(self.dataset_dir, segfilepath) - seg, seg_metainfo = read_array_normalized(segfilepath, return_metainfo=True) # (frames, C, H, W) - if seg.shape[1] != 1: - raise ValueError(f"Segmentation file must have 1 channel, got {seg.shape} in {segfilepath}") - seg = seg[:, 0, :, :] # (frames, H, W) - - if seg_metainfo is None: - raise Exception - seg_metainfos[author] = seg_metainfo - - # # FIXME: avoid enforcing resizing the mask - # seg = (Image.open(segfilepath) - # .convert('L') - # .resize((w, h), Image.Resampling.NEAREST) - # ) - # seg = np.array(seg) - - seg = torch.from_numpy(seg) - seg = seg != 0 # binary mask - # map the segmentation label to the code - if self.return_frame_by_frame: - frame_index = 0 - if seg.shape[0] != 1: - raise NotImplementedError( - "Volume segmentations are not supported yet when return_frame_by_frame is True") - seg = seg[0:1] # (#frames, H, W) -> (1, H, W) - else: - frame_index = ann.index - - if author not in segmentations.keys(): - segmentations[author] = [None] * nframes - seg_labels[author] = [None] * nframes - author_segs = segmentations[author] - author_labels = seg_labels[author] - - if frame_index is not None and ann.scope == 'frame': - seg_code = self.seglabel2code[ann.name] - if author_segs[frame_index] is None: - author_segs[frame_index] = [] - author_labels[frame_index] = [] - s = seg[0] if seg.shape[0] == 1 else seg[frame_index] - author_segs[frame_index].append(s) - author_labels[frame_index].append(seg_code) - elif frame_index is None and ann.scope == 'image': - seg_code = self.seglabel2code[ann.name] - # apply to all frames - for i in range(nframes): - if author_segs[i] is None: - author_segs[i] = [] - author_labels[i] = [] - author_segs[i].append(seg[i]) - author_labels[i].append(seg_code) - else: - raise ValueError(f"Invalid segmentation annotation: {ann}") - - # convert to tensor - for author in segmentations.keys(): - author_segs = segmentations[author] - author_labels = seg_labels[author] - for i in range(len(author_segs)): - if author_segs[i] is not None: - author_segs[i] = torch.stack(author_segs[i]) - author_labels[i] = torch.tensor(author_labels[i], dtype=torch.int32) - else: - author_segs[i] = torch.zeros((0, h, w), dtype=torch.bool) - author_labels[i] = torch.zeros(0, dtype=torch.int32) - - return segmentations, seg_labels, seg_metainfos - - def _instanceseg2semanticseg(self, - segmentations: Sequence[Tensor], - seg_labels: Sequence[Tensor]) -> Tensor: - """ - Convert instance segmentation to semantic segmentation. - - Args: - segmentations: list of `n` tensors of shape (num_instances, H, W), where `n` is the number of frames. - seg_labels: list of `n` tensors of shape (num_instances,), where `n` is the number of frames. - - Returns: - Tensor: tensor of shape (n, num_labels, H, W), where `n` is the number of frames. - """ - if segmentations is None: - return None - - if len(segmentations) != len(seg_labels): - raise ValueError("segmentations and seg_labels must have the same length") - - h, w = segmentations[0].shape[1:] - new_shape = (len(segmentations), - len(self.segmentation_labels_set)+1, # +1 for background - h, w) - new_segmentations = torch.zeros(new_shape, dtype=torch.uint8) - # for each frame - for i in range(len(segmentations)): - # for each instance - for j in range(len(segmentations[i])): - new_segmentations[i, seg_labels[i][j]] += segmentations[i][j] - new_segmentations = new_segmentations > 0 - # pixels that are not in any segmentation are labeled as background - new_segmentations[:, 0] = new_segmentations.sum(dim=1) == 0 - return new_segmentations.float() - - def apply_semantic_seg_merge_strategy(self, segmentations: dict[str, Tensor], - nframes: int, - h, w) -> Tensor | dict[str, Tensor]: - if self.semantic_seg_merge_strategy is None: - return segmentations - if len(segmentations) == 0: - segmentations = torch.zeros((nframes, len(self.segmentation_labels_set)+1, h, w), - dtype=torch.get_default_dtype()) - segmentations[:, 0, :, :] = 1 # background - return segmentations - if self.semantic_seg_merge_strategy == 'union': - merged_segs = self._apply_semantic_seg_merge_strategy_union(segmentations) - elif self.semantic_seg_merge_strategy == 'intersection': - merged_segs = self._apply_semantic_seg_merge_strategy_intersection(segmentations) - elif self.semantic_seg_merge_strategy == 'mode': - merged_segs = self._apply_semantic_seg_merge_strategy_mode(segmentations) - else: - raise ValueError(f"Unknown semantic_seg_merge_strategy: {self.semantic_seg_merge_strategy}") - return merged_segs.to(torch.get_default_dtype()) - - def _apply_semantic_seg_merge_strategy_union(self, segmentations: dict[str, torch.Tensor]) -> torch.Tensor: - new_segmentations = torch.zeros_like(list(segmentations.values())[0]) - for seg in segmentations.values(): - new_segmentations += seg - return new_segmentations.bool() - - def _apply_semantic_seg_merge_strategy_intersection(self, segmentations: dict[str, torch.Tensor]) -> torch.Tensor: - new_segmentations = torch.ones_like(list(segmentations.values())[0]) - for seg in segmentations.values(): - new_segmentations += seg - return new_segmentations.bool() - - def _apply_semantic_seg_merge_strategy_mode(self, segmentations: dict[str, torch.Tensor]) -> torch.Tensor: - new_segmentations = torch.zeros_like(list(segmentations.values())[0]) - for seg in segmentations.values(): - new_segmentations += seg - new_segmentations = new_segmentations >= len(segmentations) / 2 - return new_segmentations - - def __apply_alb_transform_segmentation(self, - img: Tensor, - segmentations: dict[str, list[Tensor]] - ) -> tuple[np.ndarray, dict[str, list]]: - all_masks_list = [] - num_masks = 0 - all_masks_key: dict[str, list] = {} - for author_name, seglist in segmentations.items(): - all_masks_key[author_name] = [] - for i, seg in enumerate(seglist): - if seg is not None: - all_masks_list.append(seg) - assert len(seg.shape) == 3, f"Segmentation must have 3 dimensions, got {seg.shape}" - all_masks_key[author_name].append([num_masks+j for j in range(seg.shape[0])]) - num_masks += seg.shape[0] - else: - all_masks_key[author_name].append(None) - - if len(all_masks_list) != 0: - all_masks_list = torch.concatenate(all_masks_list).numpy().astype(np.uint8) - else: - all_masks_list = None # np.empty((0,img.shape[-2], img.shape[-1]), dtype=np.uint8) - - augmented = self.alb_transform(image=img.numpy().transpose(1, 2, 0), - masks=all_masks_list) - - # reconstruct the segmentations - if all_masks_list is not None: - all_masks = augmented['masks'] # shape: (num_masks, H, W) - new_segmentations: dict[str, list] = {} - for author_name, seglist in all_masks_key.items(): - new_segmentations[author_name] = [] - for i in range(len(seglist)): - if seglist[i] is None: - new_segmentations[author_name].append(None) - else: - masks_i = all_masks[seglist[i]] - masks_i = np.stack(masks_i) - new_segmentations[author_name].append(masks_i) - - return augmented['image'], new_segmentations - - def _seg_labels_to_names(self, - seg_labels: dict | list | None - ) -> dict | list | None: - """ - Convert segmentation label codes to label names. - - Args: - seg_labels: Segmentation labels in various formats: - - dict[str, list[Tensor]]: author -> list of frame tensors with label codes - - dict[str, Tensor]: author -> tensor with label codes - - list[Tensor]: list of frame tensors with label codes - - Tensor: tensor with label codes - - None: when no segmentation labels are available - - Returns: - Same structure as input but with label codes converted to label names. - Returns None if input is None. - """ - if seg_labels is None: - return None - - code_to_name = self.segmentation_labels_set - if isinstance(seg_labels, dict): - # author -> list of frame tensors - seg_names = {} - for author, labels in seg_labels.items(): - if isinstance(labels, Tensor): - # single tensor for the author - seg_names[author] = [code_to_name[code.item()-1] for code in labels] - elif isinstance(labels, Sequence): - # list of frame tensors - seg_names[author] = [[code_to_name[code.item()-1] for code in frame_labels] - for frame_labels in labels] - else: - _LOGGER.warning( - f"Unexpected segmentation labels format for author {author}: {type(labels)}. Returning None") - return None - return seg_names - elif isinstance(seg_labels, list): - # list of frame tensors - return [[code_to_name[code.item()-1] for code in labels] for labels in seg_labels] - - _LOGGER.warning(f"Unexpected segmentation labels format: {type(seg_labels)}. Returning None") - return None - - def __getitem__(self, index) -> dict[str, Any]: - """ - Get the item at the given index. - - Args: - index (int): Index of the item to return. - - Returns: - dict[str, Any]: A dictionary with the following keys: - - * 'image' (Tensor): Tensor of shape (C, H, W) or (N, C, H, W), depending on `self.return_frame_by_frame`. - If `self.return_as_semantic_segmentation` is True, the image is a tensor of shape (N, L, H, W) or (L, H, W), - where `L` is the number of segmentation labels + 1 (background): ``L=len(self.segmentation_labels_set)+1``. - * 'metainfo' (dict): Dictionary with metadata information. - * 'segmentations' (dict[str, list[Tensor]] or dict[str,Tensor] or Tensor): Segmentation masks, - depending on the configuration of parameters `self.return_segmentations`, `self.return_as_semantic_segmentation`, `self.return_frame_by_frame`, `self.semantic_seg_merge_strategy`. - * 'seg_labels' (dict[str, list[Tensor]] or Tensor): Segmentation labels with the same length as `segmentations`. - * 'frame_labels' (dict[str, Tensor]): Frame-level labels. - * 'image_labels' (dict[str, Tensor]): Image-level labels. - """ - item = super().__getitem__(index) - img = item['image'] - metainfo = item['metainfo'] - annotations = item['annotations'] - - has_transformed = False # to check if albumentations transform was applied - - if self.image_transform is not None: - img = self.image_transform(img) - if isinstance(img, np.ndarray): - img = torch.from_numpy(img) - - if img.ndim == 3: - _, h, w = img.shape - nframes = 1 - elif img.ndim == 4: - nframes, _, h, w = img.shape - else: - raise ValueError(f"Image must have 3 or 4 dimensions, got {img.shape}") - - new_item = { - 'image': img, - 'metainfo': metainfo, - } - if 'dicom' in item: - new_item['dicom'] = item['dicom'] - - try: - if self.return_segmentations: - segmentations, seg_labels, seg_metainfos = self._load_segmentations(annotations, img.shape) - # seg_labels can be dict[str, list[Tensor]] - # apply mask transform - if self.mask_transform is not None: - for seglist in segmentations.values(): - for i, seg in enumerate(seglist): - if seg is not None: - seglist[i] = self.mask_transform(seg) - - if self.alb_transform is not None: - img, new_segmentations = self.__apply_alb_transform_segmentation(img, segmentations) - segmentations = new_segmentations - img = torch.from_numpy(img).permute(2, 0, 1) - new_item['image'] = img - has_transformed = True - # Update dimensions after transformation - if img.ndim == 3: - _, h, w = img.shape - elif img.ndim == 4: - nframes, _, h, w = img.shape - - if self.return_as_semantic_segmentation: - sem_segmentations: dict[str, torch.Tensor] = {} - for author in segmentations.keys(): - sem_segmentations[author] = self._instanceseg2semanticseg(segmentations[author], - seg_labels[author]) - segmentations[author] = None # free memory - segmentations = self.apply_semantic_seg_merge_strategy(sem_segmentations, - nframes, - h, w) - # In semantic segmentation, seg_labels is not needed - seg_labels = None - - if self.return_frame_by_frame: - if isinstance(segmentations, dict): # author->segmentations format - segmentations = {k: v[0] for k, v in segmentations.items()} - if seg_labels is not None: - seg_labels = {k: v[0] for k, v in seg_labels.items()} - else: - # segmentations is a tensor - segmentations = segmentations[0] - if seg_labels is not None and len(seg_labels) > 0: - seg_labels = seg_labels[0] - new_item['segmentations'] = segmentations - new_item['seg_labels'] = seg_labels - # process seg_labels to convert from code to label names - new_item['seg_labels_names'] = self._seg_labels_to_names(seg_labels) - new_item['seg_metainfo'] = {'file_metainfo': seg_metainfos} - - except Exception: - _LOGGER.error(f'Error in loading/processing segmentations of {metainfo}') - raise - - if self.alb_transform is not None and not has_transformed: - # apply albumentations transform to the image - augmented = self.alb_transform(image=img.numpy().transpose(1, 2, 0)) - img = torch.from_numpy(augmented['image']).permute(2, 0, 1) - new_item['image'] = img - - framelabel_annotations = self._get_annotations_internal(annotations, type='label', scope='frame') - framelabels = self._convert_labels_annotations(framelabel_annotations, num_frames=nframes) - # framelabels.shape: (num_frames, num_labels) - - imagelabel_annotations = self._get_annotations_internal(annotations, type='label', scope='image') - imagelabels = self._convert_labels_annotations(imagelabel_annotations) - # imagelabels.shape: (num_labels,) - - new_item['frame_labels'] = framelabels - new_item['image_labels'] = imagelabels - - # FIXME: deal with multiple annotators in instance segmentation - - return new_item - - def _convert_labels_annotations(self, - annotations: Sequence[Annotation], - num_frames: int | None = None) -> dict[str, torch.Tensor]: - """ - Converts the annotations, of the same type and scope, to tensor of shape (num_frames, num_labels) - for each annotator. - - Args: - annotations: list of Annotation objects - num_frames: number of frames in the video - - Returns: - dict[str, torch.Tensor]: dictionary of annotator_id -> tensor of shape (num_frames, num_labels) - """ - if num_frames is None: - labels_ret_size = (len(self.image_labels_set),) - label2code = self.image_lcodes['multilabel'] - should_include_label = self._should_include_image_label - else: - labels_ret_size = (num_frames, len(self.frame_labels_set)) - label2code = self.frame_lcodes['multilabel'] - should_include_label = self._should_include_frame_label - - if num_frames is not None and num_frames > 1 and self.return_frame_by_frame: - raise ValueError("num_frames must be 1 if return_frame_by_frame is True") - - frame_labels_byuser = {} # defaultdict(lambda: torch.zeros(size=labels_ret_size, dtype=torch.int32)) - if len(annotations) == 0: - return frame_labels_byuser - for ann in annotations: - user_id = ann.created_by - - frame_idx = ann.index - - if user_id not in frame_labels_byuser.keys(): - frame_labels_byuser[user_id] = torch.zeros(size=labels_ret_size, dtype=torch.int32) - labels_onehot_i = frame_labels_byuser[user_id] - code = label2code[ann.name] - if frame_idx is None: - labels_onehot_i[code] = 1 - else: - if self.return_frame_by_frame: - labels_onehot_i[0, code] = 1 - else: - labels_onehot_i[frame_idx, code] = 1 - - if self.return_frame_by_frame: - for user_id, labels_onehot_i in frame_labels_byuser.items(): - frame_labels_byuser[user_id] = labels_onehot_i[0] - return dict(frame_labels_byuser) - - def __repr__(self) -> str: - super_repr = super().__repr__() - body = [] - if self.image_transform is not None: - body.append("Image transform:") - body += [" " * 4 + line for line in repr(self.image_transform).split('\n')] - if self.mask_transform is not None: - body.append("Mask transform:") - body += [" " * 4 + line for line in repr(self.mask_transform).split('\n')] - if len(body) == 0: - return super_repr - lines = [" " * 4 + line for line in body] - return super_repr + '\n' + "\n".join(lines) diff --git a/datamint/dataset/factory.py b/datamint/dataset/factory.py index 830a15f2..8c5d652d 100644 --- a/datamint/dataset/factory.py +++ b/datamint/dataset/factory.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import warnings from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -28,8 +27,6 @@ def _classify_resource(resource: 'Resource') -> str: def build_dataset(project: str | Project | None = None, - *, - project_name: str | None = None, **kwargs: Any) -> 'DatamintBaseDataset': """Auto-detect and return the appropriate dataset class for a project. @@ -42,7 +39,6 @@ def build_dataset(project: str | Project | None = None, Args: project: Name, ID, or ``Project`` instance of the Datamint project. - project_name: (DEPRECATED) Use ``project`` instead. **kwargs: Forwarded to the dataset constructor (transforms, filters, etc.). Returns: @@ -58,11 +54,6 @@ def build_dataset(project: str | Project | None = None, ds = build_dataset('MyProject', include_unannotated=False) """ - if project_name is not None: - warnings.warn("The 'project_name' parameter is deprecated. " - "Please use 'project' instead", DeprecationWarning) - if project is None: - project = project_name if project is None: raise TypeError("build_dataset() missing required argument: 'project'") diff --git a/datamint/entities/base_entity.py b/datamint/entities/base_entity.py index da6473f6..5282347a 100644 --- a/datamint/entities/base_entity.py +++ b/datamint/entities/base_entity.py @@ -109,7 +109,7 @@ def is_attr_missing(self, attr_name: str) -> bool: raise AttributeError(f"Attribute '{attr_name}' not found in entity of type '{self.__class__.__name__}'") if not hasattr(self, attr_name): return True - return getattr(self, attr_name) == MISSING_FIELD # deprecated + return getattr(self, attr_name) == MISSING_FIELD def has_missing_attrs(self) -> bool: """Check if the entity has any attributes that are MISSING_FIELD. diff --git a/datamint/lightning/datamodule.py b/datamint/lightning/datamodule.py index 52fac031..27f5df73 100644 --- a/datamint/lightning/datamodule.py +++ b/datamint/lightning/datamodule.py @@ -58,12 +58,8 @@ class DatamintDataModule(L.LightningDataModule): split_as_of_timestamp: Historical timestamp forwarded to :meth:`DatamintBaseDataset.split` when reusing project-scoped split assignments. - use_server_splits: (DEPRECATED in favor of ``use_project_splits``) If - *True*, use server-side ``split:*`` tags instead of local random - splitting. use_project_splits: If *True*, read split assignments from the - project splits API instead of local random splitting. Preferred - over ``use_server_splits``. + project splits API instead of local random splitting. train_transform: Albumentations transform applied **only** to the training split (e.g. augmentations). Calls :meth:`~datamint.dataset.base.DatamintBaseDataset.set_transform` @@ -108,7 +104,6 @@ def __init__( split: dict[str, float] | bool | None = True, split_seed: int | None = None, split_as_of_timestamp: str | None = None, - use_server_splits: bool | None = None, use_project_splits: bool | None = None, train_transform: Callable | None = None, eval_transform: Callable | None = None, @@ -138,7 +133,6 @@ def __init__( self._split_cfg = split self._split_seed = split_seed self._split_as_of_timestamp = split_as_of_timestamp - self._use_server_splits = use_server_splits self._use_project_splits = use_project_splits self._train_transform = train_transform self._eval_transform = eval_transform @@ -172,7 +166,6 @@ def _resolve_dataset_splits(self) -> dict[str, DatamintBaseDataset | None]: parts = self.dataset.split( seed=self._split_seed, - use_server_splits=self._use_server_splits, use_project_splits=self._use_project_splits, as_of_timestamp=self._split_as_of_timestamp, **(self._split_cfg or {}), diff --git a/datamint/utils/env.py b/datamint/utils/env.py index 35bfba4e..b633a322 100644 --- a/datamint/utils/env.py +++ b/datamint/utils/env.py @@ -25,10 +25,3 @@ def ensure_asyncio_loop(): import nest_asyncio nest_asyncio.apply() _ASYNCIO_LOOP_PATCHED = True - - -def is_legacy_cli_invocation(command: str) -> bool: - """Check if the current process was launched via the deprecated `datamint-` script.""" - import os - import sys - return os.path.basename(sys.argv[0]) == f"datamint-{command}" diff --git a/docs/source/client_api_content.rst b/docs/source/client_api_content.rst index 546f77ce..7818db3e 100644 --- a/docs/source/client_api_content.rst +++ b/docs/source/client_api_content.rst @@ -435,8 +435,6 @@ ratio kwargs: Each returned subset records ``split_name``, ``split_source``, and ``split_as_of_timestamp`` for reproducibility. Local ratio splits remain available with calls such as ``dataset.split(train=0.8, val=0.2, seed=42)``. -Legacy ``split:*`` tag-based splitting is still supported for backwards -compatibility, but it is deprecated in favor of project-scoped splits. Working with Channels --------------------- @@ -488,7 +486,7 @@ Register and list models deployed_models = api.models.get_list(only_deployed=True) Models are also created automatically when you pass ``--ai-model`` to -:doc:`command_line_tools` (``datamint-upload``) with a name that doesn't +:doc:`command_line_tools` (``datamint upload``) with a name that doesn't exist yet. Inspect versions and metrics diff --git a/docs/source/command_line_tools.rst b/docs/source/command_line_tools.rst index a8730524..e00b08fb 100644 --- a/docs/source/command_line_tools.rst +++ b/docs/source/command_line_tools.rst @@ -13,13 +13,6 @@ like ``docker``, ``git``, and ``pip``: datamint config --help -.. note:: - Older versions used a separate hyphenated script per command (``datamint-config``, - ``datamint-upload``, ``datamint-init``, ``datamint-train``, ``datamint-inference``). - These still work for backward compatibility, but are deprecated — each prints a warning - telling you to switch to the ``datamint `` form, and they will be removed in a - future release. - .. note:: If the ``datamint config`` command does not work, try: diff --git a/docs/source/datamint.dataset.rst b/docs/source/datamint.dataset.rst index 22e30f1b..723923d8 100644 --- a/docs/source/datamint.dataset.rst +++ b/docs/source/datamint.dataset.rst @@ -30,18 +30,17 @@ Split Modes All dataset classes inherit :py:meth:`~datamint.dataset.base.DatamintBaseDataset.split`, which supports -three split modes: +two split modes: - Local random splitting with ratio kwargs such as ``train=0.7``. - Project-scoped split assignments resolved through :py:meth:`api.projects.get_splits() `. -- Legacy ``split:*`` resource tags, which remain available for backwards compatibility but are deprecated. When you call ``split()`` without an explicit mode, the client chooses the mode automatically: - If ratio kwargs are provided, a local random split is used. - If no ratios are provided and the dataset was loaded from a project, project-scoped splits are used. -- Otherwise, legacy ``split:*`` resource tags are used. +- Otherwise, a ``ValueError`` is raised asking for ratio kwargs or a project-backed dataset. .. code-block:: python @@ -59,9 +58,7 @@ mode automatically: # Force an ad hoc local split instead. local_parts = dataset.split(train=0.8, val=0.2, seed=42) -To override the automatic selection, pass ``use_project_splits=True`` or -``use_server_splits=True`` explicitly. ``use_server_splits`` is deprecated and -exists only for compatibility with older tag-based workflows. +To override the automatic selection, pass ``use_project_splits=True`` explicitly. Project-scoped splits require the dataset to be loaded from a project and must not be combined with ratio kwargs. Each resolved subset records @@ -131,27 +128,6 @@ SlicedVideoDataset Annotation Processing --------------------- -.. automodule:: datamint.dataset.annotation - :members: - :undoc-members: - .. automodule:: datamint.dataset.annotation_processor :members: :undoc-members: - -Legacy Classes (Deprecated) ---------------------------- - -.. deprecated:: - The classes below are kept for backwards compatibility and may be removed in a - future release. Use :class:`~datamint.dataset.image_dataset.ImageDataset` or - :class:`~datamint.dataset.volume_dataset.VolumeDataset` instead. - -.. automodule:: datamint.dataset.dataset - :members: DatamintDataset - :undoc-members: - :show-inheritance: - -.. automodule:: datamint.dataset.base_dataset - :members: - :undoc-members: diff --git a/docs/source/datamint_vs_raw_pytorch.rst b/docs/source/datamint_vs_raw_pytorch.rst index a0d26347..3ced9677 100644 --- a/docs/source/datamint_vs_raw_pytorch.rst +++ b/docs/source/datamint_vs_raw_pytorch.rst @@ -31,8 +31,8 @@ Workflow comparison at a glance | must track split manually to | ensure reproducibility with seeds. - | :py:meth:`~datamint.dataset.base.DatamintBaseDataset.split` resolves - | project-scoped splits (or falls back to legacy ``split:*`` tags) and - | returns a snapshot timestamp you can replay later. + | project-scoped splits and returns a snapshot timestamp you can + | replay later. * - | 🔌 **DataModule wiring** - | Implement ``lightning.pytorch.core.LightningDataModule`` with | ``prepare_data``, ``setup``, ``train_dataloader``, ``val_dataloader``, diff --git a/pyproject.toml b/pyproject.toml index 302ca517..8212815b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,6 @@ readme = "README.md" [project.scripts] datamint = 'datamint.__main__:main' -datamint-upload = 'datamint.client_cmd_tools.datamint_upload:main' -datamint-config = 'datamint.client_cmd_tools.datamint_config:main' -datamint-init = 'datamint.client_cmd_tools.datamint_init:main' -datamint-train = 'datamint.client_cmd_tools.datamint_train:main' -datamint-inference = 'datamint.client_cmd_tools.datamint_inference:main' [tool.poetry] # license = "Proprietary" # https://python-poetry.org/docs/pyproject/ @@ -37,7 +32,6 @@ nibabel = ">=4.0.0" pylibjpeg = { version = "^2.0.0" } pylibjpeg-libjpeg = { version = "^2.0.0" } opencv-python = ">=4.0.0" -Deprecated = ">=1.2.0" platformdirs = "^4.0.0" pandas = ">=2.0.0" matplotlib = "*" diff --git a/tests/test_annotations_api.py b/tests/test_annotations_api.py index 2d599da9..ad60a4bb 100644 --- a/tests/test_annotations_api.py +++ b/tests/test_annotations_api.py @@ -1,7 +1,6 @@ from datetime import date import httpx -import pytest from datamint.api.base_api import ApiConfig from datamint.api.endpoints.annotations_api import AnnotationsApi @@ -125,7 +124,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert annotation.geometry.point2 == (10, 30, 2) -def test_annotations_api_get_list_date_range_deprecated_alias( +def test_annotations_api_get_list_date_range( api_config: ApiConfig, api_ids, make_client, @@ -144,59 +143,6 @@ def handler(request: httpx.Request) -> httpx.Response: annotations_api = AnnotationsApi(api_config, client=client) annotations_api.get_list(from_date=date(2026, 1, 1), to_date=date(2026, 1, 31)) - with pytest.warns(DeprecationWarning, match="date_from"): - annotations_api.get_list(date_from=date(2026, 2, 1)) - with pytest.warns(DeprecationWarning, match="date_to"): - annotations_api.get_list(date_to=date(2026, 2, 28)) assert json_body(requests[0])["from"] == "2026-01-01" - assert json_body(requests[0])["to"] == "2026-01-31" - assert json_body(requests[1])["from"] == "2026-02-01" - assert json_body(requests[2])["to"] == "2026-02-28" - - -def test_annotations_api_patch_project_id_deprecated_alias( - api_config: ApiConfig, - api_ids, - make_client, - decoded_path, - json_body, -) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - if request.method == "PATCH" and decoded_path(request) == f"/annotations/{api_ids.annotation_id}": - return httpx.Response(200, json={}) - raise AssertionError(f"Unexpected request: {request.method} {request.url}") - - with make_client(handler) as client: - annotations_api = AnnotationsApi(api_config, client=client) - with pytest.warns(DeprecationWarning, match="project_id"): - annotations_api.patch(api_ids.annotation_id, project_id=api_ids.project_id) - - assert json_body(requests[0]) == {"project_id": api_ids.project_id} - - -def test_annotations_api_upload_segmentation_model_name_deprecated_alias( - api_config: ApiConfig, - api_ids, - make_client, -) -> None: - with make_client(lambda request: httpx.Response(404)) as client: - annotations_api = AnnotationsApi(api_config, client=client) - - with pytest.warns(DeprecationWarning, match="ai_model_name"): - with pytest.raises(FileNotFoundError): - annotations_api.upload_segmentations( - api_ids.resource_id, - "/nonexistent/segmentation.png", - ai_model_name="legacy-model", - ) - with pytest.warns(DeprecationWarning, match="ai_model_name"): - with pytest.raises(FileNotFoundError): - annotations_api.upload_volume_segmentation( - api_ids.resource_id, - "/nonexistent/segmentation.nii.gz", - ai_model_name="legacy-model", - ) \ No newline at end of file + assert json_body(requests[0])["to"] == "2026-01-31" \ No newline at end of file diff --git a/tests/test_annotationsets_api.py b/tests/test_annotationsets_api.py index 9260e05a..21283835 100644 --- a/tests/test_annotationsets_api.py +++ b/tests/test_annotationsets_api.py @@ -1,5 +1,4 @@ import httpx -import pytest from datamint.api.base_api import ApiConfig from datamint.api.endpoints.annotationsets_api import AnnotationWorklistApi @@ -42,69 +41,4 @@ def handler(request: httpx.Request) -> httpx.Response: renames=["old_tumor:new_tumor"], ) - assert annotation_set_id == api_ids.annotation_set_id - - -def test_annotationsets_api_deprecated_parameter_aliases( - api_config: ApiConfig, - api_ids, - make_client, - decoded_path, - json_body, -) -> None: - requests: list[httpx.Request] = [] - definitions = [{"identifier": "tumor", "color": [255, 0, 0], "index": 1}] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - path = decoded_path(request) - if request.method == "POST" and path == "/annotationsets": - return httpx.Response(200, json={"id": api_ids.annotation_set_id}) - if request.method == "PUT" and path == f"/annotationsets/{api_ids.annotation_set_id}/segmentation-group": - return httpx.Response(200, json={"updated": True}) - if request.method == "GET" and path == ( - f"/annotationsets/{api_ids.annotation_set_id}/resources/{api_ids.resource_id}/segmentations" - ): - return httpx.Response(200, json=[]) - if request.method == "POST" and path == f"/annotationsets/{api_ids.annotation_set_id}/resources": - return httpx.Response(200, json={"updated": True}) - if request.method == "GET" and path == f"/annotationsets/{api_ids.annotation_set_id}/users/{api_ids.email}": - return httpx.Response(200, json={"status": "active"}) - raise AssertionError(f"Unexpected request: {request.method} {request.url}") - - with make_client(handler) as client: - annotationsets_api = AnnotationWorklistApi(api_config, client=client) - - with pytest.warns(DeprecationWarning, match="project_id"): - annotationsets_api.create( - name="Lung Worklist", - resource_ids=[api_ids.resource_id], - project_id=api_ids.project_id, - return_entity=False, - ) - with pytest.warns(DeprecationWarning, match="annotation_worklist"): - annotationsets_api.update_segmentation_group( - annotation_worklist=api_ids.annotation_set_id, - definitions=definitions, - ) - with pytest.warns(DeprecationWarning, match="annotation_set"): - with pytest.warns(DeprecationWarning, match="resource_id"): - with pytest.warns(DeprecationWarning, match="annotator"): - annotationsets_api.get_segmentations( - annotation_set=api_ids.annotation_set_id, - resource_id=api_ids.resource_id, - annotator=api_ids.email, - ) - with pytest.warns(DeprecationWarning, match="resources_to_add"): - annotationsets_api.update_resources( - api_ids.annotation_set_id, - resources_to_add=[api_ids.resource_id], - ) - with pytest.warns(DeprecationWarning, match="email"): - annotationsets_api.get_annotator_status( - api_ids.annotation_set_id, email=api_ids.email, - ) - - assert json_body(requests[0])["project_id"] == api_ids.project_id - assert requests[2].url.params["annotator"] == api_ids.email - assert json_body(requests[3])["resource_ids_to_add"] == [api_ids.resource_id] \ No newline at end of file + assert annotation_set_id == api_ids.annotation_set_id \ No newline at end of file diff --git a/tests/test_datamint_config.py b/tests/test_datamint_config.py index dcd22112..7bd11bd7 100644 --- a/tests/test_datamint_config.py +++ b/tests/test_datamint_config.py @@ -117,24 +117,6 @@ def test_command_line_api_key_argument(self, mock_set_values) -> None: # Verify the API key was set with correct key mock_set_values.assert_called_once() - @patch('datamint.configs.set_values') - def test_legacy_hyphenated_invocation_prints_deprecation_warning(self, mock_set_values, capsys) -> None: - """Invoking via the old 'datamint-config' script name should warn.""" - with patch('sys.argv', ['datamint-config', '--api-key', 'test_key']): - from datamint.client_cmd_tools.datamint_config import main - main() - - assert 'deprecated' in capsys.readouterr().out - - @patch('datamint.configs.set_values') - def test_unified_invocation_does_not_print_deprecation_warning(self, mock_set_values, capsys) -> None: - """Invoking via the unified 'datamint config' dispatch should not warn.""" - with patch('sys.argv', ['datamint config', '--api-key', 'test_key']): - from datamint.client_cmd_tools.datamint_config import main - main() - - assert 'deprecated' not in capsys.readouterr().out - def test_show_configurations_functionality(self) -> None: """Test show_all_configurations without user interaction.""" from datamint.client_cmd_tools.datamint_config import show_all_configurations diff --git a/tests/test_datamodule_collate.py b/tests/test_datamodule_collate.py index 81203a14..554900f7 100644 --- a/tests/test_datamodule_collate.py +++ b/tests/test_datamodule_collate.py @@ -54,4 +54,3 @@ def test_use_project_splits_forwarded_to_dataset_split(): dm._resolve_dataset_splits() assert mock_dataset.split.call_args.kwargs["use_project_splits"] is True - assert mock_dataset.split.call_args.kwargs["use_server_splits"] is None diff --git a/tests/test_dataset_factory.py b/tests/test_dataset_factory.py index 8f7a3fbe..96121715 100644 --- a/tests/test_dataset_factory.py +++ b/tests/test_dataset_factory.py @@ -1,22 +1,8 @@ -from unittest.mock import MagicMock, patch - import pytest from datamint.dataset.factory import build_dataset -def test_build_dataset_project_name_deprecated_alias() -> None: - fake_api = MagicMock() - fake_api.resources.get_list.return_value = [] - - with patch("datamint.Api", return_value=fake_api): - with pytest.warns(DeprecationWarning, match="project_name"): - with pytest.raises(ValueError, match="MyProject"): - build_dataset(project_name="MyProject") - - fake_api.resources.get_list.assert_called_once_with(project_name="MyProject", limit=5) - - def test_build_dataset_requires_project() -> None: with pytest.raises(TypeError): build_dataset() diff --git a/tests/test_dataset_patient_split.py b/tests/test_dataset_patient_split.py index 9c223b17..3d7a9542 100644 --- a/tests/test_dataset_patient_split.py +++ b/tests/test_dataset_patient_split.py @@ -248,12 +248,6 @@ def test_mutual_exclusion_with_project_splits(self): with pytest.raises(ValueError, match='cannot be combined'): ds.split(train=0.7, test=0.3, by_patient=True, use_project_splits=True) - def test_mutual_exclusion_with_server_splits(self): - "Test that when splitting by patient, if use_server_splits=True is also passed, a ValueError is raised indicating that the two options cannot be combined." - ds = _make_dataset(['A', 'B']) - with pytest.raises(ValueError, match='cannot be combined'): - ds.split(train=0.7, test=0.3, by_patient=True, use_server_splits=True) - def test_requires_ratio_kwargs(self): "Test that when splitting by patient, if no ratio kwargs (train, val, test) are provided, a ValueError is raised indicating that at least one ratio must be specified." ds = _make_dataset(['A', 'B']) diff --git a/tests/test_datasetsinfo_api.py b/tests/test_datasetsinfo_api.py index 505ae8a4..313c88b2 100644 --- a/tests/test_datasetsinfo_api.py +++ b/tests/test_datasetsinfo_api.py @@ -1,7 +1,6 @@ from pathlib import Path import httpx -import pytest from datamint.api.base_api import ApiConfig from datamint.api.endpoints.datasetsinfo_api import DatasetsInfoApi @@ -82,7 +81,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert output_path.read_bytes() == archive_bytes -def test_datasets_api_update_resources_project_id_deprecated_alias( +def test_datasets_api_update_resources_project( api_config: ApiConfig, api_ids, make_client, @@ -100,7 +99,6 @@ def handler(request: httpx.Request) -> httpx.Response: with make_client(handler) as client: datasets_api = DatasetsInfoApi(api_config, client=client) - with pytest.warns(DeprecationWarning, match="project_id"): - datasets_api.update_resources(api_ids.dataset_id, project_id=api_ids.project_id) + datasets_api.update_resources(api_ids.dataset_id, project=api_ids.project_id) assert json_body(requests[0])["project_id"] == api_ids.project_id \ No newline at end of file diff --git a/tests/test_deploy_model_api.py b/tests/test_deploy_model_api.py index 47cff8dc..125f13cf 100644 --- a/tests/test_deploy_model_api.py +++ b/tests/test_deploy_model_api.py @@ -1,11 +1,10 @@ import httpx -import pytest from datamint.api.base_api import ApiConfig from datamint.api.endpoints.deploy_model_api import DeployModelApi -def test_deploy_model_api_stream_status_job_id_deprecated_alias( +def test_deploy_model_api_stream_status( api_config: ApiConfig, api_ids, make_client, @@ -25,8 +24,7 @@ def handler(request: httpx.Request) -> httpx.Response: with make_client(handler) as client: deploy_api = DeployModelApi(api_config, client=client) - with pytest.warns(DeprecationWarning, match="job_id"): - events = list(deploy_api.stream_status(job_id=api_ids.resource_id)) + events = list(deploy_api.stream_status(job=api_ids.resource_id)) assert events == [{"status": "running"}] assert len(requests) == 1 diff --git a/tests/test_imports.py b/tests/test_imports.py index 67115adc..174c0007 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -32,14 +32,6 @@ def test_dataset_imports(self) -> None: except ImportError as e: pytest.fail(f"Failed to import Dataset: {e}") - # Test importing DatamintDataset directly - try: - from datamint.dataset.dataset import DatamintDataset - assert DatamintDataset is not None - _LOGGER.info("Successfully imported DatamintDataset") - except ImportError as e: - pytest.fail(f"Failed to import DatamintDataset: {e}") - def test_api_imports(self) -> None: """Test importing API handler modules.""" # Test direct import of APIHandler diff --git a/tests/test_inference_api.py b/tests/test_inference_api.py index 1b15a240..d4be5984 100644 --- a/tests/test_inference_api.py +++ b/tests/test_inference_api.py @@ -48,7 +48,7 @@ def handler(request: httpx.Request) -> httpx.Response: inference_api.predict_image("my_model", resource_id="11111111-1111-1111-1111-111111111111") -def test_inference_api_get_status_and_stream_status_job_id_deprecated_alias( +def test_inference_api_get_status_and_stream_status( api_config: ApiConfig, api_ids, make_client, @@ -73,10 +73,8 @@ def handler(request: httpx.Request) -> httpx.Response: with make_client(handler) as client: inference_api = InferenceApi(api_config, client=client) - with pytest.warns(DeprecationWarning, match="job_id"): - job = inference_api.get_status(job_id=api_ids.resource_id) - with pytest.warns(DeprecationWarning, match="job_id"): - events = list(inference_api.stream_status(job_id=api_ids.resource_id)) + job = inference_api.get_status(job=api_ids.resource_id) + events = list(inference_api.stream_status(job=api_ids.resource_id)) assert job.id == api_ids.resource_id assert events == [{"status": "completed"}] diff --git a/tests/test_projects_api.py b/tests/test_projects_api.py index dbe3671a..00199cab 100644 --- a/tests/test_projects_api.py +++ b/tests/test_projects_api.py @@ -1,7 +1,6 @@ from pathlib import Path import httpx -import pytest from datamint.api.base_api import ApiConfig from datamint.api.endpoints.projects_api import ProjectsApi @@ -151,7 +150,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert output_path.read_bytes() == export_bytes -def test_projects_api_deprecated_parameter_aliases( +def test_projects_api_create_and_annotator_endpoints( api_config: ApiConfig, api_ids, make_client, @@ -185,25 +184,19 @@ def handler(request: httpx.Request) -> httpx.Response: with make_client(handler) as client: projects_api = ProjectsApi(api_config, client=client) - with pytest.warns(DeprecationWarning, match="resources_ids"): - projects_api.create( - name="Legacy Project", - description="desc", - resources_ids=[api_ids.resource_id], - return_entity=False, - ) - with pytest.warns(DeprecationWarning, match="resource_id"): - projects_api.get_annotation_statuses(api_ids.project_id, resource_id=api_ids.resource_id) - with pytest.warns(DeprecationWarning, match="annotator"): - projects_api.reset_annotator_status( - api_ids.resource_id, project=api_ids.project_id, annotator=api_ids.email, - ) - with pytest.warns(DeprecationWarning, match="email"): - projects_api.get_annotator_status(email=api_ids.email, project=api_ids.project_id) - with pytest.warns(DeprecationWarning, match="email"): - projects_api.get_annotators_stats(project=api_ids.project_id, email=api_ids.email) - with pytest.warns(DeprecationWarning, match="annotator"): - projects_api.get_review_messages(project=api_ids.project_id, annotator=api_ids.email) + projects_api.create( + name="Legacy Project", + description="desc", + resource_ids=[api_ids.resource_id], + return_entity=False, + ) + projects_api.get_annotation_statuses(api_ids.project_id, resource=api_ids.resource_id) + projects_api.reset_annotator_status( + api_ids.resource_id, project=api_ids.project_id, annotator_email=api_ids.email, + ) + projects_api.get_annotator_status(annotator_email=api_ids.email, project=api_ids.project_id) + projects_api.get_annotators_stats(project=api_ids.project_id, annotator_email=api_ids.email) + projects_api.get_review_messages(project=api_ids.project_id, annotator_email=api_ids.email) assert json_body(requests[1])["resource_ids"] == [api_ids.resource_id] assert requests[2].url.params["resource_id"] == api_ids.resource_id \ No newline at end of file diff --git a/tests/test_resources_api.py b/tests/test_resources_api.py index 374609b0..2bdffc9e 100644 --- a/tests/test_resources_api.py +++ b/tests/test_resources_api.py @@ -1,5 +1,4 @@ import httpx -import pytest from datamint.api.base_api import ApiConfig from datamint.api.endpoints.resources_api import ResourcesApi @@ -40,24 +39,4 @@ def handler(request: httpx.Request) -> httpx.Response: assert requests[1].url.params["offset"] == "0" assert requests[1].url.params["limit"] == "1" assert len(resources) == 1 - assert resources[0].id == sample_resource["id"] - - -def test_resources_api_upload_resources_ai_model_deprecated_alias( - api_config: ApiConfig, - api_ids, - make_client, -) -> None: - with make_client(lambda request: httpx.Response(404)) as client: - resources_api = ResourcesApi(api_config, client=client) - - # 'on_error' is intentionally invalid so the call short-circuits with a - # ValueError right after the deprecation warning fires, without needing - # to mock the full async upload pipeline. - with pytest.warns(DeprecationWarning, match="ai_model"): - with pytest.raises(ValueError): - resources_api.upload_resources( - files_path=["a.dcm", "b.dcm"], - on_error="invalid", - ai_model="legacy-model", - ) \ No newline at end of file + assert resources[0].id == sample_resource["id"] \ No newline at end of file diff --git a/tests/test_users_api.py b/tests/test_users_api.py index e743d24f..8701b8f6 100644 --- a/tests/test_users_api.py +++ b/tests/test_users_api.py @@ -1,5 +1,4 @@ import httpx -import pytest from datamint.api.base_api import ApiConfig from datamint.api.endpoints.users_api import UsersApi @@ -61,34 +60,4 @@ def handler(request: httpx.Request) -> httpx.Response: "annotation_worklist_id": api_ids.annotation_set_id, } assert requests[2].url.params["project_id"] == api_ids.project_id - assert requests[3].method == "DELETE" - - -def test_users_api_project_id_deprecated_alias( - api_config: ApiConfig, - api_ids, - make_client, - decoded_path, - json_body, -) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - path = decoded_path(request) - if request.method == "POST" and path == "/users/invite": - return httpx.Response(200, json={"status": "sent"}) - if request.method == "GET" and path == "/users/invitations": - return httpx.Response(200, json=[]) - raise AssertionError(f"Unexpected request: {request.method} {request.url}") - - with make_client(handler) as client: - users_api = UsersApi(api_config, client=client) - - with pytest.warns(DeprecationWarning, match="project_id"): - users_api.invite(api_ids.email, project_id=api_ids.project_id) - with pytest.warns(DeprecationWarning, match="project_id"): - users_api.get_invitations(project_id=api_ids.project_id) - - assert json_body(requests[0])["project_id"] == api_ids.project_id - assert requests[1].url.params["project_id"] == api_ids.project_id \ No newline at end of file + assert requests[3].method == "DELETE" \ No newline at end of file From 6bdeee12799e4f3b5ce62f3af97515c802d6cc4f Mon Sep 17 00:00:00 2001 From: luandalmazo Date: Fri, 31 Jul 2026 10:51:05 -0300 Subject: [PATCH 2/2] removed deprecated explanation from command...rst --- docs/source/command_line_tools.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/command_line_tools.rst b/docs/source/command_line_tools.rst index e00b08fb..4616afef 100644 --- a/docs/source/command_line_tools.rst +++ b/docs/source/command_line_tools.rst @@ -237,8 +237,6 @@ See all available options by running ``datamint upload --help``: --retain-pii Do not anonymize DICOMs --retain-attribute RETAIN_ATTRIBUTE Retain the value of a single attribute code specified as hexidecimal integers. Example: (0x0008, 0x0050) or just (0008, 0050) - -l LABEL, --label LABEL - Deprecated. Use --tag instead. --tag TAG A tag name to be applied to all files --publish Publish the uploaded resources, giving them the status "published" instead of "inbox" --mungfilename MUNGFILENAME