one line population of project's data (DAT-991)

datamint/__init__.py

@@ -16,7 +16,7 @@
 
     __getattr__, __dir__, __all__ = lazy.attach(
         __name__,
-        submodules=['dataset', "dataset.dataset"],
+        submodules=['dataset', "dataset.dataset", "examples"],
         submod_attrs={
             "api.client": ["Api"],
             # New modular dataset classes

datamint/__main__.py

@@ -11,6 +11,7 @@
     "init": "datamint.client_cmd_tools.datamint_init",
     "train": "datamint.client_cmd_tools.datamint_train",
     "inference": "datamint.client_cmd_tools.datamint_inference",
+    "example": "datamint.client_cmd_tools.datamint_example",
 }
 
 

datamint/client_cmd_tools/datamint_example.py

@@ -0,0 +1,77 @@
+"""datamint example command-line tool.
+
+Populates a Datamint project with a small, ready-to-use example dataset -
+no need to bring your own data first. See ``datamint example --help`` for
+the list of available datasets.
+"""
+import argparse
+import logging
+import sys
+
+from datamint.client_cmd_tools.datamint_upload import handle_api_key
+from datamint.exceptions import DatamintException
+from datamint.utils.logging_utils import load_cmdline_logging_config
+
+_LOGGER = logging.getLogger(__name__)
+_USER_LOGGER = logging.getLogger('user_logger')
+
+_DATASETS = ('bccd', 'busi', 'synapse', 'fracatlas')
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description='Populate a Datamint project with an example dataset.',
+        epilog="""
+Examples:
+  datamint example bccd                         # Blood cell detection (BCCD)
+  datamint example busi --project MyBusiProject  # Breast ultrasound segmentation (BUSI)
+  datamint example synapse                       # Multi-organ CT segmentation (Synapse)
+  datamint example fracatlas                     # Fracture classification (FracAtlas)
+
+More Documentation: https://sonanceai.github.io/datamint-python-api/command_line_tools.html
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument('dataset', choices=_DATASETS, help='Which example dataset to populate.')
+    parser.add_argument('--project', type=str, default=None,
+                        help='Name of the project to create. Defaults to a dataset-specific name.')
+    parser.add_argument('--verbose', action='store_true', default=False, help='Print debug messages.')
+
+    return parser.parse_args()
+
+
+def main() -> None:
+    load_cmdline_logging_config()
+
+    args = _parse_args()
+
+    if args.verbose:
+        logging.getLogger().handlers[0].setLevel(logging.DEBUG)
+        logging.getLogger('datamint').setLevel(logging.DEBUG)
+        _LOGGER.setLevel(logging.DEBUG)
+        _USER_LOGGER.setLevel(logging.DEBUG)
+
+    try:
+        api_key = handle_api_key()
+        if api_key is None:
+            _USER_LOGGER.error('API key not provided. Aborting.')
+            sys.exit(1)
+        import os
+
+        from datamint import configs
+        os.environ[configs.ENV_VARS[configs.APIKEY_KEY]] = api_key
+
+        from datamint import examples
+        module = getattr(examples, f'{args.dataset}_dataset')
+        kwargs = {'project_name': args.project} if args.project else {}
+        module.create(**kwargs)
+    except DatamintException as e:
+        _USER_LOGGER.error(f'❌ {e}')
+        sys.exit(1)
+    except KeyboardInterrupt:
+        _USER_LOGGER.warning('\nCancelled by user.')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

datamint/client_cmd_tools/datamint_init.py

@@ -1395,6 +1395,29 @@ def _overlay(ax, img_np, anns):
 """
 
 
+# ---------------------------------------------------------------------------
+# Example-data one-liner (replaces 01_upload_data.py when the user opts in)
+# ---------------------------------------------------------------------------
+
+_EXAMPLE_SCRIPT_01 = """\
+# Docs: https://sonanceai.github.io/datamint-python-api/command_line_tools.html
+#
+# Populates this project with the __EXAMPLE_DATASET_LABEL__ example dataset -
+# no need to bring your own data. Run this script once.
+
+from datamint.examples import __EXAMPLE_MODULE__
+
+__EXAMPLE_MODULE__.create("__PROJECT_NAME__")
+"""
+
+_EXAMPLE_DATASETS = {
+    "detection": ("bccd_dataset", "BCCD (blood cell detection)"),
+    "classification": ("fracatlas_dataset", "FracAtlas (fracture classification)"),
+    "segmentation_2d": ("busi_dataset", "BUSI (breast ultrasound segmentation)"),
+    "segmentation_3d": ("synapse_dataset", "Synapse (multi-organ CT segmentation)"),
+}
+
+
 # ---------------------------------------------------------------------------
 # Template rendering
 # ---------------------------------------------------------------------------
@@ -1403,8 +1426,15 @@ def _render(template: str, project_name: str) -> str:
     return template.replace("__PROJECT_NAME__", project_name)
 
 
-def _generate_detection_files(project_name: str) -> dict[str, str]:
-    return {
+def _render_example_script_01(project_name: str, example_module: str, dataset_label: str) -> str:
+    return (_EXAMPLE_SCRIPT_01
+           .replace("__PROJECT_NAME__", project_name)
+           .replace("__EXAMPLE_MODULE__", example_module)
+           .replace("__EXAMPLE_DATASET_LABEL__", dataset_label))
+
+
+def _generate_detection_files(project_name: str, example_key: str | None = None) -> dict[str, str]:
+    files = {
         "README.md":         _render(_README, project_name),
         "01_upload_data.py": _render(_SCRIPT_01, project_name),
         "02_explore.py":     _render(_SCRIPT_02, project_name),
@@ -1413,10 +1443,14 @@ def _generate_detection_files(project_name: str) -> dict[str, str]:
         "05_evaluate.py":    _render(_SCRIPT_05, project_name),
         "06_deploy.py":      _render(_SCRIPT_06, project_name),
     }
+    if example_key is not None:
+        module, label = _EXAMPLE_DATASETS[example_key]
+        files["01_upload_data.py"] = _render_example_script_01(project_name, module, label)
+    return files
 
 
-def _generate_classification_files(project_name: str) -> dict[str, str]:
-    return {
+def _generate_classification_files(project_name: str, example_key: str | None = None) -> dict[str, str]:
+    files = {
         "README.md":         _render(_CLS_README, project_name),
         "01_upload_data.py": _render(_CLS_SCRIPT_01, project_name),
         "02_explore.py":     _render(_CLS_SCRIPT_02, project_name),
@@ -1425,10 +1459,14 @@ def _generate_classification_files(project_name: str) -> dict[str, str]:
         "05_evaluate.py":    _render(_CLS_SCRIPT_05, project_name),
         "06_deploy.py":      _render(_CLS_SCRIPT_06, project_name),
     }
+    if example_key is not None:
+        module, label = _EXAMPLE_DATASETS[example_key]
+        files["01_upload_data.py"] = _render_example_script_01(project_name, module, label)
+    return files
 
 
-def _generate_segmentation_files(project_name: str) -> dict[str, str]:
-    return {
+def _generate_segmentation_files(project_name: str, example_key: str | None = None) -> dict[str, str]:
+    files = {
         "README.md":         _render(_SEG_README, project_name),
         "01_upload_data.py": _render(_SEG_SCRIPT_01, project_name),
         "02_explore.py":     _render(_SEG_SCRIPT_02, project_name),
@@ -1437,6 +1475,10 @@ def _generate_segmentation_files(project_name: str) -> dict[str, str]:
         "05_evaluate.py":    _render(_SEG_SCRIPT_05, project_name),
         "06_deploy.py":      _render(_SEG_SCRIPT_06, project_name),
     }
+    if example_key is not None:
+        module, label = _EXAMPLE_DATASETS[example_key]
+        files["01_upload_data.py"] = _render_example_script_01(project_name, module, label)
+    return files
 
 
 # ---------------------------------------------------------------------------
@@ -1486,6 +1528,15 @@ def main() -> None:
             choices=["detection", "segmentation", "classification"],
             console=console,
         ).strip()
+
+        example_key = None
+        if Confirm.ask(" Populate this project with example data instead of your own?",
+                       default=False, console=console):
+            if task == "segmentation":
+                dims = Prompt.ask(" 2D or 3D?", choices=["2d", "3d"], console=console).strip()
+                example_key = f"segmentation_{dims}"
+            else:
+                example_key = task
     except (KeyboardInterrupt, EOFError):
         console.print()
         sys.exit(0)
@@ -1504,11 +1555,11 @@ def main() -> None:
 
     out_dir.mkdir(exist_ok=True)
     if task == "classification":
-        files = _generate_classification_files(project_name)
+        files = _generate_classification_files(project_name, example_key)
     elif task == "segmentation":
-        files = _generate_segmentation_files(project_name)
+        files = _generate_segmentation_files(project_name, example_key)
     else:
-        files = _generate_detection_files(project_name)
+        files = _generate_detection_files(project_name, example_key)
 
     console.print()
     console.print(f" Generating scripts in [bold]./{project_name}/[/bold] ...")

datamint/examples/__init__.py

@@ -1 +1,2 @@
-from .example_projects import ProjectMR
+from .example_projects import ProjectMR
+from . import bccd_dataset, busi_dataset, synapse_dataset, fracatlas_dataset

datamint/examples/_common.py

@@ -0,0 +1,41 @@
+import logging
+from pathlib import Path
+
+from datamint import Api
+from datamint.entities import Project
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def get_or_create_project(project_name: str,
+                          description: str,
+                          api: Api) -> tuple[Project, bool]:
+    """Return (project, already_existed).
+
+    If a project with this name already exists, it is returned as-is (not modified).
+    Otherwise a new, empty project is created for the caller to populate.
+    """
+    existing = api.projects.get_by_name(project_name)
+    if existing is not None:
+        return existing, True
+
+    proj = api.projects.create(name=project_name, description=description, exists_ok=True)
+    return proj, False
+
+
+def print_skip_summary(dataset_name: str, proj: Project) -> None:
+    print(dataset_name)
+    print(f"  project '{proj.name}' already exists, skipping data population.")
+    print(f'  {proj.url}')
+
+
+def print_summary(dataset_name: str,
+                  n_files: int,
+                  n_annotated: int,
+                  cache_path: Path,
+                  proj: Project) -> None:
+    pct = (n_annotated / n_files * 100) if n_files else 0.0
+    print(dataset_name)
+    print(f'  {n_files} files uploaded, {n_annotated} annotated ({pct:.0f}%)')
+    print(f'  cached at {cache_path}')
+    print(f'  project: {proj.name} ({proj.url})')

datamint/examples/_download.py

@@ -0,0 +1,55 @@
+import logging
+import zipfile
+from pathlib import Path
+
+import requests
+from tqdm.auto import tqdm
+
+from datamint import configs
+
+_LOGGER = logging.getLogger(__name__)
+
+_DONE_MARKER = '.datamint_download_complete'
+
+
+def cache_dir(dataset_subdir: str) -> Path:
+    if configs.DATAMINT_DATA_DIR is None:
+        raise RuntimeError('Could not determine a local data directory to cache the dataset.')
+    return Path(configs.DATAMINT_DATA_DIR) / 'examples' / dataset_subdir
+
+
+def is_cached(dataset_subdir: str) -> bool:
+    return (cache_dir(dataset_subdir) / _DONE_MARKER).exists()
+
+
+def download_and_extract(url: str, dataset_subdir: str) -> Path:
+    """Download a zip file from `url` and extract it under the examples cache directory.
+
+    Idempotent: if the dataset was already downloaded and extracted, the cached
+    directory is returned without hitting the network again.
+    """
+    out_dir = cache_dir(dataset_subdir)
+    marker = out_dir / _DONE_MARKER
+    if marker.exists():
+        _LOGGER.info(f'Using cached dataset at {out_dir}')
+        return out_dir
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+    zip_path = out_dir / 'download.zip'
+
+    _LOGGER.info(f'Downloading {url}...')
+    with requests.get(url, stream=True) as resp:
+        resp.raise_for_status()
+        total = int(resp.headers.get('content-length', 0))
+        with open(zip_path, 'wb') as f, tqdm(total=total, unit='B', unit_scale=True, desc=dataset_subdir) as pbar:
+            for chunk in resp.iter_content(chunk_size=1024 * 1024):
+                f.write(chunk)
+                pbar.update(len(chunk))
+
+    _LOGGER.info(f'Extracting {zip_path}...')
+    with zipfile.ZipFile(zip_path) as zf:
+        zf.extractall(out_dir)
+    zip_path.unlink()
+    marker.touch()
+
+    return out_dir