diff --git a/datamint/api/endpoints/resources_api.py b/datamint/api/endpoints/resources_api.py index e90e8148..ded56122 100644 --- a/datamint/api/endpoints/resources_api.py +++ b/datamint/api/endpoints/resources_api.py @@ -1,5 +1,5 @@ from typing import TypeAlias, Literal, IO, overload -from collections.abc import Sequence +from collections.abc import Callable, Sequence from ..base_api import ApiConfig, BaseApi from ..entity_base_api import CreatableEntityApi, DeletableEntityApi from datamint.entities import Project, Resource @@ -1404,3 +1404,37 @@ def get_not_annotated(self, for _, items in items_gen: all_items.extend(items) return [self._init_entity_obj(**item) for item in all_items] + + def rank_resources(self, + resources: Sequence[Resource], + score_fn: Callable[[Resource], float | None], + *, + descending: bool = True, + top_k: int | None = None, + ) -> list[tuple[Resource, float]]: + """Rank resources using a custom scoring function. + + Args: + resources: The resources to rank. + score_fn: Called once per resource to produce its score. Return + ``None`` to exclude a resource (e.g. one your scoring + function can't handle). + descending: If ``True`` (default), the highest-scoring resource + comes first. + top_k: If given, only the ``top_k`` highest-ranked resources are + returned. Must be positive. + + Returns: + ``(resource, score)`` pairs sorted by score, excluding resources + whose ``score_fn`` returned ``None``. + """ + if top_k is not None and top_k <= 0: + raise ValueError(f"top_k must be positive, got {top_k}") + + scored = [(res, score_fn(res)) for res in resources] + scored = [(res, score) for res, score in scored if score is not None] + scored.sort(key=lambda pair: pair[1], reverse=descending) + + if top_k is not None: + scored = scored[:top_k] + return scored diff --git a/docs/source/client_api_content.rst b/docs/source/client_api_content.rst index b3238bd3..c34717f0 100644 --- a/docs/source/client_api_content.rst +++ b/docs/source/client_api_content.rst @@ -152,6 +152,29 @@ To delete a resource: # Delete multiple resources at once api.resources.bulk_delete(resources_to_delete) +Ranking unlabeled resources ++++++++++++++++++++++++++++ + +When deciding which unlabeled resources to send for annotation next, use +:py:meth:`api.resources.rank_resources() ` +to order them by any scoring function you provide: + +.. code-block:: python + + unlabeled = api.resources.get_not_annotated(limit=200) + + ranked = api.resources.rank_resources(unlabeled, my_score_fn, top_k=20) + for resource, score in ranked: + print(resource.filename, score) + +``rank_resources`` sorts highest score first by default (``descending=True``) +and skips any resource for which ``my_score_fn`` returns ``None``. Pass +``top_k`` to keep only the highest-ranked resources. + +A common scoring function is model uncertainty -- see +:doc:`command_line_tools` and :mod:`datamint.utils.uncertainty` for how to +compute it. + Working with Annotations ------------------------