Add native support for custom ranking of unlabeled data (DAT-1017)

datamint/api/endpoints/resources_api.py

@@ -1,5 +1,5 @@
 from typing import TypeAlias, Literal, IO, overload
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
 from ..base_api import ApiConfig, BaseApi
 from ..entity_base_api import CreatableEntityApi, DeletableEntityApi
 from datamint.entities import Project, Resource
@@ -1404,3 +1404,37 @@ def get_not_annotated(self,
         for _, items in items_gen:
             all_items.extend(items)
         return [self._init_entity_obj(**item) for item in all_items]
+
+    def rank_resources(self,
+                       resources: Sequence[Resource],
+                       score_fn: Callable[[Resource], float | None],
+                       *,
+                       descending: bool = True,
+                       top_k: int | None = None,
+                       ) -> list[tuple[Resource, float]]:
+        """Rank resources using a custom scoring function.
+
+        Args:
+            resources: The resources to rank.
+            score_fn: Called once per resource to produce its score. Return
+                ``None`` to exclude a resource (e.g. one your scoring
+                function can't handle).
+            descending: If ``True`` (default), the highest-scoring resource
+                comes first.
+            top_k: If given, only the ``top_k`` highest-ranked resources are
+                returned. Must be positive.
+
+        Returns:
+            ``(resource, score)`` pairs sorted by score, excluding resources
+            whose ``score_fn`` returned ``None``.
+        """
+        if top_k is not None and top_k <= 0:
+            raise ValueError(f"top_k must be positive, got {top_k}")
+
+        scored = [(res, score_fn(res)) for res in resources]
+        scored = [(res, score) for res, score in scored if score is not None]
+        scored.sort(key=lambda pair: pair[1], reverse=descending)
+
+        if top_k is not None:
+            scored = scored[:top_k]
+        return scored

docs/source/client_api_content.rst

@@ -152,6 +152,29 @@ To delete a resource:
     # Delete multiple resources at once
     api.resources.bulk_delete(resources_to_delete)
 
+Ranking unlabeled resources
++++++++++++++++++++++++++++
+
+When deciding which unlabeled resources to send for annotation next, use
+:py:meth:`api.resources.rank_resources() <datamint.api.endpoints.resources_api.ResourcesApi.rank_resources>`
+to order them by any scoring function you provide:
+
+.. code-block:: python
+
+    unlabeled = api.resources.get_not_annotated(limit=200)
+
+    ranked = api.resources.rank_resources(unlabeled, my_score_fn, top_k=20)
+    for resource, score in ranked:
+        print(resource.filename, score)
+
+``rank_resources`` sorts highest score first by default (``descending=True``)
+and skips any resource for which ``my_score_fn`` returns ``None``. Pass
+``top_k`` to keep only the highest-ranked resources.
+
+A common scoring function is model uncertainty -- see
+:doc:`command_line_tools` and :mod:`datamint.utils.uncertainty` for how to
+compute it.
+
 Working with Annotations
 ------------------------