diff --git a/datamint/utils/annotation_agreement.py b/datamint/utils/annotation_agreement.py
new file mode 100644
index 00000000..71c341f1
--- /dev/null
+++ b/datamint/utils/annotation_agreement.py
@@ -0,0 +1,357 @@
+"""Inter-annotator agreement metrics for Datamint annotations.
+
+Computes how well multiple annotators agree on the same resources, using a
+metric appropriate to the annotation type: Dice for segmentations, IoU for
+boxes, Cohen's/Fleiss' kappa for category/label annotations.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from itertools import combinations
+from typing import Literal, Sequence
+
+import numpy as np
+import pandas as pd
+from PIL import Image
+
+from datamint.entities.annotations import Annotation, AnnotationType, BoxAnnotation
+from datamint.entities.annotations.base_segmentation import BaseSegmentationAnnotation
+from datamint.entities.annotations.geometry import BoxGeometry
+
+MetricName = Literal['auto', 'dice', 'iou', 'cohen_kappa', 'fleiss_kappa']
+
+_PER_PAIR_GEOMETRIC_COLUMNS = [
+    'resource_id', 'identifier', 'frame_index', 'annotator_a', 'annotator_b', 'metric', 'score',
+]
+_PER_PAIR_CATEGORICAL_COLUMNS = [
+    'resource_id', 'identifier', 'frame_index', 'annotator_a', 'annotator_b',
+    'label_a', 'label_b', 'metric', 'score', 'used_in_overall',
+]
+
+
+@dataclass
+class AgreementResult:
+    """Result of :func:`compute_agreement`.
+
+    Attributes:
+        per_pair: One row per compared annotator pair per item. For
+            cohen_kappa/fleiss_kappa, includes a ``used_in_overall`` column.
+            For fleiss_kappa, an item is used only if its rater count matches
+            the most common rater count across all items (Fleiss' kappa
+            requires a consistent count, not the same rater identities every
+            time), so a low-scoring row with ``used_in_overall=False`` may
+            not be reflected in ``overall``.
+        per_resource_mean: Mean score per ``(resource_id, identifier)``.
+        overall: Summary agreement score for the whole input (mean of
+            per-resource means for Dice/IoU; a single kappa value computed
+            over all items for cohen_kappa/fleiss_kappa).
+        flagged: Rows of ``per_resource_mean`` below the requested threshold.
+            Empty (but present) when no threshold was given.
+    """
+
+    per_pair: pd.DataFrame
+    per_resource_mean: pd.DataFrame
+    overall: float
+    flagged: pd.DataFrame
+
+
+def dice_coefficient(mask_a: np.ndarray, mask_b: np.ndarray) -> float:
+    """Dice similarity coefficient between two binary masks of identical shape."""
+    if mask_a.shape != mask_b.shape:
+        raise ValueError(
+            f"Mask shapes differ: {mask_a.shape} vs {mask_b.shape}. "
+            "Masks must be spatially aligned to compute Dice."
+        )
+    mask_a = mask_a.astype(bool)
+    mask_b = mask_b.astype(bool)
+    denom = int(mask_a.sum()) + int(mask_b.sum())
+    if denom == 0:
+        return 1.0
+    intersection = int(np.logical_and(mask_a, mask_b).sum())
+    return 2 * intersection / denom
+
+
+def iou_boxes(box_a: BoxGeometry, box_b: BoxGeometry) -> float:
+    """IoU between the axis-aligned bounding rectangles of two box geometries.
+
+    Both boxes must use the same ``coordinate_system``.
+    """
+    if box_a.coordinate_system != box_b.coordinate_system:
+        raise ValueError(
+            "Cannot compare boxes in different coordinate systems: "
+            f"{box_a.coordinate_system!r} vs {box_b.coordinate_system!r}."
+        )
+    ax1, ay1, ax2, ay2 = _box_extent(box_a)
+    bx1, by1, bx2, by2 = _box_extent(box_b)
+
+    inter_w = max(0.0, min(ax2, bx2) - max(ax1, bx1))
+    inter_h = max(0.0, min(ay2, by2) - max(ay1, by1))
+    intersection = inter_w * inter_h
+
+    area_a = (ax2 - ax1) * (ay2 - ay1)
+    area_b = (bx2 - bx1) * (by2 - by1)
+    union = area_a + area_b - intersection
+    if union == 0:
+        return 1.0
+    return intersection / union
+
+
+def _box_extent(geom: BoxGeometry) -> tuple[float, float, float, float]:
+    xs = [p[0] for p in geom.points]
+    ys = [p[1] for p in geom.points]
+    return min(xs), min(ys), max(xs), max(ys)
+
+
+def cohen_kappa(labels_a: Sequence[str], labels_b: Sequence[str]) -> float:
+    """Cohen's kappa between two annotators' labels over the same items."""
+    if len(labels_a) != len(labels_b):
+        raise ValueError("labels_a and labels_b must have the same length.")
+    n = len(labels_a)
+    if n == 0:
+        raise ValueError("Cannot compute Cohen's kappa on zero items.")
+
+    categories = sorted(set(labels_a) | set(labels_b))
+    index = {category: i for i, category in enumerate(categories)}
+    confusion = np.zeros((len(categories), len(categories)))
+    for label_a, label_b in zip(labels_a, labels_b):
+        confusion[index[label_a], index[label_b]] += 1
+
+    po = np.trace(confusion) / n
+    row_marginal = confusion.sum(axis=1) / n
+    col_marginal = confusion.sum(axis=0) / n
+    pe = float(np.dot(row_marginal, col_marginal))
+    if pe == 1.0:
+        return 1.0
+    return (po - pe) / (1 - pe)
+
+
+def fleiss_kappa(ratings: Sequence[Sequence[str]]) -> float:
+    """Fleiss' kappa across 3+ annotators.
+
+    Args:
+        ratings: One entry per item, each a sequence of category labels (one
+            per annotator). Every item must have the same number of
+            annotators.
+    """
+    n_items = len(ratings)
+    if n_items == 0:
+        raise ValueError("Cannot compute Fleiss' kappa on zero items.")
+    n_raters = len(ratings[0])
+    if n_raters < 2 or any(len(item) != n_raters for item in ratings):
+        raise ValueError("Fleiss' kappa requires the same number (2+) of raters for every item.")
+
+    categories = sorted({label for item in ratings for label in item})
+    index = {category: i for i, category in enumerate(categories)}
+    counts = np.zeros((n_items, len(categories)))
+    for i, item in enumerate(ratings):
+        for label in item:
+            counts[i, index[label]] += 1
+
+    p_i = (np.sum(counts * counts, axis=1) - n_raters) / (n_raters * (n_raters - 1))
+    p_bar = float(p_i.mean())
+    p_j = counts.sum(axis=0) / (n_items * n_raters)
+    pe_bar = float(np.sum(p_j * p_j))
+    if pe_bar == 1.0:
+        return 1.0
+    return (p_bar - pe_bar) / (1 - pe_bar)
+
+
+def _kind(annotation: Annotation) -> str:
+    if isinstance(annotation, BaseSegmentationAnnotation) or annotation.annotation_type == AnnotationType.SEGMENTATION.value:
+        return 'segmentation'
+    if isinstance(annotation, BoxAnnotation) or annotation.annotation_type == AnnotationType.SQUARE.value:
+        return 'square'
+    if annotation.annotation_type in (AnnotationType.CATEGORY.value, AnnotationType.LABEL.value):
+        return 'category'
+    return 'other'
+
+
+def _group_by_item(annotations: Sequence[Annotation]) -> dict[tuple, dict[str, Annotation]]:
+    """Group annotations by (resource_id, identifier, frame_index), then by annotator."""
+    groups: dict[tuple, dict[str, Annotation]] = {}
+    for annotation in annotations:
+        if annotation.created_by is None:
+            continue
+        key = (annotation.resource_id, annotation.identifier, annotation.frame_index)
+        by_annotator = groups.setdefault(key, {})
+        if annotation.created_by in by_annotator:
+            continue  # multiple annotations from the same annotator for the same item; keep the first
+        by_annotator[annotation.created_by] = annotation
+    return groups
+
+
+def _to_binary_mask(data) -> np.ndarray:
+    if isinstance(data, Image.Image):
+        arr = np.array(data)
+    elif isinstance(data, np.ndarray):
+        arr = data
+    else:  # nibabel Nifti1Image
+        arr = np.asarray(data.dataobj)
+    return arr > 0
+
+
+def _flag(per_resource_mean: pd.DataFrame, threshold: float | None) -> pd.DataFrame:
+    if threshold is None:
+        return per_resource_mean.iloc[0:0]
+    return per_resource_mean[per_resource_mean['score'] < threshold]
+
+
+def _compute_geometric_agreement(
+    annotations: Sequence[Annotation],
+    metric: Literal['dice', 'iou'],
+    threshold: float | None,
+) -> AgreementResult:
+    groups = _group_by_item(annotations)
+    rows = []
+    for (resource_id, identifier, frame_index), by_annotator in groups.items():
+        if len(by_annotator) < 2:
+            continue
+        for annotator_a, annotator_b in combinations(sorted(by_annotator), 2):
+            ann_a, ann_b = by_annotator[annotator_a], by_annotator[annotator_b]
+            if metric == 'dice':
+                mask_a = _to_binary_mask(ann_a.fetch_file_data(use_cache=True))
+                mask_b = _to_binary_mask(ann_b.fetch_file_data(use_cache=True))
+                score = dice_coefficient(mask_a, mask_b)
+            else:
+                score = iou_boxes(ann_a.geometry, ann_b.geometry)
+            rows.append({
+                'resource_id': resource_id, 'identifier': identifier, 'frame_index': frame_index,
+                'annotator_a': annotator_a, 'annotator_b': annotator_b,
+                'metric': metric, 'score': score,
+            })
+
+    if not rows:
+        raise ValueError("No resource/label groups have 2+ distinct annotators; nothing to compare.")
+
+    per_pair = pd.DataFrame(rows, columns=_PER_PAIR_GEOMETRIC_COLUMNS)
+    per_resource_mean = per_pair.groupby(['resource_id', 'identifier'], as_index=False)['score'].mean()
+    overall = float(per_resource_mean['score'].mean())
+    return AgreementResult(
+        per_pair=per_pair,
+        per_resource_mean=per_resource_mean,
+        overall=overall,
+        flagged=_flag(per_resource_mean, threshold),
+    )
+
+
+def _compute_categorical_agreement(
+    annotations: Sequence[Annotation],
+    metric: Literal['auto', 'cohen_kappa', 'fleiss_kappa'],
+    threshold: float | None,
+) -> AgreementResult:
+    groups = _group_by_item(annotations)
+    comparable_groups = {key: by_annotator for key, by_annotator in groups.items() if len(by_annotator) >= 2}
+    if not comparable_groups:
+        raise ValueError("No resource/label groups have 2+ distinct annotators; nothing to compare.")
+
+    all_annotators = sorted({a for by_annotator in comparable_groups.values() for a in by_annotator})
+    if metric == 'auto':
+        metric = 'cohen_kappa' if len(all_annotators) == 2 else 'fleiss_kappa'
+
+    if metric == 'cohen_kappa' and len(all_annotators) != 2:
+        raise ValueError(
+            f"cohen_kappa requires exactly 2 annotators, found {len(all_annotators)}: {all_annotators}. "
+            "Use metric='fleiss_kappa' for 3+ annotators."
+        )
+
+    # Fleiss' kappa only requires a consistent count of raters per item, not the same
+    # rater identities every time
+    n_star = None
+    if metric == 'fleiss_kappa':
+        rater_counts = [len(by_annotator) for by_annotator in comparable_groups.values()]
+        
+        # Find the most common rater count (n_star) across all items
+        n_star = max(set(rater_counts), key=lambda n: (rater_counts.count(n), n))
+
+    rows = []
+    for (resource_id, identifier, frame_index), by_annotator in comparable_groups.items():
+        used_in_overall = True if n_star is None else len(by_annotator) == n_star
+        for annotator_a, annotator_b in combinations(sorted(by_annotator), 2):
+            label_a = by_annotator[annotator_a].text_value
+            label_b = by_annotator[annotator_b].text_value
+            rows.append({
+                'resource_id': resource_id, 'identifier': identifier, 'frame_index': frame_index,
+                'annotator_a': annotator_a, 'annotator_b': annotator_b,
+                'label_a': label_a, 'label_b': label_b,
+                'score': float(label_a == label_b),
+                'used_in_overall': used_in_overall,
+            })
+
+    if metric == 'cohen_kappa':
+        labels_a = [row['label_a'] for row in rows]
+        labels_b = [row['label_b'] for row in rows]
+        overall = cohen_kappa(labels_a, labels_b)
+    else:
+        ratings = [
+            [ann.text_value for ann in by_annotator.values()]
+            for by_annotator in comparable_groups.values()
+            if len(by_annotator) == n_star
+        ]
+        overall = fleiss_kappa(ratings)
+
+    for row in rows:
+        row['metric'] = metric
+    per_pair = pd.DataFrame(rows, columns=_PER_PAIR_CATEGORICAL_COLUMNS)
+    per_resource_mean = per_pair.groupby(['resource_id', 'identifier'], as_index=False).agg(
+        score=('score', 'mean'),
+        used_in_overall=('used_in_overall', 'all'),
+    )
+    return AgreementResult(
+        per_pair=per_pair,
+        per_resource_mean=per_resource_mean,
+        overall=overall,
+        flagged=_flag(per_resource_mean, threshold),
+    )
+
+
+def compute_agreement(
+    annotations: Sequence[Annotation],
+    metric: MetricName = 'auto',
+    threshold: float | None = None,
+) -> AgreementResult:
+    """Compute inter-annotator agreement over a set of annotations.
+
+    Groups annotations by ``(resource_id, identifier, frame_index)``, then
+    compares every pair of annotators on each group with a metric appropriate
+    to the annotation type.
+
+    Args:
+        annotations: Annotations to compare, typically fetched with
+            ``api.annotations.get_list(worklist_id=..., annotation_type=...)``.
+            All annotations must share the same "kind" (all segmentations, all
+            boxes, or all category/label) unless ``metric`` is given
+            explicitly, since a fair comparison metric can't be picked
+            automatically across mixed types.
+        metric: ``'auto'`` picks Dice for segmentations, IoU for boxes, and
+            Cohen's kappa (2 annotators) or Fleiss' kappa (3+) for
+            category/label annotations. Pass one explicitly to override.
+        threshold: Optional score cutoff. Rows of ``per_resource_mean`` below
+            it are returned in ``AgreementResult.flagged`` for adjudication.
+
+    Returns:
+        AgreementResult with per-pair scores, per-resource means, an overall
+        summary score, and any flagged low-agreement resources.
+    """
+    if not annotations:
+        raise ValueError("No annotations provided.")
+
+    kinds = {_kind(a) for a in annotations}
+
+    if metric in ('dice', 'iou'):
+        return _compute_geometric_agreement(annotations, metric, threshold)
+    if metric in ('cohen_kappa', 'fleiss_kappa'):
+        return _compute_categorical_agreement(annotations, metric, threshold)
+
+    # metric == 'auto': infer the family from the annotation kinds present.
+    if kinds == {'segmentation'}:
+        return _compute_geometric_agreement(annotations, 'dice', threshold)
+    if kinds == {'square'}:
+        return _compute_geometric_agreement(annotations, 'iou', threshold)
+    if kinds <= {'category'}:
+        return _compute_categorical_agreement(annotations, 'auto', threshold)
+
+    raise ValueError(
+        f"Cannot auto-select a metric for mixed/unsupported annotation kinds {sorted(kinds)}. "
+        "Filter to a single annotation_type first, or pass metric explicitly."
+    )
diff --git a/docs/source/client_api_content.rst b/docs/source/client_api_content.rst
index cc2f3921..b3238bd3 100644
--- a/docs/source/client_api_content.rst
+++ b/docs/source/client_api_content.rst
@@ -262,6 +262,48 @@ Annotation entities can fetch their own files and lazily resolve the source reso
 
     print(annotation.name, source_resource.filename)
 
+Measuring inter-annotator agreement
+++++++++++++++++++++++++++++++++++
+
+When a worklist assigns 2+ annotators to the same resources, use
+:py:func:`~datamint.utils.annotation_agreement.compute_agreement` to quantify
+how well they agree, and flag resources that need adjudication:
+
+.. code-block:: python
+
+    from datamint.utils.annotation_agreement import compute_agreement
+
+    # Fetch annotations for a worklist, filtered to a single annotation type
+    annotations = api.annotations.get_list(
+        worklist_id=worklist.id,
+        annotation_type="segmentation",
+    )
+
+    result = compute_agreement(annotations, threshold=0.7)
+
+    print(result.overall)            # summary agreement score
+    print(result.per_resource_mean)   # mean score per (resource_id, identifier)
+    print(result.flagged)             # resources below the threshold
+
+The metric is picked automatically based on the annotation type: Dice for
+segmentations, IoU for bounding boxes, and Cohen's/Fleiss' kappa for
+category/label annotations. Pass ``metric="dice"`` (or ``"iou"``,
+``"cohen_kappa"``, ``"fleiss_kappa"``) to override the automatic choice.
+
+With 3+ annotators, Fleiss' kappa requires a consistent count of raters per
+item (not the same rater identities every time, so a pool of 5 annotators
+rotating in groups of 3 per resource works fine). If rater counts vary across
+items, the most common count is used for ``overall`` and items with a
+different count are excluded from it, though they still appear in
+``per_pair``/``per_resource_mean`` (raw pairwise agreement, useful for
+flagging) marked with ``used_in_overall=False``. This means a resource can
+show up as low-agreement in the table even when ``overall`` looks high.
+
+.. automodule:: datamint.utils.annotation_agreement
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Working with Projects
 ---------------------
 
diff --git a/tests/test_annotation_agreement.py b/tests/test_annotation_agreement.py
new file mode 100644
index 00000000..10d53cd8
--- /dev/null
+++ b/tests/test_annotation_agreement.py
@@ -0,0 +1,242 @@
+import numpy as np
+import pytest
+
+from datamint.entities.annotations import BoxAnnotation, ImageClassification, ImageSegmentation
+from datamint.utils.annotation_agreement import (
+    cohen_kappa,
+    compute_agreement,
+    dice_coefficient,
+    fleiss_kappa,
+    iou_boxes,
+)
+
+
+def _segmentation(mask: np.ndarray, resource_id: str, created_by: str, identifier: str = "lesion"):
+    return ImageSegmentation(mask=mask, name=identifier, resource_id=resource_id, created_by=created_by)
+
+
+def _box(point1, point2, resource_id: str, created_by: str, identifier: str = "tumor", coords_system="pixel"):
+    return BoxAnnotation.from_points(
+        point1, point2, identifier=identifier, resource_id=resource_id,
+        created_by=created_by, coords_system=coords_system,
+    )
+
+
+def _classification(value: str, resource_id: str, created_by: str, identifier: str = "finding"):
+    return ImageClassification(name=identifier, value=value, resource_id=resource_id, created_by=created_by)
+
+
+class TestDiceCoefficient:
+    def test_identical_masks(self):
+        mask = np.zeros((10, 10), dtype=bool)
+        mask[2:6, 2:6] = True
+        assert dice_coefficient(mask, mask.copy()) == pytest.approx(1.0)
+
+    def test_partial_overlap(self):
+        mask_a = np.zeros((10, 10), dtype=np.uint8)
+        mask_a[2:6, 2:6] = 1  # 16 px
+        mask_b = np.zeros((10, 10), dtype=np.uint8)
+        mask_b[2:6, 2:8] = 1  # 24 px, overlap 16 px
+        # dice = 2*16 / (16+24) = 0.8
+        assert dice_coefficient(mask_a, mask_b) == pytest.approx(0.8)
+
+    def test_both_empty_is_perfect_agreement(self):
+        mask = np.zeros((5, 5), dtype=bool)
+        assert dice_coefficient(mask, mask.copy()) == pytest.approx(1.0)
+
+    def test_shape_mismatch_raises(self):
+        with pytest.raises(ValueError, match="shapes differ"):
+            dice_coefficient(np.zeros((5, 5)), np.zeros((6, 6)))
+
+
+class TestIouBoxes:
+    def test_partial_overlap(self):
+        box_a = _box((0, 0), (10, 10), "r1", "a@x.com").geometry
+        box_b = _box((5, 5), (15, 15), "r1", "b@x.com").geometry
+        # intersection 5x5=25, union 100+100-25=175
+        assert iou_boxes(box_a, box_b) == pytest.approx(25 / 175)
+
+    def test_mismatched_coordinate_system_raises(self):
+        box_a = _box((0, 0), (10, 10), "r1", "a@x.com", coords_system="pixel").geometry
+        box_b = BoxAnnotation(
+            geometry={"points": [(0, 0, 0), (0, 10, 0), (10, 0, 0), (10, 10, 0)],
+                      "coordinate_system": "patient"},
+            identifier="tumor", resource_id="r1", created_by="b@x.com",
+        ).geometry
+        with pytest.raises(ValueError, match="coordinate system"):
+            iou_boxes(box_a, box_b)
+
+
+class TestCohenKappa:
+    def test_known_value(self):
+        labels_a = ["yes", "yes", "no", "no"]
+        labels_b = ["yes", "no", "no", "no"]
+        assert cohen_kappa(labels_a, labels_b) == pytest.approx(0.5)
+
+    def test_perfect_agreement(self):
+        labels = ["a", "b", "a", "c"]
+        assert cohen_kappa(labels, labels) == pytest.approx(1.0)
+
+    def test_length_mismatch_raises(self):
+        with pytest.raises(ValueError, match="same length"):
+            cohen_kappa(["a"], ["a", "b"])
+
+
+class TestFleissKappa:
+    def test_perfect_agreement(self):
+        ratings = [["A", "A", "A"], ["B", "B", "B"], ["A", "A", "A"]]
+        assert fleiss_kappa(ratings) == pytest.approx(1.0)
+
+    def test_ragged_raters_raises(self):
+        with pytest.raises(ValueError, match="same number"):
+            fleiss_kappa([["A", "B", "A"], ["A", "B"]])
+
+
+class TestComputeAgreementSegmentation:
+    def test_dice_end_to_end(self):
+        mask_a = np.zeros((10, 10), dtype=np.uint8)
+        mask_a[2:6, 2:6] = 1
+        mask_b = np.zeros((10, 10), dtype=np.uint8)
+        mask_b[2:6, 2:8] = 1
+
+        result = compute_agreement([
+            _segmentation(mask_a, "r1", "alice@x.com"),
+            _segmentation(mask_b, "r1", "bob@x.com"),
+        ])
+
+        assert result.overall == pytest.approx(0.8)
+        assert len(result.per_pair) == 1
+        assert result.per_pair.iloc[0]["metric"] == "dice"
+
+    def test_threshold_flags_low_agreement_resource(self):
+        mask_a = np.zeros((10, 10), dtype=np.uint8)
+        mask_a[2:6, 2:6] = 1
+        mask_b = np.zeros((10, 10), dtype=np.uint8)
+        mask_b[2:6, 2:8] = 1
+
+        result = compute_agreement([
+            _segmentation(mask_a, "r1", "alice@x.com"),
+            _segmentation(mask_b, "r1", "bob@x.com"),
+        ], threshold=0.99)
+
+        assert len(result.flagged) == 1
+
+    def test_no_threshold_returns_empty_flagged(self):
+        mask = np.zeros((4, 4), dtype=np.uint8)
+        mask[0:2, 0:2] = 1
+        result = compute_agreement([
+            _segmentation(mask, "r1", "alice@x.com"),
+            _segmentation(mask, "r1", "bob@x.com"),
+        ])
+        assert result.flagged.empty
+
+    def test_single_annotator_raises(self):
+        mask = np.zeros((4, 4), dtype=np.uint8)
+        with pytest.raises(ValueError, match="2\\+ distinct annotators"):
+            compute_agreement([_segmentation(mask, "r1", "alice@x.com")])
+
+
+class TestComputeAgreementBox:
+    def test_iou_end_to_end(self):
+        result = compute_agreement([
+            _box((0, 0), (10, 10), "r2", "alice@x.com"),
+            _box((5, 5), (15, 15), "r2", "bob@x.com"),
+        ])
+        assert result.overall == pytest.approx(25 / 175)
+
+
+class TestComputeAgreementCategorical:
+    def test_cohen_kappa_two_annotators(self):
+        result = compute_agreement([
+            _classification("benign", "r3", "alice@x.com"),
+            _classification("malignant", "r3", "bob@x.com"),
+            _classification("benign", "r4", "alice@x.com"),
+            _classification("benign", "r4", "bob@x.com"),
+        ])
+        assert result.overall == pytest.approx(0.0)
+        assert (result.per_pair["metric"] == "cohen_kappa").all()
+
+    def test_fleiss_kappa_three_annotators(self):
+        annotations = []
+        for resource_id, value in [("r1", "A"), ("r2", "B"), ("r3", "A")]:
+            for who in ["a", "b", "c"]:
+                annotations.append(_classification(value, resource_id, f"{who}@x.com"))
+        result = compute_agreement(annotations)
+        assert result.overall == pytest.approx(1.0)
+
+    def test_explicit_cohen_kappa_with_three_annotators_raises(self):
+        annotations = []
+        for resource_id, value in [("r1", "A"), ("r2", "B")]:
+            for who in ["a", "b", "c"]:
+                annotations.append(_classification(value, resource_id, f"{who}@x.com"))
+        with pytest.raises(ValueError, match="requires exactly 2 annotators"):
+            compute_agreement(annotations, metric="cohen_kappa")
+
+    def test_fleiss_marks_partial_coverage_items_and_excludes_them_from_overall(self):
+        annotations = []
+        # r1: all 3 annotators rate it, all agree -> counts toward Fleiss' kappa
+        for who in ["a", "b", "c"]:
+            annotations.append(_classification("A", "r1", f"{who}@x.com"))
+        # r2: only 2 of 3 annotators rated it, and they disagree
+        annotations.append(_classification("A", "r2", "a@x.com"))
+        annotations.append(_classification("B", "r2", "b@x.com"))
+
+        result = compute_agreement(annotations)
+
+        # overall only reflects r1, the fully-rated item
+        assert result.overall == pytest.approx(1.0)
+
+        by_resource = result.per_resource_mean.set_index("resource_id")
+        assert bool(by_resource.loc["r1", "used_in_overall"]) is True
+        assert bool(by_resource.loc["r2", "used_in_overall"]) is False
+        assert by_resource.loc["r2", "score"] == pytest.approx(0.0)
+
+    def test_fleiss_kappa_with_rotating_annotator_pool(self):
+        # 4 annotators total, but each resource is only rated by 3 of them on
+        # rotation. Fleiss' kappa only needs a consistent *count* of raters
+        # per item, not the same identities, so this should not raise.
+        annotations = []
+        trios = [
+            ("r1", {"a": "A", "b": "A", "c": "A"}),
+            ("r2", {"a": "A", "b": "B", "d": "A"}),
+            ("r3", {"b": "A", "c": "A", "d": "A"}),
+        ]
+        for resource_id, ratings in trios:
+            for who, value in ratings.items():
+                annotations.append(_classification(value, resource_id, f"{who}@x.com"))
+
+        result = compute_agreement(annotations)
+
+        assert (result.per_pair["used_in_overall"]).all()
+        by_resource = result.per_resource_mean.set_index("resource_id")
+        assert bool(by_resource.loc["r1", "used_in_overall"]) is True
+        assert bool(by_resource.loc["r2", "used_in_overall"]) is True
+        assert bool(by_resource.loc["r3", "used_in_overall"]) is True
+
+    def test_fleiss_kappa_excludes_minority_rater_count(self):
+        # 3 items rated by 3 annotators (the common case), 1 item rated by
+        # only 2 -> the 2-rater item should be excluded from overall.
+        annotations = []
+        for resource_id in ["r1", "r2", "r3"]:
+            for who in ["a", "b", "c"]:
+                annotations.append(_classification("A", resource_id, f"{who}@x.com"))
+        annotations.append(_classification("A", "r4", "a@x.com"))
+        annotations.append(_classification("B", "r4", "b@x.com"))
+
+        result = compute_agreement(annotations)
+
+        by_resource = result.per_resource_mean.set_index("resource_id")
+        assert bool(by_resource.loc["r4", "used_in_overall"]) is False
+        assert result.overall == pytest.approx(1.0)
+
+
+class TestComputeAgreementValidation:
+    def test_mixed_kinds_without_explicit_metric_raises(self):
+        box = _box((0, 0), (10, 10), "r5", "alice@x.com")
+        classification = _classification("benign", "r5", "alice@x.com")
+        with pytest.raises(ValueError, match="mixed/unsupported annotation kinds"):
+            compute_agreement([box, classification])
+
+    def test_empty_input_raises(self):
+        with pytest.raises(ValueError, match="No annotations provided"):
+            compute_agreement([])