diff --git a/src/spatialdata/_core/operations/aggregate.py b/src/spatialdata/_core/operations/aggregate.py
index b4772df1b..88aef174e 100644
--- a/src/spatialdata/_core/operations/aggregate.py
+++ b/src/spatialdata/_core/operations/aggregate.py
@@ -317,6 +317,10 @@ def _aggregate_image_by_labels(
         zones: ArrayLike = out["zone"].to_numpy()
         outs.append(out.drop(columns=["zone"]))  # remove the 0 (background)
     df = pd.concat(outs, axis=1)
+    if len(df) == 0:
+        raise ValueError(
+            "No labelled instances (non-zero label ids) found in `by`, so there is nothing to aggregate by."
+        )
 
     X = sparse.csr_matrix(df.values)
 
diff --git a/tests/core/operations/test_aggregations.py b/tests/core/operations/test_aggregations.py
index d471b79cc..a8e346bfe 100644
--- a/tests/core/operations/test_aggregations.py
+++ b/tests/core/operations/test_aggregations.py
@@ -359,6 +359,13 @@ def test_aggregate_image_by_labels(labels_blobs, image_schema, labels_schema) ->
     assert len(out) == 3
 
 
+def test_aggregate_image_by_background_only_labels() -> None:
+    image = Image2DModel.parse(RNG.normal(size=(2, 16, 16)))
+    labels = Labels2DModel.parse(np.zeros((16, 16), dtype=np.int32))
+    with pytest.raises(ValueError, match="No labelled instances"):
+        aggregate(values=image, by=labels, agg_func="mean")
+
+
 @pytest.mark.parametrize("values", ["blobs_image", "blobs_points", "blobs_circles", "blobs_polygons"])
 @pytest.mark.parametrize("by", ["blobs_labels", "blobs_circles", "blobs_polygons"])
 def test_aggregate_requiring_alignment(sdata_blobs: SpatialData, values, by) -> None: