diff --git a/src/spatialdata/_core/query/relational_query.py b/src/spatialdata/_core/query/relational_query.py
index 1a672eefc..731b603e7 100644
--- a/src/spatialdata/_core/query/relational_query.py
+++ b/src/spatialdata/_core/query/relational_query.py
@@ -537,7 +537,9 @@ def _left_join_spatialelement_table(
         # restore the original table row order, as would be expected for a semi-join.
         if match_rows == "no":
             joined_indices = joined_indices.sort_values()
-    joined_table = table[joined_indices.tolist(), :].copy() if joined_indices is not None else None
+    joined_table = (
+        table[joined_indices.tolist(), :].copy() if joined_indices is not None and len(joined_indices) > 0 else None
+    )
     _inplace_fix_subset_categorical_obs(subset_adata=joined_table, original_adata=table)
     if joined_table is not None:
         joined_table.uns[TableModel.ATTRS_KEY][TableModel.REGION_KEY] = _region_as_str_if_list_of_len_one(
diff --git a/tests/core/query/test_spatial_query.py b/tests/core/query/test_spatial_query.py
index 3b6ff3581..724131c04 100644
--- a/tests/core/query/test_spatial_query.py
+++ b/tests/core/query/test_spatial_query.py
@@ -517,6 +517,31 @@ def test_query_spatial_data(full_sdata):
     assert_spatial_data_objects_are_identical(result0, result4, check_transformations=False)
 
 
+@pytest.mark.parametrize("with_polygon_query", [True, False])
+def test_query_labels_without_annotated_instances(with_polygon_query: bool):
+    labels = np.zeros((10, 10), dtype=np.int32)
+    labels[6:9, 6:9] = 1
+    table = AnnData(shape=(1, 0))
+    table.obs["region"] = pd.Categorical(["labels"])
+    table.obs["instance_id"] = [1]
+    table = TableModel.parse(table, region="labels", region_key="region", instance_key="instance_id")
+    sdata = SpatialData(labels={"labels": Labels2DModel.parse(labels)}, tables={"table": table})
+
+    if with_polygon_query:
+        polygon = Polygon([(0, 0), (0, 2), (2, 2), (2, 0)])
+        queried = polygon_query(sdata, polygon=polygon, target_coordinate_system="global", filter_table=True)
+    else:
+        queried = sdata.query.bounding_box(
+            axes=("x", "y"),
+            min_coordinate=np.array([0, 0]),
+            max_coordinate=np.array([2, 2]),
+            filter_table=True,
+            target_coordinate_system="global",
+        )
+    assert "labels" in queried.labels
+    assert len(queried.tables) == 0
+
+
 @pytest.mark.parametrize("with_polygon_query", [True, False])
 def test_query_filter_table(with_polygon_query: bool):
     coords0 = np.array([[10, 10], [20, 20]])