From c343d339bf1d81405b29ac7898c5620517be2e15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jesse=20Tu=C4=9Flu?= <jtuglu@netflix.com>
Date: Tue, 30 Jun 2026 15:20:12 -0700
Subject: [PATCH] perf: add sort-serving segments index for KillUnusedSegments
 find-interval query

KillUnusedSegments' per-datasource find-interval query
(SqlSegmentsMetadataQuery#retrieveUnusedSegmentIntervals) runs
`WHERE dataSource=? AND used=? AND end<=? [AND start>=?]
 AND used_status_last_updated<=? ORDER BY start, end LIMIT n`.

The existing (dataSource, used, end, start) index orders by end before
start, so it cannot serve `ORDER BY start, end`. On a large datasource
(~470k unused segments) MySQL materializes all matching rows and filesorts
them just to return LIMIT n; EXPLAIN ANALYZE measured ~11s, with the scan
dominating. With ~50 datasources this drove the duty to ~43s/cycle, almost
all in the find-interval SQL.
---
 .../druid/metadata/SQLMetadataConnector.java  | 27 +++++++++++++++++++
 .../metadata/SQLMetadataConnectorTest.java    | 12 ++++++---
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java
index 74829cddbfbf..d2b1002902cb 100644
--- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java
+++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java
@@ -404,6 +404,22 @@ tableName, getPayloadType(), getQuoteString(), getCollation()
             "start"
         )
     );
+    // Sort-serving index for KillUnusedSegments' find-interval query
+    // (WHERE dataSource=? AND used=? AND end<=? [AND start>=?] ORDER BY start, end LIMIT n).
+    // Leading (dataSource, used) equality + (start, end) matches the ORDER BY, so MySQL avoids a
+    // filesort and the LIMIT short-circuits the scan; used_status_last_updated trailing makes the
+    // query covering. See SqlSegmentsMetadataQuery#retrieveUnusedSegmentIntervals.
+    createIndex(
+        tableName,
+        "IDX_%S_DATASOURCE_USED_START_END_LASTUPDATED",
+        List.of(
+            "dataSource",
+            "used",
+            "start",
+            quoteColumn("end"),
+            "used_status_last_updated"
+        )
+    );
   }
 
   private void createUpgradeSegmentsTable(final String tableName)
@@ -663,6 +679,17 @@ protected void alterSegmentTable()
         "IDX_%S_DATASOURCE_UPGRADED_FROM_SEGMENT_ID",
         List.of("dataSource", "upgraded_from_segment_id")
     );
+    createIndex(
+        tableName,
+        "IDX_%S_DATASOURCE_USED_START_END_LASTUPDATED",
+        List.of(
+            "dataSource",
+            "used",
+            "start",
+            quoteColumn("end"),
+            "used_status_last_updated"
+        )
+    );
   }
 
   @Override
diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java
index 3fef28a963a6..1009b1865039 100644
--- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java
@@ -348,7 +348,8 @@ public void test_useShortIndexNames_true_tableIndices_areNotAdded_ifExist()
     final Set<String> expectedIndices = Sets.newHashSet(
         "IDX_DRUIDTEST_SEGMENTS_USED",
         "IDX_D011BD6ED76268701273CE512704C5AFA060D672",
-        "IDX_6381EF2DB4824C35C0E72EF9E166626ADB2B21A3"
+        "IDX_6381EF2DB4824C35C0E72EF9E166626ADB2B21A3",
+        "IDX_CF2F89C27777DAB744431580CD9C78D5365CC53A"
     );
     assertIndicesPresentOnTable(segmentsTable, expectedIndices);
 
@@ -383,7 +384,8 @@ public void test_useShortIndexNames_false_tableIndices_areNotAdded_ifExist()
     final Set<String> expectedIndices = Sets.newHashSet(
         "IDX_8FE3D20EC8C9CA932EA3FF6AC497D1A9E75ADDA0",
         "IDX_DRUIDTEST_SEGMENTS_DATASOURCE_USED_END_START",
-        "IDX_DRUIDTEST_SEGMENTS_DATASOURCE_UPGRADED_FROM_SEGMENT_ID"
+        "IDX_DRUIDTEST_SEGMENTS_DATASOURCE_UPGRADED_FROM_SEGMENT_ID",
+        "IDX_DRUIDTEST_SEGMENTS_DATASOURCE_USED_START_END_LASTUPDATED"
     );
     assertIndicesPresentOnTable(segmentsTable, expectedIndices);
 
@@ -407,7 +409,8 @@ public void test_useShortIndexNames_true_tableIndices_areAdded_IfNotExist()
     final Set<String> expectedIndices = Sets.newHashSet(
         "IDX_8FE3D20EC8C9CA932EA3FF6AC497D1A9E75ADDA0",
         "IDX_D011BD6ED76268701273CE512704C5AFA060D672",
-        "IDX_6381EF2DB4824C35C0E72EF9E166626ADB2B21A3"
+        "IDX_6381EF2DB4824C35C0E72EF9E166626ADB2B21A3",
+        "IDX_CF2F89C27777DAB744431580CD9C78D5365CC53A"
     );
 
     connector.createSegmentTable(segmentsTable);
@@ -433,7 +436,8 @@ public void test_useShortIndexNames_false_tableIndices_areAdded_IfNotExist()
     final Set<String> expectedIndices = Sets.newHashSet(
         "IDX_DRUIDTEST_SEGMENTS_USED",
         "IDX_DRUIDTEST_SEGMENTS_DATASOURCE_USED_END_START",
-        "IDX_DRUIDTEST_SEGMENTS_DATASOURCE_UPGRADED_FROM_SEGMENT_ID"
+        "IDX_DRUIDTEST_SEGMENTS_DATASOURCE_UPGRADED_FROM_SEGMENT_ID",
+        "IDX_DRUIDTEST_SEGMENTS_DATASOURCE_USED_START_END_LASTUPDATED"
     );
 
     connector.createSegmentTable(segmentsTable);