From 223e2c744110f4d147533b9b8d8300b64f0f5c87 Mon Sep 17 00:00:00 2001 From: Ashwin Tumma Date: Fri, 26 Jun 2026 15:26:56 -0700 Subject: [PATCH 1/5] Add skipOffsetFromEarliest to compaction configuration This commit adds a skipOffsetFromEarliest parameter to the compaction configuration, providing the inverse functionality of skipOffsetFromLatest. This allows users to skip re-compacting older data while experimenting with changes to recent data. Changes include: - Add skipOffsetFromEarliest field to DataSourceCompactionConfig interface with default value of Period.ZERO (no skipping) - Implement in InlineSchemaDataSourceCompactionConfig and CatalogDataSourceCompactionConfig with JSON serialization support - Add computeEarliestSkipInterval() method to DataSourceCompactibleSegmentIterator for computing skip interval from earliest timestamp - Update sortAndAddSkipIntervals() to handle both earliest and latest skip offsets - Support in CascadingReindexingTemplate with validation to ensure mutual exclusivity of skipOffsetFromNow, skipOffsetFromLatest, and skipOffsetFromEarliest - Add unit tests in DataSourceCompactibleSegmentIteratorSkipOffsetTest --- .../compact/CascadingReindexingTemplate.java | 41 ++++-- .../DataSourceCompactibleSegmentIterator.java | 69 ++++++++- .../CatalogDataSourceCompactionConfig.java | 13 +- .../DataSourceCompactionConfig.java | 3 + ...nlineSchemaDataSourceCompactionConfig.java | 21 +++ ...pactibleSegmentIteratorSkipOffsetTest.java | 137 ++++++++++++++++++ 6 files changed, 267 insertions(+), 17 deletions(-) create mode 100644 server/src/test/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIteratorSkipOffsetTest.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CascadingReindexingTemplate.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CascadingReindexingTemplate.java index 2f22f20af136..9471b4c0ee9d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CascadingReindexingTemplate.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CascadingReindexingTemplate.java @@ -101,6 +101,7 @@ public class CascadingReindexingTemplate implements CompactionJobTemplate, DataS private final int taskPriority; private final long inputSegmentSizeBytes; private final Period skipOffsetFromLatest; + private final Period skipOffsetFromEarliest; private final Period skipOffsetFromNow; private final Granularity defaultSegmentGranularity; private final PartitionsSpec defaultPartitionsSpec; @@ -118,6 +119,7 @@ public CascadingReindexingTemplate( @JsonProperty("ruleProvider") ReindexingRuleProvider ruleProvider, @JsonProperty("taskContext") @Nullable Map taskContext, @JsonProperty("skipOffsetFromLatest") @Nullable Period skipOffsetFromLatest, + @JsonProperty("skipOffsetFromEarliest") @Nullable Period skipOffsetFromEarliest, @JsonProperty("skipOffsetFromNow") @Nullable Period skipOffsetFromNow, @JsonProperty("defaultSegmentGranularity") Granularity defaultSegmentGranularity, @JsonProperty("defaultPartitionsSpec") PartitionsSpec defaultPartitionsSpec, @@ -152,11 +154,15 @@ public CascadingReindexingTemplate( } this.tuningConfig = tuningConfig; - if (skipOffsetFromNow != null && skipOffsetFromLatest != null) { - throw InvalidInput.exception("Cannot set both skipOffsetFromNow and skipOffsetFromLatest"); + int skipOffsetCount = (skipOffsetFromNow != null ? 1 : 0) + + (skipOffsetFromLatest != null ? 1 : 0) + + (skipOffsetFromEarliest != null ? 1 : 0); + if (skipOffsetCount > 1) { + throw InvalidInput.exception("Cannot set more than one of: skipOffsetFromNow, skipOffsetFromLatest, skipOffsetFromEarliest"); } this.skipOffsetFromNow = skipOffsetFromNow; this.skipOffsetFromLatest = skipOffsetFromLatest; + this.skipOffsetFromEarliest = skipOffsetFromEarliest; this.defaultPartitioningRule = ReindexingPartitioningRule.syntheticRule( defaultSegmentGranularity, @@ -220,6 +226,14 @@ public Period getSkipOffsetFromLatest() return skipOffsetFromLatest; } + @Override + @JsonProperty + @Nullable + public Period getSkipOffsetFromEarliest() + { + return skipOffsetFromEarliest; + } + @JsonProperty @Nullable private Period getSkipOffsetFromNow() @@ -366,7 +380,7 @@ public List createCompactionJobs( } // Skip offsets, if configured, can result in needing to truncate a search interval. If the truncation makes the interval invalid, skip it. - if ((skipOffsetFromNow != null || skipOffsetFromLatest != null) && + if ((skipOffsetFromNow != null || skipOffsetFromLatest != null || skipOffsetFromEarliest != null) && intervalEndsAfter(reindexingInterval, adjustedTimelineInterval.getEnd())) { DateTime alignedEnd = intervalInfo.getGranularity().bucketStart(adjustedTimelineInterval.getEnd()); @@ -422,13 +436,14 @@ protected CompactionJobTemplate createJobTemplateForInterval( } /** - * Applies the configured skip offset to an interval by adjusting its end time. Uses either - * skipOffsetFromNow (relative to reference time) or skipOffsetFromLatest (relative to interval end). - * Returns null if the adjusted end would be before the interval start. + * Applies the configured skip offset to an interval by adjusting its start or end time. Uses either + * skipOffsetFromNow (relative to reference time), skipOffsetFromLatest (relative to interval end), + * or skipOffsetFromEarliest (relative to interval start). + * Returns null if the adjusted interval would be invalid. * * @param interval the interval to adjust * @param skipFromNowReferenceTime the reference time for skipOffsetFromNow calculation - * @return the interval with adjusted end time, or null if the result would be invalid + * @return the interval with adjusted boundaries, or null if the result would be invalid */ @Nullable private Interval applySkipOffset( @@ -436,16 +451,21 @@ private Interval applySkipOffset( DateTime skipFromNowReferenceTime ) { + DateTime maybeAdjustedStart = interval.getStart(); DateTime maybeAdjustedEnd = interval.getEnd(); + if (skipOffsetFromNow != null) { maybeAdjustedEnd = skipFromNowReferenceTime.minus(skipOffsetFromNow); } else if (skipOffsetFromLatest != null) { maybeAdjustedEnd = maybeAdjustedEnd.minus(skipOffsetFromLatest); + } else if (skipOffsetFromEarliest != null) { + maybeAdjustedStart = maybeAdjustedStart.plus(skipOffsetFromEarliest); } - if (maybeAdjustedEnd.isBefore(interval.getStart())) { + + if (maybeAdjustedEnd.isBefore(maybeAdjustedStart) || maybeAdjustedEnd.equals(maybeAdjustedStart)) { return null; } else { - return new Interval(interval.getStart(), maybeAdjustedEnd); + return new Interval(maybeAdjustedStart, maybeAdjustedEnd); } } @@ -458,7 +478,8 @@ private InlineSchemaDataSourceCompactionConfig.Builder createBaseBuilder() .withInputSegmentSizeBytes(inputSegmentSizeBytes) .withEngine(CompactionEngine.MSQ) .withTaskContext(taskContext) - .withSkipOffsetFromLatest(Period.ZERO); // We handle skip offsets at the timeline level, we know we want to cover the entirety of the interval + .withSkipOffsetFromLatest(Period.ZERO) // We handle skip offsets at the timeline level + .withSkipOffsetFromEarliest(Period.ZERO); // We handle skip offsets at the timeline level } /** diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index f818d6ca97a6..2f67e3fac9ca 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -350,26 +350,36 @@ private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compacti } /** - * Returns the initial searchInterval which is {@code (timeline.first().start, timeline.last().end - skipOffset)}. + * Returns the initial searchInterval which is {@code (timeline.first().start + skipOffsetFromEarliest, timeline.last().end - skipOffsetFromLatest)}. */ private List findInitialSearchInterval( SegmentTimeline timeline, @Nullable List skipIntervals ) { - final Period skipOffset = config.getSkipOffsetFromLatest(); + final Period skipOffsetFromLatest = config.getSkipOffsetFromLatest(); + final Period skipOffsetFromEarliest = config.getSkipOffsetFromEarliest(); Preconditions.checkArgument(timeline != null && !timeline.isEmpty(), "timeline should not be null or empty"); - Preconditions.checkNotNull(skipOffset, "skipOffset"); + Preconditions.checkNotNull(skipOffsetFromLatest, "skipOffsetFromLatest"); + Preconditions.checkNotNull(skipOffsetFromEarliest, "skipOffsetFromEarliest"); final TimelineObjectHolder first = Preconditions.checkNotNull(timeline.first(), "first"); final TimelineObjectHolder last = Preconditions.checkNotNull(timeline.last(), "last"); + final Interval latestSkipInterval = computeLatestSkipInterval( config.getSegmentGranularity(), last.getInterval().getEnd(), - skipOffset + skipOffsetFromLatest + ); + + final Interval earliestSkipInterval = computeEarliestSkipInterval( + config.getSegmentGranularity(), + first.getInterval().getStart(), + skipOffsetFromEarliest ); + final List allSkipIntervals - = sortAndAddSkipIntervalFromLatest(latestSkipInterval, skipIntervals); + = sortAndAddSkipIntervals(latestSkipInterval, earliestSkipInterval, skipIntervals); // Collect stats for all skipped segments for (Interval skipInterval : allSkipIntervals) { @@ -384,7 +394,9 @@ private List findInitialSearchInterval( final CompactionStatus reason; if (compactionInterval.overlaps(latestSkipInterval)) { - reason = CompactionStatus.skipped("skip offset from latest[%s]", skipOffset); + reason = CompactionStatus.skipped("skip offset from latest[%s]", skipOffsetFromLatest); + } else if (compactionInterval.overlaps(earliestSkipInterval)) { + reason = CompactionStatus.skipped("skip offset from earliest[%s]", skipOffsetFromEarliest); } else { reason = CompactionStatus.skipped("interval locked by another task"); } @@ -454,7 +466,52 @@ static Interval computeLatestSkipInterval( } } + static Interval computeEarliestSkipInterval( + @Nullable Granularity configuredSegmentGranularity, + DateTime earliestDataTimestamp, + Period skipOffsetFromEarliest + ) + { + if (configuredSegmentGranularity == null) { + return new Interval(earliestDataTimestamp, earliestDataTimestamp.plus(skipOffsetFromEarliest)); + } else { + DateTime skipFromEarliest = new DateTime(earliestDataTimestamp, earliestDataTimestamp.getZone()).plus(skipOffsetFromEarliest); + DateTime skipOffsetBucketToSegmentGranularity = configuredSegmentGranularity.bucketStart(skipFromEarliest); + return new Interval(earliestDataTimestamp, skipOffsetBucketToSegmentGranularity); + } + } + + @VisibleForTesting + static List sortAndAddSkipIntervals( + Interval skipFromLatest, + Interval skipFromEarliest, + @Nullable List skipIntervals + ) + { + final List nonNullSkipIntervals = skipIntervals == null + ? new ArrayList<>(2) + : new ArrayList<>(skipIntervals.size() + 2); + + if (skipIntervals != null) { + nonNullSkipIntervals.addAll(skipIntervals); + } + + // Add earliest skip interval if it's not empty + if (!skipFromEarliest.getStart().equals(skipFromEarliest.getEnd())) { + nonNullSkipIntervals.add(skipFromEarliest); + } + + // Add latest skip interval + nonNullSkipIntervals.add(skipFromLatest); + + // Sort all intervals + nonNullSkipIntervals.sort(Comparators.intervalsByStartThenEnd()); + + return nonNullSkipIntervals; + } + @VisibleForTesting + @Deprecated static List sortAndAddSkipIntervalFromLatest( Interval skipFromLatest, @Nullable List skipIntervals diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfig.java index 37de2a179472..166d37540d57 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfig.java @@ -51,6 +51,7 @@ public class CatalogDataSourceCompactionConfig implements DataSourceCompactionCo @Nullable private final CompactionEngine engine; private final Period skipOffsetFromLatest; + private final Period skipOffsetFromEarliest; private final int taskPriority; @Nullable private final Map taskContext; @@ -63,6 +64,7 @@ public CatalogDataSourceCompactionConfig( @JsonProperty("dataSource") String dataSource, @JsonProperty("engine") @Nullable CompactionEngine engine, @JsonProperty("skipOffsetFromLatest") @Nullable Period skipOffsetFromLatest, + @JsonProperty("skipOffsetFromEarliest") @Nullable Period skipOffsetFromEarliest, @JsonProperty("taskPriority") @Nullable Integer taskPriority, @JsonProperty("taskContext") @Nullable Map taskContext, @JsonProperty("inputSegmentSizeBytes") @Nullable Long inputSegmentSizeBytes, @@ -72,6 +74,7 @@ public CatalogDataSourceCompactionConfig( this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); this.engine = engine; this.skipOffsetFromLatest = skipOffsetFromLatest == null ? DEFAULT_SKIP_OFFSET_FROM_LATEST : skipOffsetFromLatest; + this.skipOffsetFromEarliest = skipOffsetFromEarliest == null ? DEFAULT_SKIP_OFFSET_FROM_EARLIEST : skipOffsetFromEarliest; this.inputSegmentSizeBytes = inputSegmentSizeBytes == null ? DEFAULT_INPUT_SEGMENT_SIZE_BYTES : inputSegmentSizeBytes; @@ -103,6 +106,13 @@ public Period getSkipOffsetFromLatest() return skipOffsetFromLatest; } + @JsonProperty + @Override + public Period getSkipOffsetFromEarliest() + { + return skipOffsetFromEarliest; + } + @JsonProperty @Override public int getTaskPriority() @@ -241,12 +251,13 @@ public boolean equals(Object o) && Objects.equals(dataSource, that.dataSource) && engine == that.engine && Objects.equals(skipOffsetFromLatest, that.skipOffsetFromLatest) + && Objects.equals(skipOffsetFromEarliest, that.skipOffsetFromEarliest) && Objects.equals(taskContext, that.taskContext); } @Override public int hashCode() { - return Objects.hash(dataSource, engine, skipOffsetFromLatest, taskPriority, taskContext, inputSegmentSizeBytes); + return Objects.hash(dataSource, engine, skipOffsetFromLatest, skipOffsetFromEarliest, taskPriority, taskContext, inputSegmentSizeBytes); } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java index 4b8550ee5984..1d2325eecb9c 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java @@ -59,6 +59,7 @@ public interface DataSourceCompactionConfig // Approx. 100TB. Chosen instead of Long.MAX_VALUE to avoid overflow on web-console and other clients long DEFAULT_INPUT_SEGMENT_SIZE_BYTES = 100_000_000_000_000L; Period DEFAULT_SKIP_OFFSET_FROM_LATEST = new Period("P1D"); + Period DEFAULT_SKIP_OFFSET_FROM_EARLIEST = Period.ZERO; String getDataSource(); @@ -80,6 +81,8 @@ public interface DataSourceCompactionConfig Period getSkipOffsetFromLatest(); + Period getSkipOffsetFromEarliest(); + @Nullable UserCompactionTaskQueryTuningConfig getTuningConfig(); diff --git a/server/src/main/java/org/apache/druid/server/coordinator/InlineSchemaDataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/InlineSchemaDataSourceCompactionConfig.java index f99739d85df0..a19a68f35c7c 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/InlineSchemaDataSourceCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/InlineSchemaDataSourceCompactionConfig.java @@ -56,6 +56,7 @@ public static Builder builder() @Nullable private final Integer maxRowsPerSegment; private final Period skipOffsetFromLatest; + private final Period skipOffsetFromEarliest; @Nullable private final UserCompactionTaskQueryTuningConfig tuningConfig; @Nullable @@ -84,6 +85,7 @@ public InlineSchemaDataSourceCompactionConfig( @JsonProperty("inputSegmentSizeBytes") @Nullable Long inputSegmentSizeBytes, @JsonProperty("maxRowsPerSegment") @Deprecated @Nullable Integer maxRowsPerSegment, @JsonProperty("skipOffsetFromLatest") @Nullable Period skipOffsetFromLatest, + @JsonProperty("skipOffsetFromEarliest") @Nullable Period skipOffsetFromEarliest, @JsonProperty("tuningConfig") @Nullable UserCompactionTaskQueryTuningConfig tuningConfig, @JsonProperty("granularitySpec") @Nullable UserCompactionTaskGranularityConfig granularitySpec, @JsonProperty("dimensionsSpec") @Nullable UserCompactionTaskDimensionsConfig dimensionsSpec, @@ -105,6 +107,7 @@ public InlineSchemaDataSourceCompactionConfig( : inputSegmentSizeBytes; this.maxRowsPerSegment = maxRowsPerSegment; this.skipOffsetFromLatest = skipOffsetFromLatest == null ? DEFAULT_SKIP_OFFSET_FROM_LATEST : skipOffsetFromLatest; + this.skipOffsetFromEarliest = skipOffsetFromEarliest == null ? DEFAULT_SKIP_OFFSET_FROM_EARLIEST : skipOffsetFromEarliest; this.tuningConfig = tuningConfig; this.ioConfig = ioConfig; this.granularitySpec = granularitySpec; @@ -154,6 +157,13 @@ public Period getSkipOffsetFromLatest() return skipOffsetFromLatest; } + @JsonProperty + @Override + public Period getSkipOffsetFromEarliest() + { + return skipOffsetFromEarliest; + } + @JsonProperty @Nullable @Override @@ -263,6 +273,7 @@ public boolean equals(Object o) Objects.equals(dataSource, that.dataSource) && Objects.equals(maxRowsPerSegment, that.maxRowsPerSegment) && Objects.equals(skipOffsetFromLatest, that.skipOffsetFromLatest) && + Objects.equals(skipOffsetFromEarliest, that.skipOffsetFromEarliest) && Objects.equals(tuningConfig, that.tuningConfig) && Objects.equals(granularitySpec, that.granularitySpec) && Objects.equals(dimensionsSpec, that.dimensionsSpec) && @@ -284,6 +295,7 @@ public int hashCode() inputSegmentSizeBytes, maxRowsPerSegment, skipOffsetFromLatest, + skipOffsetFromEarliest, tuningConfig, granularitySpec, dimensionsSpec, @@ -310,6 +322,7 @@ public Builder toBuilder() .withInputSegmentSizeBytes(this.inputSegmentSizeBytes) .withMaxRowsPerSegment(this.maxRowsPerSegment) .withSkipOffsetFromLatest(this.skipOffsetFromLatest) + .withSkipOffsetFromEarliest(this.skipOffsetFromEarliest) .withTuningConfig(this.tuningConfig) .withGranularitySpec(this.granularitySpec) .withDimensionsSpec(this.dimensionsSpec) @@ -329,6 +342,7 @@ public static class Builder private Long inputSegmentSizeBytes; private Integer maxRowsPerSegment; private Period skipOffsetFromLatest; + private Period skipOffsetFromEarliest; private UserCompactionTaskQueryTuningConfig tuningConfig; private UserCompactionTaskGranularityConfig granularitySpec; private UserCompactionTaskDimensionsConfig dimensionsSpec; @@ -348,6 +362,7 @@ public InlineSchemaDataSourceCompactionConfig build() inputSegmentSizeBytes, maxRowsPerSegment, skipOffsetFromLatest, + skipOffsetFromEarliest, tuningConfig, granularitySpec, dimensionsSpec, @@ -392,6 +407,12 @@ public Builder withSkipOffsetFromLatest(Period skipOffsetFromLatest) return this; } + public Builder withSkipOffsetFromEarliest(Period skipOffsetFromEarliest) + { + this.skipOffsetFromEarliest = skipOffsetFromEarliest; + return this; + } + public Builder withTuningConfig( UserCompactionTaskQueryTuningConfig tuningConfig ) diff --git a/server/src/test/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIteratorSkipOffsetTest.java b/server/src/test/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIteratorSkipOffsetTest.java new file mode 100644 index 000000000000..7c132e88ae32 --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIteratorSkipOffsetTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.compaction; + +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Test; + +public class DataSourceCompactibleSegmentIteratorSkipOffsetTest +{ + @Test + public void test_computeEarliestSkipInterval_withoutGranularity() + { + final DateTime earliestTimestamp = DateTimes.of("2018-01-01"); + final Period skipOffset = Period.days(7); + + final Interval skipInterval = DataSourceCompactibleSegmentIterator.computeEarliestSkipInterval( + null, + earliestTimestamp, + skipOffset + ); + + // Without granularity: [2018-01-01, 2018-01-08) + Assert.assertEquals(Intervals.of("2018-01-01/2018-01-08"), skipInterval); + } + + @Test + public void test_computeEarliestSkipInterval_withGranularity() + { + final DateTime earliestTimestamp = DateTimes.of("2018-01-01"); + final Period skipOffset = Period.days(7); + + final Interval skipInterval = DataSourceCompactibleSegmentIterator.computeEarliestSkipInterval( + Granularities.DAY, + earliestTimestamp, + skipOffset + ); + + // With DAY granularity, skip offset bucket start: [2018-01-01, 2018-01-08) + Assert.assertEquals(Intervals.of("2018-01-01/2018-01-08"), skipInterval); + } + + @Test + public void test_computeEarliestSkipInterval_zeroOffset() + { + final DateTime earliestTimestamp = DateTimes.of("2018-01-01"); + final Period skipOffset = Period.ZERO; + + final Interval skipInterval = DataSourceCompactibleSegmentIterator.computeEarliestSkipInterval( + null, + earliestTimestamp, + skipOffset + ); + + // Zero offset results in an empty interval (start == end) + Assert.assertEquals(Intervals.of("2018-01-01/2018-01-01"), skipInterval); + } + + @Test + public void test_sortAndAddSkipIntervals_bothOffsets() + { + final Interval latestSkipInterval = Intervals.of("2018-12-25/2019-01-01"); + final Interval earliestSkipInterval = Intervals.of("2018-01-01/2018-01-08"); + + final java.util.List skipIntervals = DataSourceCompactibleSegmentIterator.sortAndAddSkipIntervals( + latestSkipInterval, + earliestSkipInterval, + null + ); + + // Should have both intervals, sorted + Assert.assertEquals(2, skipIntervals.size()); + Assert.assertEquals(earliestSkipInterval, skipIntervals.get(0)); + Assert.assertEquals(latestSkipInterval, skipIntervals.get(1)); + } + + @Test + public void test_sortAndAddSkipIntervals_withExistingIntervals() + { + final Interval latestSkipInterval = Intervals.of("2018-12-25/2019-01-01"); + final Interval earliestSkipInterval = Intervals.of("2018-01-01/2018-01-08"); + final java.util.List existingSkipIntervals = java.util.List.of( + Intervals.of("2018-06-01/2018-06-15") + ); + + final java.util.List skipIntervals = DataSourceCompactibleSegmentIterator.sortAndAddSkipIntervals( + latestSkipInterval, + earliestSkipInterval, + existingSkipIntervals + ); + + // Should have all three intervals, sorted + Assert.assertEquals(3, skipIntervals.size()); + Assert.assertEquals(earliestSkipInterval, skipIntervals.get(0)); + Assert.assertEquals(Intervals.of("2018-06-01/2018-06-15"), skipIntervals.get(1)); + Assert.assertEquals(latestSkipInterval, skipIntervals.get(2)); + } + + @Test + public void test_sortAndAddSkipIntervals_emptyEarliestInterval() + { + final Interval latestSkipInterval = Intervals.of("2018-12-25/2019-01-01"); + final Interval earliestSkipInterval = Intervals.of("2018-01-01/2018-01-01"); // Empty interval + + final java.util.List skipIntervals = DataSourceCompactibleSegmentIterator.sortAndAddSkipIntervals( + latestSkipInterval, + earliestSkipInterval, + null + ); + + // Empty earliest interval should not be added + Assert.assertEquals(1, skipIntervals.size()); + Assert.assertEquals(latestSkipInterval, skipIntervals.get(0)); + } +} From e0f91adf493534bf9c8dfd59f9d1cfeaf1bc72c8 Mon Sep 17 00:00:00 2001 From: Ashwin Tumma Date: Mon, 29 Jun 2026 05:16:26 -0700 Subject: [PATCH 2/5] Fix test compilation errors for skipOffsetFromEarliest Update test files to include the new skipOffsetFromEarliest parameter in CatalogDataSourceCompactionConfig constructor calls: - CatalogDataSourceCompactionConfigTest: Add null for skipOffsetFromEarliest - CompactSegmentsTest: Add null for skipOffsetFromEarliest --- .../coordinator/CatalogDataSourceCompactionConfigTest.java | 2 ++ .../druid/server/coordinator/duty/CompactSegmentsTest.java | 1 + 2 files changed, 3 insertions(+) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java index e7ed9a7d038f..83203ea5a2be 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java @@ -116,6 +116,7 @@ public void testProjections() null, null, null, + null, METADATA_CATALOG ); @@ -135,6 +136,7 @@ public void testSerde() throws JsonProcessingException null, null, null, + null, METADATA_CATALOG ); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index e3b5e7eeefc3..5329931ab968 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -1071,6 +1071,7 @@ public void testCompactWithCatalogProjections() dataSource, engine, new Period("PT0H"), + null, 0, null, 500L, From d6f9b57b1b27af5bf932beea94bd0a38fe7afb90 Mon Sep 17 00:00:00 2001 From: Ashwin Tumma Date: Mon, 29 Jun 2026 08:33:30 -0700 Subject: [PATCH 3/5] Fix CascadingReindexingTemplateTest compilation errors Add skipOffsetFromEarliest parameter (null) to all CascadingReindexingTemplate constructor calls in test file. The new parameter goes between skipOffsetFromLatest and skipOffsetFromNow parameters. --- .../CascadingReindexingTemplateTest.java | 24 + .../CascadingReindexingTemplateTest.java.bak | 2022 +++++++++++++++++ 2 files changed, 2046 insertions(+) create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java.bak diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java index af8d732311e3..7d69bd1e6aec 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java @@ -109,6 +109,7 @@ public void test_serde() throws Exception ImmutableMap.of("context_key", "context_value"), null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -148,6 +149,7 @@ public void test_serde_asDataSourceCompactionConfig() throws Exception ImmutableMap.of("key", "value"), null, null, + null, Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null, @@ -185,6 +187,7 @@ public void test_createCompactionJobs_ruleProviderNotReady() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -557,6 +560,7 @@ public void test_generateAlignedSearchIntervals_withGranularityAlignment() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -647,6 +651,7 @@ public void test_generateAlignedSearchIntervals_withNonPartitioningRuleSplits() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -742,6 +747,7 @@ public void test_generateAlignedSearchIntervals_withNoPartitioningRules() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -838,6 +844,7 @@ public void test_generateAlignedSearchIntervals_prependIntervalForShortNonPartit null, null, null, + null, Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null, @@ -946,6 +953,7 @@ public void test_generateAlignedSearchIntervals() null, null, null, + null, Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null, @@ -1017,6 +1025,7 @@ public void test_generateAlignedSearchIntervals_noRulesThrowsException() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -1084,6 +1093,7 @@ public void test_generateAlignedSearchIntervals_splitPointSnapsToExistingBoundar null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -1152,6 +1162,7 @@ public void test_generateAlignedSearchIntervals_prependAlignmentDoesNotExtendTim null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -1224,6 +1235,7 @@ public void test_generateAlignedSearchIntervals_duplicateSplitPointsFiltered() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -1288,6 +1300,7 @@ public void test_generateAlignedSearchIntervals_singleRuleOnly() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -1355,6 +1368,7 @@ public void test_generateAlignedSearchIntervals_zeroPeriodRuleAppliesImmediately null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -1447,6 +1461,7 @@ public void test_generateAlignedSearchIntervals_zeroPeriodRuleWithOtherRules() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -1519,6 +1534,7 @@ public void test_generateAlignedSearchIntervals_failsWhenDefaultGranularityIsCoa null, null, null, + null, Granularities.MONTH, // MONTH is coarser than HOUR! new DynamicPartitionsSpec(5000000, null), null, @@ -1582,6 +1598,7 @@ public void test_generateAlignedSearchIntervals_failsWhenOlderRuleHasFinerGranul null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -1629,6 +1646,7 @@ public void test_generateAlignedSearchIntervals_defaultPartitioningVirtualColumn null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), defaultVCs, @@ -1689,6 +1707,7 @@ public void test_generateAlignedSearchIntervals_defaultPartitioningVirtualColumn null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), defaultVCs, @@ -1746,6 +1765,7 @@ public void test_serde_withDefaultPartitioningVirtualColumns() throws Exception ImmutableMap.of("context_key", "context_value"), null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), defaultVCs, @@ -1788,6 +1808,7 @@ public void test_validate_returnsValid_withDynamicPartitionsSpec() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(null, null), null, @@ -1809,6 +1830,7 @@ public void test_validate_returnsInvalid_withHashedPartitionsSpec() null, null, null, + null, Granularities.DAY, new HashedPartitionsSpec(null, 3, null), null, @@ -1834,6 +1856,7 @@ public void test_validate_returnsInvalid_withMaxTotalRows() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(null, 1000L), null, @@ -1859,6 +1882,7 @@ public void test_validate_returnsInvalid_withOneMaxNumTasks() Collections.singletonMap(ClientMSQContext.CTX_MAX_NUM_TASKS, 1), null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(null, null), null, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java.bak b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java.bak new file mode 100644 index 000000000000..af8d732311e3 --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java.bak @@ -0,0 +1,2022 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.compact; + +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.client.indexing.ClientMSQContext; +import org.apache.druid.error.DruidException; +import org.apache.druid.guice.SupervisorModule; +import org.apache.druid.indexer.CompactionEngine; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.indexer.partitions.HashedPartitionsSpec; +import org.apache.druid.indexing.input.DruidInputSource; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.expression.TestExprMacroTable; +import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.virtual.ExpressionVirtualColumn; +import org.apache.druid.server.compaction.InlineReindexingRuleProvider; +import org.apache.druid.server.compaction.IntervalPartitioningInfo; +import org.apache.druid.server.compaction.ReindexingDataSchemaRule; +import org.apache.druid.server.compaction.ReindexingPartitioningRule; +import org.apache.druid.server.compaction.ReindexingRule; +import org.apache.druid.server.compaction.ReindexingRuleProvider; +import org.apache.druid.server.coordinator.ClusterCompactionConfig; +import org.apache.druid.server.coordinator.CompactionConfigValidationResult; +import org.apache.druid.server.coordinator.DataSourceCompactionConfig; +import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentTimeline; +import org.easymock.EasyMock; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.joda.time.Period; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class CascadingReindexingTemplateTest extends InitializedNullHandlingTest +{ + private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); + private static final ClusterCompactionConfig CLUSTER_CONFIG = + new ClusterCompactionConfig(null, null, null, null, null, null); + + @BeforeEach + public void setUp() + { + OBJECT_MAPPER.registerModules(new SupervisorModule().getJacksonModules()); + } + + @Test + public void test_serde() throws Exception + { + final CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDataSource", + 50, + 1000000L, + InlineReindexingRuleProvider.builder() + .partitioningRules(List.of( + new ReindexingPartitioningRule( + "hourRule", + null, + Period.days(7), + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null + ), + new ReindexingPartitioningRule( + "dayRule", + null, + Period.days(30), + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null + ) + )) + .build(), + ImmutableMap.of("context_key", "context_value"), + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + final String json = OBJECT_MAPPER.writeValueAsString(template); + final CascadingReindexingTemplate fromJson = OBJECT_MAPPER.readValue(json, CascadingReindexingTemplate.class); + + Assertions.assertEquals(template.getDataSource(), fromJson.getDataSource()); + Assertions.assertEquals(template.getTaskPriority(), fromJson.getTaskPriority()); + Assertions.assertEquals(template.getInputSegmentSizeBytes(), fromJson.getInputSegmentSizeBytes()); + Assertions.assertEquals(template.getEngine(), fromJson.getEngine()); + Assertions.assertEquals(template.getTaskContext(), fromJson.getTaskContext()); + Assertions.assertEquals(template.getType(), fromJson.getType()); + } + + @Test + public void test_serde_asDataSourceCompactionConfig() throws Exception + { + final CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDataSource", + 30, + 500000L, + InlineReindexingRuleProvider.builder() + .partitioningRules(List.of( + new ReindexingPartitioningRule( + "rule1", + null, + Period.days(7), + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null + ) + )) + .build(), + ImmutableMap.of("key", "value"), + null, + null, + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + // Serialize and deserialize as DataSourceCompactionConfig interface + final String json = OBJECT_MAPPER.writeValueAsString(template); + final DataSourceCompactionConfig fromJson = OBJECT_MAPPER.readValue(json, DataSourceCompactionConfig.class); + + Assertions.assertTrue(fromJson instanceof CascadingReindexingTemplate); + final CascadingReindexingTemplate cascadingFromJson = (CascadingReindexingTemplate) fromJson; + + Assertions.assertEquals("testDataSource", cascadingFromJson.getDataSource()); + Assertions.assertEquals(30, cascadingFromJson.getTaskPriority()); + Assertions.assertEquals(500000L, cascadingFromJson.getInputSegmentSizeBytes()); + Assertions.assertEquals(CompactionEngine.MSQ, cascadingFromJson.getEngine()); + Assertions.assertEquals(ImmutableMap.of("key", "value"), cascadingFromJson.getTaskContext()); + Assertions.assertEquals(CascadingReindexingTemplate.TYPE, cascadingFromJson.getType()); + } + + @Test + public void test_createCompactionJobs_ruleProviderNotReady() + { + final ReindexingRuleProvider notReadyProvider = EasyMock.createMock(ReindexingRuleProvider.class); + EasyMock.expect(notReadyProvider.isReady()).andReturn(false); + EasyMock.expect(notReadyProvider.getType()).andReturn("mock-provider"); + EasyMock.replay(notReadyProvider); + + final CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDataSource", + null, + null, + notReadyProvider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + // Call createCompactionJobs - should return empty list without processing + final List jobs = template.createCompactionJobs(null, null); + + Assertions.assertTrue(jobs.isEmpty()); + EasyMock.verify(notReadyProvider); + } + + @Test + public void test_constructor_setBothSkipOffsetStrategiesThrowsException() + { + final ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); + EasyMock.replay(mockProvider); + + DruidException exception = Assertions.assertThrows( + DruidException.class, + () -> new CascadingReindexingTemplate( + "testDataSource", + null, + null, + mockProvider, + null, + Period.days(7), // skipOffsetFromLatest + Period.days(3), // skipOffsetFromNow + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ) + ); + + Assertions.assertEquals("Cannot set both skipOffsetFromNow and skipOffsetFromLatest", exception.getMessage()); + EasyMock.verify(mockProvider); + } + + @Test + public void test_constructor_nullDataSourceThrowsException() + { + final ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); + EasyMock.replay(mockProvider); + + DruidException exception = Assertions.assertThrows( + DruidException.class, + () -> new CascadingReindexingTemplate( + null, // null dataSource + null, + null, + mockProvider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ) + ); + + Assertions.assertTrue(exception.getMessage().contains("'dataSource' cannot be null")); + EasyMock.verify(mockProvider); + } + + @Test + public void test_constructor_nullRuleProviderThrowsException() + { + DruidException exception = Assertions.assertThrows( + DruidException.class, + () -> new CascadingReindexingTemplate( + "testDataSource", + null, + null, + null, // null ruleProvider + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ) + ); + + Assertions.assertTrue(exception.getMessage().contains("'ruleProvider' cannot be null")); + } + + @Test + public void test_constructor_nullDefaultSegmentGranularityThrowsException() + { + final ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); + EasyMock.replay(mockProvider); + + DruidException exception = Assertions.assertThrows( + DruidException.class, + () -> new CascadingReindexingTemplate( + "testDataSource", + null, + null, + mockProvider, + null, + null, + null, + null, // null defaultSegmentGranularity + new DynamicPartitionsSpec(5000000, null), + null, + null + ) + ); + + Assertions.assertTrue(exception.getMessage().contains("'defaultSegmentGranularity' cannot be null")); + EasyMock.verify(mockProvider); + } + + @Test + public void test_constructor_tuningConfigWithPartitionsSpecThrowsException() + { + final ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); + EasyMock.replay(mockProvider); + + UserCompactionTaskQueryTuningConfig tuningWithPartitionsSpec = UserCompactionTaskQueryTuningConfig.builder() + .partitionsSpec(new DynamicPartitionsSpec(1000, null)) + .build(); + + DruidException exception = Assertions.assertThrows( + DruidException.class, + () -> new CascadingReindexingTemplate( + "testDataSource", + null, + null, + mockProvider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + tuningWithPartitionsSpec + ) + ); + + Assertions.assertTrue( + exception.getMessage().contains("Cannot set 'partitionsSpec' inside 'tuningConfig'"), + "Expected message about partitionsSpec in tuningConfig, got: " + exception.getMessage() + ); + EasyMock.verify(mockProvider); + } + + @Test + public void test_createCompactionJobs_simple() + { + DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); + SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); + ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); + CompactionJobParams mockParams = createMockParams(referenceTime, timeline); + DruidInputSource mockSource = createMockSource(); + + TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( + "testDS", null, null, mockProvider, null, null, null + ); + + template.createCompactionJobs(mockSource, mockParams); + List processedIntervals = template.getProcessedIntervals(); + + Assertions.assertEquals(2, processedIntervals.size()); + // Intervals are now in chronological order (oldest first) + Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); + Assertions.assertEquals(referenceTime.minusDays(7), processedIntervals.get(1).getEnd()); + + EasyMock.verify(mockProvider, mockParams, mockSource); + } + + @Test + public void test_createCompactionJobs_withSkipOffsetFromLatest_skipAllOfTime() + { + DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); + SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); + ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); + CompactionJobParams mockParams = createMockParams(referenceTime, timeline); + DruidInputSource mockSource = createMockSource(); + + TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( + "testDS", null, null, mockProvider, null, Period.days(100), null + ); + + List jobs = template.createCompactionJobs(mockSource, mockParams); + + Assertions.assertTrue(jobs.isEmpty()); + Assertions.assertTrue(template.getProcessedIntervals().isEmpty()); + + EasyMock.verify(mockProvider, mockParams, mockSource); + } + + @Test + public void test_createCompactionJobs_withSkipOffsetFromLatest_truncatesIntervalsExtendingPastSkipOffset() + { + DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); + SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); + ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); + CompactionJobParams mockParams = createMockParams(referenceTime, timeline); + DruidInputSource mockSource = createMockSource(); + + TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( + "testDS", null, null, mockProvider, null, Period.days(5), null + ); + + template.createCompactionJobs(mockSource, mockParams); + List processedIntervals = template.getProcessedIntervals(); + + Assertions.assertEquals(2, processedIntervals.size()); + Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); + Assertions.assertEquals(referenceTime.minusDays(15), processedIntervals.get(1).getEnd()); + + EasyMock.verify(mockProvider, mockParams, mockSource); + } + + @Test + public void test_createCompactionJobs_withSkipOffsetFromLatest_eliminatesInterval() + { + DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); + SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); + ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(3), Period.days(7), Period.days(30))); + CompactionJobParams mockParams = createMockParams(referenceTime, timeline); + DruidInputSource mockSource = createMockSource(); + + TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( + "testDS", null, null, mockProvider, null, Period.days(15), null + ); + + template.createCompactionJobs(mockSource, mockParams); + List processedIntervals = template.getProcessedIntervals(); + + Assertions.assertEquals(2, processedIntervals.size()); + Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); + Assertions.assertEquals(referenceTime.minusDays(25), processedIntervals.get(1).getEnd()); + + EasyMock.verify(mockProvider, mockParams, mockSource); + } + + @Test + public void test_createCompactionJobs_withSkipOffsetFromNow_skipAllOfTime() + { + DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); + SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); + ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); + CompactionJobParams mockParams = createMockParams(referenceTime, timeline); + DruidInputSource mockSource = createMockSource(); + + TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( + "testDS", null, null, mockProvider, null, null, Period.days(100) + ); + + List jobs = template.createCompactionJobs(mockSource, mockParams); + + Assertions.assertTrue(jobs.isEmpty()); + Assertions.assertTrue(template.getProcessedIntervals().isEmpty()); + + EasyMock.verify(mockProvider, mockParams, mockSource); + } + + @Test + public void test_createCompactionJobs_withSkipOffsetFromNow_truncatesIntervalThatExtendsPastSkipOffset() + { + DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); + SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); + ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); + CompactionJobParams mockParams = createMockParams(referenceTime, timeline); + DruidInputSource mockSource = createMockSource(); + + TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( + "testDS", null, null, mockProvider, null, null, Period.days(20) + ); + + template.createCompactionJobs(mockSource, mockParams); + List processedIntervals = template.getProcessedIntervals(); + + Assertions.assertEquals(2, processedIntervals.size()); + Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); + Assertions.assertEquals(referenceTime.minusDays(20), processedIntervals.get(1).getEnd()); + + EasyMock.verify(mockProvider, mockParams, mockSource); + } + + @Test + public void test_createCompactionJobs_withSkipOffsetFromNow_eliminatesInterval() + { + DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); + SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); + ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(3), Period.days(7), Period.days(30))); + CompactionJobParams mockParams = createMockParams(referenceTime, timeline); + DruidInputSource mockSource = createMockSource(); + + TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( + "testDS", null, null, mockProvider, null, null, Period.days(20) + ); + + template.createCompactionJobs(mockSource, mockParams); + List processedIntervals = template.getProcessedIntervals(); + + Assertions.assertEquals(2, processedIntervals.size()); + Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); + Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); + Assertions.assertEquals(referenceTime.minusDays(20), processedIntervals.get(1).getEnd()); + + EasyMock.verify(mockProvider, mockParams, mockSource); + } + + /** + * TEST: Basic timeline construction with multiple segment granularity rules + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P7D→HOUR, P1M→DAY, P3M→MONTH
    • + *
  • Other Rules: None
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: None created
    2. + *
  2. Initial Timeline: + *
      + *
    • P3M → MONTH: Raw 2024-10-29T16:15 → Aligned 2024-10-01T00:00
      • + *
    • P1M → DAY: Raw 2024-12-29T16:15 → Aligned 2024-12-29T00:00
      • + *
    • P7D → HOUR: Raw 2025-01-22T16:15 → Aligned 2025-01-22T16:00
      • + *
    + *
    3. + *
  3. Timeline Splits: None (no non-segment-gran rules)
    4. + *
+ *

+ * EXPECTED OUTPUT: 3 intervals + *

    + *
  1. [-∞, 2024-10-01T00:00:00) - MONTH
    2. + *
  2. [2024-10-01T00:00:00, 2024-12-29T00:00:00) - DAY
    3. + *
  3. [2024-12-29T00:00:00, 2025-01-22T16:00:00) - HOUR
    4. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_withGranularityAlignment() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingPartitioningRule hourRule = new ReindexingPartitioningRule("hour-rule", null, Period.days(7), Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null); + ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.months(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); + ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(3), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(hourRule, dayRule, monthRule)) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-10-01T00:00:00Z")), + monthRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-10-01T00:00:00Z"), DateTimes.of("2024-12-29T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-12-29T00:00:00Z"), DateTimes.of("2025-01-22T16:00:00Z")), + hourRule + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Timeline splitting by non-segment-granularity rules (metrics rules) + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P7D→HOUR, P1M→DAY, P3M→MONTH
    • + *
  • Other Rules: P8D-metrics, P14D-metrics, P45D-metrics, P100D-metrics
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: None (smallest segment gran rule P7D is finer than all metrics rules)
    2. + *
  2. Initial Timeline: [-∞, 2024-10-01) MONTH, [2024-10-01, 2024-12-29) DAY, [2024-12-29, 2025-01-22T16:00) HOUR
    3. + *
  3. Timeline Splits: + *
      + *
    • P100D → Raw 2024-10-21T16:15 → Falls in DAY interval → Aligned 2024-10-21T00:00 → CREATES SPLIT
      • + *
    • P45D → Raw 2024-12-15T16:15 → Falls in DAY interval → Aligned 2024-12-15T00:00 → CREATES SPLIT
      • + *
    • P14D → Raw 2025-01-15T16:15 → Falls in HOUR interval → Aligned 2025-01-15T16:00 → CREATES SPLIT
      • + *
    • P8D → Raw 2025-01-21T16:15 → Falls in HOUR interval → Aligned 2025-01-21T16:00 → CREATES SPLIT
      • + *
    + *
    4. + *
+ *

+ * EXPECTED OUTPUT: 7 intervals + *

    + *
  1. [-∞, 2024-10-01T00:00:00) - MONTH
    2. + *
  2. [2024-10-01T00:00:00, 2024-10-21T00:00:00) - DAY
    3. + *
  3. [2024-10-21T00:00:00, 2024-12-15T00:00:00) - DAY
    4. + *
  4. [2024-12-15T00:00:00, 2024-12-29T00:00:00) - DAY
    5. + *
  5. [2024-12-29T00:00:00, 2025-01-15T16:00:00) - HOUR
    6. + *
  6. [2025-01-15T16:00:00, 2025-01-21T16:00:00) - HOUR
    7. + *
  7. [2025-01-21T16:00:00, 2025-01-22T16:00:00) - HOUR
    8. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_withNonPartitioningRuleSplits() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingPartitioningRule hourRule = new ReindexingPartitioningRule("hour-rule", null, Period.days(7), Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null); + ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.months(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); + ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(3), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); + + // The data schema rules are here to trigger splits in the base timeline for granularity rules. + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(hourRule, dayRule, monthRule)) + .dataSchemaRules(List.of( + createReindexingDataSchemaRule("metrics-8d", Period.days(8)), + createReindexingDataSchemaRule("metrics-14d", Period.days(14)), + createReindexingDataSchemaRule("metrics-45d", Period.days(45)), + createReindexingDataSchemaRule("metrics-100d", Period.days(100)) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-10-01T00:00:00Z")), + monthRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-10-01T00:00:00Z"), DateTimes.of("2024-10-21T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-10-21T00:00:00Z"), DateTimes.of("2024-12-15T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-12-15T00:00:00Z"), DateTimes.of("2024-12-29T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-12-29T00:00:00Z"), DateTimes.of("2025-01-15T16:00:00Z")), + hourRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2025-01-15T16:00:00Z"), DateTimes.of("2025-01-21T16:00:00Z")), + hourRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2025-01-21T16:00:00Z"), DateTimes.of("2025-01-22T16:00:00Z")), + hourRule + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Timeline construction when NO segment granularity rules exist (Case A: default usage) + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: None
    • + *
  • Other Rules: P8D-metrics, P14D-metrics, P45D-metrics
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: Created P8D→DAY (Case A: no segment gran rules exist, use smallest rule period with default gran)
    2. + *
  2. Initial Timeline: [-∞, 2025-01-21T00:00) - DAY (from synthetic P8D rule)
    3. + *
  3. Timeline Splits: + *
      + *
    • P45D → Raw 2024-12-15T16:15 → Falls in DAY interval → Aligned 2024-12-15T00:00 → CREATES SPLIT
      • + *
    • P14D → Raw 2025-01-15T16:15 → Falls in DAY interval → Aligned 2025-01-15T00:00 → CREATES SPLIT
      • + *
    • P8D is now a segment gran rule (not processed as split)
      • + *
    + *
    4. + *
+ *

+ * EXPECTED OUTPUT: 3 intervals + *

    + *
  1. [-∞, 2024-12-15T00:00:00) - DAY
    2. + *
  2. [2024-12-15T00:00:00, 2025-01-15T00:00:00) - DAY
    3. + *
  3. [2025-01-15T00:00:00, 2025-01-21T00:00:00) - DAY
    4. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_withNoPartitioningRules() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .dataSchemaRules(List.of( + new ReindexingDataSchemaRule("metrics-8d", null, Period.days(8), null, new AggregatorFactory[0], null, null, null, null), + createReindexingDataSchemaRule("metrics-8d", Period.days(8)), + createReindexingDataSchemaRule("metrics-14d", Period.days(14)), + createReindexingDataSchemaRule("metrics-45d", Period.days(45)) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + // When no segment granularity rules exist, a synthetic rule is created with the smallest period + ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null + ); + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-12-15T00:00:00Z")), + syntheticRule, true + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-12-15T00:00:00Z"), DateTimes.of("2025-01-15T00:00:00Z")), + syntheticRule, true + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2025-01-15T00:00:00Z"), DateTimes.of("2025-01-21T00:00:00Z")), + syntheticRule, true + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Synthetic segment gran rule creation when rules are finer than smallest segment gran rule (Case B) + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P1M→DAY, P3M→MONTH
    • + *
  • Other Rules: P7D-metrics, P14D-metrics, P21D-metrics (all finer than P1M!)
    • + *
  • Default Segment Granularity: HOUR
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: Created P7D→HOUR (Case B: P7D/P14D/P21D are finer than smallest segment gran rule P1M, use finest with default gran)
    2. + *
  2. Initial Timeline: + *
      + *
    • P3M → MONTH: Raw 2024-10-29T16:15 → Aligned 2024-10-01T00:00
      • + *
    • P1M → DAY: Raw 2024-12-29T16:15 → Aligned 2024-12-29T00:00
      • + *
    • P7D → HOUR (synthetic): Raw 2025-01-22T16:15 → Aligned 2025-01-22T16:00 (PREPENDED interval!)
      • + *
    + *
    3. + *
  3. Timeline Splits: + *
      + *
    • P21D → Raw 2025-01-08T16:15 → Falls in HOUR interval → Aligned 2025-01-08T16:00 → CREATES SPLIT
      • + *
    • P14D → Raw 2025-01-15T16:15 → Falls in HOUR interval → Aligned 2025-01-15T16:00 → CREATES SPLIT
      • + *
    • P7D is now a segment gran rule (not processed as split)
      • + *
    + *
    4. + *
+ *

+ * EXPECTED OUTPUT: 5 intervals + *

    + *
  1. [-∞, 2024-10-01T00:00:00) - MONTH
    2. + *
  2. [2024-10-01T00:00:00, 2024-12-29T00:00:00) - DAY
    3. + *
  3. [2024-12-29T00:00:00, 2025-01-08T16:00:00) - HOUR (prepended)
    4. + *
  4. [2025-01-08T16:00:00, 2025-01-15T16:00:00) - HOUR (prepended)
    5. + *
  5. [2025-01-15T16:00:00, 2025-01-22T16:00:00) - HOUR (prepended)
    6. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_prependIntervalForShortNonPartitioningRules() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(3), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); + ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.months(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(monthRule, dayRule)) + .dataSchemaRules(List.of( + new ReindexingDataSchemaRule("metrics-7d", null, Period.days(7), null, new AggregatorFactory[0], null, null, null, null), + new ReindexingDataSchemaRule("metrics-14d", null, Period.days(14), null, new AggregatorFactory[0], null, null, null, null), + new ReindexingDataSchemaRule("metrics-21d", null, Period.days(21), null, new AggregatorFactory[0], null, null, null, null) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null + ); + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-10-01T00:00:00Z")), + monthRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-10-01T00:00:00Z"), DateTimes.of("2024-12-29T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-12-29T00:00:00Z"), DateTimes.of("2025-01-08T16:00:00Z")), + syntheticRule, true + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2025-01-08T16:00:00Z"), DateTimes.of("2025-01-15T16:00:00Z")), + syntheticRule, true + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2025-01-15T16:00:00Z"), DateTimes.of("2025-01-22T16:00:00Z")), + syntheticRule, true + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Comprehensive example demonstrating Case B, multiple segment gran rules, and timeline splits + *

+ * REFERENCE TIME: 2024-02-04T22:12:04.873Z (realistic messy timestamp) + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P1Y→YEAR, P1M→MONTH, P7D→DAY
    • + *
  • Other Rules: P1D-metrics, P14D-metrics, P45D-metrics (P1D is finer than P7D!)
    • + *
  • Default Segment Granularity: HOUR
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: Created P1D→HOUR (Case B: P1D is finer than smallest segment gran rule P7D)
    2. + *
  2. Initial Timeline: + *
      + *
    • P1Y → YEAR: Raw 2023-02-04T22:12:04.873 → Aligned 2023-01-01T00:00:00
      • + *
    • P1M → MONTH: Raw 2024-01-04T22:12:04.873 → Aligned 2024-01-01T00:00:00
      • + *
    • P7D → DAY: Raw 2024-01-28T22:12:04.873 → Aligned 2024-01-28T00:00:00
      • + *
    • P1D → HOUR (synthetic): Raw 2024-02-03T22:12:04.873 → Aligned 2024-02-03T22:00:00 (PREPENDED!)
      • + *
    + *
    3. + *
  3. Timeline Splits: + *
      + *
    • P45D → Raw 2023-12-21T22:12:04.873 → Falls in MONTH interval → Aligned 2023-12-01T00:00:00 → CREATES SPLIT
      • + *
    • P14D → Raw 2024-01-21T22:12:04.873 → Falls in DAY interval → Aligned 2024-01-21T00:00:00 → CREATES SPLIT
      • + *
    • P1D is now a segment gran rule (not processed as split)
      • + *
    + *
    4. + *
+ *

+ * EXPECTED OUTPUT: 6 intervals + *

    + *
  1. [-∞, 2023-01-01T00:00:00) - YEAR
    2. + *
  2. [2023-01-01T00:00:00, 2023-12-01T00:00:00) - MONTH
    3. + *
  3. [2023-12-01T00:00:00, 2024-01-01T00:00:00) - MONTH
    4. + *
  4. [2024-01-01T00:00:00, 2024-01-21T00:00:00) - DAY
    5. + *
  5. [2024-01-21T00:00:00, 2024-01-28T00:00:00) - DAY
    6. + *
  6. [2024-01-28T00:00:00, 2024-02-03T22:00:00) - HOUR (prepended, note non-midnight end)
    7. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals() + { + DateTime referenceTime = DateTimes.of("2024-02-04T22:12:04.873Z"); + + ReindexingPartitioningRule yearRule = new ReindexingPartitioningRule("year-rule", null, Period.years(1), Granularities.YEAR, new DynamicPartitionsSpec(5000000, null), null); + ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(1), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); + ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.days(7), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); + + ReindexingRuleProvider provider = + InlineReindexingRuleProvider + .builder() + .partitioningRules(List.of(yearRule, monthRule, dayRule)) + .dataSchemaRules(List.of( + new ReindexingDataSchemaRule("metrics-1d", null, Period.days(1), null, new AggregatorFactory[0], null, null, null, null), + new ReindexingDataSchemaRule("metrics-14d", null, Period.days(14), null, new AggregatorFactory[0], null, null, null, null), + new ReindexingDataSchemaRule("metrics-45d", null, Period.days(45), null, new AggregatorFactory[0], null, null, null, null) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null + ); + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2023-01-01T00:00:00Z")), + yearRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2023-01-01T00:00:00Z"), DateTimes.of("2023-12-01T00:00:00Z")), + monthRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2023-12-01T00:00:00Z"), DateTimes.of("2024-01-01T00:00:00Z")), + monthRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-01-01T00:00:00Z"), DateTimes.of("2024-01-21T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-01-21T00:00:00Z"), DateTimes.of("2024-01-28T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-01-28T00:00:00"), DateTimes.of("2024-02-03T22:00:00")), + syntheticRule, true + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: No rules at all - should throw IAE + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: None
    • + *
  • Other Rules: None
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * EXPECTED: IllegalArgumentException with message "requires at least one reindexing rule" + */ + @Test + public void test_generateAlignedSearchIntervals_noRulesThrowsException() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder().build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + DruidException exception = Assertions.assertThrows( + DruidException.class, + () -> template.generateAlignedSearchIntervals(referenceTime) + ); + + Assertions.assertTrue( + exception.getMessage().contains("requires at least one reindexing rule") + ); + } + + /** + * TEST: Split point aligns exactly to existing boundary (boundary snapping, no split created) + *

+ * REFERENCE TIME: 2025-02-01T00:00:00Z (carefully chosen for alignment) + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P1M→MONTH
    • + *
  • Other Rules: P1M-metrics (same period as segment gran rule!)
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: None
    2. + *
  2. Initial Timeline: [-∞, 2025-01-01T00:00:00) - MONTH
    3. + *
  3. Timeline Splits: + *
      + *
    • P1M metrics → Raw 2025-01-01T00:00:00 → Aligned to MONTH: 2025-01-01T00:00:00
      • + *
    • This aligns EXACTLY to the existing boundary → no split created
      • + *
    + *
    4. + *
+ *

+ * EXPECTED OUTPUT: 1 interval (no split despite having a non-segment-gran rule) + *

    + *
  1. [-∞, 2025-01-01T00:00:00) - MONTH
    2. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_splitPointSnapsToExistingBoundary() + { + DateTime referenceTime = DateTimes.of("2025-02-01T00:00:00Z"); + + ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(1), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(monthRule)) + .dataSchemaRules(List.of( + new ReindexingDataSchemaRule("metrics-1m", null, Period.months(1), null, new AggregatorFactory[0], null, null, null, null) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2025-01-01T00:00:00Z")), + monthRule + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Prepending that aligns back to last segment gran rule interval end (no prepend actually created) + *

+ * REFERENCE TIME: 2025-01-01T01:00:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P1D→DAY
    • + *
  • Other Rules: PT12H-metrics (finer than P1D, but aligns back to same interval end as the P1D rule so no prepend is done)
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Initial Timeline: [-∞, 2024-12-31T00:00:00) - DAY
    2. + *
  2. Check for prepending: + *
      + *
    • PT12H threshold: 2024-12-31T13:00:00
      • + *
    • Align to DAY (default gran): 2024-12-31T00:00:00
      • + *
    • This EQUALS the most recent segment gran rule end (2024-12-31T00:00:00) → NO PREPEND
      • + *
    + *
    3. + *
+ *

+ * EXPECTED OUTPUT: 1 interval (no split prepend despite having a finer non-segment-gran rule) + *

    + *
  1. [-∞, 2024-12-31T00:00:00) - DAY
    2. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_prependAlignmentDoesNotExtendTimeline() + { + DateTime referenceTime = DateTimes.of("2025-01-01T01:00:00Z"); + + ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.days(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(dayRule)) + .dataSchemaRules(List.of( + new ReindexingDataSchemaRule("metrics-12h", null, Period.hours(12), null, new AggregatorFactory[0], null, null, null, null) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-12-31T00:00:00Z")), + dayRule + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Multiple split points align to same timestamp (distinct() filtering removes duplicates) + *

+ * REFERENCE TIME: 2025-01-15T00:00:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P1M→DAY
    • + *
  • Other Rules: P23D+6h-metrics, P23D+18h-metrics (both align to same DAY boundary in DAY interval)
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: None
    2. + *
  2. Initial Timeline: [-∞, 2024-12-15T00:00:00) - DAY
    3. + *
  3. Timeline Splits: + *
      + *
    • P33D+6h → Raw: 2024-12-12T18:00:00 → Falls in DAY interval → Align to DAY: 2024-12-12T00:00:00
      • + *
    • P33D+18h → Raw: 2024-12-12T06:00:00 → Falls in DAY interval → Align to DAY: 2024-12-12T00:00:00
      • + *
    • Both create split point 2024-12-12T00:00:00 → distinct() removes duplicate!
      • + *
    • Only ONE split created at 2024-12-12T00:00:00
      • + *
    + *
    4. + *
+ *

+ * EXPECTED OUTPUT: 2 intervals (not 3, because duplicate split point was filtered) + *

    + *
  1. [-∞, 2024-12-12T00:00:00) - DAY
    2. + *
  2. [2024-12-12T00:00:00, 2024-12-15T00:00:00) - DAY
    3. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_duplicateSplitPointsFiltered() + { + DateTime referenceTime = DateTimes.of("2025-01-15T00:00:00Z"); + + ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.months(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(dayRule)) + .dataSchemaRules(List.of( + new ReindexingDataSchemaRule("metrics-33d-6h", null, Period.hours(33 * 24 + 6), null, new AggregatorFactory[0], null, null, null, null), + new ReindexingDataSchemaRule("metrics-33d-18h", null, Period.hours(33 * 24 + 18), null, new AggregatorFactory[0], null, null, null, null) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-12-12T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-12-12T00:00:00Z"), DateTimes.of("2024-12-15T00:00:00Z")), + dayRule + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Single rule only (minimal valid case) + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P1M→MONTH
    • + *
  • Other Rules: None
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: None
    2. + *
  2. Initial Timeline: [-∞, 2024-12-01T00:00:00) - MONTH
    3. + *
  3. Timeline Splits: None (no non-segment-gran rules)
    4. + *
+ *

+ * EXPECTED OUTPUT: 1 interval + *

    + *
  1. [-∞, 2024-12-01T00:00:00) - MONTH
    2. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_singleRuleOnly() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(1), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(monthRule)) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-12-01T00:00:00Z")), + monthRule + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Zero period rule (P0D) applies immediately to all data + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P0D→HOUR (applies to all data immediately)
    • + *
  • Other Rules: None
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: None
    2. + *
  2. Initial Timeline: [-∞, 2025-01-29T16:00:00) - HOUR (P0D means threshold equals reference time)
    3. + *
  3. Timeline Splits: None (no non-segment-gran rules)
    4. + *
+ *

+ * EXPECTED OUTPUT: 1 interval + *

    + *
  1. [-∞, 2025-01-29T16:00:00) - HOUR (aligned to hour boundary at reference time)
    2. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_zeroPeriodRuleAppliesImmediately() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingPartitioningRule hourRule = new ReindexingPartitioningRule( + "immediate-hour-rule", + "Apply HOUR granularity to all data immediately", + Period.days(0), + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null + ); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(hourRule)) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2025-01-29T16:00:00Z")), + hourRule + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Zero period rule (P0D) with other rules creates proper cascading timeline + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P0D→HOUR, P30D→DAY, P90D→MONTH
    • + *
  • Other Rules: None
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Synthetic Rules: None
    2. + *
  2. Sort rules by period: P90D (oldest), P30D (middle), P0D (newest/applies immediately)
    3. + *
  3. Initial Timeline: + *
      + *
    • P90D → MONTH: Raw 2024-10-31T16:15 → Aligned 2024-10-01T00:00
      • + *
    • P30D → DAY: Raw 2024-12-30T16:15 → Aligned 2024-12-30T00:00
      • + *
    • P0D → HOUR: Raw 2025-01-29T16:15 → Aligned 2025-01-29T16:00
      • + *
    + *
    4. + *
  4. Timeline Splits: None (no non-segment-gran rules)
    5. + *
+ *

+ * EXPECTED OUTPUT: 3 intervals + *

    + *
  1. [-∞, 2024-10-01T00:00:00) - MONTH
    2. + *
  2. [2024-10-01T00:00:00, 2024-12-30T00:00:00) - DAY
    3. + *
  3. [2024-12-30T00:00:00, 2025-01-29T16:00:00) - HOUR
    4. + *
+ */ + @Test + public void test_generateAlignedSearchIntervals_zeroPeriodRuleWithOtherRules() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule( + "month-rule", + null, + Period.days(90), + Granularities.MONTH, + new DynamicPartitionsSpec(5000000, null), + null + ); + ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule( + "day-rule", + null, + Period.days(30), + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null + ); + ReindexingPartitioningRule hourRule = new ReindexingPartitioningRule( + "immediate-hour-rule", + "Apply HOUR granularity immediately", + Period.days(0), + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null + ); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(hourRule, dayRule, monthRule)) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-10-01T00:00:00Z")), + monthRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-10-01T00:00:00Z"), DateTimes.of("2024-12-30T00:00:00Z")), + dayRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-12-30T00:00:00Z"), DateTimes.of("2025-01-29T16:00:00Z")), + hourRule + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + } + + /** + * TEST: Validation failure - default granularity is coarser than most recent segment granularity rule + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P30D→HOUR, P90D→DAY
    • + *
  • Other Rules: P7D-metrics (finer than P30D, triggers prepending with default granularity)
    • + *
  • Default Segment Granularity: MONTH (COARSER than HOUR!)
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Sort rules by period: P90D→DAY (oldest), P30D→HOUR (newest)
    2. + *
  2. P7D metrics is finer than P30D, so prepend interval with default MONTH granularity
    3. + *
  3. Timeline would be: [-∞, DAY_boundary) DAY, [DAY_boundary, HOUR_boundary) HOUR, [HOUR_boundary, MONTH_boundary) MONTH
    4. + *
  4. Validation: HOUR → MONTH progression means granularity is getting COARSER toward present
    5. + *
+ *

+ * EXPECTED: IllegalArgumentException with message about invalid granularity timeline + */ + @Test + public void test_generateAlignedSearchIntervals_failsWhenDefaultGranularityIsCoarserThanMostRecentPartitioningRule() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingRuleProvider provider = + InlineReindexingRuleProvider + .builder() + .partitioningRules(List.of( + new ReindexingPartitioningRule("hour-rule", null, Period.days(30), Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null), + new ReindexingPartitioningRule("day-rule", null, Period.days(90), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null) + )) + .dataSchemaRules(List.of( + new ReindexingDataSchemaRule("metrics-7d", null, Period.days(7), null, new AggregatorFactory[0], null, null, null, null) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.MONTH, // MONTH is coarser than HOUR! + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> template.generateAlignedSearchIntervals(referenceTime) + ); + + Assertions.assertTrue( + exception.getMessage().contains("Invalid segment granularity timeline") + ); + Assertions.assertTrue( + exception.getMessage().contains("coarser granularity") + ); + } + + /** + * TEST: Validation failure - older rule has finer granularity than newer rule + *

+ * REFERENCE TIME: 2025-01-29T16:15:00Z + *

+ * INPUT RULES: + *

    + *
  • Segment Granularity Rules: P30D→DAY, P90D→HOUR
    • + *
  • Other Rules: None
    • + *
  • Default Segment Granularity: DAY
    • + *
+ *

+ * PROCESSING: + *

    + *
  1. Sort rules by period: P90D→HOUR (oldest), P30D→DAY (newest)
    2. + *
  2. Timeline would be: [-∞, HOUR_boundary) HOUR, [HOUR_boundary, DAY_boundary) DAY
    3. + *
  3. Validation: HOUR → DAY progression means granularity is getting COARSER toward present
    4. + *
  4. This violates the constraint: older data (P90D) has HOUR granularity, newer data (P30D) has DAY granularity
    5. + *
+ *

+ * EXPECTED: IllegalArgumentException with message about invalid granularity timeline + */ + @Test + public void test_generateAlignedSearchIntervals_failsWhenOlderRuleHasFinerGranularityThanNewerRule() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + ReindexingRuleProvider provider = + InlineReindexingRuleProvider + .builder() + .partitioningRules(List.of( + new ReindexingPartitioningRule("day-rule", null, Period.days(30), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null), + new ReindexingPartitioningRule("hour-rule", null, Period.days(90), Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + null, + null + ); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> template.generateAlignedSearchIntervals(referenceTime) + ); + + Assertions.assertTrue( + exception.getMessage().contains("Invalid segment granularity timeline") + ); + Assertions.assertTrue( + exception.getMessage().contains("coarser granularity") + ); + } + + /** + * TEST: defaultPartitioningVirtualColumns flows into synthetic rules via createDefaultGranularityTimeline() + *

+ * When no partitioning rules exist, the synthetic rule should carry the default partitioning VCs. + */ + @Test + public void test_generateAlignedSearchIntervals_defaultPartitioningVirtualColumnsFlowIntoSyntheticRule() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + VirtualColumns defaultVCs = VirtualColumns.create(List.of( + new ExpressionVirtualColumn("vc_bucket", "timestamp_floor(__time, 'P1D')", ColumnType.LONG, TestExprMacroTable.INSTANCE) + )); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .dataSchemaRules(List.of( + createReindexingDataSchemaRule("metrics-14d", Period.days(14)) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + defaultVCs, + null + ); + + ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + defaultVCs + ); + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2025-01-15T00:00:00Z")), + syntheticRule, true + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + + // Also verify the VCs are accessible through the IntervalPartitioningInfo + Assertions.assertEquals(defaultVCs, actual.get(0).getVirtualColumns()); + } + + /** + * TEST: defaultPartitioningVirtualColumns flows into prepended synthetic rules via maybePrependRecentInterval() + *

+ * When partitioning rules exist but non-partitioning rules extend further, the prepended synthetic rule + * should carry the default partitioning VCs, while real rule entries should not. + */ + @Test + public void test_generateAlignedSearchIntervals_defaultPartitioningVirtualColumnsFlowIntoPrependedRule() + { + DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); + + VirtualColumns defaultVCs = VirtualColumns.create(List.of( + new ExpressionVirtualColumn("vc_bucket", "timestamp_floor(__time, 'P1D')", ColumnType.LONG, TestExprMacroTable.INSTANCE) + )); + + ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule( + "month-rule", null, Period.months(1), Granularities.MONTH, + new DynamicPartitionsSpec(5000000, null), null + ); + + ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() + .partitioningRules(List.of(monthRule)) + .dataSchemaRules(List.of( + createReindexingDataSchemaRule("metrics-7d", Period.days(7)) + )) + .build(); + + CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDS", + null, + null, + provider, + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + defaultVCs, + null + ); + + // P1M threshold: 2025-01-29 - P1M = 2024-12-29T16:15 → aligned to MONTH → 2024-12-01T00:00 + ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + defaultVCs + ); + List expected = List.of( + new IntervalPartitioningInfo( + new Interval(DateTimes.MIN, DateTimes.of("2024-12-01T00:00:00Z")), + monthRule + ), + new IntervalPartitioningInfo( + new Interval(DateTimes.of("2024-12-01T00:00:00Z"), DateTimes.of("2025-01-22T00:00:00Z")), + syntheticRule, true + ) + ); + + List actual = template.generateAlignedSearchIntervals(referenceTime); + Assertions.assertEquals(expected, actual); + + // Real rule entry should NOT have the default VCs + Assertions.assertNull(actual.get(0).getVirtualColumns()); + // Prepended synthetic entry should have the default VCs + Assertions.assertEquals(defaultVCs, actual.get(1).getVirtualColumns()); + } + + /** + * TEST: Serde round-trip with non-null defaultPartitioningVirtualColumns + */ + @Test + public void test_serde_withDefaultPartitioningVirtualColumns() throws Exception + { + VirtualColumns defaultVCs = VirtualColumns.create(List.of( + new ExpressionVirtualColumn("vc_bucket", "timestamp_floor(__time, 'P1D')", ColumnType.LONG, TestExprMacroTable.INSTANCE) + )); + + final CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDataSource", + 50, + 1000000L, + InlineReindexingRuleProvider.builder() + .partitioningRules(List.of( + new ReindexingPartitioningRule( + "hourRule", null, Period.days(7), Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), null + ) + )) + .build(), + ImmutableMap.of("context_key", "context_value"), + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), + defaultVCs, + null + ); + + // Need ExprMacroTable injectable for VirtualColumn deserialization + ObjectMapper mapper = new DefaultObjectMapper(); + mapper.registerModules(new SupervisorModule().getJacksonModules()); + mapper.setInjectableValues( + new InjectableValues.Std() + .addValue(ExprMacroTable.class, TestExprMacroTable.INSTANCE) + ); + + final String json = mapper.writeValueAsString(template); + + // Verify VCs are present in the JSON + Assertions.assertTrue(json.contains("defaultPartitioningVirtualColumns")); + Assertions.assertTrue(json.contains("vc_bucket")); + + // Verify round-trip deserialization + final CascadingReindexingTemplate fromJson = mapper.readValue(json, CascadingReindexingTemplate.class); + Assertions.assertEquals(template.getDataSource(), fromJson.getDataSource()); + Assertions.assertEquals(template.getTaskPriority(), fromJson.getTaskPriority()); + Assertions.assertEquals(template.getInputSegmentSizeBytes(), fromJson.getInputSegmentSizeBytes()); + Assertions.assertEquals( + template.getDefaultPartitioningVirtualColumns(), + fromJson.getDefaultPartitioningVirtualColumns() + ); + } + + @Test + public void test_validate_returnsValid_withDynamicPartitionsSpec() + { + final CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDataSource", + null, + null, + InlineReindexingRuleProvider.builder().build(), + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(null, null), + null, + null + ); + + CompactionConfigValidationResult result = template.validate(CLUSTER_CONFIG); + Assertions.assertTrue(result.isValid()); + } + + @Test + public void test_validate_returnsInvalid_withHashedPartitionsSpec() + { + final CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDataSource", + null, + null, + InlineReindexingRuleProvider.builder().build(), + null, + null, + null, + Granularities.DAY, + new HashedPartitionsSpec(null, 3, null), + null, + null + ); + + CompactionConfigValidationResult result = template.validate(CLUSTER_CONFIG); + Assertions.assertFalse(result.isValid()); + Assertions.assertEquals( + "MSQ: Invalid partitioning type[HashedPartitionsSpec]. Must be either 'dynamic' or 'range'", + result.getReason() + ); + } + + @Test + public void test_validate_returnsInvalid_withMaxTotalRows() + { + final CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDataSource", + null, + null, + InlineReindexingRuleProvider.builder().build(), + null, + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(null, 1000L), + null, + null + ); + + CompactionConfigValidationResult result = template.validate(CLUSTER_CONFIG); + Assertions.assertFalse(result.isValid()); + Assertions.assertEquals( + "MSQ: 'maxTotalRows' not supported with 'dynamic' partitioning", + result.getReason() + ); + } + + @Test + public void test_validate_returnsInvalid_withOneMaxNumTasks() + { + final CascadingReindexingTemplate template = new CascadingReindexingTemplate( + "testDataSource", + null, + null, + InlineReindexingRuleProvider.builder().build(), + Collections.singletonMap(ClientMSQContext.CTX_MAX_NUM_TASKS, 1), + null, + null, + Granularities.DAY, + new DynamicPartitionsSpec(null, null), + null, + null + ); + + CompactionConfigValidationResult result = template.validate(CLUSTER_CONFIG); + Assertions.assertFalse(result.isValid()); + Assertions.assertEquals( + "MSQ: Context maxNumTasks[1] must be at least 2 (1 controller + 1 worker)", + result.getReason() + ); + } + + private static class TestCascadingReindexingTemplate extends CascadingReindexingTemplate + { + // Capture intervals that were processed for assertions + private final List processedIntervals = new ArrayList<>(); + + public TestCascadingReindexingTemplate( + String dataSource, + Integer taskPriority, + Long inputSegmentSizeBytes, + ReindexingRuleProvider ruleProvider, + Map taskContext, + Period skipOffsetFromLatest, + Period skipOffsetFromNow + ) + { + super(dataSource, taskPriority, inputSegmentSizeBytes, ruleProvider, + taskContext, skipOffsetFromLatest, skipOffsetFromNow, Granularities.DAY, + new DynamicPartitionsSpec(5000000, null), null, null + ); + } + + public List getProcessedIntervals() + { + return processedIntervals; + } + + @Override + protected CompactionJobTemplate createJobTemplateForInterval( + InlineSchemaDataSourceCompactionConfig config + ) + { + return new CompactionJobTemplate() { + @Override + public String getType() + { + return "test"; + } + + @Override + @Nullable + public Granularity getSegmentGranularity() + { + return null; + } + + @Override + public List createCompactionJobs( + DruidInputSource source, + CompactionJobParams params + ) + { + // Record the interval that was processed + processedIntervals.add(source.getInterval()); + + // Return a single mock job + return List.of(); + } + }; + } + } + + private SegmentTimeline createTestTimeline(DateTime start, DateTime end) + { + DataSegment segment = DataSegment.builder() + .dataSource("testDS") + .interval(new Interval(start, end)) + .version("v1") + .size(1000) + .build(); + return SegmentTimeline.forSegments(Collections.singletonList(segment)); + } + + private ReindexingRuleProvider createMockProvider(List periods) + { + // Create partitioning rules for each period + List partitioningRules = new ArrayList<>(); + for (int i = 0; i < periods.size(); i++) { + partitioningRules.add(new ReindexingPartitioningRule( + "segment-gran-rule-" + i, + null, + periods.get(i), + Granularities.HOUR, + new DynamicPartitionsSpec(5000000, null), + null + )); + } + + ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); + EasyMock.expect(mockProvider.isReady()).andReturn(true); + EasyMock.expect(mockProvider.getPartitioningRules()).andReturn(partitioningRules).anyTimes(); + // Return a fresh stream on each call to avoid "stream has already been operated upon or closed" errors + EasyMock.expect(mockProvider.streamAllRules()).andAnswer(() -> partitioningRules.stream().map(r -> (ReindexingRule) r)).anyTimes(); + EasyMock.expect(mockProvider.getPartitioningRule(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(partitioningRules.get(0)).anyTimes(); + EasyMock.expect(mockProvider.getIndexSpecRule(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(null).anyTimes(); + EasyMock.expect(mockProvider.getDataSchemaRule(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(null).anyTimes(); + EasyMock.expect(mockProvider.getDeletionRules(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.emptyList()).anyTimes(); + EasyMock.replay(mockProvider); + return mockProvider; + } + + private CompactionJobParams createMockParams(DateTime referenceTime, SegmentTimeline timeline) + { + CompactionJobParams mockParams = EasyMock.createMock(CompactionJobParams.class); + EasyMock.expect(mockParams.getScheduleStartTime()).andReturn(referenceTime).anyTimes(); + EasyMock.expect(mockParams.getTimeline("testDS")).andReturn(timeline); + EasyMock.replay(mockParams); + return mockParams; + } + + private DruidInputSource createMockSource() + { + final Interval[] capturedInterval = new Interval[1]; + + DruidInputSource mockSource = EasyMock.createMock(DruidInputSource.class); + EasyMock.expect(mockSource.withInterval(EasyMock.anyObject(Interval.class))) + .andAnswer(() -> { + capturedInterval[0] = (Interval) EasyMock.getCurrentArguments()[0]; + return mockSource; + }) + .anyTimes(); + EasyMock.expect(mockSource.getInterval()) + .andAnswer(() -> capturedInterval[0]) + .anyTimes(); + EasyMock.replay(mockSource); + return mockSource; + } + + /** + * Helper method to create a ReindexingDataSchemaRule with minimal required fields for testing + *

+ * Helps quickly generate multiple rules to be used in testing formation of timelines and splits. + */ + private ReindexingDataSchemaRule createReindexingDataSchemaRule(String name, Period period) + { + return new ReindexingDataSchemaRule( + name, + null, + period, + null, + new AggregatorFactory[0], + null, + null, + null, + null + ); + } +} From 6b89dc5d75c062a19acc7f4997cff7f2dad2bc7e Mon Sep 17 00:00:00 2001 From: Ashwin Tumma Date: Mon, 29 Jun 2026 08:34:27 -0700 Subject: [PATCH 4/5] Remove backup file --- .../CascadingReindexingTemplateTest.java.bak | 2022 ----------------- 1 file changed, 2022 deletions(-) delete mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java.bak diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java.bak b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java.bak deleted file mode 100644 index af8d732311e3..000000000000 --- a/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java.bak +++ /dev/null @@ -1,2022 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.indexing.compact; - -import com.fasterxml.jackson.databind.InjectableValues; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableMap; -import org.apache.druid.client.indexing.ClientMSQContext; -import org.apache.druid.error.DruidException; -import org.apache.druid.guice.SupervisorModule; -import org.apache.druid.indexer.CompactionEngine; -import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; -import org.apache.druid.indexer.partitions.HashedPartitionsSpec; -import org.apache.druid.indexing.input.DruidInputSource; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.math.expr.ExprMacroTable; -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.expression.TestExprMacroTable; -import org.apache.druid.segment.VirtualColumns; -import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.virtual.ExpressionVirtualColumn; -import org.apache.druid.server.compaction.InlineReindexingRuleProvider; -import org.apache.druid.server.compaction.IntervalPartitioningInfo; -import org.apache.druid.server.compaction.ReindexingDataSchemaRule; -import org.apache.druid.server.compaction.ReindexingPartitioningRule; -import org.apache.druid.server.compaction.ReindexingRule; -import org.apache.druid.server.compaction.ReindexingRuleProvider; -import org.apache.druid.server.coordinator.ClusterCompactionConfig; -import org.apache.druid.server.coordinator.CompactionConfigValidationResult; -import org.apache.druid.server.coordinator.DataSourceCompactionConfig; -import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig; -import org.apache.druid.testing.InitializedNullHandlingTest; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentTimeline; -import org.easymock.EasyMock; -import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.joda.time.Period; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class CascadingReindexingTemplateTest extends InitializedNullHandlingTest -{ - private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); - private static final ClusterCompactionConfig CLUSTER_CONFIG = - new ClusterCompactionConfig(null, null, null, null, null, null); - - @BeforeEach - public void setUp() - { - OBJECT_MAPPER.registerModules(new SupervisorModule().getJacksonModules()); - } - - @Test - public void test_serde() throws Exception - { - final CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDataSource", - 50, - 1000000L, - InlineReindexingRuleProvider.builder() - .partitioningRules(List.of( - new ReindexingPartitioningRule( - "hourRule", - null, - Period.days(7), - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null - ), - new ReindexingPartitioningRule( - "dayRule", - null, - Period.days(30), - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null - ) - )) - .build(), - ImmutableMap.of("context_key", "context_value"), - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - final String json = OBJECT_MAPPER.writeValueAsString(template); - final CascadingReindexingTemplate fromJson = OBJECT_MAPPER.readValue(json, CascadingReindexingTemplate.class); - - Assertions.assertEquals(template.getDataSource(), fromJson.getDataSource()); - Assertions.assertEquals(template.getTaskPriority(), fromJson.getTaskPriority()); - Assertions.assertEquals(template.getInputSegmentSizeBytes(), fromJson.getInputSegmentSizeBytes()); - Assertions.assertEquals(template.getEngine(), fromJson.getEngine()); - Assertions.assertEquals(template.getTaskContext(), fromJson.getTaskContext()); - Assertions.assertEquals(template.getType(), fromJson.getType()); - } - - @Test - public void test_serde_asDataSourceCompactionConfig() throws Exception - { - final CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDataSource", - 30, - 500000L, - InlineReindexingRuleProvider.builder() - .partitioningRules(List.of( - new ReindexingPartitioningRule( - "rule1", - null, - Period.days(7), - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null - ) - )) - .build(), - ImmutableMap.of("key", "value"), - null, - null, - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - // Serialize and deserialize as DataSourceCompactionConfig interface - final String json = OBJECT_MAPPER.writeValueAsString(template); - final DataSourceCompactionConfig fromJson = OBJECT_MAPPER.readValue(json, DataSourceCompactionConfig.class); - - Assertions.assertTrue(fromJson instanceof CascadingReindexingTemplate); - final CascadingReindexingTemplate cascadingFromJson = (CascadingReindexingTemplate) fromJson; - - Assertions.assertEquals("testDataSource", cascadingFromJson.getDataSource()); - Assertions.assertEquals(30, cascadingFromJson.getTaskPriority()); - Assertions.assertEquals(500000L, cascadingFromJson.getInputSegmentSizeBytes()); - Assertions.assertEquals(CompactionEngine.MSQ, cascadingFromJson.getEngine()); - Assertions.assertEquals(ImmutableMap.of("key", "value"), cascadingFromJson.getTaskContext()); - Assertions.assertEquals(CascadingReindexingTemplate.TYPE, cascadingFromJson.getType()); - } - - @Test - public void test_createCompactionJobs_ruleProviderNotReady() - { - final ReindexingRuleProvider notReadyProvider = EasyMock.createMock(ReindexingRuleProvider.class); - EasyMock.expect(notReadyProvider.isReady()).andReturn(false); - EasyMock.expect(notReadyProvider.getType()).andReturn("mock-provider"); - EasyMock.replay(notReadyProvider); - - final CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDataSource", - null, - null, - notReadyProvider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - // Call createCompactionJobs - should return empty list without processing - final List jobs = template.createCompactionJobs(null, null); - - Assertions.assertTrue(jobs.isEmpty()); - EasyMock.verify(notReadyProvider); - } - - @Test - public void test_constructor_setBothSkipOffsetStrategiesThrowsException() - { - final ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); - EasyMock.replay(mockProvider); - - DruidException exception = Assertions.assertThrows( - DruidException.class, - () -> new CascadingReindexingTemplate( - "testDataSource", - null, - null, - mockProvider, - null, - Period.days(7), // skipOffsetFromLatest - Period.days(3), // skipOffsetFromNow - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ) - ); - - Assertions.assertEquals("Cannot set both skipOffsetFromNow and skipOffsetFromLatest", exception.getMessage()); - EasyMock.verify(mockProvider); - } - - @Test - public void test_constructor_nullDataSourceThrowsException() - { - final ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); - EasyMock.replay(mockProvider); - - DruidException exception = Assertions.assertThrows( - DruidException.class, - () -> new CascadingReindexingTemplate( - null, // null dataSource - null, - null, - mockProvider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ) - ); - - Assertions.assertTrue(exception.getMessage().contains("'dataSource' cannot be null")); - EasyMock.verify(mockProvider); - } - - @Test - public void test_constructor_nullRuleProviderThrowsException() - { - DruidException exception = Assertions.assertThrows( - DruidException.class, - () -> new CascadingReindexingTemplate( - "testDataSource", - null, - null, - null, // null ruleProvider - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ) - ); - - Assertions.assertTrue(exception.getMessage().contains("'ruleProvider' cannot be null")); - } - - @Test - public void test_constructor_nullDefaultSegmentGranularityThrowsException() - { - final ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); - EasyMock.replay(mockProvider); - - DruidException exception = Assertions.assertThrows( - DruidException.class, - () -> new CascadingReindexingTemplate( - "testDataSource", - null, - null, - mockProvider, - null, - null, - null, - null, // null defaultSegmentGranularity - new DynamicPartitionsSpec(5000000, null), - null, - null - ) - ); - - Assertions.assertTrue(exception.getMessage().contains("'defaultSegmentGranularity' cannot be null")); - EasyMock.verify(mockProvider); - } - - @Test - public void test_constructor_tuningConfigWithPartitionsSpecThrowsException() - { - final ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); - EasyMock.replay(mockProvider); - - UserCompactionTaskQueryTuningConfig tuningWithPartitionsSpec = UserCompactionTaskQueryTuningConfig.builder() - .partitionsSpec(new DynamicPartitionsSpec(1000, null)) - .build(); - - DruidException exception = Assertions.assertThrows( - DruidException.class, - () -> new CascadingReindexingTemplate( - "testDataSource", - null, - null, - mockProvider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - tuningWithPartitionsSpec - ) - ); - - Assertions.assertTrue( - exception.getMessage().contains("Cannot set 'partitionsSpec' inside 'tuningConfig'"), - "Expected message about partitionsSpec in tuningConfig, got: " + exception.getMessage() - ); - EasyMock.verify(mockProvider); - } - - @Test - public void test_createCompactionJobs_simple() - { - DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); - SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); - ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); - CompactionJobParams mockParams = createMockParams(referenceTime, timeline); - DruidInputSource mockSource = createMockSource(); - - TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( - "testDS", null, null, mockProvider, null, null, null - ); - - template.createCompactionJobs(mockSource, mockParams); - List processedIntervals = template.getProcessedIntervals(); - - Assertions.assertEquals(2, processedIntervals.size()); - // Intervals are now in chronological order (oldest first) - Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); - Assertions.assertEquals(referenceTime.minusDays(7), processedIntervals.get(1).getEnd()); - - EasyMock.verify(mockProvider, mockParams, mockSource); - } - - @Test - public void test_createCompactionJobs_withSkipOffsetFromLatest_skipAllOfTime() - { - DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); - SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); - ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); - CompactionJobParams mockParams = createMockParams(referenceTime, timeline); - DruidInputSource mockSource = createMockSource(); - - TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( - "testDS", null, null, mockProvider, null, Period.days(100), null - ); - - List jobs = template.createCompactionJobs(mockSource, mockParams); - - Assertions.assertTrue(jobs.isEmpty()); - Assertions.assertTrue(template.getProcessedIntervals().isEmpty()); - - EasyMock.verify(mockProvider, mockParams, mockSource); - } - - @Test - public void test_createCompactionJobs_withSkipOffsetFromLatest_truncatesIntervalsExtendingPastSkipOffset() - { - DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); - SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); - ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); - CompactionJobParams mockParams = createMockParams(referenceTime, timeline); - DruidInputSource mockSource = createMockSource(); - - TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( - "testDS", null, null, mockProvider, null, Period.days(5), null - ); - - template.createCompactionJobs(mockSource, mockParams); - List processedIntervals = template.getProcessedIntervals(); - - Assertions.assertEquals(2, processedIntervals.size()); - Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); - Assertions.assertEquals(referenceTime.minusDays(15), processedIntervals.get(1).getEnd()); - - EasyMock.verify(mockProvider, mockParams, mockSource); - } - - @Test - public void test_createCompactionJobs_withSkipOffsetFromLatest_eliminatesInterval() - { - DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); - SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); - ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(3), Period.days(7), Period.days(30))); - CompactionJobParams mockParams = createMockParams(referenceTime, timeline); - DruidInputSource mockSource = createMockSource(); - - TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( - "testDS", null, null, mockProvider, null, Period.days(15), null - ); - - template.createCompactionJobs(mockSource, mockParams); - List processedIntervals = template.getProcessedIntervals(); - - Assertions.assertEquals(2, processedIntervals.size()); - Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); - Assertions.assertEquals(referenceTime.minusDays(25), processedIntervals.get(1).getEnd()); - - EasyMock.verify(mockProvider, mockParams, mockSource); - } - - @Test - public void test_createCompactionJobs_withSkipOffsetFromNow_skipAllOfTime() - { - DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); - SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); - ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); - CompactionJobParams mockParams = createMockParams(referenceTime, timeline); - DruidInputSource mockSource = createMockSource(); - - TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( - "testDS", null, null, mockProvider, null, null, Period.days(100) - ); - - List jobs = template.createCompactionJobs(mockSource, mockParams); - - Assertions.assertTrue(jobs.isEmpty()); - Assertions.assertTrue(template.getProcessedIntervals().isEmpty()); - - EasyMock.verify(mockProvider, mockParams, mockSource); - } - - @Test - public void test_createCompactionJobs_withSkipOffsetFromNow_truncatesIntervalThatExtendsPastSkipOffset() - { - DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); - SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); - ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(7), Period.days(30))); - CompactionJobParams mockParams = createMockParams(referenceTime, timeline); - DruidInputSource mockSource = createMockSource(); - - TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( - "testDS", null, null, mockProvider, null, null, Period.days(20) - ); - - template.createCompactionJobs(mockSource, mockParams); - List processedIntervals = template.getProcessedIntervals(); - - Assertions.assertEquals(2, processedIntervals.size()); - Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); - Assertions.assertEquals(referenceTime.minusDays(20), processedIntervals.get(1).getEnd()); - - EasyMock.verify(mockProvider, mockParams, mockSource); - } - - @Test - public void test_createCompactionJobs_withSkipOffsetFromNow_eliminatesInterval() - { - DateTime referenceTime = DateTimes.of("2024-01-15T00:00:00Z"); - SegmentTimeline timeline = createTestTimeline(referenceTime.minusDays(90), referenceTime.minusDays(10)); - ReindexingRuleProvider mockProvider = createMockProvider(List.of(Period.days(3), Period.days(7), Period.days(30))); - CompactionJobParams mockParams = createMockParams(referenceTime, timeline); - DruidInputSource mockSource = createMockSource(); - - TestCascadingReindexingTemplate template = new TestCascadingReindexingTemplate( - "testDS", null, null, mockProvider, null, null, Period.days(20) - ); - - template.createCompactionJobs(mockSource, mockParams); - List processedIntervals = template.getProcessedIntervals(); - - Assertions.assertEquals(2, processedIntervals.size()); - Assertions.assertEquals(DateTimes.MIN, processedIntervals.get(0).getStart()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(0).getEnd()); - Assertions.assertEquals(referenceTime.minusDays(30), processedIntervals.get(1).getStart()); - Assertions.assertEquals(referenceTime.minusDays(20), processedIntervals.get(1).getEnd()); - - EasyMock.verify(mockProvider, mockParams, mockSource); - } - - /** - * TEST: Basic timeline construction with multiple segment granularity rules - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P7D→HOUR, P1M→DAY, P3M→MONTH
    • - *
  • Other Rules: None
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: None created
    2. - *
  2. Initial Timeline: - *
      - *
    • P3M → MONTH: Raw 2024-10-29T16:15 → Aligned 2024-10-01T00:00
      • - *
    • P1M → DAY: Raw 2024-12-29T16:15 → Aligned 2024-12-29T00:00
      • - *
    • P7D → HOUR: Raw 2025-01-22T16:15 → Aligned 2025-01-22T16:00
      • - *
    - *
    3. - *
  3. Timeline Splits: None (no non-segment-gran rules)
    4. - *
- *

- * EXPECTED OUTPUT: 3 intervals - *

    - *
  1. [-∞, 2024-10-01T00:00:00) - MONTH
    2. - *
  2. [2024-10-01T00:00:00, 2024-12-29T00:00:00) - DAY
    3. - *
  3. [2024-12-29T00:00:00, 2025-01-22T16:00:00) - HOUR
    4. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_withGranularityAlignment() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingPartitioningRule hourRule = new ReindexingPartitioningRule("hour-rule", null, Period.days(7), Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null); - ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.months(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); - ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(3), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(hourRule, dayRule, monthRule)) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-10-01T00:00:00Z")), - monthRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-10-01T00:00:00Z"), DateTimes.of("2024-12-29T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-12-29T00:00:00Z"), DateTimes.of("2025-01-22T16:00:00Z")), - hourRule - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Timeline splitting by non-segment-granularity rules (metrics rules) - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P7D→HOUR, P1M→DAY, P3M→MONTH
    • - *
  • Other Rules: P8D-metrics, P14D-metrics, P45D-metrics, P100D-metrics
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: None (smallest segment gran rule P7D is finer than all metrics rules)
    2. - *
  2. Initial Timeline: [-∞, 2024-10-01) MONTH, [2024-10-01, 2024-12-29) DAY, [2024-12-29, 2025-01-22T16:00) HOUR
    3. - *
  3. Timeline Splits: - *
      - *
    • P100D → Raw 2024-10-21T16:15 → Falls in DAY interval → Aligned 2024-10-21T00:00 → CREATES SPLIT
      • - *
    • P45D → Raw 2024-12-15T16:15 → Falls in DAY interval → Aligned 2024-12-15T00:00 → CREATES SPLIT
      • - *
    • P14D → Raw 2025-01-15T16:15 → Falls in HOUR interval → Aligned 2025-01-15T16:00 → CREATES SPLIT
      • - *
    • P8D → Raw 2025-01-21T16:15 → Falls in HOUR interval → Aligned 2025-01-21T16:00 → CREATES SPLIT
      • - *
    - *
    4. - *
- *

- * EXPECTED OUTPUT: 7 intervals - *

    - *
  1. [-∞, 2024-10-01T00:00:00) - MONTH
    2. - *
  2. [2024-10-01T00:00:00, 2024-10-21T00:00:00) - DAY
    3. - *
  3. [2024-10-21T00:00:00, 2024-12-15T00:00:00) - DAY
    4. - *
  4. [2024-12-15T00:00:00, 2024-12-29T00:00:00) - DAY
    5. - *
  5. [2024-12-29T00:00:00, 2025-01-15T16:00:00) - HOUR
    6. - *
  6. [2025-01-15T16:00:00, 2025-01-21T16:00:00) - HOUR
    7. - *
  7. [2025-01-21T16:00:00, 2025-01-22T16:00:00) - HOUR
    8. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_withNonPartitioningRuleSplits() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingPartitioningRule hourRule = new ReindexingPartitioningRule("hour-rule", null, Period.days(7), Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null); - ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.months(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); - ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(3), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); - - // The data schema rules are here to trigger splits in the base timeline for granularity rules. - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(hourRule, dayRule, monthRule)) - .dataSchemaRules(List.of( - createReindexingDataSchemaRule("metrics-8d", Period.days(8)), - createReindexingDataSchemaRule("metrics-14d", Period.days(14)), - createReindexingDataSchemaRule("metrics-45d", Period.days(45)), - createReindexingDataSchemaRule("metrics-100d", Period.days(100)) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-10-01T00:00:00Z")), - monthRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-10-01T00:00:00Z"), DateTimes.of("2024-10-21T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-10-21T00:00:00Z"), DateTimes.of("2024-12-15T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-12-15T00:00:00Z"), DateTimes.of("2024-12-29T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-12-29T00:00:00Z"), DateTimes.of("2025-01-15T16:00:00Z")), - hourRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2025-01-15T16:00:00Z"), DateTimes.of("2025-01-21T16:00:00Z")), - hourRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2025-01-21T16:00:00Z"), DateTimes.of("2025-01-22T16:00:00Z")), - hourRule - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Timeline construction when NO segment granularity rules exist (Case A: default usage) - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: None
    • - *
  • Other Rules: P8D-metrics, P14D-metrics, P45D-metrics
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: Created P8D→DAY (Case A: no segment gran rules exist, use smallest rule period with default gran)
    2. - *
  2. Initial Timeline: [-∞, 2025-01-21T00:00) - DAY (from synthetic P8D rule)
    3. - *
  3. Timeline Splits: - *
      - *
    • P45D → Raw 2024-12-15T16:15 → Falls in DAY interval → Aligned 2024-12-15T00:00 → CREATES SPLIT
      • - *
    • P14D → Raw 2025-01-15T16:15 → Falls in DAY interval → Aligned 2025-01-15T00:00 → CREATES SPLIT
      • - *
    • P8D is now a segment gran rule (not processed as split)
      • - *
    - *
    4. - *
- *

- * EXPECTED OUTPUT: 3 intervals - *

    - *
  1. [-∞, 2024-12-15T00:00:00) - DAY
    2. - *
  2. [2024-12-15T00:00:00, 2025-01-15T00:00:00) - DAY
    3. - *
  3. [2025-01-15T00:00:00, 2025-01-21T00:00:00) - DAY
    4. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_withNoPartitioningRules() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .dataSchemaRules(List.of( - new ReindexingDataSchemaRule("metrics-8d", null, Period.days(8), null, new AggregatorFactory[0], null, null, null, null), - createReindexingDataSchemaRule("metrics-8d", Period.days(8)), - createReindexingDataSchemaRule("metrics-14d", Period.days(14)), - createReindexingDataSchemaRule("metrics-45d", Period.days(45)) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - // When no segment granularity rules exist, a synthetic rule is created with the smallest period - ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null - ); - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-12-15T00:00:00Z")), - syntheticRule, true - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-12-15T00:00:00Z"), DateTimes.of("2025-01-15T00:00:00Z")), - syntheticRule, true - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2025-01-15T00:00:00Z"), DateTimes.of("2025-01-21T00:00:00Z")), - syntheticRule, true - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Synthetic segment gran rule creation when rules are finer than smallest segment gran rule (Case B) - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P1M→DAY, P3M→MONTH
    • - *
  • Other Rules: P7D-metrics, P14D-metrics, P21D-metrics (all finer than P1M!)
    • - *
  • Default Segment Granularity: HOUR
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: Created P7D→HOUR (Case B: P7D/P14D/P21D are finer than smallest segment gran rule P1M, use finest with default gran)
    2. - *
  2. Initial Timeline: - *
      - *
    • P3M → MONTH: Raw 2024-10-29T16:15 → Aligned 2024-10-01T00:00
      • - *
    • P1M → DAY: Raw 2024-12-29T16:15 → Aligned 2024-12-29T00:00
      • - *
    • P7D → HOUR (synthetic): Raw 2025-01-22T16:15 → Aligned 2025-01-22T16:00 (PREPENDED interval!)
      • - *
    - *
    3. - *
  3. Timeline Splits: - *
      - *
    • P21D → Raw 2025-01-08T16:15 → Falls in HOUR interval → Aligned 2025-01-08T16:00 → CREATES SPLIT
      • - *
    • P14D → Raw 2025-01-15T16:15 → Falls in HOUR interval → Aligned 2025-01-15T16:00 → CREATES SPLIT
      • - *
    • P7D is now a segment gran rule (not processed as split)
      • - *
    - *
    4. - *
- *

- * EXPECTED OUTPUT: 5 intervals - *

    - *
  1. [-∞, 2024-10-01T00:00:00) - MONTH
    2. - *
  2. [2024-10-01T00:00:00, 2024-12-29T00:00:00) - DAY
    3. - *
  3. [2024-12-29T00:00:00, 2025-01-08T16:00:00) - HOUR (prepended)
    4. - *
  4. [2025-01-08T16:00:00, 2025-01-15T16:00:00) - HOUR (prepended)
    5. - *
  5. [2025-01-15T16:00:00, 2025-01-22T16:00:00) - HOUR (prepended)
    6. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_prependIntervalForShortNonPartitioningRules() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(3), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); - ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.months(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(monthRule, dayRule)) - .dataSchemaRules(List.of( - new ReindexingDataSchemaRule("metrics-7d", null, Period.days(7), null, new AggregatorFactory[0], null, null, null, null), - new ReindexingDataSchemaRule("metrics-14d", null, Period.days(14), null, new AggregatorFactory[0], null, null, null, null), - new ReindexingDataSchemaRule("metrics-21d", null, Period.days(21), null, new AggregatorFactory[0], null, null, null, null) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null - ); - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-10-01T00:00:00Z")), - monthRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-10-01T00:00:00Z"), DateTimes.of("2024-12-29T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-12-29T00:00:00Z"), DateTimes.of("2025-01-08T16:00:00Z")), - syntheticRule, true - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2025-01-08T16:00:00Z"), DateTimes.of("2025-01-15T16:00:00Z")), - syntheticRule, true - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2025-01-15T16:00:00Z"), DateTimes.of("2025-01-22T16:00:00Z")), - syntheticRule, true - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Comprehensive example demonstrating Case B, multiple segment gran rules, and timeline splits - *

- * REFERENCE TIME: 2024-02-04T22:12:04.873Z (realistic messy timestamp) - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P1Y→YEAR, P1M→MONTH, P7D→DAY
    • - *
  • Other Rules: P1D-metrics, P14D-metrics, P45D-metrics (P1D is finer than P7D!)
    • - *
  • Default Segment Granularity: HOUR
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: Created P1D→HOUR (Case B: P1D is finer than smallest segment gran rule P7D)
    2. - *
  2. Initial Timeline: - *
      - *
    • P1Y → YEAR: Raw 2023-02-04T22:12:04.873 → Aligned 2023-01-01T00:00:00
      • - *
    • P1M → MONTH: Raw 2024-01-04T22:12:04.873 → Aligned 2024-01-01T00:00:00
      • - *
    • P7D → DAY: Raw 2024-01-28T22:12:04.873 → Aligned 2024-01-28T00:00:00
      • - *
    • P1D → HOUR (synthetic): Raw 2024-02-03T22:12:04.873 → Aligned 2024-02-03T22:00:00 (PREPENDED!)
      • - *
    - *
    3. - *
  3. Timeline Splits: - *
      - *
    • P45D → Raw 2023-12-21T22:12:04.873 → Falls in MONTH interval → Aligned 2023-12-01T00:00:00 → CREATES SPLIT
      • - *
    • P14D → Raw 2024-01-21T22:12:04.873 → Falls in DAY interval → Aligned 2024-01-21T00:00:00 → CREATES SPLIT
      • - *
    • P1D is now a segment gran rule (not processed as split)
      • - *
    - *
    4. - *
- *

- * EXPECTED OUTPUT: 6 intervals - *

    - *
  1. [-∞, 2023-01-01T00:00:00) - YEAR
    2. - *
  2. [2023-01-01T00:00:00, 2023-12-01T00:00:00) - MONTH
    3. - *
  3. [2023-12-01T00:00:00, 2024-01-01T00:00:00) - MONTH
    4. - *
  4. [2024-01-01T00:00:00, 2024-01-21T00:00:00) - DAY
    5. - *
  5. [2024-01-21T00:00:00, 2024-01-28T00:00:00) - DAY
    6. - *
  6. [2024-01-28T00:00:00, 2024-02-03T22:00:00) - HOUR (prepended, note non-midnight end)
    7. - *
- */ - @Test - public void test_generateAlignedSearchIntervals() - { - DateTime referenceTime = DateTimes.of("2024-02-04T22:12:04.873Z"); - - ReindexingPartitioningRule yearRule = new ReindexingPartitioningRule("year-rule", null, Period.years(1), Granularities.YEAR, new DynamicPartitionsSpec(5000000, null), null); - ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(1), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); - ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.days(7), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); - - ReindexingRuleProvider provider = - InlineReindexingRuleProvider - .builder() - .partitioningRules(List.of(yearRule, monthRule, dayRule)) - .dataSchemaRules(List.of( - new ReindexingDataSchemaRule("metrics-1d", null, Period.days(1), null, new AggregatorFactory[0], null, null, null, null), - new ReindexingDataSchemaRule("metrics-14d", null, Period.days(14), null, new AggregatorFactory[0], null, null, null, null), - new ReindexingDataSchemaRule("metrics-45d", null, Period.days(45), null, new AggregatorFactory[0], null, null, null, null) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null - ); - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2023-01-01T00:00:00Z")), - yearRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2023-01-01T00:00:00Z"), DateTimes.of("2023-12-01T00:00:00Z")), - monthRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2023-12-01T00:00:00Z"), DateTimes.of("2024-01-01T00:00:00Z")), - monthRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-01-01T00:00:00Z"), DateTimes.of("2024-01-21T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-01-21T00:00:00Z"), DateTimes.of("2024-01-28T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-01-28T00:00:00"), DateTimes.of("2024-02-03T22:00:00")), - syntheticRule, true - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: No rules at all - should throw IAE - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: None
    • - *
  • Other Rules: None
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * EXPECTED: IllegalArgumentException with message "requires at least one reindexing rule" - */ - @Test - public void test_generateAlignedSearchIntervals_noRulesThrowsException() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder().build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - DruidException exception = Assertions.assertThrows( - DruidException.class, - () -> template.generateAlignedSearchIntervals(referenceTime) - ); - - Assertions.assertTrue( - exception.getMessage().contains("requires at least one reindexing rule") - ); - } - - /** - * TEST: Split point aligns exactly to existing boundary (boundary snapping, no split created) - *

- * REFERENCE TIME: 2025-02-01T00:00:00Z (carefully chosen for alignment) - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P1M→MONTH
    • - *
  • Other Rules: P1M-metrics (same period as segment gran rule!)
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: None
    2. - *
  2. Initial Timeline: [-∞, 2025-01-01T00:00:00) - MONTH
    3. - *
  3. Timeline Splits: - *
      - *
    • P1M metrics → Raw 2025-01-01T00:00:00 → Aligned to MONTH: 2025-01-01T00:00:00
      • - *
    • This aligns EXACTLY to the existing boundary → no split created
      • - *
    - *
    4. - *
- *

- * EXPECTED OUTPUT: 1 interval (no split despite having a non-segment-gran rule) - *

    - *
  1. [-∞, 2025-01-01T00:00:00) - MONTH
    2. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_splitPointSnapsToExistingBoundary() - { - DateTime referenceTime = DateTimes.of("2025-02-01T00:00:00Z"); - - ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(1), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(monthRule)) - .dataSchemaRules(List.of( - new ReindexingDataSchemaRule("metrics-1m", null, Period.months(1), null, new AggregatorFactory[0], null, null, null, null) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2025-01-01T00:00:00Z")), - monthRule - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Prepending that aligns back to last segment gran rule interval end (no prepend actually created) - *

- * REFERENCE TIME: 2025-01-01T01:00:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P1D→DAY
    • - *
  • Other Rules: PT12H-metrics (finer than P1D, but aligns back to same interval end as the P1D rule so no prepend is done)
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Initial Timeline: [-∞, 2024-12-31T00:00:00) - DAY
    2. - *
  2. Check for prepending: - *
      - *
    • PT12H threshold: 2024-12-31T13:00:00
      • - *
    • Align to DAY (default gran): 2024-12-31T00:00:00
      • - *
    • This EQUALS the most recent segment gran rule end (2024-12-31T00:00:00) → NO PREPEND
      • - *
    - *
    3. - *
- *

- * EXPECTED OUTPUT: 1 interval (no split prepend despite having a finer non-segment-gran rule) - *

    - *
  1. [-∞, 2024-12-31T00:00:00) - DAY
    2. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_prependAlignmentDoesNotExtendTimeline() - { - DateTime referenceTime = DateTimes.of("2025-01-01T01:00:00Z"); - - ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.days(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(dayRule)) - .dataSchemaRules(List.of( - new ReindexingDataSchemaRule("metrics-12h", null, Period.hours(12), null, new AggregatorFactory[0], null, null, null, null) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-12-31T00:00:00Z")), - dayRule - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Multiple split points align to same timestamp (distinct() filtering removes duplicates) - *

- * REFERENCE TIME: 2025-01-15T00:00:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P1M→DAY
    • - *
  • Other Rules: P23D+6h-metrics, P23D+18h-metrics (both align to same DAY boundary in DAY interval)
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: None
    2. - *
  2. Initial Timeline: [-∞, 2024-12-15T00:00:00) - DAY
    3. - *
  3. Timeline Splits: - *
      - *
    • P33D+6h → Raw: 2024-12-12T18:00:00 → Falls in DAY interval → Align to DAY: 2024-12-12T00:00:00
      • - *
    • P33D+18h → Raw: 2024-12-12T06:00:00 → Falls in DAY interval → Align to DAY: 2024-12-12T00:00:00
      • - *
    • Both create split point 2024-12-12T00:00:00 → distinct() removes duplicate!
      • - *
    • Only ONE split created at 2024-12-12T00:00:00
      • - *
    - *
    4. - *
- *

- * EXPECTED OUTPUT: 2 intervals (not 3, because duplicate split point was filtered) - *

    - *
  1. [-∞, 2024-12-12T00:00:00) - DAY
    2. - *
  2. [2024-12-12T00:00:00, 2024-12-15T00:00:00) - DAY
    3. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_duplicateSplitPointsFiltered() - { - DateTime referenceTime = DateTimes.of("2025-01-15T00:00:00Z"); - - ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule("day-rule", null, Period.months(1), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(dayRule)) - .dataSchemaRules(List.of( - new ReindexingDataSchemaRule("metrics-33d-6h", null, Period.hours(33 * 24 + 6), null, new AggregatorFactory[0], null, null, null, null), - new ReindexingDataSchemaRule("metrics-33d-18h", null, Period.hours(33 * 24 + 18), null, new AggregatorFactory[0], null, null, null, null) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-12-12T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-12-12T00:00:00Z"), DateTimes.of("2024-12-15T00:00:00Z")), - dayRule - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Single rule only (minimal valid case) - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P1M→MONTH
    • - *
  • Other Rules: None
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: None
    2. - *
  2. Initial Timeline: [-∞, 2024-12-01T00:00:00) - MONTH
    3. - *
  3. Timeline Splits: None (no non-segment-gran rules)
    4. - *
- *

- * EXPECTED OUTPUT: 1 interval - *

    - *
  1. [-∞, 2024-12-01T00:00:00) - MONTH
    2. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_singleRuleOnly() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule("month-rule", null, Period.months(1), Granularities.MONTH, new DynamicPartitionsSpec(5000000, null), null); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(monthRule)) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-12-01T00:00:00Z")), - monthRule - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Zero period rule (P0D) applies immediately to all data - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P0D→HOUR (applies to all data immediately)
    • - *
  • Other Rules: None
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: None
    2. - *
  2. Initial Timeline: [-∞, 2025-01-29T16:00:00) - HOUR (P0D means threshold equals reference time)
    3. - *
  3. Timeline Splits: None (no non-segment-gran rules)
    4. - *
- *

- * EXPECTED OUTPUT: 1 interval - *

    - *
  1. [-∞, 2025-01-29T16:00:00) - HOUR (aligned to hour boundary at reference time)
    2. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_zeroPeriodRuleAppliesImmediately() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingPartitioningRule hourRule = new ReindexingPartitioningRule( - "immediate-hour-rule", - "Apply HOUR granularity to all data immediately", - Period.days(0), - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null - ); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(hourRule)) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2025-01-29T16:00:00Z")), - hourRule - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Zero period rule (P0D) with other rules creates proper cascading timeline - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P0D→HOUR, P30D→DAY, P90D→MONTH
    • - *
  • Other Rules: None
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Synthetic Rules: None
    2. - *
  2. Sort rules by period: P90D (oldest), P30D (middle), P0D (newest/applies immediately)
    3. - *
  3. Initial Timeline: - *
      - *
    • P90D → MONTH: Raw 2024-10-31T16:15 → Aligned 2024-10-01T00:00
      • - *
    • P30D → DAY: Raw 2024-12-30T16:15 → Aligned 2024-12-30T00:00
      • - *
    • P0D → HOUR: Raw 2025-01-29T16:15 → Aligned 2025-01-29T16:00
      • - *
    - *
    4. - *
  4. Timeline Splits: None (no non-segment-gran rules)
    5. - *
- *

- * EXPECTED OUTPUT: 3 intervals - *

    - *
  1. [-∞, 2024-10-01T00:00:00) - MONTH
    2. - *
  2. [2024-10-01T00:00:00, 2024-12-30T00:00:00) - DAY
    3. - *
  3. [2024-12-30T00:00:00, 2025-01-29T16:00:00) - HOUR
    4. - *
- */ - @Test - public void test_generateAlignedSearchIntervals_zeroPeriodRuleWithOtherRules() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule( - "month-rule", - null, - Period.days(90), - Granularities.MONTH, - new DynamicPartitionsSpec(5000000, null), - null - ); - ReindexingPartitioningRule dayRule = new ReindexingPartitioningRule( - "day-rule", - null, - Period.days(30), - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null - ); - ReindexingPartitioningRule hourRule = new ReindexingPartitioningRule( - "immediate-hour-rule", - "Apply HOUR granularity immediately", - Period.days(0), - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null - ); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(hourRule, dayRule, monthRule)) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-10-01T00:00:00Z")), - monthRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-10-01T00:00:00Z"), DateTimes.of("2024-12-30T00:00:00Z")), - dayRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-12-30T00:00:00Z"), DateTimes.of("2025-01-29T16:00:00Z")), - hourRule - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - } - - /** - * TEST: Validation failure - default granularity is coarser than most recent segment granularity rule - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P30D→HOUR, P90D→DAY
    • - *
  • Other Rules: P7D-metrics (finer than P30D, triggers prepending with default granularity)
    • - *
  • Default Segment Granularity: MONTH (COARSER than HOUR!)
    • - *
- *

- * PROCESSING: - *

    - *
  1. Sort rules by period: P90D→DAY (oldest), P30D→HOUR (newest)
    2. - *
  2. P7D metrics is finer than P30D, so prepend interval with default MONTH granularity
    3. - *
  3. Timeline would be: [-∞, DAY_boundary) DAY, [DAY_boundary, HOUR_boundary) HOUR, [HOUR_boundary, MONTH_boundary) MONTH
    4. - *
  4. Validation: HOUR → MONTH progression means granularity is getting COARSER toward present
    5. - *
- *

- * EXPECTED: IllegalArgumentException with message about invalid granularity timeline - */ - @Test - public void test_generateAlignedSearchIntervals_failsWhenDefaultGranularityIsCoarserThanMostRecentPartitioningRule() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingRuleProvider provider = - InlineReindexingRuleProvider - .builder() - .partitioningRules(List.of( - new ReindexingPartitioningRule("hour-rule", null, Period.days(30), Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null), - new ReindexingPartitioningRule("day-rule", null, Period.days(90), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null) - )) - .dataSchemaRules(List.of( - new ReindexingDataSchemaRule("metrics-7d", null, Period.days(7), null, new AggregatorFactory[0], null, null, null, null) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.MONTH, // MONTH is coarser than HOUR! - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - IllegalArgumentException exception = Assertions.assertThrows( - IllegalArgumentException.class, - () -> template.generateAlignedSearchIntervals(referenceTime) - ); - - Assertions.assertTrue( - exception.getMessage().contains("Invalid segment granularity timeline") - ); - Assertions.assertTrue( - exception.getMessage().contains("coarser granularity") - ); - } - - /** - * TEST: Validation failure - older rule has finer granularity than newer rule - *

- * REFERENCE TIME: 2025-01-29T16:15:00Z - *

- * INPUT RULES: - *

    - *
  • Segment Granularity Rules: P30D→DAY, P90D→HOUR
    • - *
  • Other Rules: None
    • - *
  • Default Segment Granularity: DAY
    • - *
- *

- * PROCESSING: - *

    - *
  1. Sort rules by period: P90D→HOUR (oldest), P30D→DAY (newest)
    2. - *
  2. Timeline would be: [-∞, HOUR_boundary) HOUR, [HOUR_boundary, DAY_boundary) DAY
    3. - *
  3. Validation: HOUR → DAY progression means granularity is getting COARSER toward present
    4. - *
  4. This violates the constraint: older data (P90D) has HOUR granularity, newer data (P30D) has DAY granularity
    5. - *
- *

- * EXPECTED: IllegalArgumentException with message about invalid granularity timeline - */ - @Test - public void test_generateAlignedSearchIntervals_failsWhenOlderRuleHasFinerGranularityThanNewerRule() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - ReindexingRuleProvider provider = - InlineReindexingRuleProvider - .builder() - .partitioningRules(List.of( - new ReindexingPartitioningRule("day-rule", null, Period.days(30), Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null), - new ReindexingPartitioningRule("hour-rule", null, Period.days(90), Granularities.HOUR, new DynamicPartitionsSpec(5000000, null), null) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - null, - null - ); - - IllegalArgumentException exception = Assertions.assertThrows( - IllegalArgumentException.class, - () -> template.generateAlignedSearchIntervals(referenceTime) - ); - - Assertions.assertTrue( - exception.getMessage().contains("Invalid segment granularity timeline") - ); - Assertions.assertTrue( - exception.getMessage().contains("coarser granularity") - ); - } - - /** - * TEST: defaultPartitioningVirtualColumns flows into synthetic rules via createDefaultGranularityTimeline() - *

- * When no partitioning rules exist, the synthetic rule should carry the default partitioning VCs. - */ - @Test - public void test_generateAlignedSearchIntervals_defaultPartitioningVirtualColumnsFlowIntoSyntheticRule() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - VirtualColumns defaultVCs = VirtualColumns.create(List.of( - new ExpressionVirtualColumn("vc_bucket", "timestamp_floor(__time, 'P1D')", ColumnType.LONG, TestExprMacroTable.INSTANCE) - )); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .dataSchemaRules(List.of( - createReindexingDataSchemaRule("metrics-14d", Period.days(14)) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - defaultVCs, - null - ); - - ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - defaultVCs - ); - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2025-01-15T00:00:00Z")), - syntheticRule, true - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - - // Also verify the VCs are accessible through the IntervalPartitioningInfo - Assertions.assertEquals(defaultVCs, actual.get(0).getVirtualColumns()); - } - - /** - * TEST: defaultPartitioningVirtualColumns flows into prepended synthetic rules via maybePrependRecentInterval() - *

- * When partitioning rules exist but non-partitioning rules extend further, the prepended synthetic rule - * should carry the default partitioning VCs, while real rule entries should not. - */ - @Test - public void test_generateAlignedSearchIntervals_defaultPartitioningVirtualColumnsFlowIntoPrependedRule() - { - DateTime referenceTime = DateTimes.of("2025-01-29T16:15:00Z"); - - VirtualColumns defaultVCs = VirtualColumns.create(List.of( - new ExpressionVirtualColumn("vc_bucket", "timestamp_floor(__time, 'P1D')", ColumnType.LONG, TestExprMacroTable.INSTANCE) - )); - - ReindexingPartitioningRule monthRule = new ReindexingPartitioningRule( - "month-rule", null, Period.months(1), Granularities.MONTH, - new DynamicPartitionsSpec(5000000, null), null - ); - - ReindexingRuleProvider provider = InlineReindexingRuleProvider.builder() - .partitioningRules(List.of(monthRule)) - .dataSchemaRules(List.of( - createReindexingDataSchemaRule("metrics-7d", Period.days(7)) - )) - .build(); - - CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDS", - null, - null, - provider, - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - defaultVCs, - null - ); - - // P1M threshold: 2025-01-29 - P1M = 2024-12-29T16:15 → aligned to MONTH → 2024-12-01T00:00 - ReindexingPartitioningRule syntheticRule = ReindexingPartitioningRule.syntheticRule( - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - defaultVCs - ); - List expected = List.of( - new IntervalPartitioningInfo( - new Interval(DateTimes.MIN, DateTimes.of("2024-12-01T00:00:00Z")), - monthRule - ), - new IntervalPartitioningInfo( - new Interval(DateTimes.of("2024-12-01T00:00:00Z"), DateTimes.of("2025-01-22T00:00:00Z")), - syntheticRule, true - ) - ); - - List actual = template.generateAlignedSearchIntervals(referenceTime); - Assertions.assertEquals(expected, actual); - - // Real rule entry should NOT have the default VCs - Assertions.assertNull(actual.get(0).getVirtualColumns()); - // Prepended synthetic entry should have the default VCs - Assertions.assertEquals(defaultVCs, actual.get(1).getVirtualColumns()); - } - - /** - * TEST: Serde round-trip with non-null defaultPartitioningVirtualColumns - */ - @Test - public void test_serde_withDefaultPartitioningVirtualColumns() throws Exception - { - VirtualColumns defaultVCs = VirtualColumns.create(List.of( - new ExpressionVirtualColumn("vc_bucket", "timestamp_floor(__time, 'P1D')", ColumnType.LONG, TestExprMacroTable.INSTANCE) - )); - - final CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDataSource", - 50, - 1000000L, - InlineReindexingRuleProvider.builder() - .partitioningRules(List.of( - new ReindexingPartitioningRule( - "hourRule", null, Period.days(7), Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), null - ) - )) - .build(), - ImmutableMap.of("context_key", "context_value"), - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), - defaultVCs, - null - ); - - // Need ExprMacroTable injectable for VirtualColumn deserialization - ObjectMapper mapper = new DefaultObjectMapper(); - mapper.registerModules(new SupervisorModule().getJacksonModules()); - mapper.setInjectableValues( - new InjectableValues.Std() - .addValue(ExprMacroTable.class, TestExprMacroTable.INSTANCE) - ); - - final String json = mapper.writeValueAsString(template); - - // Verify VCs are present in the JSON - Assertions.assertTrue(json.contains("defaultPartitioningVirtualColumns")); - Assertions.assertTrue(json.contains("vc_bucket")); - - // Verify round-trip deserialization - final CascadingReindexingTemplate fromJson = mapper.readValue(json, CascadingReindexingTemplate.class); - Assertions.assertEquals(template.getDataSource(), fromJson.getDataSource()); - Assertions.assertEquals(template.getTaskPriority(), fromJson.getTaskPriority()); - Assertions.assertEquals(template.getInputSegmentSizeBytes(), fromJson.getInputSegmentSizeBytes()); - Assertions.assertEquals( - template.getDefaultPartitioningVirtualColumns(), - fromJson.getDefaultPartitioningVirtualColumns() - ); - } - - @Test - public void test_validate_returnsValid_withDynamicPartitionsSpec() - { - final CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDataSource", - null, - null, - InlineReindexingRuleProvider.builder().build(), - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(null, null), - null, - null - ); - - CompactionConfigValidationResult result = template.validate(CLUSTER_CONFIG); - Assertions.assertTrue(result.isValid()); - } - - @Test - public void test_validate_returnsInvalid_withHashedPartitionsSpec() - { - final CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDataSource", - null, - null, - InlineReindexingRuleProvider.builder().build(), - null, - null, - null, - Granularities.DAY, - new HashedPartitionsSpec(null, 3, null), - null, - null - ); - - CompactionConfigValidationResult result = template.validate(CLUSTER_CONFIG); - Assertions.assertFalse(result.isValid()); - Assertions.assertEquals( - "MSQ: Invalid partitioning type[HashedPartitionsSpec]. Must be either 'dynamic' or 'range'", - result.getReason() - ); - } - - @Test - public void test_validate_returnsInvalid_withMaxTotalRows() - { - final CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDataSource", - null, - null, - InlineReindexingRuleProvider.builder().build(), - null, - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(null, 1000L), - null, - null - ); - - CompactionConfigValidationResult result = template.validate(CLUSTER_CONFIG); - Assertions.assertFalse(result.isValid()); - Assertions.assertEquals( - "MSQ: 'maxTotalRows' not supported with 'dynamic' partitioning", - result.getReason() - ); - } - - @Test - public void test_validate_returnsInvalid_withOneMaxNumTasks() - { - final CascadingReindexingTemplate template = new CascadingReindexingTemplate( - "testDataSource", - null, - null, - InlineReindexingRuleProvider.builder().build(), - Collections.singletonMap(ClientMSQContext.CTX_MAX_NUM_TASKS, 1), - null, - null, - Granularities.DAY, - new DynamicPartitionsSpec(null, null), - null, - null - ); - - CompactionConfigValidationResult result = template.validate(CLUSTER_CONFIG); - Assertions.assertFalse(result.isValid()); - Assertions.assertEquals( - "MSQ: Context maxNumTasks[1] must be at least 2 (1 controller + 1 worker)", - result.getReason() - ); - } - - private static class TestCascadingReindexingTemplate extends CascadingReindexingTemplate - { - // Capture intervals that were processed for assertions - private final List processedIntervals = new ArrayList<>(); - - public TestCascadingReindexingTemplate( - String dataSource, - Integer taskPriority, - Long inputSegmentSizeBytes, - ReindexingRuleProvider ruleProvider, - Map taskContext, - Period skipOffsetFromLatest, - Period skipOffsetFromNow - ) - { - super(dataSource, taskPriority, inputSegmentSizeBytes, ruleProvider, - taskContext, skipOffsetFromLatest, skipOffsetFromNow, Granularities.DAY, - new DynamicPartitionsSpec(5000000, null), null, null - ); - } - - public List getProcessedIntervals() - { - return processedIntervals; - } - - @Override - protected CompactionJobTemplate createJobTemplateForInterval( - InlineSchemaDataSourceCompactionConfig config - ) - { - return new CompactionJobTemplate() { - @Override - public String getType() - { - return "test"; - } - - @Override - @Nullable - public Granularity getSegmentGranularity() - { - return null; - } - - @Override - public List createCompactionJobs( - DruidInputSource source, - CompactionJobParams params - ) - { - // Record the interval that was processed - processedIntervals.add(source.getInterval()); - - // Return a single mock job - return List.of(); - } - }; - } - } - - private SegmentTimeline createTestTimeline(DateTime start, DateTime end) - { - DataSegment segment = DataSegment.builder() - .dataSource("testDS") - .interval(new Interval(start, end)) - .version("v1") - .size(1000) - .build(); - return SegmentTimeline.forSegments(Collections.singletonList(segment)); - } - - private ReindexingRuleProvider createMockProvider(List periods) - { - // Create partitioning rules for each period - List partitioningRules = new ArrayList<>(); - for (int i = 0; i < periods.size(); i++) { - partitioningRules.add(new ReindexingPartitioningRule( - "segment-gran-rule-" + i, - null, - periods.get(i), - Granularities.HOUR, - new DynamicPartitionsSpec(5000000, null), - null - )); - } - - ReindexingRuleProvider mockProvider = EasyMock.createMock(ReindexingRuleProvider.class); - EasyMock.expect(mockProvider.isReady()).andReturn(true); - EasyMock.expect(mockProvider.getPartitioningRules()).andReturn(partitioningRules).anyTimes(); - // Return a fresh stream on each call to avoid "stream has already been operated upon or closed" errors - EasyMock.expect(mockProvider.streamAllRules()).andAnswer(() -> partitioningRules.stream().map(r -> (ReindexingRule) r)).anyTimes(); - EasyMock.expect(mockProvider.getPartitioningRule(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(partitioningRules.get(0)).anyTimes(); - EasyMock.expect(mockProvider.getIndexSpecRule(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(null).anyTimes(); - EasyMock.expect(mockProvider.getDataSchemaRule(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(null).anyTimes(); - EasyMock.expect(mockProvider.getDeletionRules(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.emptyList()).anyTimes(); - EasyMock.replay(mockProvider); - return mockProvider; - } - - private CompactionJobParams createMockParams(DateTime referenceTime, SegmentTimeline timeline) - { - CompactionJobParams mockParams = EasyMock.createMock(CompactionJobParams.class); - EasyMock.expect(mockParams.getScheduleStartTime()).andReturn(referenceTime).anyTimes(); - EasyMock.expect(mockParams.getTimeline("testDS")).andReturn(timeline); - EasyMock.replay(mockParams); - return mockParams; - } - - private DruidInputSource createMockSource() - { - final Interval[] capturedInterval = new Interval[1]; - - DruidInputSource mockSource = EasyMock.createMock(DruidInputSource.class); - EasyMock.expect(mockSource.withInterval(EasyMock.anyObject(Interval.class))) - .andAnswer(() -> { - capturedInterval[0] = (Interval) EasyMock.getCurrentArguments()[0]; - return mockSource; - }) - .anyTimes(); - EasyMock.expect(mockSource.getInterval()) - .andAnswer(() -> capturedInterval[0]) - .anyTimes(); - EasyMock.replay(mockSource); - return mockSource; - } - - /** - * Helper method to create a ReindexingDataSchemaRule with minimal required fields for testing - *

- * Helps quickly generate multiple rules to be used in testing formation of timelines and splits. - */ - private ReindexingDataSchemaRule createReindexingDataSchemaRule(String name, Period period) - { - return new ReindexingDataSchemaRule( - name, - null, - period, - null, - new AggregatorFactory[0], - null, - null, - null, - null - ); - } -} From fd6136ca2a658cf3b3da43b32d04eca6ea06a070 Mon Sep 17 00:00:00 2001 From: Ashwin Tumma Date: Mon, 29 Jun 2026 08:47:43 -0700 Subject: [PATCH 5/5] Fix remaining CascadingReindexingTemplateTest constructors Add missing skipOffsetFromEarliest parameter to exception test constructors: - test_constructor_setBothSkipOffsetStrategiesThrowsException - test_constructor_nullDataSourceThrowsException - test_constructor_nullRuleProviderThrowsException - test_constructor_nullDefaultSegmentGranularityThrowsException - test_constructor_tuningConfigWithPartitionsSpecThrowsException Also update assertion for setBothSkipOffsetStrategiesThrowsException to match the new error message format. --- .../indexing/compact/CascadingReindexingTemplateTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java index 7d69bd1e6aec..0c6e8feea73d 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/compact/CascadingReindexingTemplateTest.java @@ -216,6 +216,7 @@ public void test_constructor_setBothSkipOffsetStrategiesThrowsException() mockProvider, null, Period.days(7), // skipOffsetFromLatest + null, // skipOffsetFromEarliest Period.days(3), // skipOffsetFromNow Granularities.DAY, new DynamicPartitionsSpec(5000000, null), @@ -224,7 +225,7 @@ public void test_constructor_setBothSkipOffsetStrategiesThrowsException() ) ); - Assertions.assertEquals("Cannot set both skipOffsetFromNow and skipOffsetFromLatest", exception.getMessage()); + Assertions.assertTrue(exception.getMessage().contains("Cannot set more than one of")); EasyMock.verify(mockProvider); } @@ -244,6 +245,7 @@ public void test_constructor_nullDataSourceThrowsException() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -268,6 +270,7 @@ public void test_constructor_nullRuleProviderThrowsException() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null, @@ -294,6 +297,7 @@ public void test_constructor_nullDefaultSegmentGranularityThrowsException() null, null, null, + null, null, // null defaultSegmentGranularity new DynamicPartitionsSpec(5000000, null), null, @@ -325,6 +329,7 @@ public void test_constructor_tuningConfigWithPartitionsSpecThrowsException() null, null, null, + null, Granularities.DAY, new DynamicPartitionsSpec(5000000, null), null,