Skip to content

Fix: plan unexecuted migrations regardless of major.minor order #27703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sonika-shah merged 6 commits into from
Apr 28, 2026
Merged

Fix: plan unexecuted migrations regardless of major.minor order #27703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b7686ba
Fix: backfill historical gaps in SERVER_CHANGE_LOG during migrate
sonika-shah Apr 24, 2026
2b89929
Merge branch 'main' into fix/migration-backfill-historical-gaps
sonika-shah Apr 24, 2026
588af5e
Merge branch 'main' into fix/migration-backfill-historical-gaps
mohityadav766 Apr 27, 2026
6293fbc
Merge branch 'main' into fix/migration-backfill-historical-gaps
sonika-shah Apr 27, 2026
f021717
Merge branch 'main' into fix/migration-backfill-historical-gaps
sonika-shah Apr 28, 2026
fb294c4
Merge branch 'main' into fix/migration-backfill-historical-gaps
sonika-shah Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 1 addition & 9 deletions ...adata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,13 +263,6 @@ private static int[] parseVersion(String version) {
return numbers;
}

static boolean sameOrHigherMajorMinor(String version, String maxVersion) {
int[] v = parseVersion(version);
int[] max = parseVersion(maxVersion);
if (v[0] != max[0]) return v[0] > max[0];
return v[1] >= max[1];
}

// Package-private for testing
List<MigrationFile> resolveApplyMigrations(List<MigrationFile> availableMigrations) {
LOG.debug("Filtering Server Migrations");
Expand Down Expand Up @@ -321,8 +314,7 @@ private List<MigrationFile> processNativeMigrations(
for (MigrationFile migration : nativeMigrations) {
if (migration.version.equals(maxVer)) {
result.add(migration.copyWithReprocessing(true));
} else if (!executedMigrations.contains(migration.version)
&& sameOrHigherMajorMinor(migration.version, maxVer)) {
} else if (!executedMigrations.contains(migration.version)) {
result.add(migration.copyWithReprocessing(false));
}
}
Expand Down
93 changes: 72 additions & 21 deletions ...a-service/src/test/java/org/openmetadata/service/migration/api/MigrationWorkflowTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ void runMigrationWorkflowsExecutesDataMigrationForReprocessing() throws Exceptio
}

@Test
void getMigrationsToApplyOlderMinorBackfillsExcludedWhenOnHigherMinor() throws Exception {
void getMigrationsToApplyBackportedLowerMinorVersionsAreIncluded() throws Exception {
List<String> executedMigrations = List.of("1.11.10", "1.12.0", "1.12.1");
List<MigrationFile> availableMigrations =
List.of(
Expand All @@ -429,9 +429,31 @@ void getMigrationsToApplyOlderMinorBackfillsExcludedWhenOnHigherMinor() throws E
workflow.getMigrationsToApply(executedMigrations, availableMigrations);

List<String> versions = result.stream().map(m -> m.version).toList();
assertEquals(List.of("1.12.1", "1.12.2"), versions);
assertTrue(result.get(0).isReprocessing());
assertFalse(result.get(1).isReprocessing());
assertEquals(List.of("1.11.11", "1.11.12", "1.12.1", "1.12.2"), versions);
assertFalse(
result.stream()
.filter(m -> m.version.equals("1.11.11"))
.findFirst()
.orElseThrow()
.isReprocessing());
assertFalse(
result.stream()
.filter(m -> m.version.equals("1.11.12"))
.findFirst()
.orElseThrow()
.isReprocessing());
assertTrue(
result.stream()
.filter(m -> m.version.equals("1.12.1"))
.findFirst()
.orElseThrow()
.isReprocessing());
assertFalse(
result.stream()
.filter(m -> m.version.equals("1.12.2"))
.findFirst()
.orElseThrow()
.isReprocessing());
}

@Test
Expand All @@ -458,7 +480,7 @@ void getMigrationsToApplyCollateVersionsAreIncluded() throws Exception {
List<String> extensionVersions =
result.stream().filter(m -> m.isExtension).map(m -> m.version).toList();

assertEquals(List.of("1.12.1", "1.12.2"), nativeVersions);
assertEquals(List.of("1.11.11", "1.12.1", "1.12.2"), nativeVersions);
assertEquals(List.of("1.12.1-collate"), extensionVersions);
assertTrue(result.stream().anyMatch(m -> m.version.equals("1.12.1") && m.isReprocessing()));
assertTrue(
Expand Down Expand Up @@ -606,7 +628,7 @@ void getMigrationsToApplyAllMigrationsAlreadyExecuted() throws Exception {
}

@Test
void getMigrationsToApplyMultipleBackportedMinorVersionsExcluded() throws Exception {
void getMigrationsToApplyMultipleBackportedMinorVersionsAreIncluded() throws Exception {
List<String> executedMigrations = List.of("1.10.5", "1.11.0", "1.12.0", "1.12.1");
List<MigrationFile> availableMigrations =
List.of(
Expand All @@ -627,8 +649,50 @@ void getMigrationsToApplyMultipleBackportedMinorVersionsExcluded() throws Except
workflow.getMigrationsToApply(executedMigrations, availableMigrations);

List<String> versions = result.stream().map(m -> m.version).toList();
assertEquals(List.of("1.12.1", "1.12.2"), versions);
assertTrue(result.get(0).isReprocessing());
assertEquals(List.of("1.10.6", "1.11.1", "1.11.1-collate", "1.12.1", "1.12.2"), versions);
assertTrue(
result.stream()
.filter(m -> m.version.equals("1.12.1"))
.findFirst()
.orElseThrow()
.isReprocessing());
}

@Test
void getMigrationsToApplyHistoricalGapBelowMaxIsBackfilled() throws Exception {
List<String> executedMigrations =
List.of("1.5.0", "1.5.11", "1.9.0", "1.9.1", "1.10.0", "1.10.5");
List<MigrationFile> availableMigrations =
List.of(
createMigrationFile("1.5.0", false),
createMigrationFile("1.5.11", false),
createMigrationFile("1.5.15", false),
createMigrationFile("1.9.0", false),
createMigrationFile("1.9.1", false),
createMigrationFile("1.10.0", false),
createMigrationFile("1.10.5", false));

MigrationWorkflow workflow =
new MigrationWorkflow(
jdbi, tempDir.toString(), ConnectionType.MYSQL, null, null, config, false);

List<MigrationFile> result =
workflow.getMigrationsToApply(executedMigrations, availableMigrations);

List<String> versions = result.stream().map(m -> m.version).toList();
assertEquals(List.of("1.5.15", "1.10.5"), versions);
assertFalse(
result.stream()
.filter(m -> m.version.equals("1.5.15"))
.findFirst()
.orElseThrow()
.isReprocessing());
assertTrue(
result.stream()
.filter(m -> m.version.equals("1.10.5"))
.findFirst()
.orElseThrow()
.isReprocessing());
}

@Test
Expand Down Expand Up @@ -696,19 +760,6 @@ void resolveApplyMigrationsAllExecutedNoDuplicates() throws Exception {
assertTrue(result.get(0).isReprocessing());
}

@Test
void sameOrHigherMajorMinorComparisons() {
assertTrue(MigrationWorkflow.sameOrHigherMajorMinor("1.12.0", "1.12.1"));
assertTrue(MigrationWorkflow.sameOrHigherMajorMinor("1.12.5", "1.12.1"));
assertTrue(MigrationWorkflow.sameOrHigherMajorMinor("1.13.0", "1.12.1"));
assertTrue(MigrationWorkflow.sameOrHigherMajorMinor("2.0.0", "1.12.1"));
assertFalse(MigrationWorkflow.sameOrHigherMajorMinor("1.11.15", "1.12.1"));
assertFalse(MigrationWorkflow.sameOrHigherMajorMinor("1.10.6", "1.12.1"));
assertFalse(MigrationWorkflow.sameOrHigherMajorMinor("0.13.0", "1.12.1"));
assertTrue(MigrationWorkflow.sameOrHigherMajorMinor("1.12.1-collate", "1.12.1"));
assertFalse(MigrationWorkflow.sameOrHigherMajorMinor("1.11.1-collate", "1.12.1"));
}

private void mockContext(
MigrationWorkflowContext contextMock, org.mockito.MockedConstruction.Context context) {
HashMap<String, MigrationContext> contexts = new HashMap<>();
Expand Down
Loading