From aa544a8ee004af3dcd0ea4a18307c90495eb367a Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 10 Sep 2025 17:58:44 +0200 Subject: [PATCH 01/38] MINOR - Data Contracts only execute tests without results --- .../service/jdbi3/DataContractRepository.java | 203 +++++++--- .../data/DataContractResourceTest.java | 370 ++++++++++++++++++ 2 files changed, 529 insertions(+), 44 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index 2d3bbb858d5d..e129cc74335e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -17,6 +17,7 @@ import static org.openmetadata.schema.type.EventType.ENTITY_CREATED; import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.Entity.TEST_CASE_RESULT; import jakarta.ws.rs.core.Response; import java.util.ArrayList; @@ -51,6 +52,7 @@ import org.openmetadata.schema.metadataIngestion.TestSuitePipeline; import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; import org.openmetadata.schema.tests.ResultSummary; +import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.TestCaseStatus; import org.openmetadata.schema.type.Column; @@ -459,7 +461,7 @@ public static String getTestSuiteName(DataContract dataContract) { return dataContract.getId().toString(); } - private TestSuite createOrUpdateDataContractTestSuite(DataContract dataContract, boolean update) { + private void createOrUpdateDataContractTestSuite(DataContract dataContract, boolean update) { try { if (update) { // If we're running an update, fetch the existing test suite information restoreExistingDataContract(dataContract); @@ -468,28 +470,37 @@ private TestSuite createOrUpdateDataContractTestSuite(DataContract dataContract, // If we don't have quality expectations or a test suite, we don't need to create one if (nullOrEmpty(dataContract.getQualityExpectations()) && !contractHasTestSuite(dataContract)) { - return null; + return; } // If we had a test suite from older tests, but we removed them, we can delete the suite if (nullOrEmpty(dataContract.getQualityExpectations())) { deleteTestSuite(dataContract); dataContract.setTestSuite(null); - return null; + return; } - TestSuite testSuite = getOrCreateTestSuite(dataContract); - updateTestSuiteTests(dataContract, testSuite); + // Create the test suite with only the tests pending to be executed + List tests = getTestsWithResults(dataContract); + List testsWithoutResults = filterTestsWithoutResults(tests); - // Add the test suite to the data contract - dataContract.setTestSuite( - new EntityReference() - .withId(testSuite.getId()) - .withFullyQualifiedName(testSuite.getFullyQualifiedName()) - .withType(Entity.TEST_SUITE)); + if (!nullOrEmpty(testsWithoutResults)) { + TestSuite testSuite = getOrCreateTestSuite(dataContract); + updateTestSuiteTests(dataContract, testSuite, testsWithoutResults); - return testSuite; + // Add the test suite to the data contract + dataContract.setTestSuite( + new EntityReference() + .withId(testSuite.getId()) + .withFullyQualifiedName(testSuite.getFullyQualifiedName()) + .withType(Entity.TEST_SUITE)); + } + // If we already have a test suite but no tests pending to execute, remove it + if (contractHasTestSuite(dataContract) && nullOrEmpty(testsWithoutResults)) { + deleteTestSuite(dataContract); + dataContract.setTestSuite(null); + } } catch (Exception e) { LOG.error("Error creating/updating test suite for data contract", e); throw e; @@ -510,7 +521,8 @@ private void restoreExistingDataContract(DataContract dataContract) { dataContract.setId(existing.map(DataContract::getId).orElse(dataContract.getId())); } - private void updateTestSuiteTests(DataContract dataContract, TestSuite testSuite) { + private void updateTestSuiteTests( + DataContract dataContract, TestSuite testSuite, List testsWithoutResults) { TestCaseRepository testCaseRepository = (TestCaseRepository) Entity.getEntityRepository(Entity.TEST_CASE); @@ -521,17 +533,17 @@ private void updateTestSuiteTests(DataContract dataContract, TestSuite testSuite testSuite.getTests() != null ? testSuite.getTests().stream().map(EntityReference::getId).toList() : Collections.emptyList(); + List testsToAdd = testsWithoutResults.stream().map(TestCase::getId).toList(); - // Add only new tests to the test suite - List newTestCases = - testCaseRefs.stream().filter(testCaseRef -> !currentTests.contains(testCaseRef)).toList(); - if (!nullOrEmpty(newTestCases)) { - testCaseRepository.addTestCasesToLogicalTestSuite(testSuite, newTestCases); + if (!nullOrEmpty(testsWithoutResults)) { + testCaseRepository.addTestCasesToLogicalTestSuite(testSuite, testsToAdd); } - // Then, remove any tests that are no longer in the quality expectations + // Remove tests that are no longer in the quality expectations or already have results List testsToRemove = - currentTests.stream().filter(testId -> !testCaseRefs.contains(testId)).toList(); + currentTests.stream() + .filter(testId -> !testCaseRefs.contains(testId) || !testsToAdd.contains(testId)) + .toList(); if (!nullOrEmpty(testsToRemove)) { testsToRemove.forEach( test -> { @@ -656,20 +668,41 @@ public RestUtil.PutResponse validateContract(DataContract da } // If we don't have quality expectations, flag the results based on schema and semantics - // Otherwise, keep it Running and wait for the DQ results to kick in + // Otherwise, check if we need to run tests or use existing results if (!nullOrEmpty(dataContract.getQualityExpectations())) { - try { - deployAndTriggerDQValidation(dataContract); - compileResult(result, ContractExecutionStatus.Running); - } catch (Exception e) { - LOG.error( - "Failed to trigger DQ validation for data contract {}: {}", - dataContract.getFullyQualifiedName(), - e.getMessage()); - result - .withContractExecutionStatus(ContractExecutionStatus.Aborted) - .withResult("Failed to trigger DQ validation: " + e.getMessage()); - compileResult(result, ContractExecutionStatus.Aborted); + // Check if all tests already have results + List tests = getTestsWithResults(dataContract); + List testsWithoutResults = filterTestsWithoutResults(tests); + List testsWithResults = filterTestsWithResults(tests); + + // Initialize the quality validation results + result.withQualityValidation(initDQValidation(dataContract)); + + if (!nullOrEmpty(testsWithoutResults)) { + try { + deployAndTriggerDQValidation(dataContract); + compileResult(result, ContractExecutionStatus.Running); + } catch (Exception e) { + LOG.error( + "Failed to trigger DQ validation for data contract {}: {}", + dataContract.getFullyQualifiedName(), + e.getMessage()); + result + .withContractExecutionStatus(ContractExecutionStatus.Aborted) + .withResult("Failed to trigger DQ validation: " + e.getMessage()); + compileResult(result, ContractExecutionStatus.Aborted); + } + } + if (!nullOrEmpty(testsWithResults)) { + QualityValidation qualityValidation = + getExistingTestResults(dataContract, testsWithResults); + result.withQualityValidation(qualityValidation); + // Fallback to running if we're still waiting to some tests to report back their status + compileResult( + result, + !nullOrEmpty(testsWithoutResults) + ? ContractExecutionStatus.Running + : ContractExecutionStatus.Success); } } else { compileResult(result, ContractExecutionStatus.Success); @@ -734,6 +767,84 @@ private void prepareAndRunIngestionPipeline(IngestionPipeline pipeline, TestSuit pipelineServiceClient.runPipeline(pipeline, testSuite); } + private QualityValidation initDQValidation(DataContract dataContract) { + QualityValidation validation = new QualityValidation(); + int totalTests = dataContract.getQualityExpectations().size(); + validation.withTotal(totalTests).withPassed(0).withFailed(0).withQualityScore(0.0); + return validation; + } + + private List getTestsWithResults(DataContract dataContract) { + TestCaseRepository testCaseRepository = + (TestCaseRepository) Entity.getEntityRepository(Entity.TEST_CASE); + return dataContract.getQualityExpectations().stream() + .map( + testRef -> { + try { + return testCaseRepository.get( + null, testRef.getId(), new Fields(Set.of(TEST_CASE_RESULT))); + } catch (EntityNotFoundException e) { + LOG.warn("Test case {} not found: {}", testRef.getId(), e.getMessage()); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private List filterTestsWithoutResults(List tests) { + return tests.stream() + .filter( + test -> + test.getTestCaseResult() == null + || test.getTestCaseResult().getTestCaseStatus() == null) + .collect(Collectors.toList()); + } + + private List filterTestsWithResults(List tests) { + return tests.stream() + .filter( + test -> + test.getTestCaseResult() != null + && test.getTestCaseResult().getTestCaseStatus() != null) + .collect(Collectors.toList()); + } + + private QualityValidation getExistingTestResults( + DataContract dataContract, List testsWithResults) { + QualityValidation validation = new QualityValidation(); + + if (nullOrEmpty(testsWithResults)) { + return validation; + } + + int totalTests = dataContract.getQualityExpectations().size(); + int passedTests = 0; + int failedTests = 0; + + for (TestCase test : testsWithResults) { + try { + if (FAILED_DQ_STATUSES.contains(test.getTestCaseResult().getTestCaseStatus())) { + failedTests++; + } else { + passedTests++; + } + } catch (Exception e) { + LOG.warn("Failed to get test result for test {}: {}", test.getId(), e.getMessage()); + } + } + + double qualityScore = totalTests > 0 ? (passedTests / (double) totalTests) * 100 : 0.0; + + validation + .withTotal(totalTests) + .withPassed(passedTests) + .withFailed(failedTests) + .withQualityScore(qualityScore); + + return validation; + } + private SemanticsValidation validateSemantics(DataContract dataContract) { SemanticsValidation validation = new SemanticsValidation(); @@ -772,10 +883,12 @@ private SemanticsValidation validateSemantics(DataContract dataContract) { return validation; } - private QualityValidation validateDQ(TestSuite testSuite) { - QualityValidation validation = new QualityValidation(); + private QualityValidation validateDQ(TestSuite testSuite, QualityValidation existingValidation) { + if (existingValidation == null) { + existingValidation = new QualityValidation(); + } if (nullOrEmpty(testSuite.getTestCaseResultSummary())) { - return validation; // return the existing result without updates + return existingValidation; // return the existing result without updates } List currentTests = @@ -791,14 +904,16 @@ private QualityValidation validateDQ(TestSuite testSuite) { List failedTests = testSummary.stream().filter(test -> FAILED_DQ_STATUSES.contains(test.getStatus())).toList(); - validation - .withFailed(failedTests.size()) - .withPassed(testSummary.size() - failedTests.size()) - .withTotal(testSummary.size()) - .withQualityScore( - (((testSummary.size() - failedTests.size()) / (double) testSummary.size())) * 100); + existingValidation + .withFailed(existingValidation.getFailed() + failedTests.size()) + .withPassed(existingValidation.getPassed() + (testSummary.size() - failedTests.size())); - return validation; + existingValidation.withQualityScore( + (((existingValidation.getPassed() - existingValidation.getFailed()) + / (double) existingValidation.getTotal())) + * 100); + + return existingValidation; } public void compileResult(DataContractResult result, ContractExecutionStatus fallbackStatus) { @@ -880,7 +995,7 @@ public DataContractResult updateContractDQResults( // Get the latest result or throw if none exists DataContractResult result = getLatestResult(dataContract); - QualityValidation validation = validateDQ(testSuite); + QualityValidation validation = validateDQ(testSuite, result.getQualityValidation()); result.withQualityValidation(validation); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java index 6cf3ba752135..9d598be18d46 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java @@ -70,6 +70,7 @@ import org.openmetadata.schema.api.services.CreateMessagingService; import org.openmetadata.schema.api.services.DatabaseConnection; import org.openmetadata.schema.api.tests.CreateTestCase; +import org.openmetadata.schema.api.tests.CreateTestCaseResult; import org.openmetadata.schema.entity.data.APIEndpoint; import org.openmetadata.schema.entity.data.Chart; import org.openmetadata.schema.entity.data.Dashboard; @@ -91,6 +92,7 @@ import org.openmetadata.schema.tests.ResultSummary; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.schema.tests.type.TestCaseResult; import org.openmetadata.schema.tests.type.TestCaseStatus; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; @@ -939,6 +941,19 @@ private String getTableUri() { return String.format("http://localhost:%s/api/v1/tables", APP.getLocalPort()); } + private TestCaseResult postTestCaseResult( + String testCaseFQN, CreateTestCaseResult createTestCaseResult) throws HttpResponseException { + WebTarget target = + APP.client() + .target( + String.format( + "http://localhost:%s/api/v1/dataQuality/testCases/testCaseResults/%s", + APP.getLocalPort(), testCaseFQN)); + Response response = + SecurityUtil.addHeaders(target, ADMIN_AUTH_HEADERS).post(Entity.json(createTestCaseResult)); + return TestUtils.readResponse(response, TestCaseResult.class, Status.CREATED.getStatusCode()); + } + /** * Creates and ensures messaging service for topic creation */ @@ -5532,4 +5547,359 @@ void testContractSecurityConsumerDataIntegrityOnUpdates(TestInfo test) throws IO assertEquals(1, finalState.getSecurity().getConsumers().size()); assertEquals("policy-2", finalState.getSecurity().getConsumers().get(0).getAccessPolicy()); } + + // ===================== Execute Summary Tests for Tests Without Results ===================== + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testValidateContractWithNoTestResults(TestInfo test) throws IOException { + // Test scenario 1: No tests have results - Test suite is created with all tests + Table table = createUniqueTable(test.getDisplayName()); + + // Create multiple test cases for quality expectations + String tableLink = String.format("<#E::table::%s>", table.getFullyQualifiedName()); + + CreateTestCase createTestCase1 = + testCaseResourceTest + .createRequest("test_case_1_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase1 = + testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase2 = + testCaseResourceTest + .createRequest("test_case_2_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase2 = + testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase3 = + testCaseResourceTest + .createRequest("test_case_3_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase3 = + testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); + + List qualityExpectations = + List.of( + testCase1.getEntityReference(), + testCase2.getEntityReference(), + testCase3.getEntityReference()); + + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table) + .withQualityExpectations(qualityExpectations); + + DataContract dataContract = createDataContract(create); + assertNotNull(dataContract.getTestSuite()); + + // Verify test suite was created and contains all tests (no results exist) + TestSuite testSuite = + testSuiteResourceTest.getEntity( + dataContract.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); + assertNotNull(testSuite.getTests()); + assertEquals(3, testSuite.getTests().size()); + + // Call validate endpoint - should trigger DQ validation since no tests have results + DataContractResult result = runValidate(dataContract); + assertNotNull(result); + assertEquals(ContractExecutionStatus.Running, result.getContractExecutionStatus()); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testValidateContractWithAllTestResultsCompileExisting(TestInfo test) throws IOException { + // Test scenario: All tests have results - Test suite should be empty and results compiled from + // existing data + Table table = createUniqueTable(test.getDisplayName()); + + String tableLink = String.format("<#E::table::%s>", table.getFullyQualifiedName()); + + // Create two test cases + CreateTestCase createTestCase1 = + testCaseResourceTest + .createRequest("test_case_all_results_1_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase1 = + testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase2 = + testCaseResourceTest + .createRequest("test_case_all_results_2_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase2 = + testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); + + List qualityExpectations = + List.of(testCase1.getEntityReference(), testCase2.getEntityReference()); + + // Create the contract first + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table) + .withQualityExpectations(qualityExpectations); + + DataContract dataContract = createDataContract(create); + + // Run validate to create initial result for the contract + DataContractResult initialResult = runValidate(dataContract); + assertNotNull(initialResult); + + // Now post test results to simulate they have been executed + // This creates time series data that will be picked up by the DataContract filtering logic + CreateTestCaseResult createTestCaseResult1 = + new CreateTestCaseResult() + .withResult("Test passed") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase1.getFullyQualifiedName(), createTestCaseResult1); + + CreateTestCaseResult createTestCaseResult2 = + new CreateTestCaseResult() + .withResult("Test failed") + .withTestCaseStatus(TestCaseStatus.Failed) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase2.getFullyQualifiedName(), createTestCaseResult2); + + // Update the contract - this should remove all tests from test suite since all have results + DataContract updatedContract = updateDataContract(create); + + // Verify test suite is null or empty since all tests have results + if (updatedContract.getTestSuite() != null) { + TestSuite testSuite = + testSuiteResourceTest.getEntity( + updatedContract.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); + assertTrue( + testSuite.getTests() == null || testSuite.getTests().isEmpty(), + "Test suite should be empty when all tests have results"); + } + + // Call validate endpoint - should compile existing results without triggering new DQ validation + DataContractResult result = runValidate(dataContract); + assertNotNull(result); + assertEquals( + ContractExecutionStatus.Failed, + result.getContractExecutionStatus(), + "Contract should fail because one test failed"); + + // Verify quality validation was compiled from existing results + assertNotNull(result.getQualityValidation()); + assertEquals(2, result.getQualityValidation().getTotal().intValue()); + assertEquals(1, result.getQualityValidation().getPassed().intValue()); + assertEquals(1, result.getQualityValidation().getFailed().intValue()); + assertEquals(50.0, result.getQualityValidation().getQualityScore(), 0.01); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testUpdateContractFiltersTestsWithResults(TestInfo test) throws IOException { + // Test that when updating a contract, tests with results are filtered out of the test suite + Table table = createUniqueTable(test.getDisplayName()); + + String tableLink = String.format("<#E::table::%s>", table.getFullyQualifiedName()); + + // Create three test cases + CreateTestCase createTestCase1 = + testCaseResourceTest + .createRequest("test_case_update_1_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase1 = + testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase2 = + testCaseResourceTest + .createRequest("test_case_update_2_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase2 = + testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase3 = + testCaseResourceTest + .createRequest("test_case_update_3_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase3 = + testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); + + List qualityExpectations = + List.of( + testCase1.getEntityReference(), + testCase2.getEntityReference(), + testCase3.getEntityReference()); + + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table) + .withQualityExpectations(qualityExpectations); + + // Create initial contract - should create test suite with all tests since no results exist + DataContract dataContract = createDataContract(create); + assertNotNull(dataContract.getTestSuite()); + + TestSuite initialTestSuite = + testSuiteResourceTest.getEntity( + dataContract.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); + assertNotNull(initialTestSuite.getTests()); + assertEquals(3, initialTestSuite.getTests().size(), "All tests should be in suite initially"); + + // Run validate to create initial result for the contract + DataContractResult initialResult = runValidate(dataContract); + assertNotNull(initialResult); + + // Post test results for testCase1 and testCase2 to simulate they have been executed + // This creates time series data that will be picked up by the DataContract filtering logic + CreateTestCaseResult createTestCaseResult1 = + new CreateTestCaseResult() + .withResult("Test passed") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase1.getFullyQualifiedName(), createTestCaseResult1); + + CreateTestCaseResult createTestCaseResult2 = + new CreateTestCaseResult() + .withResult("Test failed") + .withTestCaseStatus(TestCaseStatus.Failed) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase2.getFullyQualifiedName(), createTestCaseResult2); + + // Now update the contract - this should filter out tests with results + DataContract updated = updateDataContract(create); + assertNotNull(updated.getTestSuite()); + + // After update, the test suite should only contain testCase3 (the one without results) + TestSuite updatedTestSuite = + testSuiteResourceTest.getEntity(updated.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); + assertNotNull(updatedTestSuite.getTests()); + assertEquals( + 1, + updatedTestSuite.getTests().size(), + "Test suite should only contain tests without results after update"); + assertEquals( + testCase3.getId(), + updatedTestSuite.getTests().get(0).getId(), + "Test suite should contain only testCase3 which has no results"); + + // Validate the contract - should trigger DQ for remaining test and compile existing results + DataContractResult validationResult = runValidate(updated); + assertEquals( + ContractExecutionStatus.Failed, + validationResult.getContractExecutionStatus(), + "Should be Failed because testCase2 already failed"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testCompleteWorkflowFromNoResultsToAllResults(TestInfo test) throws IOException { + // Comprehensive test: Start with no results, gradually add results, and verify filtering + // behavior + Table table = createUniqueTable(test.getDisplayName()); + + String tableLink = String.format("<#E::table::%s>", table.getFullyQualifiedName()); + + // Create three test cases + CreateTestCase createTestCase1 = + testCaseResourceTest + .createRequest("test_case_workflow_1_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase1 = + testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase2 = + testCaseResourceTest + .createRequest("test_case_workflow_2_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase2 = + testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase3 = + testCaseResourceTest + .createRequest("test_case_workflow_3_" + test.getDisplayName()) + .withEntityLink(tableLink); + TestCase testCase3 = + testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); + + List qualityExpectations = + List.of( + testCase1.getEntityReference(), + testCase2.getEntityReference(), + testCase3.getEntityReference()); + + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table) + .withQualityExpectations(qualityExpectations); + + // Phase 1: Create contract - all tests should be in test suite + DataContract dataContract = createDataContract(create); + TestSuite initialTestSuite = + testSuiteResourceTest.getEntity( + dataContract.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); + assertEquals(3, initialTestSuite.getTests().size(), "All tests should be in suite initially"); + + // Run validate to create initial result for the contract + DataContractResult initialResult = runValidate(dataContract); + assertNotNull(initialResult); + + // Phase 2: Execute first test to generate results + // Post result for testCase1 only + CreateTestCaseResult firstTestResult = + new CreateTestCaseResult() + .withResult("Test passed") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase1.getFullyQualifiedName(), firstTestResult); + + // Update contract and verify filtering + DataContract afterFirstResult = updateDataContract(create); + TestSuite suiteAfterFirst = + testSuiteResourceTest.getEntity( + afterFirstResult.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); + assertEquals(2, suiteAfterFirst.getTests().size(), "Should have 2 tests without results"); + + // Phase 3: Execute second test to generate more results + // Post result for testCase2 (testCase1 already has results from Phase 2) + CreateTestCaseResult secondTestResult = + new CreateTestCaseResult() + .withResult("Test failed") + .withTestCaseStatus(TestCaseStatus.Failed) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase2.getFullyQualifiedName(), secondTestResult); + + // Update contract and verify filtering + DataContract afterSecondResult = updateDataContract(create); + TestSuite suiteAfterSecond = + testSuiteResourceTest.getEntity( + afterSecondResult.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); + assertEquals(1, suiteAfterSecond.getTests().size(), "Should have 1 test without results"); + assertEquals(testCase3.getId(), suiteAfterSecond.getTests().get(0).getId()); + + // Phase 4: Execute final test + // Post result for testCase3 (testCase1 and testCase2 already have results) + CreateTestCaseResult thirdTestResult = + new CreateTestCaseResult() + .withResult("Test passed") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase3.getFullyQualifiedName(), thirdTestResult); + + // Final update and validation + DataContract finalContract = updateDataContract(create); + TestSuite finalSuite = + testSuiteResourceTest.getEntity( + finalContract.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); + assertTrue( + finalSuite.getTests() == null || finalSuite.getTests().isEmpty(), + "Test suite should be empty when all tests have results"); + + // Final validation - should compile all existing results + DataContractResult finalResult = runValidate(finalContract); + assertEquals( + ContractExecutionStatus.Failed, + finalResult.getContractExecutionStatus(), + "Contract should fail due to one failed test"); + + assertNotNull(finalResult.getQualityValidation()); + assertEquals(3, finalResult.getQualityValidation().getTotal().intValue()); + assertEquals(2, finalResult.getQualityValidation().getPassed().intValue()); + assertEquals(1, finalResult.getQualityValidation().getFailed().intValue()); + + double expectedScore = (2.0 / 3.0) * 100; + assertEquals(expectedScore, finalResult.getQualityValidation().getQualityScore(), 0.01); + } } From bfa0e23761c44024fffcc6d13f7931a7614a18e4 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 10 Sep 2025 18:12:45 +0200 Subject: [PATCH 02/38] MINOR - Data Contracts only execute tests without results --- .../service/resources/data/DataContractResourceTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java index 9d598be18d46..bd039600c581 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java @@ -5880,12 +5880,8 @@ void testCompleteWorkflowFromNoResultsToAllResults(TestInfo test) throws IOExcep // Final update and validation DataContract finalContract = updateDataContract(create); - TestSuite finalSuite = - testSuiteResourceTest.getEntity( - finalContract.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); - assertTrue( - finalSuite.getTests() == null || finalSuite.getTests().isEmpty(), - "Test suite should be empty when all tests have results"); + assertNull( + finalContract.getTestSuite(), "Test suite should be null when all tests have results"); // Final validation - should compile all existing results DataContractResult finalResult = runValidate(finalContract); From 088ef378e9edc34fae5a4f697f0a37c3df62df9f Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 22 Sep 2025 12:49:47 +0200 Subject: [PATCH 03/38] MINOR - Data Contracts only execute tests without results --- .../service/jdbi3/DataContractRepository.java | 35 +++++++++++-------- .../data/DataContractResourceTest.java | 13 +++---- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index e129cc74335e..cc601c808a25 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -193,20 +193,27 @@ protected void postDelete(DataContract dataContract, boolean hardDelete) { private void postCreateOrUpdate(DataContract dataContract) { if (!nullOrEmpty(dataContract.getQualityExpectations())) { - TestSuite testSuite = getOrCreateTestSuite(dataContract); - // Create the ingestion pipeline only if needed - if (testSuite != null && nullOrEmpty(testSuite.getPipelines())) { - IngestionPipeline pipeline = createIngestionPipeline(testSuite); - EntityReference pipelineRef = - Entity.getEntityReference( - new EntityReference().withId(pipeline.getId()).withType(Entity.INGESTION_PIPELINE), - Include.NON_DELETED); - testSuite.setPipelines(List.of(pipelineRef)); - TestSuiteRepository testSuiteRepository = - (TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE); - testSuiteRepository.createOrUpdate(null, testSuite, ADMIN_USER_NAME); - if (!pipeline.getDeployed()) { - prepareAndDeployIngestionPipeline(pipeline, testSuite); + // Create the test suite with only the tests pending to be executed + List tests = getTestsWithResults(dataContract); + List testsWithoutResults = filterTestsWithoutResults(tests); + if (!nullOrEmpty(testsWithoutResults)) { + TestSuite testSuite = getOrCreateTestSuite(dataContract); + // Create the ingestion pipeline only if needed + if (testSuite != null && nullOrEmpty(testSuite.getPipelines())) { + IngestionPipeline pipeline = createIngestionPipeline(testSuite); + EntityReference pipelineRef = + Entity.getEntityReference( + new EntityReference() + .withId(pipeline.getId()) + .withType(Entity.INGESTION_PIPELINE), + Include.NON_DELETED); + testSuite.setPipelines(List.of(pipelineRef)); + TestSuiteRepository testSuiteRepository = + (TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE); + testSuiteRepository.createOrUpdate(null, testSuite, ADMIN_USER_NAME); + if (!pipeline.getDeployed()) { + prepareAndDeployIngestionPipeline(pipeline, testSuite); + } } } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java index bd039600c581..200a5b81411d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java @@ -5663,15 +5663,10 @@ void testValidateContractWithAllTestResultsCompileExisting(TestInfo test) throws // Update the contract - this should remove all tests from test suite since all have results DataContract updatedContract = updateDataContract(create); - // Verify test suite is null or empty since all tests have results - if (updatedContract.getTestSuite() != null) { - TestSuite testSuite = - testSuiteResourceTest.getEntity( - updatedContract.getTestSuite().getId(), "*", ADMIN_AUTH_HEADERS); - assertTrue( - testSuite.getTests() == null || testSuite.getTests().isEmpty(), - "Test suite should be empty when all tests have results"); - } + // Verify test suite is null since all tests have results - no test suite should be created + assertNull( + updatedContract.getTestSuite(), + "Test suite should be null when all tests already have results - no test suite should be created"); // Call validate endpoint - should compile existing results without triggering new DQ validation DataContractResult result = runValidate(dataContract); From 031b7e1336e03ababb3108661fb454b8879f684e Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 23 Sep 2025 11:52:06 +0200 Subject: [PATCH 04/38] add data contract to test cases --- .../resources/dqtests/TestCaseMapper.java | 35 ++- .../dqtests/TestCaseResultResource.java | 32 ++- .../service/search/SearchListFilter.java | 3 + .../search/indexes/TestCaseResultIndex.java | 8 +- .../data/DataContractResourceTest.java | 231 ++++++++++++++++++ .../dqtests/TestCaseResourceTest.java | 166 +++++++++++++ .../en/test_case_result_index_mapping.json | 46 ++++ .../jp/test_case_result_index_mapping.json | 80 ++++-- .../ru/test_case_result_index_mapping.json | 46 ++++ .../zh/test_case_result_index_mapping.json | 45 ++++ .../json/schema/api/tests/createTestCase.json | 4 + .../resources/json/schema/tests/testCase.json | 4 + 12 files changed, 666 insertions(+), 34 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java index ef011484bb11..0d4d951c9a8a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java @@ -4,6 +4,7 @@ import org.openmetadata.schema.api.tests.CreateTestCase; import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.mapper.EntityMapper; import org.openmetadata.service.resources.feeds.MessageParser; @@ -12,17 +13,27 @@ public class TestCaseMapper implements EntityMapper { @Override public TestCase createToEntity(CreateTestCase create, String user) { MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(create.getEntityLink()); - return copy(new TestCase(), create, user) - .withDescription(create.getDescription()) - .withName(create.getName()) - .withDisplayName(create.getDisplayName()) - .withParameterValues(create.getParameterValues()) - .withEntityLink(create.getEntityLink()) - .withComputePassedFailedRowCount(create.getComputePassedFailedRowCount()) - .withUseDynamicAssertion(create.getUseDynamicAssertion()) - .withEntityFQN(entityLink.getFullyQualifiedFieldValue()) - .withTestDefinition(getEntityReference(Entity.TEST_DEFINITION, create.getTestDefinition())) - .withTags(create.getTags()) - .withCreatedBy(user); + TestCase testCase = + copy(new TestCase(), create, user) + .withDescription(create.getDescription()) + .withName(create.getName()) + .withDisplayName(create.getDisplayName()) + .withParameterValues(create.getParameterValues()) + .withEntityLink(create.getEntityLink()) + .withComputePassedFailedRowCount(create.getComputePassedFailedRowCount()) + .withUseDynamicAssertion(create.getUseDynamicAssertion()) + .withEntityFQN(entityLink.getFullyQualifiedFieldValue()) + .withTestDefinition( + getEntityReference(Entity.TEST_DEFINITION, create.getTestDefinition())) + .withTags(create.getTags()) + .withCreatedBy(user); + + if (create.getDataContract() != null) { + testCase.setDataContract( + Entity.getEntityReferenceByName( + Entity.DATA_CONTRACT, create.getDataContract(), Include.NON_DELETED)); + } + + return testCase; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java index de3159c9d16d..d485f536083a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java @@ -1,5 +1,6 @@ package org.openmetadata.service.resources.dqtests; +import static org.openmetadata.service.Entity.DATA_CONTRACT; import static org.openmetadata.service.Entity.TABLE; import static org.openmetadata.service.Entity.TEST_CASE; import static org.openmetadata.service.Entity.TEST_SUITE; @@ -262,6 +263,11 @@ public ResultList listTestCaseResultsFromSearch( schema = @Schema(type = "string")) @QueryParam("entityFQN") String entityFQN, + @Parameter( + description = "Data Contract Id the test case belongs to", + schema = @Schema(type = "string")) + @QueryParam("dataContractId") + String dataContractId, @Parameter( description = "Get the latest test case result for each test case -- requires `testSuiteId`. Offset and limit are ignored", @@ -327,11 +333,14 @@ public ResultList listTestCaseResultsFromSearch( Optional.ofNullable(testSuiteId) .ifPresent(tsi -> searchListFilter.addQueryParam("testSuiteId", tsi)); Optional.ofNullable(entityFQN).ifPresent(ef -> searchListFilter.addQueryParam("entityFQN", ef)); + Optional.ofNullable(dataContractId) + .ifPresent(dci -> searchListFilter.addQueryParam("dataContractId", dci)); Optional.ofNullable(type).ifPresent(t -> searchListFilter.addQueryParam("testCaseType", t)); Optional.ofNullable(dataQualityDimension) .ifPresent(dqd -> searchListFilter.addQueryParam("dataQualityDimension", dqd)); - List authRequests = getAuthRequestsForListOps(testCaseFQN, testSuiteId); + List authRequests = + getAuthRequestsForListOps(testCaseFQN, testSuiteId, dataContractId); if (latest.equals("true")) { return listLatestFromSearch( securityContext, @@ -397,6 +406,11 @@ public TestCaseResult latestTestCaseResultFromSearch( schema = @Schema(type = "string")) @QueryParam("testSuiteId") String testSuiteId, + @Parameter( + description = "Data Contract Id the test case belongs to", + schema = @Schema(type = "string")) + @QueryParam("dataContractId") + String dataContractId, @Parameter( description = "search query term to use in list", schema = @Schema(type = "string")) @@ -411,8 +425,11 @@ public TestCaseResult latestTestCaseResultFromSearch( .ifPresent(tcf -> searchListFilter.addQueryParam("testCaseFQN", tcf)); Optional.ofNullable(testSuiteId) .ifPresent(tsi -> searchListFilter.addQueryParam("testSuiteId", tsi)); + Optional.ofNullable(dataContractId) + .ifPresent(dci -> searchListFilter.addQueryParam("dataContractId", dci)); - List authRequests = getAuthRequestsForListOps(testCaseFQN, testSuiteId); + List authRequests = + getAuthRequestsForListOps(testCaseFQN, testSuiteId, dataContractId); return super.latestInternalFromSearch( securityContext, fields, searchListFilter, q, authRequests, AuthorizationLogic.ANY); @@ -530,7 +547,8 @@ private ResourceContextInterface getResourceContext(String testCaseFQN) { return resourceContext; } - private List getAuthRequestsForListOps(String testCaseFQN, String testSuiteId) { + private List getAuthRequestsForListOps( + String testCaseFQN, String testSuiteId, String dataContractId) { List authRequests = new ArrayList<>(); if (testCaseFQN != null) { TestCase testCase = getTestCase(testCaseFQN); @@ -561,6 +579,14 @@ private List getAuthRequestsForListOps(String testCaseFQN, String t authRequests.add(new AuthRequest(testSuiteOperationContext, testSuiteResourceContext)); } + if (dataContractId != null) { + ResourceContextInterface dataContractResourceContext = + new ResourceContext<>(DATA_CONTRACT, UUID.fromString(dataContractId), null); + OperationContext dataContractOperationContext = + new OperationContext(DATA_CONTRACT, MetadataOperation.VIEW_ALL); + authRequests.add(new AuthRequest(dataContractOperationContext, dataContractResourceContext)); + } + return authRequests; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java index 13eee6e82f62..97e28967106a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java @@ -231,6 +231,7 @@ private String getTestCaseResultCondition() { String testCaseFQN = getQueryParam("testCaseFQN"); String testCaseStatus = getQueryParam("testCaseStatus"); String testSuiteId = getQueryParam("testSuiteId"); + String dataContractId = getQueryParam("dataContractId"); if (entityFQN != null) conditions.add(getTestCaseForEntityCondition(entityFQN, "testCase.entityFQN")); @@ -255,6 +256,8 @@ private String getTestCaseResultCondition() { conditions.add( getDataQualityDimensionCondition( dataQualityDimension, "testDefinition.dataQualityDimension")); + if (dataContractId != null) + conditions.add(String.format("{\"term\": {\"dataContract.id\": \"%s\"}}", dataContractId)); return addCondition(conditions); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResultIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResultIndex.java index 80c80e50fd26..d6d6d80e42db 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResultIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResultIndex.java @@ -47,7 +47,7 @@ public Map buildSearchIndexDocInternal(Map esDoc Entity.getEntityByName( Entity.TEST_CASE, testCaseResult.getTestCaseFQN(), - "testSuites,testSuite,testDefinition,entityLink", + "testSuites,testSuite,testDefinition,entityLink,dataContract", Include.ALL); // Load TestDefinition with only required fields @@ -72,12 +72,16 @@ public Map buildSearchIndexDocInternal(Map esDoc "testSuites", "testSuite", "testCaseResult", - "testDefinition")); // remove testCase fields not needed + "testDefinition", + "dataContract")); // remove testCase fields not needed esDoc.put("testCase", testCaseMap); esDoc.put("@timestamp", testCaseResult.getTimestamp()); if (testDefinition != null) { esDoc.put("testDefinition", JsonUtils.getMap(testDefinition)); } + if (testCase.getDataContract() != null) { + esDoc.put("dataContract", JsonUtils.getMap(testCase.getDataContract())); + } setParentRelationships(testCase, esDoc); return esDoc; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java index 200a5b81411d..5b2b14a28421 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java @@ -16,6 +16,7 @@ import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -116,6 +117,7 @@ import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.datamodels.DashboardDataModelResourceTest; import org.openmetadata.service.resources.dqtests.TestCaseResourceTest; +import org.openmetadata.service.resources.dqtests.TestCaseResultResource; import org.openmetadata.service.resources.dqtests.TestSuiteResourceTest; import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineResourceTest; import org.openmetadata.service.resources.topics.TopicResourceTest; @@ -5893,4 +5895,233 @@ void testCompleteWorkflowFromNoResultsToAllResults(TestInfo test) throws IOExcep double expectedScore = (2.0 / 3.0) * 100; assertEquals(expectedScore, finalResult.getQualityValidation().getQualityScore(), 0.01); } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testSearchListEndpointWithDataContractIdFilter(TestInfo test) throws IOException { + // Create two unique tables for test isolation + Table table1 = createUniqueTable(test.getDisplayName() + "_1"); + Table table2 = createUniqueTable(test.getDisplayName() + "_2"); + + String table1Link = String.format("<#E::table::%s>", table1.getFullyQualifiedName()); + String table2Link = String.format("<#E::table::%s>", table2.getFullyQualifiedName()); + + // Create first data contract with test cases + CreateDataContract createContract1 = + createDataContractRequest("testSearchContract1_" + test.getDisplayName(), table1); + + // Create second data contract with different test cases + CreateDataContract createContract2 = + createDataContractRequest("testSearchContract2_" + test.getDisplayName(), table2); + + // First create the data contracts + DataContract dataContract1 = createDataContract(createContract1); + DataContract dataContract2 = createDataContract(createContract2); + + // Then create test cases associated with the data contracts + CreateTestCase createTestCase1 = + testCaseResourceTest + .createRequest("search_test_case_1_" + test.getDisplayName()) + .withEntityLink(table1Link) + .withDataContract(dataContract1.getFullyQualifiedName()); + TestCase testCase1 = + testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase2 = + testCaseResourceTest + .createRequest("search_test_case_2_" + test.getDisplayName()) + .withEntityLink(table1Link) + .withDataContract(dataContract1.getFullyQualifiedName()); + TestCase testCase2 = + testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase3 = + testCaseResourceTest + .createRequest("search_test_case_3_" + test.getDisplayName()) + .withEntityLink(table2Link) + .withDataContract(dataContract2.getFullyQualifiedName()); + TestCase testCase3 = + testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); + + // Create test case results for all test cases + CreateTestCaseResult result1 = + new CreateTestCaseResult() + .withResult("Test passed successfully") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase1.getFullyQualifiedName(), result1); + + CreateTestCaseResult result2 = + new CreateTestCaseResult() + .withResult("Test failed due to data quality issue") + .withTestCaseStatus(TestCaseStatus.Failed) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase2.getFullyQualifiedName(), result2); + + CreateTestCaseResult result3 = + new CreateTestCaseResult() + .withResult("Test passed with warnings") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(System.currentTimeMillis()); + postTestCaseResult(testCase3.getFullyQualifiedName(), result3); + + // Test 1: Filter by dataContract1 ID - should return results for testCase1 and testCase2 + Map queryParams = new HashMap<>(); + queryParams.put("dataContractId", dataContract1.getId().toString()); + ResultList testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + assertEquals( + 2, testCaseResults.getData().size(), "Should return exactly 2 results for dataContract1"); + + List returnedTestCaseFQNs = + testCaseResults.getData().stream() + .map(TestCaseResult::getTestCaseFQN) + .collect(Collectors.toList()); + + assertTrue( + returnedTestCaseFQNs.contains(testCase1.getFullyQualifiedName()), + "Results should include testCase1"); + assertTrue( + returnedTestCaseFQNs.contains(testCase2.getFullyQualifiedName()), + "Results should include testCase2"); + + // Verify result content and completeness + for (TestCaseResult result : testCaseResults.getData()) { + assertNotNull(result.getId(), "Result should have an ID"); + assertNotNull(result.getTestCaseFQN(), "Result should have test case FQN"); + assertNotNull(result.getTestCaseStatus(), "Result should have status"); + assertNotNull(result.getTimestamp(), "Result should have timestamp"); + assertNotNull(result.getResult(), "Result should have result content"); + } + + // Test 2: Filter by dataContract2 ID - should return result for testCase3 only + queryParams.clear(); + queryParams.put("dataContractId", dataContract2.getId().toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + assertEquals( + 1, testCaseResults.getData().size(), "Should return exactly 1 result for dataContract2"); + assertEquals( + testCase3.getFullyQualifiedName(), + testCaseResults.getData().get(0).getTestCaseFQN(), + "Should return result for testCase3"); + assertEquals( + TestCaseStatus.Success, + testCaseResults.getData().get(0).getTestCaseStatus(), + "testCase3 result should have Success status"); + + // Test 3: Edge case - filter with non-existent dataContractId + queryParams.clear(); + queryParams.put("dataContractId", UUID.randomUUID().toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + assertEquals( + 0, + testCaseResults.getData().size(), + "Should return no results for non-existent dataContractId"); + + // Test 4: Combine dataContractId filter with status filter + queryParams.clear(); + queryParams.put("dataContractId", dataContract1.getId().toString()); + queryParams.put("testCaseStatus", TestCaseStatus.Success.toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + assertEquals( + 1, testCaseResults.getData().size(), "Should return 1 successful result for dataContract1"); + assertEquals( + testCase1.getFullyQualifiedName(), + testCaseResults.getData().get(0).getTestCaseFQN(), + "Should return successful testCase1 result"); + assertEquals( + TestCaseStatus.Success, + testCaseResults.getData().get(0).getTestCaseStatus(), + "Returned result should have Success status"); + + // Test 5: Combine dataContractId filter with conflicting status filter + queryParams.clear(); + queryParams.put("dataContractId", dataContract2.getId().toString()); + queryParams.put("testCaseStatus", TestCaseStatus.Failed.toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + assertEquals( + 0, + testCaseResults.getData().size(), + "Should return no results when dataContract2 has no failed tests"); + + // Test 6: Test with fields parameter to verify returned data is complete + queryParams.clear(); + queryParams.put("dataContractId", dataContract1.getId().toString()); + queryParams.put("fields", "testCase,testDefinition"); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + assertEquals( + 2, testCaseResults.getData().size(), "Should return 2 results with requested fields"); + + for (TestCaseResult result : testCaseResults.getData()) { + assertNotNull(result.getTestCase(), "Result should include testCase field"); + assertNotNull(result.getTestDefinition(), "Result should include testDefinition field"); + } + + // Test 7: Test pagination with dataContractId filter + queryParams.clear(); + queryParams.put("dataContractId", dataContract1.getId().toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 1, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + assertEquals(1, testCaseResults.getData().size(), "Should return 1 result with limit=1"); + assertNotNull(testCaseResults.getPaging(), "Should include paging information"); + + ResultList secondPage = + listTestCaseResultsFromSearch( + queryParams, 1, 1, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + assertEquals(1, secondPage.getData().size(), "Should return 1 result on second page"); + + // Verify different results on different pages + assertNotEquals( + testCaseResults.getData().get(0).getId(), + secondPage.getData().get(0).getId(), + "Different pages should return different results"); + } + + private ResultList listTestCaseResultsFromSearch( + Map queryParams, + Integer limit, + Integer offset, + String path, + Map authHeader) + throws HttpResponseException { + WebTarget target = + APP.client() + .target( + String.format( + "http://localhost:%s/api/v1/dataQuality/testCases/", APP.getLocalPort())) + .path(path); + + for (Map.Entry entry : queryParams.entrySet()) { + target = target.queryParam(entry.getKey(), entry.getValue()); + } + + if (limit != null) { + target = target.queryParam("limit", limit); + } + if (offset != null) { + target = target.queryParam("offset", offset); + } + + return TestUtils.get(target, TestCaseResultResource.TestCaseResultList.class, authHeader); + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index 809c58c428cc..150be94d9e61 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -77,6 +77,7 @@ import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; +import org.openmetadata.schema.api.data.CreateDataContract; import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.feed.CloseTask; import org.openmetadata.schema.api.feed.ResolveTask; @@ -88,6 +89,7 @@ import org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus; import org.openmetadata.schema.api.tests.CreateTestCaseResult; import org.openmetadata.schema.api.tests.CreateTestSuite; +import org.openmetadata.schema.entity.data.DataContract; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.feed.Thread; import org.openmetadata.schema.entity.policies.Policy; @@ -128,6 +130,7 @@ import org.openmetadata.search.IndexMapping; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.EntityResourceTest; +import org.openmetadata.service.resources.data.DataContractResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.feeds.FeedResourceTest; import org.openmetadata.service.resources.feeds.MessageParser; @@ -4041,4 +4044,167 @@ void test_listTestCasesFilterByCreatedBy(TestInfo testInfo) throws IOException { "List endpoint should also filter by createdBy"); } } + + @Test + void test_listTestCaseResultsFromSearchWithDataContractIdFilter(TestInfo testInfo) + throws HttpResponseException, ParseException, IOException { + // Create two tables for test case setup (each data contract needs its own table) + TableResourceTest tableResourceTest = new TableResourceTest(); + CreateTable createTable1 = tableResourceTest.createRequest(testInfo, 1); + Table table1 = tableResourceTest.createEntity(createTable1, ADMIN_AUTH_HEADERS); + + CreateTable createTable2 = tableResourceTest.createRequest(testInfo, 2); + Table table2 = tableResourceTest.createEntity(createTable2, ADMIN_AUTH_HEADERS); + + // Create two data contracts - one for each table + DataContractResourceTest dataContractResourceTest = new DataContractResourceTest(); + CreateDataContract createDataContract1 = + dataContractResourceTest.createRequest(testInfo, 1).withEntity(table1.getEntityReference()); + DataContract dataContract1 = + dataContractResourceTest.createEntity(createDataContract1, ADMIN_AUTH_HEADERS); + + CreateDataContract createDataContract2 = + dataContractResourceTest.createRequest(testInfo, 2).withEntity(table2.getEntityReference()); + DataContract dataContract2 = + dataContractResourceTest.createEntity(createDataContract2, ADMIN_AUTH_HEADERS); + + // Create test cases - one for each data contract and one without data contract + CreateTestCase createTestCase1 = + createRequest("test_case_1_" + testInfo.getDisplayName()) + .withEntityLink(String.format("<#E::table::%s>", table1.getFullyQualifiedName())) + .withDataContract(dataContract1.getFullyQualifiedName()); + TestCase testCase1 = createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase2 = + createRequest("test_case_2_" + testInfo.getDisplayName()) + .withEntityLink(String.format("<#E::table::%s>", table2.getFullyQualifiedName())) + .withDataContract(dataContract2.getFullyQualifiedName()); + TestCase testCase2 = createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase3 = + createRequest("test_case_3_" + testInfo.getDisplayName()) + .withEntityLink(String.format("<#E::table::%s>", table1.getFullyQualifiedName())); + TestCase testCase3 = createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); + + // Create test case results for all test cases + Long currentTimestamp = System.currentTimeMillis(); + + // Create results for test case 1 (associated with dataContract1) + CreateTestCaseResult createTestCaseResult1 = + new CreateTestCaseResult() + .withResult("tested") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(currentTimestamp); + postTestCaseResult( + testCase1.getFullyQualifiedName(), createTestCaseResult1, ADMIN_AUTH_HEADERS); + + // Create results for test case 2 (associated with dataContract2) + CreateTestCaseResult createTestCaseResult2 = + new CreateTestCaseResult() + .withResult("tested") + .withTestCaseStatus(TestCaseStatus.Failed) + .withTimestamp(currentTimestamp + 1000); + postTestCaseResult( + testCase2.getFullyQualifiedName(), createTestCaseResult2, ADMIN_AUTH_HEADERS); + + // Create results for test case 3 (no data contract) + CreateTestCaseResult createTestCaseResult3 = + new CreateTestCaseResult() + .withResult("tested") + .withTestCaseStatus(TestCaseStatus.Success) + .withTimestamp(currentTimestamp + 2000); + postTestCaseResult( + testCase3.getFullyQualifiedName(), createTestCaseResult3, ADMIN_AUTH_HEADERS); + + // Wait for search index update + try { + java.lang.Thread.sleep(1000); + } catch (InterruptedException e) { + java.lang.Thread.currentThread().interrupt(); + } + + // Test filtering by dataContract1 ID + Map queryParams = new HashMap<>(); + queryParams.put("dataContractId", dataContract1.getId().toString()); + ResultList testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + // Should only return results for test case 1 + assertEquals(1, testCaseResults.getData().size()); + assertEquals( + testCase1.getFullyQualifiedName(), testCaseResults.getData().get(0).getTestCaseFQN()); + assertEquals(TestCaseStatus.Success, testCaseResults.getData().get(0).getTestCaseStatus()); + + // Test filtering by dataContract2 ID + queryParams.clear(); + queryParams.put("dataContractId", dataContract2.getId().toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + // Should only return results for test case 2 + assertEquals(1, testCaseResults.getData().size()); + assertEquals( + testCase2.getFullyQualifiedName(), testCaseResults.getData().get(0).getTestCaseFQN()); + assertEquals(TestCaseStatus.Failed, testCaseResults.getData().get(0).getTestCaseStatus()); + + // Test filtering by non-existent data contract ID + queryParams.clear(); + queryParams.put("dataContractId", UUID.randomUUID().toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + // Should return no results + assertEquals(0, testCaseResults.getData().size()); + + // Test combining dataContractId with other filters + queryParams.clear(); + queryParams.put("dataContractId", dataContract1.getId().toString()); + queryParams.put("testCaseStatus", TestCaseStatus.Success.toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + // Should return the successful result for test case 1 + assertEquals(1, testCaseResults.getData().size()); + assertEquals( + testCase1.getFullyQualifiedName(), testCaseResults.getData().get(0).getTestCaseFQN()); + assertEquals(TestCaseStatus.Success, testCaseResults.getData().get(0).getTestCaseStatus()); + + // Test combining dataContractId with conflicting status filter + queryParams.clear(); + queryParams.put("dataContractId", dataContract1.getId().toString()); + queryParams.put("testCaseStatus", TestCaseStatus.Failed.toString()); + testCaseResults = + listTestCaseResultsFromSearch( + queryParams, 10, 0, "/testCaseResults/search/list", ADMIN_AUTH_HEADERS); + + // Should return no results since dataContract1's test case succeeded + assertEquals(0, testCaseResults.getData().size()); + + // Test latest endpoint with dataContractId filter + queryParams.clear(); + queryParams.put("dataContractId", dataContract1.getId().toString()); + TestCaseResult latestResult = + latestTestCaseResultFromSearch( + queryParams, dataContract1.getId().toString(), ADMIN_AUTH_HEADERS); + + // Should return the latest result for dataContract1 + assertNotNull(latestResult); + assertEquals(testCase1.getFullyQualifiedName(), latestResult.getTestCaseFQN()); + assertEquals(TestCaseStatus.Success, latestResult.getTestCaseStatus()); + } + + public TestCaseResult latestTestCaseResultFromSearch( + Map queryParams, String dataContractId, Map authHeaders) + throws HttpResponseException { + WebTarget target = getCollection().path("/testCaseResults/search/latest"); + for (Map.Entry entry : queryParams.entrySet()) { + target = target.queryParam(entry.getKey(), entry.getValue()); + } + target = target.queryParam("dataContractId", dataContractId); + return TestUtils.get(target, TestCaseResult.class, authHeaders); + } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json index 390fc0f06544..8fb5560a7195 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json @@ -493,6 +493,52 @@ "ignore_above": 256 } } + }, + "dataContract": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "term_vector": "with_positions_offsets" + } + } } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json index fb21e7b9ea5c..ceddda62bd06 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json @@ -93,24 +93,25 @@ } }, "testCase": { - "id": { - "type": "text" - }, - "name": { - "type": "text", - "analyzer": "om_analyzer_jp", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256, - "normalizer": "lowercase_normalizer" - }, - "ngram": { - "type": "text", - "analyzer": "om_ngram" + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } } - } - }, + }, "fullyQualifiedName": { "type": "text", "analyzer": "om_analyzer_jp", @@ -325,6 +326,51 @@ } } }, + "dataContract": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer_jp" + } + } + }, "testDefinition": { "properties": { "id": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_result_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_result_index_mapping.json index 733430232078..ab4b10c09fd0 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_result_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_result_index_mapping.json @@ -512,6 +512,52 @@ "ignore_above": 256 } } + }, + "dataContract": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "term_vector": "with_positions_offsets" + } + } } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json index 212ad06a228a..8838b77ea2c5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json @@ -455,6 +455,51 @@ "type": "text" } } + }, + "dataContract": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "text", + "analyzer": "ik_max_word", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "ik_max_word", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + } + } + }, + "description": { + "type": "text", + "analyzer": "ik_max_word" + } + } } } }, diff --git a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCase.json b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCase.json index 0610e75ff2f2..7f9813e1fe95 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCase.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCase.json @@ -36,6 +36,10 @@ "description": "Owners of this test", "$ref": "../../type/entityReferenceList.json" }, + "dataContract": { + "description": "Data Contract that this test case is associated with. Fully qualified name of the data contract.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, "computePassedFailedRowCount": { "description": "Compute the passed and failed row count for the test case.", "type": "boolean", diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json b/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json index 96cad0eca568..20b28b080ce7 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json @@ -66,6 +66,10 @@ "$ref": "./testSuite.json" } }, + "dataContract": { + "description": "Data Contract that this test case is associated with.", + "$ref": "../type/entityReference.json" + }, "parameterValues": { "type": "array", "items": { From 4cdcfaff00025ad966bfb1f0e707116522246b42 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 23 Sep 2025 12:21:16 +0200 Subject: [PATCH 05/38] test case data contract migration --- .../migration/mysql/v1100/Migration.java | 1 + .../migration/postgres/v1100/Migration.java | 1 + .../migration/utils/v1100/MigrationUtil.java | 116 ++++++++++++++++++ 3 files changed, 118 insertions(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1100/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1100/Migration.java index 0d43444393c5..ab24e161e09f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1100/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1100/Migration.java @@ -16,5 +16,6 @@ public Migration(MigrationFile migrationFile) { public void runDataMigration() { MigrationUtil migrationUtil = new MigrationUtil(collectionDAO); migrationUtil.migrateEntityStatusForExistingEntities(handle); + migrationUtil.migrateTestCaseDataContractReferences(handle); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1100/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1100/Migration.java index 20bb06da5a7d..74764e37d897 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1100/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1100/Migration.java @@ -16,5 +16,6 @@ public Migration(MigrationFile migrationFile) { public void runDataMigration() { MigrationUtil migrationUtil = new MigrationUtil(collectionDAO); migrationUtil.migrateEntityStatusForExistingEntities(handle); + migrationUtil.migrateTestCaseDataContractReferences(handle); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1100/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1100/MigrationUtil.java index 4a8556483b81..30886ec8ca21 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1100/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1100/MigrationUtil.java @@ -1,10 +1,18 @@ package org.openmetadata.service.migration.utils.v1100; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.SQLException; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.core.Handle; +import org.openmetadata.schema.entity.data.DataContract; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; @Slf4j @@ -309,4 +317,112 @@ private String buildCountQuery(String tableName) { tableName); } } + + public void migrateTestCaseDataContractReferences(Handle handle) { + LOG.info("===== STARTING TEST CASE DATA CONTRACT MIGRATION ====="); + + int totalTestCasesMigrated = 0; + int dataContractsProcessed = 0; + int pageSize = 1000; + int offset = 0; + + try { + // Step 1: Paginate through all data contracts using DAO + while (true) { + List dataContractJsons = + collectionDAO.dataContractDAO().listAfterWithOffset(pageSize, offset); + if (dataContractJsons.isEmpty()) { + break; + } + offset += pageSize; + + LOG.info( + "Processing {} data contracts in batch (offset: {})", + dataContractJsons.size(), + offset - pageSize); + + for (String dataContractJson : dataContractJsons) { + try { + DataContract dataContract = JsonUtils.readValue(dataContractJson, DataContract.class); + + // Step 2: Filter - only process contracts with quality expectations + if (nullOrEmpty(dataContract.getQualityExpectations())) { + LOG.debug( + "Data contract {} has no quality expectations, skipping", + dataContract.getFullyQualifiedName()); + continue; + } + + LOG.debug( + "Processing data contract: {} (ID: {}) with {} quality expectations", + dataContract.getFullyQualifiedName(), + dataContract.getId(), + dataContract.getQualityExpectations().size()); + dataContractsProcessed++; + + // Step 3: Process each test case in quality expectations + int testCasesUpdated = 0; + for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { + try { + // Get test case using DAO + TestCase testCase = collectionDAO.testCaseDAO().findEntityById(testCaseRef.getId()); + if (testCase == null) { + LOG.debug("Test case not found: {}", testCaseRef.getId()); + continue; + } + + // Check if test case already has dataContract reference + if (testCase.getDataContract() != null) { + LOG.debug( + "Test case {} already has dataContract reference", + testCase.getFullyQualifiedName()); + continue; + } + + // Step 4: Update test case with dataContract reference using DAO + testCase.setDataContract( + new EntityReference() + .withId(dataContract.getId()) + .withType(Entity.DATA_CONTRACT) + .withFullyQualifiedName(dataContract.getFullyQualifiedName())); + + // Update the test case using DAO + collectionDAO.testCaseDAO().update(testCase); + testCasesUpdated++; + + LOG.debug( + "Updated test case {} with dataContract reference to {}", + testCase.getFullyQualifiedName(), + dataContract.getFullyQualifiedName()); + + } catch (Exception e) { + LOG.warn("Failed to update test case {}: {}", testCaseRef.getId(), e.getMessage()); + } + } + + totalTestCasesMigrated += testCasesUpdated; + + if (testCasesUpdated > 0) { + LOG.info( + "Updated {} test cases for data contract: {}", + testCasesUpdated, + dataContract.getFullyQualifiedName()); + } + + } catch (Exception e) { + LOG.error("Failed to process data contract: {}", e.getMessage(), e); + } + } + } + + } catch (Exception e) { + LOG.error("Error during test case dataContract migration: {}", e.getMessage(), e); + throw new RuntimeException("Migration failed", e); + } + + LOG.info("===== TEST CASE DATA CONTRACT MIGRATION SUMMARY ====="); + LOG.info("Data contracts processed: {}", dataContractsProcessed); + LOG.info("Total test cases updated with dataContract reference: {}", totalTestCasesMigrated); + LOG.info("===== MIGRATION COMPLETE ====="); + } } From 56ee55d513329153ac281d68aae35695ddb2771e Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 23 Sep 2025 14:38:10 +0200 Subject: [PATCH 06/38] fix tests --- .../service/jdbi3/CollectionDAO.java | 30 +++ .../service/jdbi3/DataContractRepository.java | 118 ++++++++++++ .../data/DataContractResourceTest.java | 171 ++++++++++++++++++ 3 files changed, 319 insertions(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index b367278d054a..04a672ada769 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -5928,6 +5928,36 @@ public TestCaseRecord map(ResultSet rs, StatementContext ctx) throws SQLExceptio return new TestCaseRecord(rs.getString("json"), rs.getInt("ranked")); } } + + @ConnectionAwareSqlUpdate( + value = + "UPDATE test_case SET json = JSON_SET(json, '$.dataContract', CAST(:dataContractJson AS JSON)) WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE test_case SET json = jsonb_set(json, '{dataContract}', to_jsonb(:dataContractJson::text), false) WHERE id = :id", + connectionType = POSTGRES) + void updateTestCaseDataContract( + @Bind("id") String id, @Bind("dataContractJson") String dataContractJson); + + @ConnectionAwareSqlUpdate( + value = "UPDATE test_case SET json = JSON_REMOVE(json, '$.dataContract') WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE test_case SET json = json - 'dataContract' WHERE id = :id", + connectionType = POSTGRES) + void removeTestCaseDataContract(@Bind("id") String id); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE test_case SET json = JSON_REMOVE(json, '$.dataContract') WHERE id = :id AND JSON_EXTRACT(json, '$.dataContract.id') = :dataContractId", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE test_case SET json = json - 'dataContract' WHERE id = :id AND json->'dataContract'->>'id' = :dataContractId", + connectionType = POSTGRES) + void removeTestCaseDataContractForSpecificContract( + @Bind("id") String id, @Bind("dataContractId") String dataContractId); } interface WebAnalyticEventDAO extends EntityDAO { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index cc601c808a25..989bc7bfe474 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -170,6 +170,8 @@ protected void setDefaultStatus(DataContract entity, boolean update) { protected void postCreate(DataContract dataContract) { super.postCreate(dataContract); postCreateOrUpdate(dataContract); + // Update test cases with dataContract reference + updateTestCasesWithDataContract(dataContract); } // If we update the contract adding DQ validation, add the pipeline if needed @@ -177,6 +179,10 @@ protected void postCreate(DataContract dataContract) { protected void postUpdate(DataContract original, DataContract updated) { super.postUpdate(original, updated); postCreateOrUpdate(updated); + // Update test cases with dataContract reference for new/updated quality expectations + updateTestCasesWithDataContract(updated); + // Remove dataContract reference from test cases no longer in quality expectations + removeDataContractFromOldTestCases(original, updated); } @Override @@ -185,6 +191,8 @@ protected void postDelete(DataContract dataContract, boolean hardDelete) { if (!nullOrEmpty(dataContract.getQualityExpectations())) { deleteTestSuite(dataContract); } + // Remove dataContract reference from all associated test cases + removeDataContractFromTestCases(dataContract); // Clean status daoCollection .entityExtensionTimeSeriesDao() @@ -1190,4 +1198,114 @@ private void validateEntityReference(EntityReference entity) { entity.getType(), entity.getId())); } } + + private void updateTestCasesWithDataContract(DataContract dataContract) { + if (nullOrEmpty(dataContract.getQualityExpectations())) { + return; + } + + for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { + try { + // Get the existing test case to check if it already has a dataContract + TestCase existingTestCase = daoCollection.testCaseDAO().findEntityById(testCaseRef.getId()); + if (existingTestCase == null) { + LOG.warn("Test case {} not found, skipping dataContract update", testCaseRef.getId()); + continue; + } + + // Only update if the test case doesn't have a dataContract or has a different dataContract + // ID + boolean shouldUpdate = + existingTestCase.getDataContract() == null + || !existingTestCase.getDataContract().getId().equals(dataContract.getId()); + + if (shouldUpdate) { + // Create the dataContract EntityReference + EntityReference dataContractRef = + new EntityReference() + .withId(dataContract.getId()) + .withType(Entity.DATA_CONTRACT) + .withFullyQualifiedName(dataContract.getFullyQualifiedName()); + + // Use testCase DAO to update the dataContract field directly + daoCollection + .testCaseDAO() + .updateTestCaseDataContract( + testCaseRef.getId().toString(), JsonUtils.pojoToJson(dataContractRef)); + + LOG.debug( + "Updated test case {} with dataContract reference to {}", + testCaseRef.getId(), + dataContract.getFullyQualifiedName()); + } else { + LOG.debug( + "Test case {} already has the same dataContract reference, skipping update", + testCaseRef.getId()); + } + } catch (Exception e) { + LOG.warn( + "Failed to update test case {} with dataContract reference: {}", + testCaseRef.getId(), + e.getMessage()); + } + } + } + + private void removeDataContractFromTestCases(DataContract dataContract) { + if (nullOrEmpty(dataContract.getQualityExpectations())) { + return; + } + + for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { + try { + // Use testCase DAO to remove the dataContract field + daoCollection.testCaseDAO().removeTestCaseDataContract(testCaseRef.getId().toString()); + + LOG.debug("Removed dataContract reference from test case {}", testCaseRef.getId()); + } catch (Exception e) { + LOG.warn( + "Failed to remove dataContract reference from test case {}: {}", + testCaseRef.getId(), + e.getMessage()); + } + } + } + + private void removeDataContractFromOldTestCases(DataContract original, DataContract updated) { + // Find test cases that were in the original but not in the updated quality expectations + if (nullOrEmpty(original.getQualityExpectations())) { + return; + } + + Set updatedTestCaseIds = + nullOrEmpty(updated.getQualityExpectations()) + ? Collections.emptySet() + : updated.getQualityExpectations().stream() + .map(EntityReference::getId) + .collect(Collectors.toSet()); + + for (EntityReference testCaseRef : original.getQualityExpectations()) { + // If this test case is no longer in the updated quality expectations, remove dataContract + // reference + if (!updatedTestCaseIds.contains(testCaseRef.getId())) { + try { + // Use testCase DAO to remove the dataContract field only if it points to this data + // contract + daoCollection + .testCaseDAO() + .removeTestCaseDataContractForSpecificContract( + testCaseRef.getId().toString(), original.getId().toString()); + + LOG.debug( + "Removed dataContract reference from test case {} (no longer in quality expectations)", + testCaseRef.getId()); + } catch (Exception e) { + LOG.warn( + "Failed to remove dataContract reference from old test case {}: {}", + testCaseRef.getId(), + e.getMessage()); + } + } + } + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java index 5b2b14a28421..d97c1dc40411 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java @@ -2743,6 +2743,7 @@ void testUpdateDataContractQualityExpectations_TestSuiteUpdated(TestInfo test) CreateTestCase createTestCase2 = testCaseResourceTest .createRequest("test_case_2_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -3017,6 +3018,7 @@ void testValidateDataContractWithPassingSemanticsAndDQExpectations(TestInfo test CreateTestCase createTestCase1 = testCaseResourceTest .createRequest("test_case_completeness_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase1 = testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); @@ -3024,6 +3026,7 @@ void testValidateDataContractWithPassingSemanticsAndDQExpectations(TestInfo test CreateTestCase createTestCase2 = testCaseResourceTest .createRequest("test_case_validity_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -3140,6 +3143,7 @@ void testValidateDataContractWithPassingSemanticsButFailingDQExpectations(TestIn CreateTestCase createTestCase1 = testCaseResourceTest .createRequest("test_case_completeness_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase1 = testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); @@ -3147,6 +3151,7 @@ void testValidateDataContractWithPassingSemanticsButFailingDQExpectations(TestIn CreateTestCase createTestCase2 = testCaseResourceTest .createRequest("test_case_validity_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -3479,6 +3484,7 @@ void testDeleteDataContractWithDQExpectationsDoesNotDeleteTestCases(TestInfo tes CreateTestCase createTestCase1 = testCaseResourceTest .createRequest("test_case_completeness_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase1 = testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); @@ -3486,6 +3492,7 @@ void testDeleteDataContractWithDQExpectationsDoesNotDeleteTestCases(TestInfo tes CreateTestCase createTestCase2 = testCaseResourceTest .createRequest("test_case_validity_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -3608,6 +3615,7 @@ void testCreateContractWithSemanticThenAddQualityExpectationCreatesTestSuite(Tes CreateTestCase createTestCase = testCaseResourceTest .createRequest("test_case_quality_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase = testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); @@ -3836,6 +3844,7 @@ void testDashboardEntityConstraints(TestInfo test) throws IOException { CreateTestCase createTestCase = testCaseResourceTest .createRequest("test_case_dashboard_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase = testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); @@ -4558,6 +4567,7 @@ void testTableEntityConstraints(TestInfo test) throws IOException { CreateTestCase createTestCase = testCaseResourceTest .createRequest("test_case_table_all_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase = testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); @@ -4657,6 +4667,7 @@ void testQualityExpectationConstraints(TestInfo test) throws IOException { CreateTestCase createTestCase = testCaseResourceTest .createRequest("test_case_quality_constraint_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase = testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); @@ -4677,6 +4688,7 @@ void testQualityExpectationConstraints(TestInfo test) throws IOException { CreateTestCase tableTestCase = testCaseResourceTest .createRequest("test_case_table_quality_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(realTableLink); TestCase tableTest = testCaseResourceTest.createAndCheckEntity(tableTestCase, ADMIN_AUTH_HEADERS); @@ -4783,6 +4795,7 @@ void testMultipleValidationErrors(TestInfo test) throws IOException { CreateTestCase createTestCase = testCaseResourceTest .createRequest("test_case_multiple_errors_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase = testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); @@ -5564,6 +5577,7 @@ void testValidateContractWithNoTestResults(TestInfo test) throws IOException { CreateTestCase createTestCase1 = testCaseResourceTest .createRequest("test_case_1_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase1 = testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); @@ -5571,6 +5585,7 @@ void testValidateContractWithNoTestResults(TestInfo test) throws IOException { CreateTestCase createTestCase2 = testCaseResourceTest .createRequest("test_case_2_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -5578,6 +5593,7 @@ void testValidateContractWithNoTestResults(TestInfo test) throws IOException { CreateTestCase createTestCase3 = testCaseResourceTest .createRequest("test_case_3_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase3 = testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); @@ -5621,6 +5637,7 @@ void testValidateContractWithAllTestResultsCompileExisting(TestInfo test) throws CreateTestCase createTestCase1 = testCaseResourceTest .createRequest("test_case_all_results_1_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase1 = testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); @@ -5628,6 +5645,7 @@ void testValidateContractWithAllTestResultsCompileExisting(TestInfo test) throws CreateTestCase createTestCase2 = testCaseResourceTest .createRequest("test_case_all_results_2_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -5698,6 +5716,7 @@ void testUpdateContractFiltersTestsWithResults(TestInfo test) throws IOException CreateTestCase createTestCase1 = testCaseResourceTest .createRequest("test_case_update_1_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase1 = testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); @@ -5705,6 +5724,7 @@ void testUpdateContractFiltersTestsWithResults(TestInfo test) throws IOException CreateTestCase createTestCase2 = testCaseResourceTest .createRequest("test_case_update_2_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -5712,6 +5732,7 @@ void testUpdateContractFiltersTestsWithResults(TestInfo test) throws IOException CreateTestCase createTestCase3 = testCaseResourceTest .createRequest("test_case_update_3_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase3 = testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); @@ -5794,6 +5815,7 @@ void testCompleteWorkflowFromNoResultsToAllResults(TestInfo test) throws IOExcep CreateTestCase createTestCase1 = testCaseResourceTest .createRequest("test_case_workflow_1_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase1 = testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); @@ -5801,6 +5823,7 @@ void testCompleteWorkflowFromNoResultsToAllResults(TestInfo test) throws IOExcep CreateTestCase createTestCase2 = testCaseResourceTest .createRequest("test_case_workflow_2_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -5808,6 +5831,7 @@ void testCompleteWorkflowFromNoResultsToAllResults(TestInfo test) throws IOExcep CreateTestCase createTestCase3 = testCaseResourceTest .createRequest("test_case_workflow_3_" + test.getDisplayName()) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withEntityLink(tableLink); TestCase testCase3 = testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); @@ -5923,6 +5947,7 @@ void testSearchListEndpointWithDataContractIdFilter(TestInfo test) throws IOExce testCaseResourceTest .createRequest("search_test_case_1_" + test.getDisplayName()) .withEntityLink(table1Link) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withDataContract(dataContract1.getFullyQualifiedName()); TestCase testCase1 = testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); @@ -5931,6 +5956,7 @@ void testSearchListEndpointWithDataContractIdFilter(TestInfo test) throws IOExce testCaseResourceTest .createRequest("search_test_case_2_" + test.getDisplayName()) .withEntityLink(table1Link) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withDataContract(dataContract1.getFullyQualifiedName()); TestCase testCase2 = testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); @@ -5939,6 +5965,7 @@ void testSearchListEndpointWithDataContractIdFilter(TestInfo test) throws IOExce testCaseResourceTest .createRequest("search_test_case_3_" + test.getDisplayName()) .withEntityLink(table2Link) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()) .withDataContract(dataContract2.getFullyQualifiedName()); TestCase testCase3 = testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); @@ -6124,4 +6151,148 @@ private ResultList listTestCaseResultsFromSearch( return TestUtils.get(target, TestCaseResultResource.TestCaseResultList.class, authHeader); } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataContractTestCaseReferencesLifecycle(TestInfo test) throws IOException { + // Test that test cases get dataContract references when contracts are created/updated/deleted + Table table = createUniqueTable(test.getDisplayName()); + String tableLink = String.format("<#E::table::%s>", table.getFullyQualifiedName()); + + // Create test cases + CreateTestCase createTestCase1 = + testCaseResourceTest + .createRequest("test_case_lifecycle_1_" + test.getDisplayName()) + .withEntityLink(tableLink) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()); + TestCase testCase1 = + testCaseResourceTest.createAndCheckEntity(createTestCase1, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase2 = + testCaseResourceTest + .createRequest("test_case_lifecycle_2_" + test.getDisplayName()) + .withEntityLink(tableLink) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()); + TestCase testCase2 = + testCaseResourceTest.createAndCheckEntity(createTestCase2, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase3 = + testCaseResourceTest + .createRequest("test_case_lifecycle_3_" + test.getDisplayName()) + .withEntityLink(tableLink) + .withTestDefinition(TEST_DEFINITION4.getFullyQualifiedName()); + TestCase testCase3 = + testCaseResourceTest.createAndCheckEntity(createTestCase3, ADMIN_AUTH_HEADERS); + + // Verify test cases initially don't have dataContract references + TestCase initialTestCase1 = + testCaseResourceTest.getEntity(testCase1.getId(), "dataContract", ADMIN_AUTH_HEADERS); + TestCase initialTestCase2 = + testCaseResourceTest.getEntity(testCase2.getId(), "dataContract", ADMIN_AUTH_HEADERS); + TestCase initialTestCase3 = + testCaseResourceTest.getEntity(testCase3.getId(), "dataContract", ADMIN_AUTH_HEADERS); + + assertNull( + initialTestCase1.getDataContract(), + "Test case 1 should not have dataContract reference initially"); + assertNull( + initialTestCase2.getDataContract(), + "Test case 2 should not have dataContract reference initially"); + assertNull( + initialTestCase3.getDataContract(), + "Test case 3 should not have dataContract reference initially"); + + // Step 1: Create data contract with quality expectations (testCase1, testCase2) + List initialQualityExpectations = + List.of(testCase1.getEntityReference(), testCase2.getEntityReference()); + + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table) + .withQualityExpectations(initialQualityExpectations); + + DataContract dataContract = createDataContract(create); + + // Verify test cases 1 and 2 now have dataContract references after creation + TestCase afterCreateTestCase1 = + testCaseResourceTest.getEntity(testCase1.getId(), "dataContract", ADMIN_AUTH_HEADERS); + TestCase afterCreateTestCase2 = + testCaseResourceTest.getEntity(testCase2.getId(), "dataContract", ADMIN_AUTH_HEADERS); + TestCase afterCreateTestCase3 = + testCaseResourceTest.getEntity(testCase3.getId(), "dataContract", ADMIN_AUTH_HEADERS); + + assertNotNull( + afterCreateTestCase1.getDataContract(), + "Test case 1 should have dataContract reference after contract creation"); + assertEquals( + dataContract.getId(), + afterCreateTestCase1.getDataContract().getId(), + "Test case 1 should reference the correct data contract"); + assertEquals( + dataContract.getFullyQualifiedName(), + afterCreateTestCase1.getDataContract().getFullyQualifiedName()); + + assertNotNull( + afterCreateTestCase2.getDataContract(), + "Test case 2 should have dataContract reference after contract creation"); + assertEquals( + dataContract.getId(), + afterCreateTestCase2.getDataContract().getId(), + "Test case 2 should reference the correct data contract"); + + assertNull( + afterCreateTestCase3.getDataContract(), + "Test case 3 should not have dataContract reference (not in quality expectations)"); + + // Step 2: Update data contract to include testCase3 and remove testCase1 + List updatedQualityExpectations = + List.of(testCase2.getEntityReference(), testCase3.getEntityReference()); + + String originalJson = JsonUtils.pojoToJson(dataContract); + dataContract.setQualityExpectations(updatedQualityExpectations); + DataContract updatedDataContract = + patchDataContract(dataContract.getId(), originalJson, dataContract); + + // Verify test case references after update + TestCase afterUpdateTestCase1 = + testCaseResourceTest.getEntity(testCase1.getId(), "dataContract", ADMIN_AUTH_HEADERS); + TestCase afterUpdateTestCase2 = + testCaseResourceTest.getEntity(testCase2.getId(), "dataContract", ADMIN_AUTH_HEADERS); + TestCase afterUpdateTestCase3 = + testCaseResourceTest.getEntity(testCase3.getId(), "dataContract", ADMIN_AUTH_HEADERS); + + assertNull( + afterUpdateTestCase1.getDataContract(), + "Test case 1 should no longer have dataContract reference (removed from quality expectations)"); + + assertNotNull( + afterUpdateTestCase2.getDataContract(), + "Test case 2 should still have dataContract reference"); + assertEquals(dataContract.getId(), afterUpdateTestCase2.getDataContract().getId()); + + assertNotNull( + afterUpdateTestCase3.getDataContract(), + "Test case 3 should now have dataContract reference (added to quality expectations)"); + assertEquals(dataContract.getId(), afterUpdateTestCase3.getDataContract().getId()); + + // Step 3: Delete data contract + deleteDataContract(dataContract.getId()); + + // Verify all test case references are removed after deletion + TestCase afterDeleteTestCase1 = + testCaseResourceTest.getEntity(testCase1.getId(), "dataContract", ADMIN_AUTH_HEADERS); + TestCase afterDeleteTestCase2 = + testCaseResourceTest.getEntity(testCase2.getId(), "dataContract", ADMIN_AUTH_HEADERS); + TestCase afterDeleteTestCase3 = + testCaseResourceTest.getEntity(testCase3.getId(), "dataContract", ADMIN_AUTH_HEADERS); + + assertNull( + afterDeleteTestCase1.getDataContract(), + "Test case 1 should not have dataContract reference after contract deletion"); + assertNull( + afterDeleteTestCase2.getDataContract(), + "Test case 2 should not have dataContract reference after contract deletion"); + assertNull( + afterDeleteTestCase3.getDataContract(), + "Test case 3 should not have dataContract reference after contract deletion"); + } } From 2e1dd7488db9ec810967d4eb37777f6227c8720d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 23 Sep 2025 19:31:41 +0000 Subject: [PATCH 07/38] Update generated TypeScript types --- .../ui/src/generated/api/tests/createTestCase.ts | 5 +++++ .../resources/ui/src/generated/tests/testCase.ts | 14 ++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestCase.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestCase.ts index 3f8525811aa0..a84a30276e26 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestCase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestCase.ts @@ -18,6 +18,11 @@ export interface CreateTestCase { * Compute the passed and failed row count for the test case. */ computePassedFailedRowCount?: boolean; + /** + * Data Contract that this test case is associated with. Fully qualified name of the data + * contract. + */ + dataContract?: string; /** * Description of the testcase. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts index b9ed818dfe25..f0cd5cb3100d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts @@ -27,6 +27,10 @@ export interface TestCase { * User who made the update. */ createdBy?: string; + /** + * Data Contract that this test case is associated with. + */ + dataContract?: EntityReference; /** * When `true` indicates the entity has been soft deleted. */ @@ -209,15 +213,17 @@ export interface FieldChange { } /** - * Domains the test case belongs to. When not set, the test case inherits the domain from - * the table it belongs to. + * Data Contract that this test case is associated with. * - * This schema defines the EntityReferenceList type used for referencing an entity. + * This schema defines the EntityReference type used for referencing an entity. * EntityReference is used for capturing relationships from one entity to another. For * example, a table has an attribute called database of type EntityReference that captures * the relationship of a table `belongs to a` database. * - * This schema defines the EntityReference type used for referencing an entity. + * Domains the test case belongs to. When not set, the test case inherits the domain from + * the table it belongs to. + * + * This schema defines the EntityReferenceList type used for referencing an entity. * EntityReference is used for capturing relationships from one entity to another. For * example, a table has an attribute called database of type EntityReference that captures * the relationship of a table `belongs to a` database. From b1f39b6075707c33ae401c34ad9527de2e1c4868 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Thu, 25 Sep 2025 19:04:18 +0530 Subject: [PATCH 08/38] replace testsuiteid with dataContractId --- .../ContractQualityCard/ContractQualityCard.component.tsx | 2 +- openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx index 49e756acb732..204ce9846c28 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx @@ -55,7 +55,7 @@ const ContractQualityCard: React.FC<{ setIsTestCaseLoading(true); try { const response = await getListTestCaseBySearch({ - testSuiteId: contract?.testSuite?.id, + dataContractId: contract.id, ...DEFAULT_SORT_ORDER, limit: ES_MAX_PAGE_SIZE, }); diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts index bfa6d1f7751c..534e2555b0a5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts @@ -58,6 +58,7 @@ export type ListTestCaseParams = ListParams & { includeAllTests?: boolean; testCaseStatus?: TestCaseStatus; testCaseType?: TestCaseType; + dataContractId?: string; }; export type ListTestCaseParamsBySearch = ListTestCaseParams & { q?: string; From a6da51ede3e4eb72979008137096f785a33d9e94 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 25 Sep 2025 18:08:00 +0200 Subject: [PATCH 09/38] fix migration after merge --- .../migration/mysql/v1100/Migration.java | 4 ++-- .../migration/postgres/v1100/Migration.java | 4 ++-- .../migration/utils/v1100/MigrationUtil.java | 20 +++++-------------- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1100/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1100/Migration.java index eafe56454881..f29badb49312 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1100/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1100/Migration.java @@ -15,8 +15,8 @@ public Migration(MigrationFile migrationFile) { @Override @SneakyThrows public void runDataMigration() { - MigrationUtil migrationUtil = new MigrationUtil(handle, ConnectionType.MYSQL); + MigrationUtil migrationUtil = new MigrationUtil(collectionDAO, handle, ConnectionType.MYSQL); migrationUtil.migrateEntityStatusForExistingEntities(); - migrationUtil.migrateTestCaseDataContractReferences(handle); + migrationUtil.migrateTestCaseDataContractReferences(); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1100/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1100/Migration.java index 5010a498f6b9..8ccdda464486 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1100/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1100/Migration.java @@ -15,8 +15,8 @@ public Migration(MigrationFile migrationFile) { @Override @SneakyThrows public void runDataMigration() { - MigrationUtil migrationUtil = new MigrationUtil(handle, ConnectionType.POSTGRES); + MigrationUtil migrationUtil = new MigrationUtil(collectionDAO, handle, ConnectionType.POSTGRES); migrationUtil.migrateEntityStatusForExistingEntities(); - migrationUtil.migrateTestCaseDataContractReferences(handle); + migrationUtil.migrateTestCaseDataContractReferences(); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1100/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1100/MigrationUtil.java index a02280d4e95c..8d60bbab36c1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1100/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1100/MigrationUtil.java @@ -2,9 +2,6 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.SQLException; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.core.Handle; @@ -13,27 +10,20 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.locator.ConnectionType; - -import java.util.List; - -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import org.openmetadata.schema.entity.data.DataContract; -import org.openmetadata.schema.tests.TestCase; -import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.utils.JsonUtils; -import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.locator.ConnectionType; @Slf4j public class MigrationUtil { private static final int BATCH_SIZE = 500; private final Handle handle; private final ConnectionType connectionType; + private final CollectionDAO collectionDAO; - public MigrationUtil(Handle handle, ConnectionType connectionType) { + public MigrationUtil(CollectionDAO collectionDAO, Handle handle, ConnectionType connectionType) { this.handle = handle; this.connectionType = connectionType; + this.collectionDAO = collectionDAO; } public void migrateEntityStatusForExistingEntities() { @@ -326,7 +316,7 @@ private int migrateDataContractMySQLBatch(int totalToMigrate) throws Interrupted return totalMigrated; } - public void migrateTestCaseDataContractReferences(Handle handle) { + public void migrateTestCaseDataContractReferences() { LOG.info("===== STARTING TEST CASE DATA CONTRACT MIGRATION ====="); int totalTestCasesMigrated = 0; From dcd39645a20ac8e53076335ba672d5b1cb2c902b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 25 Sep 2025 18:28:03 +0000 Subject: [PATCH 10/38] Update generated TypeScript types --- .../main/resources/ui/src/generated/entity/data/spreadsheet.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/spreadsheet.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/spreadsheet.ts index e6270e5e4e62..ef3310a5e055 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/spreadsheet.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/spreadsheet.ts @@ -409,6 +409,7 @@ export enum EntityStatus { Draft = "Draft", InReview = "In Review", Rejected = "Rejected", + Unprocessed = "Unprocessed", } /** From 6a563b93328433ed13611714135b002947445b0e Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 26 Sep 2025 00:56:06 +0530 Subject: [PATCH 11/38] change the api to get the testCaseResult and need to work on the executionSummary --- .../ContractDetailTab/ContractDetail.tsx | 2 +- .../ContractQualityCard.component.tsx | 65 +++++++++++-------- .../src/main/resources/ui/src/rest/testAPI.ts | 16 ++++- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx index 8911ff381cd4..6bea459d35a6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx @@ -455,7 +455,7 @@ const ContractDetail: React.FC<{ )} {/* Quality Component */} - {contract?.testSuite?.id && ( + {!isEmpty(contract?.qualityExpectations) && ( = ({ contract, contractStatus }) => { const { t } = useTranslation(); + const { fqn } = useFqn(); const [isTestCaseLoading, setIsTestCaseLoading] = useState(false); const [testCaseSummary, setTestCaseSummary] = useState(); - const [testCaseResult, setTestCaseResult] = useState([]); + const [testCaseResult, setTestCaseResult] = useState([]); const fetchTestCaseSummary = async () => { try { @@ -54,7 +60,7 @@ const ContractQualityCard: React.FC<{ const fetchTestCases = async () => { setIsTestCaseLoading(true); try { - const response = await getListTestCaseBySearch({ + const response = await getListTestCasResultsBySearch({ dataContractId: contract.id, ...DEFAULT_SORT_ORDER, limit: ES_MAX_PAGE_SIZE, @@ -91,24 +97,30 @@ const ContractQualityCard: React.FC<{ }; }, [testCaseSummary]); - const getTestCaseStatusIcon = (record: TestCase) => ( - - ); + const processedQualityExpectations = useMemo(() => { + const testCaseResultsMap = new Map( + testCaseResult.map((result) => [ + result.testCaseFQN?.split('.').pop(), // Use the last segment as the key (name) + result, + ]) + ); + + const mergedData = contract.qualityExpectations?.map((item) => ({ + id: item.id, + name: item.name, + fullyQualifiedName: `${fqn}.${item.name}`, + testCaseStatus: + testCaseResultsMap.get(item.name)?.testCaseStatus ?? + TestCaseStatus.Queued, + })); + + return mergedData ?? []; + }, [contract, testCaseResult]); useEffect(() => { - if (contract?.testSuite?.id) { - fetchTestCaseSummary(); - fetchTestCases(); - } - }, [contract]); + // fetchTestCaseSummary(); + fetchTestCases(); + }, []); if (isTestCaseLoading) { return ; @@ -124,7 +136,7 @@ const ContractQualityCard: React.FC<{ entity: t('label.test'), })}:`}{' '} - {testCaseSummary?.total || 8000} + {testCaseSummary?.total} @@ -185,18 +197,19 @@ const ContractQualityCard: React.FC<{ className="data-quality-test-item-container" direction="vertical" size={14}> - {testCaseResult.map((item) => { + {processedQualityExpectations.map((item) => { return (
- {getTestCaseStatusIcon(item)} +
+ to={getTestCaseDetailPagePath(item.fullyQualifiedName)}> {item.name}
diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts index 534e2555b0a5..eb9efa7acbed 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts @@ -118,6 +118,7 @@ export type DataQualityReportParamsType = { const testCaseUrl = '/dataQuality/testCases'; const testSuiteUrl = '/dataQuality/testSuites'; +const testResultUrl = '/dataQuality/testCases/testCaseResults'; const testDefinitionUrl = '/dataQuality/testDefinitions'; // testCase section @@ -134,11 +135,24 @@ export const getListTestCaseBySearch = async ( return response.data; }; +export const getListTestCasResultsBySearch = async ( + params?: ListTestCaseParamsBySearch +) => { + const response = await APIClient.get>( + `${testResultUrl}/search/list`, + { + params, + } + ); + + return response.data; +}; + export const getListTestCaseResults = async ( fqn: string, params?: ListTestCaseResultsParams ) => { - const url = `${testCaseUrl}/testCaseResults/${getEncodedFqn(fqn)}`; + const url = `${testResultUrl}/${getEncodedFqn(fqn)}`; const response = await APIClient.get<{ data: TestCaseResult[]; paging: Paging; From 45064d191d4b372b7a23c1d3c289f02e7def60ad Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 26 Sep 2025 11:50:55 +0530 Subject: [PATCH 12/38] updated the quality execution status with the contract last result --- .../ContractDetailTab/ContractDetail.tsx | 1 + .../ContractQualityCard.component.tsx | 80 +++--- .../ContractQualityCard.test.tsx | 255 ++++++------------ 3 files changed, 119 insertions(+), 217 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx index 6bea459d35a6..df4a80e4e3a5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx @@ -470,6 +470,7 @@ const ContractDetail: React.FC<{ )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx index 7e8172835071..935438a6b2d7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx @@ -19,16 +19,13 @@ import { ES_MAX_PAGE_SIZE } from '../../../constants/constants'; import { TEST_CASE_STATUS_ICON } from '../../../constants/DataQuality.constants'; import { DEFAULT_SORT_ORDER } from '../../../constants/profiler.constant'; import { DataContract } from '../../../generated/entity/data/dataContract'; +import { DataContractResult } from '../../../generated/entity/datacontract/dataContractResult'; import { TestCaseResult, TestCaseStatus, - TestSummary, } from '../../../generated/tests/testCase'; import { useFqn } from '../../../hooks/useFqn'; -import { - getListTestCasResultsBySearch, - getTestCaseExecutionSummary, -} from '../../../rest/testAPI'; +import { getListTestCasResultsBySearch } from '../../../rest/testAPI'; import { getContractStatusType } from '../../../utils/DataContract/DataContractUtils'; import { getTestCaseDetailPagePath } from '../../../utils/RouterUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; @@ -39,24 +36,13 @@ import './contract-quality-card.less'; const ContractQualityCard: React.FC<{ contract: DataContract; contractStatus?: string; -}> = ({ contract, contractStatus }) => { + latestContractResults?: DataContractResult; +}> = ({ contract, contractStatus, latestContractResults }) => { const { t } = useTranslation(); const { fqn } = useFqn(); const [isTestCaseLoading, setIsTestCaseLoading] = useState(false); - const [testCaseSummary, setTestCaseSummary] = useState(); const [testCaseResult, setTestCaseResult] = useState([]); - const fetchTestCaseSummary = async () => { - try { - const response = await getTestCaseExecutionSummary( - contract?.testSuite?.id - ); - setTestCaseSummary(response); - } catch { - // silent fail - } - }; - const fetchTestCases = async () => { setIsTestCaseLoading(true); try { @@ -77,25 +63,52 @@ const ContractQualityCard: React.FC<{ } }; - const { showTestCaseSummaryChart, segmentWidths } = useMemo(() => { - const total = testCaseSummary?.total ?? 0; - const success = testCaseSummary?.success ?? 0; - const failed = testCaseSummary?.failed ?? 0; - const aborted = testCaseSummary?.aborted ?? 0; + const { + showTestCaseSummaryChart, + segmentWidths, + total, + success, + failed, + aborted, + } = useMemo(() => { + if (!latestContractResults?.qualityValidation) { + return { + showTestCaseSummaryChart: false, + segmentWidths: { + successPercent: 0, + failedPercent: 0, + abortedPercent: 0, + }, + total: 0, + success: 0, + failed: 0, + aborted: 0, + }; + } + + const { qualityValidation } = latestContractResults; + const total = qualityValidation?.total ?? 0; + const success = qualityValidation?.passed ?? 0; + const failed = qualityValidation?.failed ?? 0; + const aborted = total - success - failed; const successPercent = (success / total) * 100; const failedPercent = (failed / total) * 100; const abortedPercent = (aborted / total) * 100; return { - showTestCaseSummaryChart: Boolean(total), + showTestCaseSummaryChart: true, segmentWidths: { successPercent, failedPercent, abortedPercent, }, + total, + success, + failed, + aborted, }; - }, [testCaseSummary]); + }, [latestContractResults?.qualityValidation]); const processedQualityExpectations = useMemo(() => { const testCaseResultsMap = new Map( @@ -118,7 +131,6 @@ const ContractQualityCard: React.FC<{ }, [contract, testCaseResult]); useEffect(() => { - // fetchTestCaseSummary(); fetchTestCases(); }, []); @@ -135,9 +147,7 @@ const ContractQualityCard: React.FC<{ {`${t('label.total-entity', { entity: t('label.test'), })}:`}{' '} - - {testCaseSummary?.total} - + {total}
@@ -166,27 +176,21 @@ const ContractQualityCard: React.FC<{
{`${t('label.success')}:`}{' '} - - {testCaseSummary?.success} - + {success}
{`${t('label.failed')}:`}{' '} - - {testCaseSummary?.failed} - + {failed}
{`${t('label.aborted')}:`}{' '} - - {testCaseSummary?.aborted} - + {aborted}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx index 4b1693f416b8..b579a07edd20 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx @@ -12,23 +12,24 @@ */ import { render, screen, waitFor } from '@testing-library/react'; import { MemoryRouter } from 'react-router-dom'; -import { DataContract } from '../../../generated/entity/data/dataContract'; import { - TestCase, - TestCaseStatus, - TestSummary, -} from '../../../generated/tests/testCase'; + ContractExecutionStatus, + DataContractResult, +} from '../../../generated/entity/datacontract/dataContractResult'; +import { TestCase, TestCaseStatus } from '../../../generated/tests/testCase'; import { MOCK_DATA_CONTRACT } from '../../../mocks/DataContract.mock'; -import { - getListTestCaseBySearch, - getTestCaseExecutionSummary, -} from '../../../rest/testAPI'; +import { getListTestCasResultsBySearch } from '../../../rest/testAPI'; import { showErrorToast } from '../../../utils/ToastUtils'; import ContractQualityCard from './ContractQualityCard.component'; +jest.mock('../../../hooks/useFqn', () => ({ + useFqn: jest.fn(() => ({ + fqn: 'fqn', + })), +})); + jest.mock('../../../rest/testAPI', () => ({ - getTestCaseExecutionSummary: jest.fn(), - getListTestCaseBySearch: jest.fn(), + getListTestCasResultsBySearch: jest.fn(), })); jest.mock('../../../utils/ToastUtils', () => ({ @@ -67,11 +68,18 @@ jest.mock('../../../utils/RouterUtils', () => ({ getTestCaseDetailPagePath: jest.fn((fqn) => `/test-case/${fqn}`), })); -const mockTestSummary: TestSummary = { - total: 100, - success: 70, - failed: 20, - aborted: 10, +const mockLatestContractResults: DataContractResult = { + id: '0cdc53d6-0711-4776-b005-444100c76b4d', + dataContractFQN: + 'Configure.default.openmetadata_db.ACT_EVT_LOG.dataContract_Banking Sectors', + timestamp: 1758865987530, + contractExecutionStatus: ContractExecutionStatus.Success, + qualityValidation: { + passed: 70, + failed: 20, + total: 100, + qualityScore: 100, + }, }; const commonCaseMock = { @@ -101,7 +109,7 @@ const commonCaseMock = { const mockTestCases: TestCase[] = [ { id: 'test-case-1', - name: 'Test Case 1', + name: 'CLV Must be Positive', fullyQualifiedName: 'table.test_case_1', testCaseResult: { testCaseStatus: TestCaseStatus.Success, @@ -115,7 +123,7 @@ const mockTestCases: TestCase[] = [ }, { id: 'test-case-2', - name: 'Test Case 2', + name: 'Customer ID To Be Unique', fullyQualifiedName: 'table.test_case_2', testCaseResult: { testCaseStatus: TestCaseStatus.Failed, @@ -129,7 +137,7 @@ const mockTestCases: TestCase[] = [ }, { id: 'test-case-3', - name: 'Test Case 3', + name: 'Table Row Count To Equal', fullyQualifiedName: 'table.test_case_3', testCaseResult: { testCaseStatus: TestCaseStatus.Aborted, @@ -161,46 +169,43 @@ describe('ContractQualityCard', () => { }); it('should fetch and display test case summary and test cases', async () => { - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - mockTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ + (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); render( - + ); await waitFor(() => { - expect(screen.getByText('Test Case 1')).toBeInTheDocument(); - expect(screen.getByText('Test Case 2')).toBeInTheDocument(); - expect(screen.getByText('Test Case 3')).toBeInTheDocument(); + expect(screen.getByText('CLV Must be Positive')).toBeInTheDocument(); + expect(screen.getByText('Customer ID To Be Unique')).toBeInTheDocument(); + expect(screen.getByText('Table Row Count To Equal')).toBeInTheDocument(); }); - expect(getTestCaseExecutionSummary).toHaveBeenCalledWith( - MOCK_DATA_CONTRACT.testSuite.id - ); - expect(getListTestCaseBySearch).toHaveBeenCalledWith( + expect(getListTestCasResultsBySearch).toHaveBeenCalledWith( expect.objectContaining({ - testSuiteId: MOCK_DATA_CONTRACT.testSuite.id, + dataContractId: MOCK_DATA_CONTRACT.id, }) ); }); it('should display test summary chart when data is available', async () => { - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - mockTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ + (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); render( - + ); @@ -215,10 +220,7 @@ describe('ContractQualityCard', () => { }); it('should display contract status when provided', async () => { - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - mockTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ + (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); @@ -227,6 +229,7 @@ describe('ContractQualityCard', () => { ); @@ -239,32 +242,8 @@ describe('ContractQualityCard', () => { }); }); - it('should handle test case summary fetch error silently', async () => { - (getTestCaseExecutionSummary as jest.Mock).mockRejectedValue( - new Error('API Error') - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ - data: mockTestCases, - }); - - render( - - - - ); - - await waitFor(() => { - expect(screen.getByText('Test Case 1')).toBeInTheDocument(); - }); - - expect(showErrorToast).not.toHaveBeenCalled(); - }); - it('should show error toast when test cases fetch fails', async () => { - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - mockTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockRejectedValue( + (getListTestCasResultsBySearch as jest.Mock).mockRejectedValue( new Error('API Error') ); @@ -281,29 +260,8 @@ describe('ContractQualityCard', () => { }); }); - it('should not fetch data when testSuite id is not available', async () => { - const contractWithoutTestSuite: DataContract = { - ...MOCK_DATA_CONTRACT, - testSuite: undefined, - }; - - render( - - - - ); - - await waitFor(() => { - expect(getTestCaseExecutionSummary).not.toHaveBeenCalled(); - expect(getListTestCaseBySearch).not.toHaveBeenCalled(); - }); - }); - it('should render test case links correctly', async () => { - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - mockTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ + (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); @@ -316,50 +274,32 @@ describe('ContractQualityCard', () => { await waitFor(() => { const links = screen.getAllByRole('link'); - expect(links[0]).toHaveAttribute('href', '/test-case/table.test_case_1'); - expect(links[1]).toHaveAttribute('href', '/test-case/table.test_case_2'); - expect(links[2]).toHaveAttribute('href', '/test-case/table.test_case_3'); - }); - }); - - it('should handle test cases with missing test results', async () => { - const testCasesWithoutResults: TestCase[] = [ - { - id: 'test-case-4', - name: 'Test Case 4', - fullyQualifiedName: 'table.test_case_4', - } as TestCase, - ]; - - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - mockTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ - data: testCasesWithoutResults, - }); - - render( - - - - ); - - await waitFor(() => { - expect(screen.getByText('Test Case 4')).toBeInTheDocument(); + expect(links[0]).toHaveAttribute( + 'href', + '/test-case/fqn.CLV Must be Positive' + ); + expect(links[1]).toHaveAttribute( + 'href', + '/test-case/fqn.Customer ID To Be Unique' + ); + expect(links[2]).toHaveAttribute( + 'href', + '/test-case/fqn.Table Row Count To Equal' + ); }); }); it('should calculate segment widths correctly', async () => { - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - mockTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ + (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); const { container } = render( - + ); @@ -381,23 +321,24 @@ describe('ContractQualityCard', () => { }); it('should not show test summary chart when total is 0', async () => { - const emptyTestSummary: TestSummary = { - total: 0, - success: 0, - failed: 0, - aborted: 0, - }; - - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - emptyTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ + (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ data: [], }); const { container } = render( - + ); @@ -409,48 +350,4 @@ describe('ContractQualityCard', () => { expect(chartContainer).not.toBeInTheDocument(); }); }); - - it('should re-fetch data when contract changes', async () => { - (getTestCaseExecutionSummary as jest.Mock).mockResolvedValue( - mockTestSummary - ); - (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ - data: mockTestCases, - }); - - const { rerender } = render( - - - - ); - - await waitFor(() => { - expect(getTestCaseExecutionSummary).toHaveBeenCalledTimes(1); - expect(getListTestCaseBySearch).toHaveBeenCalledTimes(1); - }); - - const updatedContract: DataContract = { - ...MOCK_DATA_CONTRACT, - testSuite: { - id: 'test-suite-2', - name: 'Updated Test Suite', - fullyQualifiedName: 'test.suite.updated', - type: 'testSuite', - }, - }; - - rerender( - - - - ); - - await waitFor(() => { - expect(getTestCaseExecutionSummary).toHaveBeenCalledTimes(2); - expect(getListTestCaseBySearch).toHaveBeenCalledTimes(2); - expect(getTestCaseExecutionSummary).toHaveBeenLastCalledWith( - 'test-suite-2' - ); - }); - }); }); From c7d9ed4a66555d1f04bbd5ba782b7d5b2a41243c Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 15 Oct 2025 08:33:37 +0200 Subject: [PATCH 13/38] bump main --- .../service/jdbi3/DataContractRepository.java | 2 +- .../migration/mysql/v1110/Migration.java | 9 ++ .../migration/postgres/v1110/Migration.java | 9 ++ .../migration/utils/v1110/MigrationUtil.java | 124 ++++++++++++++++++ 4 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index b820b264e0e5..775f8c8683bb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -19,10 +19,10 @@ import static org.openmetadata.service.Entity.ADMIN_USER_NAME; import static org.openmetadata.service.Entity.DATA_CONTRACT; import static org.openmetadata.service.Entity.TEAM; +import static org.openmetadata.service.Entity.TEST_CASE_RESULT; import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; -import static org.openmetadata.service.Entity.TEST_CASE_RESULT; import jakarta.ws.rs.core.Response; import java.util.ArrayList; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java index a59bd950e134..cfc77a31c833 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java @@ -1,5 +1,8 @@ package org.openmetadata.service.migration.mysql.v1110; +import static org.openmetadata.service.migration.utils.v1110.MigrationUtil.migrateTestCaseDataContractReferences; + +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.migration.utils.MigrationFile; import org.openmetadata.service.migration.utils.v1110.MigrationProcessBase; @@ -15,4 +18,10 @@ protected String getQueryFormat() { return "UPDATE tag SET json = JSON_SET(json, '$.recognizers', CAST('%s' AS JSON)) " + "WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') = '%s'"; } + + @Override + @SneakyThrows + public void runDataMigration() { + migrateTestCaseDataContractReferences(collectionDAO); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java index e5a6d6cdab8c..bc66d8e92981 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java @@ -1,5 +1,8 @@ package org.openmetadata.service.migration.postgres.v1110; +import static org.openmetadata.service.migration.utils.v1110.MigrationUtil.migrateTestCaseDataContractReferences; + +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.migration.utils.MigrationFile; import org.openmetadata.service.migration.utils.v1110.MigrationProcessBase; @@ -15,4 +18,10 @@ protected String getQueryFormat() { return "UPDATE tag SET json = jsonb_set(json, '{recognizers}', '%s'::jsonb) " + "WHERE json->>'fullyQualifiedName' = '%s'"; } + + @Override + @SneakyThrows + public void runDataMigration() { + migrateTestCaseDataContractReferences(collectionDAO); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java new file mode 100644 index 000000000000..fa0973cc1311 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java @@ -0,0 +1,124 @@ +package org.openmetadata.service.migration.utils.v1110; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.entity.data.DataContract; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; + +@Slf4j +public class MigrationUtil { + + public static void migrateTestCaseDataContractReferences(CollectionDAO collectionDAO) { + LOG.info("===== STARTING TEST CASE DATA CONTRACT MIGRATION ====="); + + int totalTestCasesMigrated = 0; + int dataContractsProcessed = 0; + int pageSize = 1000; + int offset = 0; + + try { + // Step 1: Paginate through all data contracts using DAO + while (true) { + List dataContractJsons = + collectionDAO.dataContractDAO().listAfterWithOffset(pageSize, offset); + if (dataContractJsons.isEmpty()) { + break; + } + offset += pageSize; + + LOG.info( + "Processing {} data contracts in batch (offset: {})", + dataContractJsons.size(), + offset - pageSize); + + for (String dataContractJson : dataContractJsons) { + try { + DataContract dataContract = JsonUtils.readValue(dataContractJson, DataContract.class); + + // Step 2: Filter - only process contracts with quality expectations + if (nullOrEmpty(dataContract.getQualityExpectations())) { + LOG.debug( + "Data contract {} has no quality expectations, skipping", + dataContract.getFullyQualifiedName()); + continue; + } + + LOG.debug( + "Processing data contract: {} (ID: {}) with {} quality expectations", + dataContract.getFullyQualifiedName(), + dataContract.getId(), + dataContract.getQualityExpectations().size()); + dataContractsProcessed++; + + // Step 3: Process each test case in quality expectations + int testCasesUpdated = 0; + for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { + try { + // Get test case using DAO + TestCase testCase = collectionDAO.testCaseDAO().findEntityById(testCaseRef.getId()); + if (testCase == null) { + LOG.debug("Test case not found: {}", testCaseRef.getId()); + continue; + } + + // Check if test case already has dataContract reference + if (testCase.getDataContract() != null) { + LOG.debug( + "Test case {} already has dataContract reference", + testCase.getFullyQualifiedName()); + continue; + } + + // Step 4: Update test case with dataContract reference using DAO + testCase.setDataContract( + new EntityReference() + .withId(dataContract.getId()) + .withType(Entity.DATA_CONTRACT) + .withFullyQualifiedName(dataContract.getFullyQualifiedName())); + + // Update the test case using DAO + collectionDAO.testCaseDAO().update(testCase); + testCasesUpdated++; + + LOG.debug( + "Updated test case {} with dataContract reference to {}", + testCase.getFullyQualifiedName(), + dataContract.getFullyQualifiedName()); + + } catch (Exception e) { + LOG.warn("Failed to update test case {}: {}", testCaseRef.getId(), e.getMessage()); + } + } + + totalTestCasesMigrated += testCasesUpdated; + + if (testCasesUpdated > 0) { + LOG.info( + "Updated {} test cases for data contract: {}", + testCasesUpdated, + dataContract.getFullyQualifiedName()); + } + + } catch (Exception e) { + LOG.error("Failed to process data contract: {}", e.getMessage(), e); + } + } + } + + } catch (Exception e) { + LOG.error("Error during test case dataContract migration: {}", e.getMessage(), e); + throw new RuntimeException("Migration failed", e); + } + + LOG.info("===== TEST CASE DATA CONTRACT MIGRATION SUMMARY ====="); + LOG.info("Data contracts processed: {}", dataContractsProcessed); + LOG.info("Total test cases updated with dataContract reference: {}", totalTestCasesMigrated); + LOG.info("===== MIGRATION COMPLETE ====="); + } +} From 190cc2c1c4c7203dd9d3de043cb0169020a25ba4 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Thu, 16 Oct 2025 20:04:00 +0530 Subject: [PATCH 14/38] change the api to get all the testResult now and start picking from them --- .../ContractQualityCard.component.tsx | 31 +++++++++---------- .../ContractQualityCard.test.tsx | 27 +++++++++------- .../src/main/resources/ui/src/rest/testAPI.ts | 13 -------- 3 files changed, 31 insertions(+), 40 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx index 935438a6b2d7..0dca2c094fa9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx @@ -20,14 +20,13 @@ import { TEST_CASE_STATUS_ICON } from '../../../constants/DataQuality.constants' import { DEFAULT_SORT_ORDER } from '../../../constants/profiler.constant'; import { DataContract } from '../../../generated/entity/data/dataContract'; import { DataContractResult } from '../../../generated/entity/datacontract/dataContractResult'; -import { - TestCaseResult, - TestCaseStatus, -} from '../../../generated/tests/testCase'; +import { TestCase, TestCaseStatus } from '../../../generated/tests/testCase'; +import { Include } from '../../../generated/type/include'; import { useFqn } from '../../../hooks/useFqn'; -import { getListTestCasResultsBySearch } from '../../../rest/testAPI'; +import { getListTestCaseBySearch } from '../../../rest/testAPI'; import { getContractStatusType } from '../../../utils/DataContract/DataContractUtils'; import { getTestCaseDetailPagePath } from '../../../utils/RouterUtils'; +import { generateEntityLink } from '../../../utils/TableUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import Loader from '../../common/Loader/Loader'; import StatusBadgeV2 from '../../common/StatusBadge/StatusBadgeV2.component'; @@ -41,18 +40,21 @@ const ContractQualityCard: React.FC<{ const { t } = useTranslation(); const { fqn } = useFqn(); const [isTestCaseLoading, setIsTestCaseLoading] = useState(false); - const [testCaseResult, setTestCaseResult] = useState([]); + const [testCase, setTestCase] = useState([]); const fetchTestCases = async () => { setIsTestCaseLoading(true); try { - const response = await getListTestCasResultsBySearch({ - dataContractId: contract.id, + const { data } = await getListTestCaseBySearch({ ...DEFAULT_SORT_ORDER, + entityLink: generateEntityLink(fqn ?? ''), + includeAllTests: true, limit: ES_MAX_PAGE_SIZE, + include: Include.NonDeleted, }); - setTestCaseResult(response.data); - } catch { + + setTestCase(data); + } catch (error) { showErrorToast( t('server.entity-fetch-error', { entity: t('label.test-case-plural'), @@ -112,10 +114,7 @@ const ContractQualityCard: React.FC<{ const processedQualityExpectations = useMemo(() => { const testCaseResultsMap = new Map( - testCaseResult.map((result) => [ - result.testCaseFQN?.split('.').pop(), // Use the last segment as the key (name) - result, - ]) + testCase.map((result) => [result.id, result]) ); const mergedData = contract.qualityExpectations?.map((item) => ({ @@ -123,12 +122,12 @@ const ContractQualityCard: React.FC<{ name: item.name, fullyQualifiedName: `${fqn}.${item.name}`, testCaseStatus: - testCaseResultsMap.get(item.name)?.testCaseStatus ?? + testCaseResultsMap.get(item.id)?.testCaseStatus ?? TestCaseStatus.Queued, })); return mergedData ?? []; - }, [contract, testCaseResult]); + }, [contract, testCase]); useEffect(() => { fetchTestCases(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx index b579a07edd20..0dad06e5e986 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx @@ -18,7 +18,7 @@ import { } from '../../../generated/entity/datacontract/dataContractResult'; import { TestCase, TestCaseStatus } from '../../../generated/tests/testCase'; import { MOCK_DATA_CONTRACT } from '../../../mocks/DataContract.mock'; -import { getListTestCasResultsBySearch } from '../../../rest/testAPI'; +import { getListTestCaseBySearch } from '../../../rest/testAPI'; import { showErrorToast } from '../../../utils/ToastUtils'; import ContractQualityCard from './ContractQualityCard.component'; @@ -29,7 +29,7 @@ jest.mock('../../../hooks/useFqn', () => ({ })); jest.mock('../../../rest/testAPI', () => ({ - getListTestCasResultsBySearch: jest.fn(), + getListTestCaseBySearch: jest.fn(), })); jest.mock('../../../utils/ToastUtils', () => ({ @@ -169,7 +169,7 @@ describe('ContractQualityCard', () => { }); it('should fetch and display test case summary and test cases', async () => { - (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ + (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); @@ -188,15 +188,20 @@ describe('ContractQualityCard', () => { expect(screen.getByText('Table Row Count To Equal')).toBeInTheDocument(); }); - expect(getListTestCasResultsBySearch).toHaveBeenCalledWith( + expect(getListTestCaseBySearch).toHaveBeenCalledWith( expect.objectContaining({ - dataContractId: MOCK_DATA_CONTRACT.id, + entityLink: '<#E::table::fqn>', + include: 'non-deleted', + includeAllTests: true, + limit: 10000, + sortField: 'testCaseResult.timestamp', + sortType: 'desc', }) ); }); it('should display test summary chart when data is available', async () => { - (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ + (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); @@ -220,7 +225,7 @@ describe('ContractQualityCard', () => { }); it('should display contract status when provided', async () => { - (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ + (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); @@ -243,7 +248,7 @@ describe('ContractQualityCard', () => { }); it('should show error toast when test cases fetch fails', async () => { - (getListTestCasResultsBySearch as jest.Mock).mockRejectedValue( + (getListTestCaseBySearch as jest.Mock).mockRejectedValue( new Error('API Error') ); @@ -261,7 +266,7 @@ describe('ContractQualityCard', () => { }); it('should render test case links correctly', async () => { - (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ + (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); @@ -290,7 +295,7 @@ describe('ContractQualityCard', () => { }); it('should calculate segment widths correctly', async () => { - (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ + (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ data: mockTestCases, }); @@ -321,7 +326,7 @@ describe('ContractQualityCard', () => { }); it('should not show test summary chart when total is 0', async () => { - (getListTestCasResultsBySearch as jest.Mock).mockResolvedValue({ + (getListTestCaseBySearch as jest.Mock).mockResolvedValue({ data: [], }); diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts index eb9efa7acbed..183aa136d3ae 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts @@ -135,19 +135,6 @@ export const getListTestCaseBySearch = async ( return response.data; }; -export const getListTestCasResultsBySearch = async ( - params?: ListTestCaseParamsBySearch -) => { - const response = await APIClient.get>( - `${testResultUrl}/search/list`, - { - params, - } - ); - - return response.data; -}; - export const getListTestCaseResults = async ( fqn: string, params?: ListTestCaseResultsParams From 554d673156987e8eafdbf89323c8541ed9b8088b Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Thu, 16 Oct 2025 20:22:57 +0530 Subject: [PATCH 15/38] remove some unwanted changes --- openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts index 183aa136d3ae..bfa6d1f7751c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts @@ -58,7 +58,6 @@ export type ListTestCaseParams = ListParams & { includeAllTests?: boolean; testCaseStatus?: TestCaseStatus; testCaseType?: TestCaseType; - dataContractId?: string; }; export type ListTestCaseParamsBySearch = ListTestCaseParams & { q?: string; @@ -118,7 +117,6 @@ export type DataQualityReportParamsType = { const testCaseUrl = '/dataQuality/testCases'; const testSuiteUrl = '/dataQuality/testSuites'; -const testResultUrl = '/dataQuality/testCases/testCaseResults'; const testDefinitionUrl = '/dataQuality/testDefinitions'; // testCase section @@ -139,7 +137,7 @@ export const getListTestCaseResults = async ( fqn: string, params?: ListTestCaseResultsParams ) => { - const url = `${testResultUrl}/${getEncodedFqn(fqn)}`; + const url = `${testCaseUrl}/testCaseResults/${getEncodedFqn(fqn)}`; const response = await APIClient.get<{ data: TestCaseResult[]; paging: Paging; From 607b5db8a806536d383224a7d8f23a1c504d13cc Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Thu, 16 Oct 2025 21:29:21 +0530 Subject: [PATCH 16/38] fix sonar --- .../DataContract/ContractDetailTab/ContractDetail.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx index 15f138a2d75d..e4e436253815 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx @@ -435,10 +435,10 @@ describe('ContractDetail', () => { expect(screen.queryByText('ContractQualityCard')).toBeInTheDocument(); }); - it('should not display test cases when contract has quality expectations', async () => { + it("should not display test cases when contract doesn't have quality expectations", async () => { render( , From 59d859f4b85fbbce3e068c247daa81cd1813eea3 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 31 Oct 2025 12:10:53 +0100 Subject: [PATCH 17/38] fix merge --- .../migration/mysql/v1110/Migration.java | 1 + .../migration/postgres/v1110/Migration.java | 1 + .../migration/utils/v1110/MigrationUtil.java | 114 ++++++++++++++++++ .../dqtests/TestCaseResourceTest.java | 1 + 4 files changed, 117 insertions(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java index 0c19e83c7984..3925891cfcb5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java @@ -35,5 +35,6 @@ public Map runPostDDLScripts(boolean isForceMigration) { @SneakyThrows public void runDataMigration() { this.migrationUtil.migrateFlywayHistory(handle); + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java index cb47e6046e89..64fcffd2b8aa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java @@ -35,5 +35,6 @@ public Map runPostDDLScripts(boolean isForceMigration) { @SneakyThrows public void runDataMigration() { this.migrationUtil.migrateFlywayHistory(handle); + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java index cd86ff380c15..4babab0793f5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java @@ -1,5 +1,6 @@ package org.openmetadata.service.migration.utils.v1110; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.CLASSIFICATION; import static org.openmetadata.service.util.EntityUtil.hash; @@ -11,8 +12,13 @@ import org.jdbi.v3.core.Handle; import org.openmetadata.schema.api.classification.CreateTag; import org.openmetadata.schema.api.classification.LoadTags; +import org.openmetadata.schema.entity.data.DataContract; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Recognizer; import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.MigrationDAO; import org.openmetadata.service.jdbi3.locator.ConnectionType; @@ -35,6 +41,114 @@ public MigrationUtil(ConnectionType connectionType, MigrationFile migrationFile) this.migrationFile = migrationFile; } + public static void migrateTestCaseDataContractReferences(CollectionDAO collectionDAO) { + LOG.info("===== STARTING TEST CASE DATA CONTRACT MIGRATION ====="); + + int totalTestCasesMigrated = 0; + int dataContractsProcessed = 0; + int pageSize = 1000; + int offset = 0; + + try { + // Step 1: Paginate through all data contracts using DAO + while (true) { + List dataContractJsons = + collectionDAO.dataContractDAO().listAfterWithOffset(pageSize, offset); + if (dataContractJsons.isEmpty()) { + break; + } + offset += pageSize; + + LOG.info( + "Processing {} data contracts in batch (offset: {})", + dataContractJsons.size(), + offset - pageSize); + + for (String dataContractJson : dataContractJsons) { + try { + DataContract dataContract = JsonUtils.readValue(dataContractJson, DataContract.class); + + // Step 2: Filter - only process contracts with quality expectations + if (nullOrEmpty(dataContract.getQualityExpectations())) { + LOG.debug( + "Data contract {} has no quality expectations, skipping", + dataContract.getFullyQualifiedName()); + continue; + } + + LOG.debug( + "Processing data contract: {} (ID: {}) with {} quality expectations", + dataContract.getFullyQualifiedName(), + dataContract.getId(), + dataContract.getQualityExpectations().size()); + dataContractsProcessed++; + + // Step 3: Process each test case in quality expectations + int testCasesUpdated = 0; + for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { + try { + // Get test case using DAO + TestCase testCase = collectionDAO.testCaseDAO().findEntityById(testCaseRef.getId()); + if (testCase == null) { + LOG.debug("Test case not found: {}", testCaseRef.getId()); + continue; + } + + // Check if test case already has dataContract reference + if (testCase.getDataContract() != null) { + LOG.debug( + "Test case {} already has dataContract reference", + testCase.getFullyQualifiedName()); + continue; + } + + // Step 4: Update test case with dataContract reference using DAO + testCase.setDataContract( + new EntityReference() + .withId(dataContract.getId()) + .withType(Entity.DATA_CONTRACT) + .withFullyQualifiedName(dataContract.getFullyQualifiedName())); + + // Update the test case using DAO + collectionDAO.testCaseDAO().update(testCase); + testCasesUpdated++; + + LOG.debug( + "Updated test case {} with dataContract reference to {}", + testCase.getFullyQualifiedName(), + dataContract.getFullyQualifiedName()); + + } catch (Exception e) { + LOG.warn("Failed to update test case {}: {}", testCaseRef.getId(), e.getMessage()); + } + } + + totalTestCasesMigrated += testCasesUpdated; + + if (testCasesUpdated > 0) { + LOG.info( + "Updated {} test cases for data contract: {}", + testCasesUpdated, + dataContract.getFullyQualifiedName()); + } + + } catch (Exception e) { + LOG.error("Failed to process data contract: {}", e.getMessage(), e); + } + } + } + + } catch (Exception e) { + LOG.error("Error during test case dataContract migration: {}", e.getMessage(), e); + throw new RuntimeException("Migration failed", e); + } + + LOG.info("===== TEST CASE DATA CONTRACT MIGRATION SUMMARY ====="); + LOG.info("Data contracts processed: {}", dataContractsProcessed); + LOG.info("Total test cases updated with dataContract reference: {}", totalTestCasesMigrated); + LOG.info("===== MIGRATION COMPLETE ====="); + } + public Map setRecognizersForSensitiveTags( String queryTemplate, Handle handle, MigrationDAO migrationDAO, boolean isForceMigration) { Map result = new HashMap<>(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index d6d7f4ce588c..ccbc9368d2f2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -4354,6 +4354,7 @@ void test_listTestCasesFilterByCreatedBy(TestInfo testInfo) throws IOException { "List endpoint should also filter by createdBy"); } } + @Test void test_listTestCaseResultsFromSearchWithDataContractIdFilter(TestInfo testInfo) throws HttpResponseException, ParseException, IOException { From 4c5e799876b2aa40c581f7ad05804b73314334e2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 31 Oct 2025 11:14:23 +0000 Subject: [PATCH 18/38] Update generated TypeScript types --- .../ingestionPipelines/createIngestionPipeline.ts | 10 ++++++++++ .../ui/src/generated/entity/applications/app.ts | 10 ++++++++++ .../internal/dataRetentionConfiguration.ts | 10 ++++++++++ .../marketplace/appMarketPlaceDefinition.ts | 10 ++++++++++ .../marketplace/createAppMarketPlaceDefinitionReq.ts | 10 ++++++++++ .../services/ingestionPipelines/ingestionPipeline.ts | 10 ++++++++++ .../ui/src/generated/metadataIngestion/application.ts | 10 ++++++++++ .../generated/metadataIngestion/applicationPipeline.ts | 10 ++++++++++ .../ui/src/generated/metadataIngestion/workflow.ts | 10 ++++++++++ 9 files changed, 90 insertions(+) diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts index bc6b12e61a13..a9eb60042c9f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts @@ -1025,6 +1025,16 @@ export interface CollateAIAppConfig { * one month). */ changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; /** * Service Entity Link for which to trigger the application. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts index 5ea97b4ab948..0c06a4f55575 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts @@ -342,6 +342,16 @@ export interface CollateAIAppConfig { * one month). */ changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; /** * Service Entity Link for which to trigger the application. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/dataRetentionConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/dataRetentionConfiguration.ts index e48c99db0b2f..844de2ed3bfe 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/dataRetentionConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/dataRetentionConfiguration.ts @@ -21,4 +21,14 @@ export interface DataRetentionConfigurationClass { * one month). */ changeEventRetentionPeriod: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod: number; } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts index 2dc4846b7c29..86afb6b8100c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts @@ -323,6 +323,16 @@ export interface CollateAIAppConfig { * one month). */ changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; /** * Service Entity Link for which to trigger the application. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts index 0547205d3cf2..4735b96eb3a5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts @@ -280,6 +280,16 @@ export interface CollateAIAppConfig { * one month). */ changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; /** * Service Entity Link for which to trigger the application. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts index 8bda2ebc75c2..782ce3303e3a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts @@ -1536,6 +1536,16 @@ export interface CollateAIAppConfig { * one month). */ changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; /** * Service Entity Link for which to trigger the application. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts index d34227b998a3..1ba6bd6fe7a9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts @@ -190,6 +190,16 @@ export interface CollateAIAppConfig { * one month). */ changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; /** * Service Entity Link for which to trigger the application. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts index 4679a3383db3..6e08090214eb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts @@ -175,6 +175,16 @@ export interface CollateAIAppConfig { * one month). */ changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; /** * Service Entity Link for which to trigger the application. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts index ffcb4b3a2a1f..42720947aa1f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts @@ -5018,6 +5018,16 @@ export interface CollateAIAppConfig { * one month). */ changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; /** * Service Entity Link for which to trigger the application. */ From 25f3ada759d83cf8619b1ff83249dc225642fbba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 21 Nov 2025 15:36:39 +0000 Subject: [PATCH 19/38] Update generated TypeScript types --- .../main/resources/ui/src/generated/api/policies/createPolicy.ts | 1 + .../entity/policies/accessControl/resourceDescriptor.ts | 1 + .../entity/policies/accessControl/resourcePermission.ts | 1 + .../ui/src/generated/entity/policies/accessControl/rule.ts | 1 + .../main/resources/ui/src/generated/entity/policies/policy.ts | 1 + 5 files changed, 5 insertions(+) diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/policies/createPolicy.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/policies/createPolicy.ts index c57850170a6a..4b4b08a742dc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/policies/createPolicy.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/policies/createPolicy.ts @@ -153,6 +153,7 @@ export enum Operation { Create = "Create", CreateIngestionPipelineAutomator = "CreateIngestionPipelineAutomator", CreateScim = "CreateScim", + CreateTests = "CreateTests", Delete = "Delete", DeleteScim = "DeleteScim", DeleteTestCaseFailedRowsSample = "DeleteTestCaseFailedRowsSample", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/resourceDescriptor.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/resourceDescriptor.ts index c6a011f9933d..b43d5b5b41c3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/resourceDescriptor.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/resourceDescriptor.ts @@ -35,6 +35,7 @@ export enum Operation { Create = "Create", CreateIngestionPipelineAutomator = "CreateIngestionPipelineAutomator", CreateScim = "CreateScim", + CreateTests = "CreateTests", Delete = "Delete", DeleteScim = "DeleteScim", DeleteTestCaseFailedRowsSample = "DeleteTestCaseFailedRowsSample", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/resourcePermission.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/resourcePermission.ts index 2dc808a026ec..c7e186e0194f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/resourcePermission.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/resourcePermission.ts @@ -77,6 +77,7 @@ export enum Operation { Create = "Create", CreateIngestionPipelineAutomator = "CreateIngestionPipelineAutomator", CreateScim = "CreateScim", + CreateTests = "CreateTests", Delete = "Delete", DeleteScim = "DeleteScim", DeleteTestCaseFailedRowsSample = "DeleteTestCaseFailedRowsSample", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/rule.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/rule.ts index a759a3004112..b74833ac50e1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/rule.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/accessControl/rule.ts @@ -60,6 +60,7 @@ export enum Operation { Create = "Create", CreateIngestionPipelineAutomator = "CreateIngestionPipelineAutomator", CreateScim = "CreateScim", + CreateTests = "CreateTests", Delete = "Delete", DeleteScim = "DeleteScim", DeleteTestCaseFailedRowsSample = "DeleteTestCaseFailedRowsSample", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/policy.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/policy.ts index e92827e0c1f2..a2faea2aa631 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/policy.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/policies/policy.ts @@ -305,6 +305,7 @@ export enum Operation { Create = "Create", CreateIngestionPipelineAutomator = "CreateIngestionPipelineAutomator", CreateScim = "CreateScim", + CreateTests = "CreateTests", Delete = "Delete", DeleteScim = "DeleteScim", DeleteTestCaseFailedRowsSample = "DeleteTestCaseFailedRowsSample", From d774e432b78c719d75e21e1183bb98635fd3f22d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Mar 2026 10:02:15 +0000 Subject: [PATCH 20/38] Update generated TypeScript types --- .../events/notificationTemplateSendRequest.ts | 57 +++++++++++++++-- .../createIngestionPipeline.ts | 2 +- .../ui/src/generated/api/teams/createTeam.ts | 57 +++++++++++++++-- .../ui/src/generated/api/teams/createUser.ts | 57 +++++++++++++++-- .../marketplace/appMarketPlaceDefinition.ts | 57 +++++++++++++++-- .../createAppMarketPlaceDefinitionReq.ts | 57 +++++++++++++++-- .../authentication/webhookBearerAuth.ts | 33 ++++++++++ .../events/authentication/webhookNoAuth.ts | 28 +++++++++ .../authentication/webhookOAuth2Config.ts | 44 +++++++++++++ .../ui/src/generated/entity/events/webhook.ts | 63 +++++++++++++------ .../ingestionPipelines/ingestionPipeline.ts | 2 +- .../ui/src/generated/entity/teams/team.ts | 57 +++++++++++++++-- .../ui/src/generated/entity/teams/user.ts | 57 +++++++++++++++-- .../events/api/createEventSubscription.ts | 57 +++++++++++++++-- .../api/testEventSubscriptionDestination.ts | 57 +++++++++++++++-- .../src/generated/events/eventSubscription.ts | 63 +++++++++++++------ .../databaseServiceProfilerPipeline.ts | 2 +- .../generated/metadataIngestion/workflow.ts | 2 +- .../ui/src/generated/type/profile.ts | 57 +++++++++++++++-- 19 files changed, 717 insertions(+), 92 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookBearerAuth.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookNoAuth.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookOAuth2Config.ts diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/events/notificationTemplateSendRequest.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/events/notificationTemplateSendRequest.ts index cfec6aaf59a5..e011208a75e1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/events/notificationTemplateSendRequest.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/events/notificationTemplateSendRequest.ts @@ -85,6 +85,11 @@ export enum SubscriptionCategory { * A generic map that can be deserialized later. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -105,11 +110,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins * @@ -131,6 +131,53 @@ export interface Webhook { [property: string]: any; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts index 6f9a9643d012..8673b9293838 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts @@ -580,7 +580,7 @@ export interface Pipeline { /** * Number of threads to use during metric computations */ - threadCount?: number; + threadCount?: number | null; /** * Profiler Timeout in Seconds */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/teams/createTeam.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/teams/createTeam.ts index c0fd261984ec..1162ff3891df 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/teams/createTeam.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/teams/createTeam.ts @@ -177,6 +177,11 @@ export interface MessagingProvider { * This schema defines webhook for receiving events from OpenMetadata. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -197,11 +202,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins */ @@ -216,6 +216,53 @@ export interface Webhook { sendToOwners?: boolean; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/teams/createUser.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/teams/createUser.ts index f98404b4df29..e441f62998b8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/teams/createUser.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/teams/createUser.ts @@ -468,6 +468,11 @@ export interface MessagingProvider { * This schema defines webhook for receiving events from OpenMetadata. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -488,11 +493,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins */ @@ -507,6 +507,53 @@ export interface Webhook { sendToOwners?: boolean; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts index e3457a01b077..e68d2be4004d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts @@ -1490,6 +1490,11 @@ export enum SubscriptionCategory { * A generic map that can be deserialized later. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -1510,11 +1515,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins * @@ -1536,6 +1536,53 @@ export interface Webhook { [property: string]: any; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: AuthenticationConfigurationTypeType; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum AuthenticationConfigurationTypeType { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts index ee42ee807ae7..7eddb8189f45 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts @@ -1383,6 +1383,11 @@ export enum SubscriptionCategory { * A generic map that can be deserialized later. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -1403,11 +1408,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins * @@ -1429,6 +1429,53 @@ export interface Webhook { [property: string]: any; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: AuthenticationConfigurationTypeType; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum AuthenticationConfigurationTypeType { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookBearerAuth.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookBearerAuth.ts new file mode 100644 index 000000000000..8ee12ddfc404 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookBearerAuth.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Bearer token authentication for webhook endpoints. + */ +export interface WebhookBearerAuth { + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey: string; + /** + * Authentication type discriminator. + */ + type: Type; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookNoAuth.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookNoAuth.ts new file mode 100644 index 000000000000..3b15a4f6f0ef --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookNoAuth.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * No authentication. + */ +export interface WebhookNoAuth { + /** + * Authentication type discriminator. + */ + type: Type; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + None = "none", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookOAuth2Config.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookOAuth2Config.ts new file mode 100644 index 000000000000..9f1ef9715330 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/authentication/webhookOAuth2Config.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface WebhookOAuth2Config { + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl: string; + /** + * Authentication type discriminator. + */ + type: Type; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Oauth2 = "oauth2", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/webhook.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/webhook.ts index 6d0c4b699d15..fbd66b5d0b3f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/webhook.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/events/webhook.ts @@ -15,9 +15,10 @@ */ export interface Webhook { /** - * Authentication configuration for the webhook. + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. */ - authType?: WebhookNoAuth | WebhookBearerAuth | WebhookOAuth2Config; + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -52,27 +53,51 @@ export interface Webhook { sendToOwners?: boolean; } -export interface WebhookNoAuth { - type: WebhookAuthType.None; -} - -export interface WebhookBearerAuth { - type: WebhookAuthType.Bearer; - secretKey: string; -} - -export interface WebhookOAuth2Config { - type: WebhookAuthType.OAuth2; - tokenUrl: string; - clientId: string; - clientSecret: string; +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; } -export enum WebhookAuthType { - None = "none", +/** + * Authentication type discriminator. + */ +export enum Type { Bearer = "bearer", - OAuth2 = "oauth2", + None = "none", + Oauth2 = "oauth2", } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts index 9b34a8748afc..0bb02f59241f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts @@ -1252,7 +1252,7 @@ export interface Pipeline { /** * Number of threads to use during metric computations */ - threadCount?: number; + threadCount?: number | null; /** * Profiler Timeout in Seconds */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/teams/team.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/teams/team.ts index 0c97ec8858f2..29435cd27f28 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/teams/team.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/teams/team.ts @@ -309,6 +309,11 @@ export interface MessagingProvider { * This schema defines webhook for receiving events from OpenMetadata. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -329,11 +334,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins */ @@ -348,6 +348,53 @@ export interface Webhook { sendToOwners?: boolean; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/teams/user.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/teams/user.ts index e4acd9d15077..e08aa7d4bf26 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/teams/user.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/teams/user.ts @@ -627,6 +627,11 @@ export interface MessagingProvider { * This schema defines webhook for receiving events from OpenMetadata. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -647,11 +652,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins */ @@ -666,6 +666,53 @@ export interface Webhook { sendToOwners?: boolean; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/events/api/createEventSubscription.ts b/openmetadata-ui/src/main/resources/ui/src/generated/events/api/createEventSubscription.ts index 4c9d65a3f879..b544d27fdda0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/events/api/createEventSubscription.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/events/api/createEventSubscription.ts @@ -152,6 +152,11 @@ export enum SubscriptionCategory { * A generic map that can be deserialized later. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -172,11 +177,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins * @@ -198,6 +198,53 @@ export interface Webhook { [property: string]: any; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/events/api/testEventSubscriptionDestination.ts b/openmetadata-ui/src/main/resources/ui/src/generated/events/api/testEventSubscriptionDestination.ts index 11a4790d564f..f47f7868d18f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/events/api/testEventSubscriptionDestination.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/events/api/testEventSubscriptionDestination.ts @@ -79,6 +79,11 @@ export enum SubscriptionCategory { * A generic map that can be deserialized later. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -99,11 +104,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins * @@ -125,6 +125,53 @@ export interface Webhook { [property: string]: any; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/events/eventSubscription.ts b/openmetadata-ui/src/main/resources/ui/src/generated/events/eventSubscription.ts index 88d9306cd2a7..b3d7ccf5f3b3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/events/eventSubscription.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/events/eventSubscription.ts @@ -259,9 +259,10 @@ export enum SubscriptionCategory { */ export interface Webhook { /** - * Authentication configuration for the webhook. + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. */ - authType?: WebhookNoAuth | WebhookBearerAuth | WebhookOAuth2Config; + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -303,27 +304,51 @@ export interface Webhook { [property: string]: any; } -export interface WebhookNoAuth { - type: WebhookAuthType.None; -} - -export interface WebhookBearerAuth { - type: WebhookAuthType.Bearer; - secretKey: string; -} - -export interface WebhookOAuth2Config { - type: WebhookAuthType.OAuth2; - tokenUrl: string; - clientId: string; - clientSecret: string; +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; } -export enum WebhookAuthType { - None = "none", +/** + * Authentication type discriminator. + */ +export enum Type { Bearer = "bearer", - OAuth2 = "oauth2", + None = "none", + Oauth2 = "oauth2", } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts index ce0fe32a7c3d..72c2886cbed5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/databaseServiceProfilerPipeline.ts @@ -64,7 +64,7 @@ export interface DatabaseServiceProfilerPipeline { /** * Number of threads to use during metric computations */ - threadCount?: number; + threadCount?: number | null; /** * Profiler Timeout in Seconds */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts index 0c8c10093b0e..8f4ed04a4aa8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts @@ -5208,7 +5208,7 @@ export interface Pipeline { /** * Number of threads to use during metric computations */ - threadCount?: number; + threadCount?: number | null; /** * Profiler Timeout in Seconds */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/type/profile.ts b/openmetadata-ui/src/main/resources/ui/src/generated/type/profile.ts index 431811e10db8..4abcd6033342 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/type/profile.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/type/profile.ts @@ -45,6 +45,11 @@ export interface MessagingProvider { * This schema defines webhook for receiving events from OpenMetadata. */ export interface Webhook { + /** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + */ + authType?: AuthenticationConfigurationType; /** * Endpoint to receive the webhook events over POST requests. */ @@ -65,11 +70,6 @@ export interface Webhook { * List of receivers to send mail to */ receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; /** * Send the Event to Admins */ @@ -84,6 +84,53 @@ export interface Webhook { sendToOwners?: boolean; } +/** + * Authentication configuration for the webhook. If not specified, the webhook will be sent + * without authentication. + * + * No authentication. + * + * Bearer token authentication for webhook endpoints. + * + * OAuth2 Client Credentials configuration for webhook authentication. + */ +export interface AuthenticationConfigurationType { + /** + * Authentication type discriminator. + */ + type: Type; + /** + * Secret key used for computing HMAC SHA256 signature of webhook payload, sent in the + * X-OM-Signature header. + */ + secretKey?: string; + /** + * OAuth2 client identifier. Stored encrypted via Fernet. + */ + clientId?: string; + /** + * OAuth2 client secret. Stored encrypted via Fernet. + */ + clientSecret?: string; + /** + * Optional OAuth2 scopes to request (space-separated). + */ + scope?: string; + /** + * Token endpoint URL to obtain access tokens. + */ + tokenUrl?: string; +} + +/** + * Authentication type discriminator. + */ +export enum Type { + Bearer = "bearer", + None = "none", + Oauth2 = "oauth2", +} + /** * HTTP operation to send the webhook request. Supports POST or PUT. */ From 3a18beb1fe7b0cad7e2cf6c1b2c710fa27a0b7d1 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 12 Mar 2026 11:02:13 +0100 Subject: [PATCH 21/38] Move migrateTestCaseDataContractReferences to v1130 migration Co-Authored-By: Claude Opus 4.6 --- .../migration/mysql/v1110/Migration.java | 1 - .../migration/mysql/v1130/Migration.java | 1 + .../migration/postgres/v1110/Migration.java | 1 - .../migration/postgres/v1130/Migration.java | 1 + .../migration/utils/v1110/MigrationUtil.java | 114 ------------------ .../migration/utils/v1130/MigrationUtil.java | 112 +++++++++++++++++ 6 files changed, 114 insertions(+), 116 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java index 8a9bd19f0da1..a4fd83e6dbc7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1110/Migration.java @@ -36,6 +36,5 @@ public Map runPostDDLScripts(boolean isForceMigration) { public void runDataMigration() { // Flyway history migration is now handled in MigrationWorkflow.loadMigrations() // before parsing SQL files, to ensure it runs before flyway migrations in force mode - MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1130/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1130/Migration.java index d146890469f6..3b014768fec1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1130/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1130/Migration.java @@ -15,5 +15,6 @@ public Migration(MigrationFile migrationFile) { @SneakyThrows public void runDataMigration() { MigrationUtil.updateOwnerChartFormulas(); + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java index c2e03af2461f..9ab0455a3fc9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1110/Migration.java @@ -36,6 +36,5 @@ public Map runPostDDLScripts(boolean isForceMigration) { public void runDataMigration() { // Flyway history migration is now handled in MigrationWorkflow.loadMigrations() // before parsing SQL files, to ensure it runs before flyway migrations in force mode - MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1130/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1130/Migration.java index 909ea509319d..49a329966bbe 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1130/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1130/Migration.java @@ -15,5 +15,6 @@ public Migration(MigrationFile migrationFile) { @SneakyThrows public void runDataMigration() { MigrationUtil.updateOwnerChartFormulas(); + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java index aad21ca4ddcf..9bd511513e83 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1110/MigrationUtil.java @@ -1,6 +1,5 @@ package org.openmetadata.service.migration.utils.v1110; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.CLASSIFICATION; import static org.openmetadata.service.util.EntityUtil.hash; @@ -12,13 +11,8 @@ import org.jdbi.v3.core.Handle; import org.openmetadata.schema.api.classification.CreateTag; import org.openmetadata.schema.api.classification.LoadTags; -import org.openmetadata.schema.entity.data.DataContract; -import org.openmetadata.schema.tests.TestCase; -import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Recognizer; import org.openmetadata.schema.utils.JsonUtils; -import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.MigrationDAO; import org.openmetadata.service.jdbi3.locator.ConnectionType; @@ -41,114 +35,6 @@ public MigrationUtil(ConnectionType connectionType, MigrationFile migrationFile) this.migrationFile = migrationFile; } - public static void migrateTestCaseDataContractReferences(CollectionDAO collectionDAO) { - LOG.info("===== STARTING TEST CASE DATA CONTRACT MIGRATION ====="); - - int totalTestCasesMigrated = 0; - int dataContractsProcessed = 0; - int pageSize = 1000; - int offset = 0; - - try { - // Step 1: Paginate through all data contracts using DAO - while (true) { - List dataContractJsons = - collectionDAO.dataContractDAO().listAfterWithOffset(pageSize, offset); - if (dataContractJsons.isEmpty()) { - break; - } - offset += pageSize; - - LOG.info( - "Processing {} data contracts in batch (offset: {})", - dataContractJsons.size(), - offset - pageSize); - - for (String dataContractJson : dataContractJsons) { - try { - DataContract dataContract = JsonUtils.readValue(dataContractJson, DataContract.class); - - // Step 2: Filter - only process contracts with quality expectations - if (nullOrEmpty(dataContract.getQualityExpectations())) { - LOG.debug( - "Data contract {} has no quality expectations, skipping", - dataContract.getFullyQualifiedName()); - continue; - } - - LOG.debug( - "Processing data contract: {} (ID: {}) with {} quality expectations", - dataContract.getFullyQualifiedName(), - dataContract.getId(), - dataContract.getQualityExpectations().size()); - dataContractsProcessed++; - - // Step 3: Process each test case in quality expectations - int testCasesUpdated = 0; - for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { - try { - // Get test case using DAO - TestCase testCase = collectionDAO.testCaseDAO().findEntityById(testCaseRef.getId()); - if (testCase == null) { - LOG.debug("Test case not found: {}", testCaseRef.getId()); - continue; - } - - // Check if test case already has dataContract reference - if (testCase.getDataContract() != null) { - LOG.debug( - "Test case {} already has dataContract reference", - testCase.getFullyQualifiedName()); - continue; - } - - // Step 4: Update test case with dataContract reference using DAO - testCase.setDataContract( - new EntityReference() - .withId(dataContract.getId()) - .withType(Entity.DATA_CONTRACT) - .withFullyQualifiedName(dataContract.getFullyQualifiedName())); - - // Update the test case using DAO - collectionDAO.testCaseDAO().update(testCase); - testCasesUpdated++; - - LOG.debug( - "Updated test case {} with dataContract reference to {}", - testCase.getFullyQualifiedName(), - dataContract.getFullyQualifiedName()); - - } catch (Exception e) { - LOG.warn("Failed to update test case {}: {}", testCaseRef.getId(), e.getMessage()); - } - } - - totalTestCasesMigrated += testCasesUpdated; - - if (testCasesUpdated > 0) { - LOG.info( - "Updated {} test cases for data contract: {}", - testCasesUpdated, - dataContract.getFullyQualifiedName()); - } - - } catch (Exception e) { - LOG.error("Failed to process data contract: {}", e.getMessage(), e); - } - } - } - - } catch (Exception e) { - LOG.error("Error during test case dataContract migration: {}", e.getMessage(), e); - throw new RuntimeException("Migration failed", e); - } - - LOG.info("===== TEST CASE DATA CONTRACT MIGRATION SUMMARY ====="); - LOG.info("Data contracts processed: {}", dataContractsProcessed); - LOG.info("Total test cases updated with dataContract reference: {}", totalTestCasesMigrated); - LOG.info("===== MIGRATION COMPLETE ====="); - } - public Map setRecognizersForSensitiveTags( String queryTemplate, Handle handle, MigrationDAO migrationDAO, boolean isForceMigration) { Map result = new HashMap<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java index f5005e76f4b3..e06cdd878e84 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java @@ -1,7 +1,16 @@ package org.openmetadata.service.migration.utils.v1130; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; +import org.openmetadata.schema.entity.data.DataContract; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository; import org.openmetadata.service.util.EntityUtil; @@ -48,4 +57,107 @@ public static void updateOwnerChartFormulas() { } } } + + public static void migrateTestCaseDataContractReferences(CollectionDAO collectionDAO) { + LOG.info("===== STARTING TEST CASE DATA CONTRACT MIGRATION ====="); + + int totalTestCasesMigrated = 0; + int dataContractsProcessed = 0; + int pageSize = 1000; + int offset = 0; + + try { + while (true) { + List dataContractJsons = + collectionDAO.dataContractDAO().listAfterWithOffset(pageSize, offset); + if (dataContractJsons.isEmpty()) { + break; + } + offset += pageSize; + + LOG.info( + "Processing {} data contracts in batch (offset: {})", + dataContractJsons.size(), + offset - pageSize); + + for (String dataContractJson : dataContractJsons) { + try { + DataContract dataContract = JsonUtils.readValue(dataContractJson, DataContract.class); + + if (nullOrEmpty(dataContract.getQualityExpectations())) { + LOG.debug( + "Data contract {} has no quality expectations, skipping", + dataContract.getFullyQualifiedName()); + continue; + } + + LOG.debug( + "Processing data contract: {} (ID: {}) with {} quality expectations", + dataContract.getFullyQualifiedName(), + dataContract.getId(), + dataContract.getQualityExpectations().size()); + dataContractsProcessed++; + + int testCasesUpdated = 0; + for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { + try { + TestCase testCase = + collectionDAO.testCaseDAO().findEntityById(testCaseRef.getId()); + if (testCase == null) { + LOG.debug("Test case not found: {}", testCaseRef.getId()); + continue; + } + + if (testCase.getDataContract() != null) { + LOG.debug( + "Test case {} already has dataContract reference", + testCase.getFullyQualifiedName()); + continue; + } + + testCase.setDataContract( + new EntityReference() + .withId(dataContract.getId()) + .withType(Entity.DATA_CONTRACT) + .withFullyQualifiedName(dataContract.getFullyQualifiedName())); + + collectionDAO.testCaseDAO().update(testCase); + testCasesUpdated++; + + LOG.debug( + "Updated test case {} with dataContract reference to {}", + testCase.getFullyQualifiedName(), + dataContract.getFullyQualifiedName()); + + } catch (Exception e) { + LOG.warn( + "Failed to update test case {}: {}", testCaseRef.getId(), e.getMessage()); + } + } + + totalTestCasesMigrated += testCasesUpdated; + + if (testCasesUpdated > 0) { + LOG.info( + "Updated {} test cases for data contract: {}", + testCasesUpdated, + dataContract.getFullyQualifiedName()); + } + + } catch (Exception e) { + LOG.error("Failed to process data contract: {}", e.getMessage(), e); + } + } + } + + } catch (Exception e) { + LOG.error("Error during test case dataContract migration: {}", e.getMessage(), e); + throw new RuntimeException("Migration failed", e); + } + + LOG.info("===== TEST CASE DATA CONTRACT MIGRATION SUMMARY ====="); + LOG.info("Data contracts processed: {}", dataContractsProcessed); + LOG.info("Total test cases updated with dataContract reference: {}", totalTestCasesMigrated); + LOG.info("===== MIGRATION COMPLETE ====="); + } } From 7eeb433579b0a9a952726ef7f3f739a62d96580b Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 12 Mar 2026 11:10:46 +0100 Subject: [PATCH 22/38] Fix Postgres SQL bugs, duplicate test addition, and dead null check - Fix Postgres updateTestCaseDataContract: use ::jsonb cast instead of to_jsonb(::text) and set create_missing=true so the key is created - Simplify testsToRemove filter to !testsToAdd.contains (equivalent but clearer since testsToAdd is a subset of testCaseRefs) - Filter out tests already in currentTests before calling addTestCasesToLogicalTestSuite to avoid unnecessary postUpdate events - Replace dead null check after findEntityById with explicit EntityNotFoundException catch Co-Authored-By: Claude Opus 4.6 --- .../org/openmetadata/service/jdbi3/CollectionDAO.java | 2 +- .../service/jdbi3/DataContractRepository.java | 10 +++++----- .../service/migration/utils/v1130/MigrationUtil.java | 7 +++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index ed69d6b0a494..3662673ab7cc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -7025,7 +7025,7 @@ public TestCaseRecord map(ResultSet rs, StatementContext ctx) throws SQLExceptio connectionType = MYSQL) @ConnectionAwareSqlUpdate( value = - "UPDATE test_case SET json = jsonb_set(json, '{dataContract}', to_jsonb(:dataContractJson::text), false) WHERE id = :id", + "UPDATE test_case SET json = jsonb_set(json, '{dataContract}', :dataContractJson::jsonb, true) WHERE id = :id", connectionType = POSTGRES) void updateTestCaseDataContract( @Bind("id") String id, @Bind("dataContractJson") String dataContractJson); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index 8ef5055566dd..10ef50f62481 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -909,15 +909,15 @@ private void updateTestSuiteTests( : Collections.emptyList(); List testsToAdd = testsWithoutResults.stream().map(TestCase::getId).toList(); - if (!nullOrEmpty(testsWithoutResults)) { - testCaseRepository.addTestCasesToLogicalTestSuite(testSuite, testsToAdd); + List newTestCases = + testsToAdd.stream().filter(testId -> !currentTests.contains(testId)).toList(); + if (!nullOrEmpty(newTestCases)) { + testCaseRepository.addTestCasesToLogicalTestSuite(testSuite, newTestCases); } // Remove tests that are no longer in the quality expectations or already have results List testsToRemove = - currentTests.stream() - .filter(testId -> !testCaseRefs.contains(testId) || !testsToAdd.contains(testId)) - .toList(); + currentTests.stream().filter(testId -> !testsToAdd.contains(testId)).toList(); if (!nullOrEmpty(testsToRemove)) { testsToRemove.forEach( test -> { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java index e06cdd878e84..faddcc608a93 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java @@ -10,6 +10,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository; import org.openmetadata.service.util.EntityUtil; @@ -103,10 +104,6 @@ public static void migrateTestCaseDataContractReferences(CollectionDAO collectio try { TestCase testCase = collectionDAO.testCaseDAO().findEntityById(testCaseRef.getId()); - if (testCase == null) { - LOG.debug("Test case not found: {}", testCaseRef.getId()); - continue; - } if (testCase.getDataContract() != null) { LOG.debug( @@ -129,6 +126,8 @@ public static void migrateTestCaseDataContractReferences(CollectionDAO collectio testCase.getFullyQualifiedName(), dataContract.getFullyQualifiedName()); + } catch (EntityNotFoundException e) { + LOG.debug("Test case not found: {}", testCaseRef.getId()); } catch (Exception e) { LOG.warn( "Failed to update test case {}: {}", testCaseRef.getId(), e.getMessage()); From ca268193c497ff653b8a4b22a5fb207fb1dd2360 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 12 Mar 2026 11:14:14 +0100 Subject: [PATCH 23/38] format --- .../service/migration/utils/v1130/MigrationUtil.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java index faddcc608a93..73ac75e00fe0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java @@ -102,8 +102,7 @@ public static void migrateTestCaseDataContractReferences(CollectionDAO collectio int testCasesUpdated = 0; for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { try { - TestCase testCase = - collectionDAO.testCaseDAO().findEntityById(testCaseRef.getId()); + TestCase testCase = collectionDAO.testCaseDAO().findEntityById(testCaseRef.getId()); if (testCase.getDataContract() != null) { LOG.debug( @@ -129,8 +128,7 @@ public static void migrateTestCaseDataContractReferences(CollectionDAO collectio } catch (EntityNotFoundException e) { LOG.debug("Test case not found: {}", testCaseRef.getId()); } catch (Exception e) { - LOG.warn( - "Failed to update test case {}: {}", testCaseRef.getId(), e.getMessage()); + LOG.warn("Failed to update test case {}: {}", testCaseRef.getId(), e.getMessage()); } } From c66a99939642e7bea5699afdf7a3e6f6e8de3162 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 12 Mar 2026 11:15:13 +0100 Subject: [PATCH 24/38] Escape dataContractId in search filter and fix dead null check in updateTestCasesWithDataContract Co-Authored-By: Claude Opus 4.6 --- .../service/jdbi3/DataContractRepository.java | 11 ++--------- .../openmetadata/service/search/SearchListFilter.java | 4 +++- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index 10ef50f62481..408bf9f050ee 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -1831,28 +1831,19 @@ private void updateTestCasesWithDataContract(DataContract dataContract) { for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { try { - // Get the existing test case to check if it already has a dataContract TestCase existingTestCase = daoCollection.testCaseDAO().findEntityById(testCaseRef.getId()); - if (existingTestCase == null) { - LOG.warn("Test case {} not found, skipping dataContract update", testCaseRef.getId()); - continue; - } - // Only update if the test case doesn't have a dataContract or has a different dataContract - // ID boolean shouldUpdate = existingTestCase.getDataContract() == null || !existingTestCase.getDataContract().getId().equals(dataContract.getId()); if (shouldUpdate) { - // Create the dataContract EntityReference EntityReference dataContractRef = new EntityReference() .withId(dataContract.getId()) .withType(Entity.DATA_CONTRACT) .withFullyQualifiedName(dataContract.getFullyQualifiedName()); - // Use testCase DAO to update the dataContract field directly daoCollection .testCaseDAO() .updateTestCaseDataContract( @@ -1867,6 +1858,8 @@ private void updateTestCasesWithDataContract(DataContract dataContract) { "Test case {} already has the same dataContract reference, skipping update", testCaseRef.getId()); } + } catch (EntityNotFoundException e) { + LOG.warn("Test case {} not found, skipping dataContract update", testCaseRef.getId()); } catch (Exception e) { LOG.warn( "Failed to update test case {} with dataContract reference: {}", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java index e141b76e4b5f..9a9789098117 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java @@ -317,7 +317,9 @@ private String getTestCaseResultCondition() { getDataQualityDimensionCondition( dataQualityDimension, "testDefinition.dataQualityDimension")); if (dataContractId != null) - conditions.add(String.format("{\"term\": {\"dataContract.id\": \"%s\"}}", dataContractId)); + conditions.add( + String.format( + "{\"term\": {\"dataContract.id\": \"%s\"}}", escapeDoubleQuotes(dataContractId))); return addCondition(conditions); } From bc7ea9cd526fe4eedad0935a22544d77aa64ef2b Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Thu, 12 Mar 2026 11:44:08 +0100 Subject: [PATCH 25/38] format --- .../it/tests/DataContractResourceIT.java | 251 ++++++++++++++++++ .../it/tests/TestCaseResourceIT.java | 92 +++++++ 2 files changed, 343 insertions(+) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataContractResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataContractResourceIT.java index a5fa74bb37d5..06eb11070d64 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataContractResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataContractResourceIT.java @@ -31,6 +31,7 @@ import org.openmetadata.schema.entity.datacontract.odcs.ODCSSlaProperty; import org.openmetadata.schema.entity.datacontract.odcs.ODCSTeamMember; import org.openmetadata.schema.entity.services.DatabaseService; +import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.type.ContractExecutionStatus; @@ -41,6 +42,7 @@ import org.openmetadata.sdk.exceptions.OpenMetadataException; import org.openmetadata.sdk.fluent.DataContracts; import org.openmetadata.sdk.fluent.DataContracts.FluentDataContract; +import org.openmetadata.sdk.fluent.builders.TestCaseBuilder; import org.openmetadata.sdk.models.ListParams; import org.openmetadata.sdk.models.ListResponse; import org.openmetadata.service.resources.data.DataContractResource; @@ -6664,4 +6666,253 @@ void testImportODCSYamlWithAllV310Features(TestNamespace ns) { .getIdentities() .contains("manager@company.com")); } + + // =================================================================== + // QUALITY EXPECTATIONS TESTS + // =================================================================== + + private TestCase createTestCaseForTable(TestNamespace ns, Table table, String suffix) { + return TestCaseBuilder.create(SdkClients.adminClient()) + .name(ns.prefix("tc_" + suffix)) + .forTable(table) + .testDefinition("tableRowCountToEqual") + .parameter("value", "100") + .create(); + } + + @Test + void testCreateContractWithQualityExpectations_TestCasesLinked(TestNamespace ns) { + Table table = createTestTable(ns); + TestCase tc1 = createTestCaseForTable(ns, table, "qe1"); + TestCase tc2 = createTestCaseForTable(ns, table, "qe2"); + + CreateDataContract request = + new CreateDataContract() + .withName(ns.prefix("qe_create")) + .withEntity(table.getEntityReference()) + .withDescription("Contract with quality expectations") + .withQualityExpectations( + List.of( + new EntityReference().withId(tc1.getId()).withType("testCase"), + new EntityReference().withId(tc2.getId()).withType("testCase"))); + + DataContract contract = createEntity(request); + + assertNotNull(contract.getQualityExpectations()); + assertEquals(2, contract.getQualityExpectations().size()); + + // Verify test cases now have dataContract reference + TestCase fetchedTc1 = SdkClients.adminClient().testCases().get(tc1.getId().toString()); + assertNotNull( + fetchedTc1.getDataContract(), "Test case should have dataContract reference after create"); + assertEquals(contract.getId(), fetchedTc1.getDataContract().getId()); + + TestCase fetchedTc2 = SdkClients.adminClient().testCases().get(tc2.getId().toString()); + assertNotNull(fetchedTc2.getDataContract()); + assertEquals(contract.getId(), fetchedTc2.getDataContract().getId()); + } + + @Test + void testUpdateContractQualityExpectations_TestCasesUpdated(TestNamespace ns) { + Table table = createTestTable(ns); + TestCase tc1 = createTestCaseForTable(ns, table, "upd1"); + TestCase tc2 = createTestCaseForTable(ns, table, "upd2"); + TestCase tc3 = createTestCaseForTable(ns, table, "upd3"); + + // Create contract with tc1 and tc2 + CreateDataContract request = + new CreateDataContract() + .withName(ns.prefix("qe_update")) + .withEntity(table.getEntityReference()) + .withDescription("Contract for update test") + .withQualityExpectations( + List.of( + new EntityReference().withId(tc1.getId()).withType("testCase"), + new EntityReference().withId(tc2.getId()).withType("testCase"))); + + DataContract contract = createEntity(request); + + // Update: remove tc1, keep tc2, add tc3 + contract.setQualityExpectations( + List.of( + new EntityReference().withId(tc2.getId()).withType("testCase"), + new EntityReference().withId(tc3.getId()).withType("testCase"))); + DataContract updated = patchEntity(contract.getId().toString(), contract); + + assertEquals(2, updated.getQualityExpectations().size()); + + // tc1 should have lost its dataContract reference + TestCase fetchedTc1 = SdkClients.adminClient().testCases().get(tc1.getId().toString()); + assertTrue( + fetchedTc1.getDataContract() == null + || !fetchedTc1.getDataContract().getId().equals(contract.getId()), + "Removed test case should no longer reference this contract"); + + // tc2 should still reference the contract + TestCase fetchedTc2 = SdkClients.adminClient().testCases().get(tc2.getId().toString()); + assertNotNull(fetchedTc2.getDataContract()); + assertEquals(contract.getId(), fetchedTc2.getDataContract().getId()); + + // tc3 should now reference the contract + TestCase fetchedTc3 = SdkClients.adminClient().testCases().get(tc3.getId().toString()); + assertNotNull( + fetchedTc3.getDataContract(), "Newly added test case should have dataContract reference"); + assertEquals(contract.getId(), fetchedTc3.getDataContract().getId()); + } + + @Test + void testDeleteContractWithQualityExpectations_TestCasesUnlinked(TestNamespace ns) { + Table table = createTestTable(ns); + TestCase tc1 = createTestCaseForTable(ns, table, "del1"); + TestCase tc2 = createTestCaseForTable(ns, table, "del2"); + + CreateDataContract request = + new CreateDataContract() + .withName(ns.prefix("qe_delete")) + .withEntity(table.getEntityReference()) + .withDescription("Contract for delete test") + .withQualityExpectations( + List.of( + new EntityReference().withId(tc1.getId()).withType("testCase"), + new EntityReference().withId(tc2.getId()).withType("testCase"))); + + DataContract contract = createEntity(request); + + // Verify test cases are linked + TestCase beforeDelete = SdkClients.adminClient().testCases().get(tc1.getId().toString()); + assertNotNull(beforeDelete.getDataContract()); + + // Hard delete the contract + hardDeleteEntity(contract.getId().toString()); + + // Test cases should no longer reference the contract + TestCase afterDelete1 = SdkClients.adminClient().testCases().get(tc1.getId().toString()); + assertTrue( + afterDelete1.getDataContract() == null + || !afterDelete1.getDataContract().getId().equals(contract.getId()), + "Test case should lose dataContract reference after contract deletion"); + + TestCase afterDelete2 = SdkClients.adminClient().testCases().get(tc2.getId().toString()); + assertTrue( + afterDelete2.getDataContract() == null + || !afterDelete2.getDataContract().getId().equals(contract.getId()), + "Test case should lose dataContract reference after contract deletion"); + } + + @Test + void testContractWithoutQualityExpectations_NoTestSuite(TestNamespace ns) { + Table table = createTestTable(ns); + + CreateDataContract request = + new CreateDataContract() + .withName(ns.prefix("no_qe")) + .withEntity(table.getEntityReference()) + .withDescription("Contract without quality expectations"); + + DataContract contract = createEntity(request); + + assertNull( + contract.getQualityExpectations(), + "Contract without quality expectations should have null quality expectations"); + assertNull( + contract.getTestSuite(), "Contract without quality expectations should have no test suite"); + } + + @Test + void testValidateContractWithTestResultsCompilesQualityScore(TestNamespace ns) { + Table table = createTestTable(ns); + TestCase tc1 = createTestCaseForTable(ns, table, "val1"); + TestCase tc2 = createTestCaseForTable(ns, table, "val2"); + + // Add test results so validation can use them instead of triggering pipeline + SdkClients.adminClient() + .testCaseResults() + .forTestCase(tc1.getFullyQualifiedName()) + .passed() + .result("Row count matches") + .create(); + + SdkClients.adminClient() + .testCaseResults() + .forTestCase(tc2.getFullyQualifiedName()) + .failed() + .result("Row count mismatch") + .create(); + + CreateDataContract request = + new CreateDataContract() + .withName(ns.prefix("qe_validate")) + .withEntity(table.getEntityReference()) + .withDescription("Contract for validation test") + .withQualityExpectations( + List.of( + new EntityReference().withId(tc1.getId()).withType("testCase"), + new EntityReference().withId(tc2.getId()).withType("testCase"))); + + DataContract contract = createEntity(request); + + // Validate: since both tests have results, quality score should be computed + DataContractResult result = SdkClients.adminClient().dataContracts().validate(contract.getId()); + + assertNotNull(result); + assertNotNull(result.getQualityValidation(), "Validation result should have quality section"); + } + + @Test + void testCreateContractWithQualityExpectations_TestSuiteCreated(TestNamespace ns) { + Table table = createTestTable(ns); + TestCase tc1 = createTestCaseForTable(ns, table, "ts1"); + + CreateDataContract request = + new CreateDataContract() + .withName(ns.prefix("qe_testsuite")) + .withEntity(table.getEntityReference()) + .withDescription("Contract to verify test suite creation") + .withQualityExpectations( + List.of(new EntityReference().withId(tc1.getId()).withType("testCase"))); + + DataContract contract = createEntity(request); + + // Re-fetch with fields to get testSuite + DataContract fetched = + SdkClients.adminClient() + .dataContracts() + .get(contract.getId().toString(), "testSuite,qualityExpectations"); + + assertNotNull(fetched.getQualityExpectations()); + assertEquals(1, fetched.getQualityExpectations().size()); + } + + @Test + void testCreateOrUpdateContractWithQualityExpectations(TestNamespace ns) { + Table table = createTestTable(ns); + TestCase tc1 = createTestCaseForTable(ns, table, "cou1"); + TestCase tc2 = createTestCaseForTable(ns, table, "cou2"); + + // Create contract without quality expectations + CreateDataContract request = + new CreateDataContract() + .withName(ns.prefix("qe_cou")) + .withEntity(table.getEntityReference()) + .withDescription("Contract for createOrUpdate test"); + + DataContract contract = SdkClients.adminClient().dataContracts().createOrUpdate(request); + assertNull(contract.getQualityExpectations()); + + // Update with quality expectations via createOrUpdate + request.setQualityExpectations( + List.of( + new EntityReference().withId(tc1.getId()).withType("testCase"), + new EntityReference().withId(tc2.getId()).withType("testCase"))); + + DataContract updated = SdkClients.adminClient().dataContracts().createOrUpdate(request); + assertEquals(contract.getId(), updated.getId()); + assertNotNull(updated.getQualityExpectations()); + assertEquals(2, updated.getQualityExpectations().size()); + + // Verify test cases got linked + TestCase fetchedTc1 = SdkClients.adminClient().testCases().get(tc1.getId().toString()); + assertNotNull(fetchedTc1.getDataContract()); + assertEquals(contract.getId(), fetchedTc1.getDataContract().getId()); + } } diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java index 8de5a88e148b..9c746aee57f0 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java @@ -20,11 +20,13 @@ import org.openmetadata.it.util.TestNamespace; import org.openmetadata.schema.api.classification.CreateClassification; import org.openmetadata.schema.api.classification.CreateTag; +import org.openmetadata.schema.api.data.CreateDataContract; import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.tests.CreateTestCase; import org.openmetadata.schema.api.tests.CreateTestSuite; import org.openmetadata.schema.entity.classification.Classification; import org.openmetadata.schema.entity.classification.Tag; +import org.openmetadata.schema.entity.data.DataContract; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.services.DatabaseService; @@ -35,6 +37,7 @@ import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.schema.utils.JsonUtils; @@ -3231,4 +3234,93 @@ private String escapeCSVValue(String value) { } return value; } + + // =================================================================== + // DATA CONTRACT REFERENCE TESTS + // =================================================================== + + @Test + void test_createTestCase_withDataContractReference(TestNamespace ns) { + Table table = createTable(ns); + + // Create a data contract for the table + CreateDataContract dcRequest = + new CreateDataContract() + .withName(ns.prefix("tc_dc")) + .withEntity(table.getEntityReference()) + .withDescription("Contract for test case reference test"); + DataContract contract = SdkClients.adminClient().dataContracts().create(dcRequest); + + // Create a test case with dataContract FQN (the mapper resolves it to EntityReference) + CreateTestCase request = + TestCaseBuilder.create(SdkClients.adminClient()) + .name(ns.prefix("tc_with_dc")) + .forTable(table) + .testDefinition("tableRowCountToEqual") + .parameter("value", "100") + .build(); + request.setDataContract(contract.getFullyQualifiedName()); + + TestCase testCase = createEntity(request); + + assertNotNull(testCase.getDataContract(), "Test case should have dataContract reference"); + assertEquals(contract.getId(), testCase.getDataContract().getId()); + assertEquals("dataContract", testCase.getDataContract().getType()); + } + + @Test + void test_createTestCase_withInvalidDataContractReference(TestNamespace ns) { + Table table = createTable(ns); + + CreateTestCase request = + TestCaseBuilder.create(SdkClients.adminClient()) + .name(ns.prefix("tc_bad_dc")) + .forTable(table) + .testDefinition("tableRowCountToEqual") + .parameter("value", "100") + .build(); + request.setDataContract("non.existent.data.contract"); + + assertThrows( + Exception.class, + () -> createEntity(request), + "Should fail with nonexistent data contract FQN"); + } + + @Test + void test_testCase_dataContractLinkedViaContractCreate(TestNamespace ns) { + Table table = createTable(ns); + + // Create test case first without dataContract + TestCase testCase = + TestCaseBuilder.create(SdkClients.adminClient()) + .name(ns.prefix("tc_linked")) + .forTable(table) + .testDefinition("tableRowCountToEqual") + .parameter("value", "100") + .create(); + + // Verify no dataContract initially + TestCase beforeContract = SdkClients.adminClient().testCases().get(testCase.getId().toString()); + assertTrue( + beforeContract.getDataContract() == null, + "Test case should have no dataContract before contract creation"); + + // Now create a data contract that references this test case + CreateDataContract dcRequest = + new CreateDataContract() + .withName(ns.prefix("dc_link")) + .withEntity(table.getEntityReference()) + .withDescription("Contract linking to existing test case") + .withQualityExpectations( + List.of(new EntityReference().withId(testCase.getId()).withType("testCase"))); + DataContract contract = SdkClients.adminClient().dataContracts().create(dcRequest); + + // Verify test case now has dataContract reference (set by postCreate lifecycle hook) + TestCase afterContract = SdkClients.adminClient().testCases().get(testCase.getId().toString()); + assertNotNull( + afterContract.getDataContract(), + "Test case should have dataContract after contract creation"); + assertEquals(contract.getId(), afterContract.getDataContract().getId()); + } } From 27626a209b20f91a3e8557aa7aa0eec95ce1f76e Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sat, 14 Mar 2026 12:22:44 +0100 Subject: [PATCH 26/38] Fix data contract reference integrity, ES injection, and UI stale fetch - Use removeTestCaseDataContractForSpecificContract in postDelete to avoid wiping another contract's reference on shared test cases - Add .withName() to dataContract EntityReference in repository and migration - Validate dataContractId as UUID before adding to ES search filter - Guard fetchTestCases against undefined fqn and add proper useEffect deps - Add integration tests for cross-contract reference integrity Co-Authored-By: Claude Opus 4.6 (1M context) --- .../it/tests/DataContractResourceIT.java | 136 ++++++++++++++++++ .../service/jdbi3/DataContractRepository.java | 7 +- .../migration/utils/v1130/MigrationUtil.java | 1 + .../dqtests/TestCaseResultResource.java | 12 +- .../ContractQualityCard.component.tsx | 13 +- 5 files changed, 160 insertions(+), 9 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataContractResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataContractResourceIT.java index 06eb11070d64..73f24ffb8148 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataContractResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataContractResourceIT.java @@ -6915,4 +6915,140 @@ void testCreateOrUpdateContractWithQualityExpectations(TestNamespace ns) { assertNotNull(fetchedTc1.getDataContract()); assertEquals(contract.getId(), fetchedTc1.getDataContract().getId()); } + + // =================================================================== + // DATA CONTRACT REFERENCE INTEGRITY TESTS + // =================================================================== + + @Test + void testDeleteContract_DoesNotWipeOtherContractRef(TestNamespace ns) { + Table table1 = createTestTable(ns); + Table table2 = createTestTable(ns); + TestCase tc = createTestCaseForTable(ns, table1, "shared"); + + // Create contractA with tc + CreateDataContract requestA = + new CreateDataContract() + .withName(ns.prefix("contractA")) + .withEntity(table1.getEntityReference()) + .withDescription("Contract A") + .withQualityExpectations( + List.of(new EntityReference().withId(tc.getId()).withType("testCase"))); + + DataContract contractA = createEntity(requestA); + + // Verify tc references contractA + TestCase fetched = SdkClients.adminClient().testCases().get(tc.getId().toString()); + assertNotNull(fetched.getDataContract()); + assertEquals(contractA.getId(), fetched.getDataContract().getId()); + + // Create contractB for a different table, and reassign tc to contractB + CreateDataContract requestB = + new CreateDataContract() + .withName(ns.prefix("contractB")) + .withEntity(table2.getEntityReference()) + .withDescription("Contract B") + .withQualityExpectations( + List.of(new EntityReference().withId(tc.getId()).withType("testCase"))); + + DataContract contractB = createEntity(requestB); + + // tc should now reference contractB (the latest contract that claimed it) + fetched = SdkClients.adminClient().testCases().get(tc.getId().toString()); + assertNotNull(fetched.getDataContract()); + assertEquals(contractB.getId(), fetched.getDataContract().getId()); + + // Delete contractA - this should NOT wipe tc's reference to contractB + hardDeleteEntity(contractA.getId().toString()); + + // tc should still reference contractB + fetched = SdkClients.adminClient().testCases().get(tc.getId().toString()); + assertNotNull( + fetched.getDataContract(), + "Deleting contractA should not wipe tc's reference to contractB"); + assertEquals( + contractB.getId(), + fetched.getDataContract().getId(), + "tc should still reference contractB after contractA deletion"); + } + + @Test + void testDataContractRef_IncludesName(TestNamespace ns) { + Table table = createTestTable(ns); + TestCase tc = createTestCaseForTable(ns, table, "name_check"); + + CreateDataContract request = + new CreateDataContract() + .withName(ns.prefix("ref_name")) + .withEntity(table.getEntityReference()) + .withDescription("Contract ref name test") + .withQualityExpectations( + List.of(new EntityReference().withId(tc.getId()).withType("testCase"))); + + DataContract contract = createEntity(request); + + TestCase fetched = SdkClients.adminClient().testCases().get(tc.getId().toString()); + assertNotNull(fetched.getDataContract()); + assertEquals(contract.getId(), fetched.getDataContract().getId()); + assertNotNull( + fetched.getDataContract().getName(), "dataContract EntityReference should include name"); + assertEquals(contract.getName(), fetched.getDataContract().getName()); + assertNotNull(fetched.getDataContract().getFullyQualifiedName()); + } + + @Test + void testUpdateContract_RemoveTestCase_OnlyRemovesOwnRef(TestNamespace ns) { + Table table1 = createTestTable(ns); + Table table2 = createTestTable(ns); + TestCase tc1 = createTestCaseForTable(ns, table1, "own1"); + TestCase tc2 = createTestCaseForTable(ns, table1, "own2"); + + // Create contractA with tc1 and tc2 + CreateDataContract requestA = + new CreateDataContract() + .withName(ns.prefix("own_refA")) + .withEntity(table1.getEntityReference()) + .withDescription("Contract A owns tc1 and tc2") + .withQualityExpectations( + List.of( + new EntityReference().withId(tc1.getId()).withType("testCase"), + new EntityReference().withId(tc2.getId()).withType("testCase"))); + + DataContract contractA = createEntity(requestA); + + // Create contractB and reassign tc1 to it + CreateDataContract requestB = + new CreateDataContract() + .withName(ns.prefix("own_refB")) + .withEntity(table2.getEntityReference()) + .withDescription("Contract B takes tc1") + .withQualityExpectations( + List.of(new EntityReference().withId(tc1.getId()).withType("testCase"))); + + DataContract contractB = createEntity(requestB); + + // tc1 should now point to contractB + TestCase fetchedTc1 = SdkClients.adminClient().testCases().get(tc1.getId().toString()); + assertEquals(contractB.getId(), fetchedTc1.getDataContract().getId()); + + // Update contractA: remove tc1 from its quality expectations + contractA.setQualityExpectations( + List.of(new EntityReference().withId(tc2.getId()).withType("testCase"))); + patchEntity(contractA.getId().toString(), contractA); + + // tc1 should still reference contractB (removeDataContractFromOldTestCases is contract-aware) + fetchedTc1 = SdkClients.adminClient().testCases().get(tc1.getId().toString()); + assertNotNull( + fetchedTc1.getDataContract(), + "tc1 should still have dataContract ref after being removed from contractA"); + assertEquals( + contractB.getId(), + fetchedTc1.getDataContract().getId(), + "tc1 should still reference contractB"); + + // tc2 should still reference contractA + TestCase fetchedTc2 = SdkClients.adminClient().testCases().get(tc2.getId().toString()); + assertNotNull(fetchedTc2.getDataContract()); + assertEquals(contractA.getId(), fetchedTc2.getDataContract().getId()); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index 408bf9f050ee..8d7aa5b605df 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -1842,6 +1842,7 @@ private void updateTestCasesWithDataContract(DataContract dataContract) { new EntityReference() .withId(dataContract.getId()) .withType(Entity.DATA_CONTRACT) + .withName(dataContract.getName()) .withFullyQualifiedName(dataContract.getFullyQualifiedName()); daoCollection @@ -1876,8 +1877,10 @@ private void removeDataContractFromTestCases(DataContract dataContract) { for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { try { - // Use testCase DAO to remove the dataContract field - daoCollection.testCaseDAO().removeTestCaseDataContract(testCaseRef.getId().toString()); + daoCollection + .testCaseDAO() + .removeTestCaseDataContractForSpecificContract( + testCaseRef.getId().toString(), dataContract.getId().toString()); LOG.debug("Removed dataContract reference from test case {}", testCaseRef.getId()); } catch (Exception e) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java index 73ac75e00fe0..3f76fe4f4c9a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java @@ -115,6 +115,7 @@ public static void migrateTestCaseDataContractReferences(CollectionDAO collectio new EntityReference() .withId(dataContract.getId()) .withType(Entity.DATA_CONTRACT) + .withName(dataContract.getName()) .withFullyQualifiedName(dataContract.getFullyQualifiedName())); collectionDAO.testCaseDAO().update(testCase); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java index 1cb1a3ac5f16..6d453289e930 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java @@ -335,7 +335,11 @@ public ResultList listTestCaseResultsFromSearch( .ifPresent(tsi -> searchListFilter.addQueryParam("testSuiteId", tsi)); Optional.ofNullable(entityFQN).ifPresent(ef -> searchListFilter.addQueryParam("entityFQN", ef)); Optional.ofNullable(dataContractId) - .ifPresent(dci -> searchListFilter.addQueryParam("dataContractId", dci)); + .ifPresent( + dci -> { + UUID.fromString(dci); + searchListFilter.addQueryParam("dataContractId", dci); + }); Optional.ofNullable(type).ifPresent(t -> searchListFilter.addQueryParam("testCaseType", t)); Optional.ofNullable(dataQualityDimension) .ifPresent(dqd -> searchListFilter.addQueryParam("dataQualityDimension", dqd)); @@ -431,7 +435,11 @@ public TestCaseResult latestTestCaseResultFromSearch( Optional.ofNullable(testSuiteId) .ifPresent(tsi -> searchListFilter.addQueryParam("testSuiteId", tsi)); Optional.ofNullable(dataContractId) - .ifPresent(dci -> searchListFilter.addQueryParam("dataContractId", dci)); + .ifPresent( + dci -> { + UUID.fromString(dci); + searchListFilter.addQueryParam("dataContractId", dci); + }); List authRequests = getAuthRequestsForListOps(testCaseFQN, testSuiteId, dataContractId); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx index 0dca2c094fa9..15b9652f7455 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx @@ -12,7 +12,7 @@ */ import Icon from '@ant-design/icons'; import { Col, Row, Space, Typography } from 'antd'; -import { useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; import { ES_MAX_PAGE_SIZE } from '../../../constants/constants'; @@ -42,12 +42,15 @@ const ContractQualityCard: React.FC<{ const [isTestCaseLoading, setIsTestCaseLoading] = useState(false); const [testCase, setTestCase] = useState([]); - const fetchTestCases = async () => { + const fetchTestCases = useCallback(async () => { + if (!fqn) { + return; + } setIsTestCaseLoading(true); try { const { data } = await getListTestCaseBySearch({ ...DEFAULT_SORT_ORDER, - entityLink: generateEntityLink(fqn ?? ''), + entityLink: generateEntityLink(fqn), includeAllTests: true, limit: ES_MAX_PAGE_SIZE, include: Include.NonDeleted, @@ -63,7 +66,7 @@ const ContractQualityCard: React.FC<{ } finally { setIsTestCaseLoading(false); } - }; + }, [fqn]); const { showTestCaseSummaryChart, @@ -131,7 +134,7 @@ const ContractQualityCard: React.FC<{ useEffect(() => { fetchTestCases(); - }, []); + }, [fetchTestCases]); if (isTestCaseLoading) { return ; From 5adf633de3cd79b867734b3399b68f8f96450f72 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sat, 14 Mar 2026 13:26:22 +0100 Subject: [PATCH 27/38] Guard pipelineServiceClient null in DataContractRepository postCreateOrUpdate and triggerDataQualityValidation both called pipelineServiceClient without null checks, causing NPE in test environments where no pipeline service is configured. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../openmetadata/service/jdbi3/DataContractRepository.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index 8d7aa5b605df..4b3055c02826 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -279,7 +279,7 @@ private void postCreateOrUpdate(DataContract dataContract) { TestSuiteRepository testSuiteRepository = (TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE); testSuiteRepository.createOrUpdate(null, testSuite, ADMIN_USER_NAME); - if (!pipeline.getDeployed()) { + if (!pipeline.getDeployed() && pipelineServiceClient != null) { prepareAndDeployIngestionPipeline(pipeline, testSuite); } } @@ -1185,6 +1185,10 @@ public void deployAndTriggerDQValidation(DataContract dataContract) { IngestionPipeline pipeline = Entity.getEntity(testSuite.getPipelines().get(0), "*", Include.NON_DELETED); + if (pipelineServiceClient == null) { + throw DataContractValidationException.byMessage( + "Pipeline service client is not configured, cannot trigger DQ validation"); + } // ensure pipeline is deployed before running // we deploy the pipeline during post create if (!pipeline.getDeployed()) { From 8ddd0dc2e2ea09df5649ceda45d1774b5b877dfe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 15 Mar 2026 07:01:30 +0000 Subject: [PATCH 28/38] Update generated TypeScript types --- .../ui/src/generated/api/automations/createWorkflow.ts | 4 ---- .../ui/src/generated/api/services/createDatabaseService.ts | 4 ---- .../services/ingestionPipelines/createIngestionPipeline.ts | 4 ---- .../generated/entity/automations/testServiceConnection.ts | 4 ---- .../ui/src/generated/entity/automations/workflow.ts | 4 ---- .../services/connections/database/informixConnection.ts | 6 ++---- .../entity/services/connections/serviceConnection.ts | 4 ---- .../ui/src/generated/entity/services/databaseService.ts | 4 ---- .../entity/services/ingestionPipelines/ingestionPipeline.ts | 4 ---- .../ui/src/generated/metadataIngestion/testSuitePipeline.ts | 4 ---- .../ui/src/generated/metadataIngestion/workflow.ts | 4 ---- 11 files changed, 2 insertions(+), 44 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts index be2a784b06f2..f54e29c7eb04 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts @@ -792,10 +792,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts index ea34670e6d94..32fbd54f3c0b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts @@ -393,10 +393,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts index cdc8cc2d12c4..b2a3403ee493 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts @@ -3891,10 +3891,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts index 107e7b0db7de..97dd6965cd57 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts @@ -674,10 +674,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts index 6aa828f0303c..0ccdbf4fb605 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts @@ -1322,10 +1322,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/informixConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/informixConnection.ts index 9ae6b792c3eb..ac4ecb0d3afc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/informixConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/informixConnection.ts @@ -17,11 +17,9 @@ export interface InformixConnection { connectionArguments?: { [key: string]: any }; connectionOptions?: { [key: string]: string }; /** - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. + * Database of the data source. */ - database?: string; + database: string; /** * Regex to only include/exclude databases that matches the pattern. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts index 4b49a7da536d..a15d5bf9d5a3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts @@ -1183,10 +1183,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts index 29d4c5c0d123..40d76c5b2a84 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts @@ -520,10 +520,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts index 03cc333bf5ff..accaaa5311ef 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts @@ -4474,10 +4474,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts index deb1e89c2a78..de6fdf758ceb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts @@ -1227,10 +1227,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts index 5e7c1afc061d..37d7e6e05957 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts @@ -1272,10 +1272,6 @@ export interface ConfigObject { * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This * is optional parameter, if you would like to restrict the metadata reading to a single * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. - * - * Database of the data source. This is an optional parameter, if you would like to restrict - * the metadata reading to a single database. If left blank, OpenMetadata ingestion attempts - * to scan all the databases. */ database?: string; /** From 165fe28d3e35911942e7dbfe1e697f06f9c28ef8 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 15 Mar 2026 11:12:20 +0100 Subject: [PATCH 29/38] trigger From 99ccd11a5878c14e0c0bc6ef38fd6d2df4f23de5 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 18 Mar 2026 10:03:52 +0100 Subject: [PATCH 30/38] trigger From ee46b86fa9636ef6627f3ffcced1c2f753b6643a Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 23 Mar 2026 07:46:59 +0100 Subject: [PATCH 31/38] Apply Copilot review fixes: division-by-zero guard, FQN fallback, status fallback, quality score formula, MySQL JSON_UNQUOTE Co-Authored-By: Claude Opus 4.6 --- .../service/jdbi3/CollectionDAO.java | 2 +- .../service/jdbi3/DataContractRepository.java | 4 +-- .../ContractQualityCard.component.tsx | 32 +++++++++++-------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 697dfde7a049..570ef63f7a55 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -7392,7 +7392,7 @@ void updateTestCaseDataContract( @ConnectionAwareSqlUpdate( value = - "UPDATE test_case SET json = JSON_REMOVE(json, '$.dataContract') WHERE id = :id AND JSON_EXTRACT(json, '$.dataContract.id') = :dataContractId", + "UPDATE test_case SET json = JSON_REMOVE(json, '$.dataContract') WHERE id = :id AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.dataContract.id')) = :dataContractId", connectionType = MYSQL) @ConnectionAwareSqlUpdate( value = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index 7b61be73ccf2..94dd9f43079a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -1366,9 +1366,7 @@ private QualityValidation validateDQ(TestSuite testSuite, QualityValidation exis .withPassed(existingValidation.getPassed() + (testSummary.size() - failedTests.size())); existingValidation.withQualityScore( - (((existingValidation.getPassed() - existingValidation.getFailed()) - / (double) existingValidation.getTotal())) - * 100); + (existingValidation.getPassed() / (double) existingValidation.getTotal()) * 100); return existingValidation; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx index 15b9652f7455..1b7d59b9fd33 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx @@ -97,12 +97,12 @@ const ContractQualityCard: React.FC<{ const failed = qualityValidation?.failed ?? 0; const aborted = total - success - failed; - const successPercent = (success / total) * 100; - const failedPercent = (failed / total) * 100; - const abortedPercent = (aborted / total) * 100; + const successPercent = total ? (success / total) * 100 : 0; + const failedPercent = total ? (failed / total) * 100 : 0; + const abortedPercent = total ? (aborted / total) * 100 : 0; return { - showTestCaseSummaryChart: true, + showTestCaseSummaryChart: Boolean(total), segmentWidths: { successPercent, failedPercent, @@ -120,17 +120,23 @@ const ContractQualityCard: React.FC<{ testCase.map((result) => [result.id, result]) ); - const mergedData = contract.qualityExpectations?.map((item) => ({ - id: item.id, - name: item.name, - fullyQualifiedName: `${fqn}.${item.name}`, - testCaseStatus: - testCaseResultsMap.get(item.id)?.testCaseStatus ?? - TestCaseStatus.Queued, - })); + const mergedData = contract.qualityExpectations?.map((item) => { + const matchedTestCase = testCaseResultsMap.get(item.id); + + return { + id: item.id, + name: item.name, + fullyQualifiedName: + item.fullyQualifiedName ?? (fqn ? `${fqn}.${item.name}` : item.name), + testCaseStatus: + matchedTestCase?.testCaseStatus ?? + matchedTestCase?.testCaseResult?.testCaseStatus ?? + TestCaseStatus.Queued, + }; + }); return mergedData ?? []; - }, [contract, testCase]); + }, [contract, testCase, fqn]); useEffect(() => { fetchTestCases(); From a0f7503fccfec6e2c55a054278768fd25bbc0256 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 25 Mar 2026 18:17:49 +0100 Subject: [PATCH 32/38] Add unit tests for data contract coverage (SearchListFilter, DataContractRepository, MigrationUtil, TestCaseResultIndex) Co-Authored-By: Claude Opus 4.6 --- .../jdbi3/DataContractRepositoryTest.java | 786 ++++++++++++++++++ .../service/jdbi3/SearchListFilterTest.java | 40 + .../utils/v1130/MigrationUtilTest.java | 241 ++++++ .../indexes/TestCaseResultIndexTest.java | 102 +++ 4 files changed, 1169 insertions(+) create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v1130/MigrationUtilTest.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseResultIndexTest.java diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java new file mode 100644 index 000000000000..c80b6a691711 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java @@ -0,0 +1,786 @@ +package org.openmetadata.service.jdbi3; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.mock; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.data.DataContract; +import org.openmetadata.schema.entity.datacontract.DataContractResult; +import org.openmetadata.schema.entity.datacontract.QualityValidation; +import org.openmetadata.schema.entity.datacontract.SchemaValidation; +import org.openmetadata.schema.entity.datacontract.SemanticsValidation; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.type.TestCaseResult; +import org.openmetadata.schema.tests.type.TestCaseStatus; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.ColumnDataType; +import org.openmetadata.schema.type.ContractExecutionStatus; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.EntityStatus; +import org.openmetadata.schema.type.SemanticsRule; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.BadRequestException; + +class DataContractRepositoryTest { + private static DataContractRepository repository; + + @BeforeAll + static void setup() { + repository = mock(DataContractRepository.class); + } + + @SuppressWarnings("unchecked") + private T invoke(String methodName, Class[] paramTypes, Object... args) throws Exception { + Method method = DataContractRepository.class.getDeclaredMethod(methodName, paramTypes); + method.setAccessible(true); + return (T) method.invoke(repository, args); + } + + // --- getTestSuiteName --- + + @Test + void testGetTestSuiteName() { + DataContract contract = new DataContract(); + UUID id = UUID.randomUUID(); + contract.setId(id); + assertEquals(id.toString(), DataContractRepository.getTestSuiteName(contract)); + } + + // --- areTypesCompatible --- + + @Test + void testAreTypesCompatible_sameType() throws Exception { + assertTrue(invokeAreTypesCompatible(ColumnDataType.STRING, ColumnDataType.STRING)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.INT, ColumnDataType.INT)); + } + + @Test + void testAreTypesCompatible_nullTypes() throws Exception { + assertTrue(invokeAreTypesCompatible(null, ColumnDataType.STRING)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.STRING, null)); + assertTrue(invokeAreTypesCompatible(null, null)); + } + + @Test + void testAreTypesCompatible_stringFamily() throws Exception { + assertTrue(invokeAreTypesCompatible(ColumnDataType.STRING, ColumnDataType.VARCHAR)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.CHAR, ColumnDataType.TEXT)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.MEDIUMTEXT, ColumnDataType.CLOB)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.NTEXT, ColumnDataType.STRING)); + } + + @Test + void testAreTypesCompatible_integerFamily() throws Exception { + assertTrue(invokeAreTypesCompatible(ColumnDataType.INT, ColumnDataType.BIGINT)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.SMALLINT, ColumnDataType.TINYINT)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.BYTEINT, ColumnDataType.LONG)); + } + + @Test + void testAreTypesCompatible_decimalFamily() throws Exception { + assertTrue(invokeAreTypesCompatible(ColumnDataType.DECIMAL, ColumnDataType.NUMERIC)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.NUMBER, ColumnDataType.DOUBLE)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.FLOAT, ColumnDataType.MONEY)); + } + + @Test + void testAreTypesCompatible_booleanFamily() throws Exception { + assertTrue(invokeAreTypesCompatible(ColumnDataType.BOOLEAN, ColumnDataType.BOOLEAN)); + } + + @Test + void testAreTypesCompatible_dateTimeFamily() throws Exception { + assertTrue(invokeAreTypesCompatible(ColumnDataType.DATE, ColumnDataType.DATETIME)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.TIMESTAMP, ColumnDataType.TIMESTAMPZ)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.TIME, ColumnDataType.DATE)); + } + + @Test + void testAreTypesCompatible_binaryFamily() throws Exception { + assertTrue(invokeAreTypesCompatible(ColumnDataType.BINARY, ColumnDataType.VARBINARY)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.BLOB, ColumnDataType.BYTEA)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.BYTES, ColumnDataType.LONGBLOB)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.MEDIUMBLOB, ColumnDataType.BINARY)); + } + + @Test + void testAreTypesCompatible_complexFamily() throws Exception { + assertTrue(invokeAreTypesCompatible(ColumnDataType.ARRAY, ColumnDataType.MAP)); + assertTrue(invokeAreTypesCompatible(ColumnDataType.STRUCT, ColumnDataType.JSON)); + } + + @Test + void testAreTypesCompatible_incompatibleTypes() throws Exception { + assertFalse(invokeAreTypesCompatible(ColumnDataType.STRING, ColumnDataType.INT)); + assertFalse(invokeAreTypesCompatible(ColumnDataType.BOOLEAN, ColumnDataType.DATE)); + assertFalse(invokeAreTypesCompatible(ColumnDataType.DECIMAL, ColumnDataType.BINARY)); + assertFalse(invokeAreTypesCompatible(ColumnDataType.ARRAY, ColumnDataType.STRING)); + assertFalse(invokeAreTypesCompatible(ColumnDataType.INT, ColumnDataType.DOUBLE)); + } + + private boolean invokeAreTypesCompatible(ColumnDataType t1, ColumnDataType t2) throws Exception { + return invoke( + "areTypesCompatible", new Class[] {ColumnDataType.class, ColumnDataType.class}, t1, t2); + } + + // --- findDuplicateColumnNames --- + + @Test + void testFindDuplicateColumnNames_noDuplicates() throws Exception { + DataContract contract = contractWithSchema("col_a", "col_b", "col_c"); + List result = + invoke("findDuplicateColumnNames", new Class[] {DataContract.class}, contract); + assertTrue(result.isEmpty()); + } + + @Test + void testFindDuplicateColumnNames_withDuplicates() throws Exception { + DataContract contract = contractWithSchema("col_a", "col_b", "col_a", "col_c", "col_b"); + List result = + invoke("findDuplicateColumnNames", new Class[] {DataContract.class}, contract); + assertEquals(2, result.size()); + assertTrue(result.contains("col_a")); + assertTrue(result.contains("col_b")); + } + + @Test + void testFindDuplicateColumnNames_tripleDuplicate() throws Exception { + DataContract contract = contractWithSchema("col_a", "col_a", "col_a"); + List result = + invoke("findDuplicateColumnNames", new Class[] {DataContract.class}, contract); + assertEquals(1, result.size()); + assertEquals("col_a", result.get(0)); + } + + // --- buildColumnMap --- + + @Test + void testBuildColumnMap_flat() throws Exception { + List columns = + List.of(column("id", ColumnDataType.INT), column("name", ColumnDataType.STRING)); + Map result = invoke("buildColumnMap", new Class[] {List.class}, columns); + assertEquals(2, result.size()); + assertNotNull(result.get("id")); + assertNotNull(result.get("name")); + } + + @Test + void testBuildColumnMap_nested() throws Exception { + Column child = column("nested_col", ColumnDataType.STRING); + Column parent = column("parent", ColumnDataType.STRUCT); + parent.setChildren(List.of(child)); + List columns = List.of(parent); + Map result = invoke("buildColumnMap", new Class[] {List.class}, columns); + assertEquals(2, result.size()); + assertNotNull(result.get("parent")); + assertNotNull(result.get("nested_col")); + } + + @Test + void testBuildColumnMap_nullOrEmpty() throws Exception { + Map result1 = invoke("buildColumnMap", new Class[] {List.class}, (Object) null); + assertTrue(result1.isEmpty()); + Map result2 = + invoke("buildColumnMap", new Class[] {List.class}, Collections.emptyList()); + assertTrue(result2.isEmpty()); + } + + // --- extractFieldNames --- + + @Test + void testExtractFieldNames_flat() throws Exception { + List fields = List.of(field("field1"), field("field2")); + Set result = invoke("extractFieldNames", new Class[] {List.class}, fields); + assertEquals(Set.of("field1", "field2"), result); + } + + @Test + void testExtractFieldNames_nested() throws Exception { + org.openmetadata.schema.type.Field child = field("child"); + org.openmetadata.schema.type.Field parent = field("parent"); + parent.setChildren(List.of(child)); + List fields = List.of(parent); + Set result = invoke("extractFieldNames", new Class[] {List.class}, fields); + assertEquals(Set.of("parent", "child"), result); + } + + @Test + void testExtractFieldNames_nullOrEmpty() throws Exception { + Set result1 = invoke("extractFieldNames", new Class[] {List.class}, (Object) null); + assertTrue(result1.isEmpty()); + Set result2 = + invoke("extractFieldNames", new Class[] {List.class}, Collections.emptyList()); + assertTrue(result2.isEmpty()); + } + + // --- extractColumnNames --- + + @Test + void testExtractColumnNames_flat() throws Exception { + List columns = + List.of(column("col1", ColumnDataType.INT), column("col2", ColumnDataType.STRING)); + Set result = invoke("extractColumnNames", new Class[] {List.class}, columns); + assertEquals(Set.of("col1", "col2"), result); + } + + @Test + void testExtractColumnNames_nested() throws Exception { + Column child = column("child_col", ColumnDataType.INT); + Column parent = column("parent_col", ColumnDataType.STRUCT); + parent.setChildren(List.of(child)); + Set result = invoke("extractColumnNames", new Class[] {List.class}, List.of(parent)); + assertEquals(Set.of("parent_col", "child_col"), result); + } + + @Test + void testExtractColumnNames_nullOrEmpty() throws Exception { + Set result1 = invoke("extractColumnNames", new Class[] {List.class}, (Object) null); + assertTrue(result1.isEmpty()); + Set result2 = + invoke("extractColumnNames", new Class[] {List.class}, Collections.emptyList()); + assertTrue(result2.isEmpty()); + } + + // --- getAllContractFieldNames --- + + @Test + void testGetAllContractFieldNames() throws Exception { + DataContract contract = contractWithSchema("a", "b", "c"); + List result = + invoke("getAllContractFieldNames", new Class[] {DataContract.class}, contract); + assertEquals(List.of("a", "b", "c"), result); + } + + // --- validateContractFieldsAgainstNames --- + + @Test + void testValidateContractFieldsAgainstNames_allPresent() throws Exception { + DataContract contract = contractWithSchema("col1", "col2"); + Set entityFields = Set.of("col1", "col2", "col3"); + List result = + invoke( + "validateContractFieldsAgainstNames", + new Class[] {DataContract.class, Set.class}, + contract, + entityFields); + assertTrue(result.isEmpty()); + } + + @Test + void testValidateContractFieldsAgainstNames_missingFields() throws Exception { + DataContract contract = contractWithSchema("col1", "col2", "col3"); + Set entityFields = Set.of("col1"); + List result = + invoke( + "validateContractFieldsAgainstNames", + new Class[] {DataContract.class, Set.class}, + contract, + entityFields); + assertEquals(2, result.size()); + assertTrue(result.contains("col2")); + assertTrue(result.contains("col3")); + } + + // --- isSupportedEntityType --- + + @Test + void testIsSupportedEntityType_supported() throws Exception { + assertTrue(invokeIsSupportedEntityType(Entity.TABLE)); + assertTrue(invokeIsSupportedEntityType(Entity.TOPIC)); + assertTrue(invokeIsSupportedEntityType(Entity.PIPELINE)); + assertTrue(invokeIsSupportedEntityType(Entity.DASHBOARD)); + assertTrue(invokeIsSupportedEntityType(Entity.API_ENDPOINT)); + assertTrue(invokeIsSupportedEntityType(Entity.DASHBOARD_DATA_MODEL)); + assertTrue(invokeIsSupportedEntityType(Entity.MLMODEL)); + assertTrue(invokeIsSupportedEntityType(Entity.CONTAINER)); + assertTrue(invokeIsSupportedEntityType(Entity.STORED_PROCEDURE)); + assertTrue(invokeIsSupportedEntityType(Entity.DATABASE)); + assertTrue(invokeIsSupportedEntityType(Entity.DATABASE_SCHEMA)); + assertTrue(invokeIsSupportedEntityType(Entity.SEARCH_INDEX)); + assertTrue(invokeIsSupportedEntityType(Entity.API_COLLECTION)); + assertTrue(invokeIsSupportedEntityType(Entity.API)); + assertTrue(invokeIsSupportedEntityType(Entity.DIRECTORY)); + assertTrue(invokeIsSupportedEntityType(Entity.FILE)); + assertTrue(invokeIsSupportedEntityType(Entity.SPREADSHEET)); + assertTrue(invokeIsSupportedEntityType(Entity.WORKSHEET)); + assertTrue(invokeIsSupportedEntityType(Entity.DATA_PRODUCT)); + assertTrue(invokeIsSupportedEntityType(Entity.CHART)); + } + + @Test + void testIsSupportedEntityType_unsupported() throws Exception { + assertFalse(invokeIsSupportedEntityType(Entity.USER)); + assertFalse(invokeIsSupportedEntityType(Entity.TEAM)); + assertFalse(invokeIsSupportedEntityType("unknown_entity")); + } + + private boolean invokeIsSupportedEntityType(String entityType) throws Exception { + return invoke("isSupportedEntityType", new Class[] {String.class}, entityType); + } + + // --- supportsSchemaValidation --- + + @Test + void testSupportsSchemaValidation() throws Exception { + assertTrue(invokeSupportsSchemaValidation(Entity.TABLE)); + assertTrue(invokeSupportsSchemaValidation(Entity.TOPIC)); + assertTrue(invokeSupportsSchemaValidation(Entity.API_ENDPOINT)); + assertTrue(invokeSupportsSchemaValidation(Entity.DASHBOARD_DATA_MODEL)); + assertFalse(invokeSupportsSchemaValidation(Entity.PIPELINE)); + assertFalse(invokeSupportsSchemaValidation(Entity.DASHBOARD)); + } + + private boolean invokeSupportsSchemaValidation(String entityType) throws Exception { + return invoke("supportsSchemaValidation", new Class[] {String.class}, entityType); + } + + // --- supportsQualityValidation --- + + @Test + void testSupportsQualityValidation() throws Exception { + assertTrue(invokeSupportsQualityValidation(Entity.TABLE)); + assertFalse(invokeSupportsQualityValidation(Entity.TOPIC)); + assertFalse(invokeSupportsQualityValidation(Entity.PIPELINE)); + } + + private boolean invokeSupportsQualityValidation(String entityType) throws Exception { + return invoke("supportsQualityValidation", new Class[] {String.class}, entityType); + } + + // --- validateEntitySpecificConstraints --- + + @Test + void testValidateEntitySpecificConstraints_unsupportedEntityType() { + DataContract contract = new DataContract(); + EntityReference entityRef = new EntityReference().withType(Entity.USER); + + InvocationTargetException ex = + assertThrows( + InvocationTargetException.class, + () -> + invoke( + "validateEntitySpecificConstraints", + new Class[] {DataContract.class, EntityReference.class}, + contract, + entityRef)); + assertTrue(ex.getCause() instanceof BadRequestException); + assertTrue(ex.getCause().getMessage().contains("not supported for data contracts")); + } + + @Test + void testValidateEntitySpecificConstraints_schemaNotSupportedForPipeline() { + DataContract contract = contractWithSchema("col1"); + EntityReference entityRef = new EntityReference().withType(Entity.PIPELINE); + + InvocationTargetException ex = + assertThrows( + InvocationTargetException.class, + () -> + invoke( + "validateEntitySpecificConstraints", + new Class[] {DataContract.class, EntityReference.class}, + contract, + entityRef)); + assertTrue(ex.getCause() instanceof BadRequestException); + assertTrue(ex.getCause().getMessage().contains("Schema validation is not supported")); + } + + @Test + void testValidateEntitySpecificConstraints_qualityNotSupportedForTopic() { + DataContract contract = new DataContract(); + contract.setQualityExpectations(List.of(new EntityReference().withId(UUID.randomUUID()))); + EntityReference entityRef = new EntityReference().withType(Entity.TOPIC); + + InvocationTargetException ex = + assertThrows( + InvocationTargetException.class, + () -> + invoke( + "validateEntitySpecificConstraints", + new Class[] {DataContract.class, EntityReference.class}, + contract, + entityRef)); + assertTrue(ex.getCause() instanceof BadRequestException); + assertTrue(ex.getCause().getMessage().contains("Quality expectations are not supported")); + } + + @Test + void testValidateEntitySpecificConstraints_tableSupportsAll() { + DataContract contract = contractWithSchema("col1"); + contract.setQualityExpectations(List.of(new EntityReference().withId(UUID.randomUUID()))); + EntityReference entityRef = new EntityReference().withType(Entity.TABLE); + + assertDoesNotThrow( + () -> + invoke( + "validateEntitySpecificConstraints", + new Class[] {DataContract.class, EntityReference.class}, + contract, + entityRef)); + } + + @Test + void testValidateEntitySpecificConstraints_emptySchemaAndQuality() { + DataContract contract = new DataContract(); + EntityReference entityRef = new EntityReference().withType(Entity.PIPELINE); + + assertDoesNotThrow( + () -> + invoke( + "validateEntitySpecificConstraints", + new Class[] {DataContract.class, EntityReference.class}, + contract, + entityRef)); + } + + // --- contractHasTestSuite --- + + @Test + void testContractHasTestSuite() throws Exception { + DataContract withSuite = new DataContract(); + withSuite.setTestSuite(new EntityReference().withId(UUID.randomUUID())); + Boolean hasSuite = invoke("contractHasTestSuite", new Class[] {DataContract.class}, withSuite); + assertTrue(hasSuite); + + DataContract withoutSuite = new DataContract(); + Boolean noSuite = + invoke("contractHasTestSuite", new Class[] {DataContract.class}, withoutSuite); + assertFalse(noSuite); + } + + // --- filterTestsWithoutResults / filterTestsWithResults --- + + @Test + void testFilterTestsWithoutResults() throws Exception { + TestCase withResult = new TestCase(); + withResult.setTestCaseResult(new TestCaseResult().withTestCaseStatus(TestCaseStatus.Success)); + + TestCase withoutResult = new TestCase(); + + TestCase withNullStatus = new TestCase(); + withNullStatus.setTestCaseResult(new TestCaseResult()); + + List tests = List.of(withResult, withoutResult, withNullStatus); + List result = invoke("filterTestsWithoutResults", new Class[] {List.class}, tests); + assertEquals(2, result.size()); + assertTrue(result.contains(withoutResult)); + assertTrue(result.contains(withNullStatus)); + } + + @Test + void testFilterTestsWithResults() throws Exception { + TestCase withResult = new TestCase(); + withResult.setTestCaseResult(new TestCaseResult().withTestCaseStatus(TestCaseStatus.Success)); + + TestCase withoutResult = new TestCase(); + + List tests = List.of(withResult, withoutResult); + List result = invoke("filterTestsWithResults", new Class[] {List.class}, tests); + assertEquals(1, result.size()); + assertTrue(result.contains(withResult)); + } + + // --- initDQValidation --- + + @Test + void testInitDQValidation() throws Exception { + DataContract contract = new DataContract(); + contract.setQualityExpectations( + List.of( + new EntityReference().withId(UUID.randomUUID()), + new EntityReference().withId(UUID.randomUUID()), + new EntityReference().withId(UUID.randomUUID()))); + + QualityValidation result = + invoke("initDQValidation", new Class[] {DataContract.class}, contract); + + assertEquals(3, result.getTotal()); + assertEquals(0, result.getPassed()); + assertEquals(0, result.getFailed()); + assertEquals(0.0, result.getQualityScore()); + } + + // --- getExistingTestResults --- + + @Test + void testGetExistingTestResults_mixedResults() throws Exception { + DataContract contract = new DataContract(); + contract.setQualityExpectations( + List.of( + new EntityReference().withId(UUID.randomUUID()), + new EntityReference().withId(UUID.randomUUID()), + new EntityReference().withId(UUID.randomUUID()))); + + TestCase passed = new TestCase(); + passed.setTestCaseResult(new TestCaseResult().withTestCaseStatus(TestCaseStatus.Success)); + TestCase failed = new TestCase(); + failed.setTestCaseResult(new TestCaseResult().withTestCaseStatus(TestCaseStatus.Failed)); + TestCase aborted = new TestCase(); + aborted.setTestCaseResult(new TestCaseResult().withTestCaseStatus(TestCaseStatus.Aborted)); + + QualityValidation result = + invoke( + "getExistingTestResults", + new Class[] {DataContract.class, List.class}, + contract, + List.of(passed, failed, aborted)); + + assertEquals(3, result.getTotal()); + assertEquals(1, result.getPassed()); + assertEquals(2, result.getFailed()); + assertTrue(result.getQualityScore() > 0); + } + + @Test + void testGetExistingTestResults_emptyList() throws Exception { + DataContract contract = new DataContract(); + contract.setQualityExpectations(List.of()); + + QualityValidation result = + invoke( + "getExistingTestResults", + new Class[] {DataContract.class, List.class}, + contract, + Collections.emptyList()); + + assertNotNull(result); + } + + // --- compileResult --- + + @Test + void testCompileResult_noValidationIssues() { + doCallRealMethod() + .when(repository) + .compileResult( + org.mockito.ArgumentMatchers.any(DataContractResult.class), + org.mockito.ArgumentMatchers.any(ContractExecutionStatus.class)); + + DataContractResult result = new DataContractResult(); + repository.compileResult(result, ContractExecutionStatus.Success); + assertEquals(ContractExecutionStatus.Success, result.getContractExecutionStatus()); + } + + @Test + void testCompileResult_schemaValidationFailed() { + doCallRealMethod() + .when(repository) + .compileResult( + org.mockito.ArgumentMatchers.any(DataContractResult.class), + org.mockito.ArgumentMatchers.any(ContractExecutionStatus.class)); + + DataContractResult result = new DataContractResult(); + result.setSchemaValidation(new SchemaValidation().withFailed(2).withPassed(3).withTotal(5)); + repository.compileResult(result, ContractExecutionStatus.Success); + assertEquals(ContractExecutionStatus.Failed, result.getContractExecutionStatus()); + } + + @Test + void testCompileResult_semanticsValidationFailed() { + doCallRealMethod() + .when(repository) + .compileResult( + org.mockito.ArgumentMatchers.any(DataContractResult.class), + org.mockito.ArgumentMatchers.any(ContractExecutionStatus.class)); + + DataContractResult result = new DataContractResult(); + result.setSemanticsValidation( + new SemanticsValidation().withFailed(1).withPassed(2).withTotal(3)); + repository.compileResult(result, ContractExecutionStatus.Success); + assertEquals(ContractExecutionStatus.Failed, result.getContractExecutionStatus()); + } + + @Test + void testCompileResult_qualityValidationFailed() { + doCallRealMethod() + .when(repository) + .compileResult( + org.mockito.ArgumentMatchers.any(DataContractResult.class), + org.mockito.ArgumentMatchers.any(ContractExecutionStatus.class)); + + DataContractResult result = new DataContractResult(); + result.setQualityValidation(new QualityValidation().withFailed(1).withPassed(2).withTotal(3)); + repository.compileResult(result, ContractExecutionStatus.Success); + assertEquals(ContractExecutionStatus.Failed, result.getContractExecutionStatus()); + } + + @Test + void testCompileResult_allValidationsPass() { + doCallRealMethod() + .when(repository) + .compileResult( + org.mockito.ArgumentMatchers.any(DataContractResult.class), + org.mockito.ArgumentMatchers.any(ContractExecutionStatus.class)); + + DataContractResult result = new DataContractResult(); + result.setSchemaValidation(new SchemaValidation().withFailed(0).withPassed(5).withTotal(5)); + result.setSemanticsValidation( + new SemanticsValidation().withFailed(0).withPassed(3).withTotal(3)); + result.setQualityValidation(new QualityValidation().withFailed(0).withPassed(2).withTotal(2)); + repository.compileResult(result, ContractExecutionStatus.Success); + assertEquals(ContractExecutionStatus.Success, result.getContractExecutionStatus()); + } + + // --- inheritFromDataProductContract --- + + @Test + void testInheritFromDataProductContract() throws Exception { + EntityReference entityRef = + new EntityReference().withId(UUID.randomUUID()).withType(Entity.TABLE); + org.openmetadata.schema.EntityInterface entity = + mock(org.openmetadata.schema.EntityInterface.class); + org.mockito.Mockito.when(entity.getEntityReference()).thenReturn(entityRef); + + DataContract dpContract = new DataContract(); + dpContract.setId(UUID.randomUUID()); + dpContract.setName("dp-contract"); + dpContract.setEntityStatus(EntityStatus.APPROVED); + dpContract.setSemantics(List.of(new SemanticsRule().withName("rule1"))); + dpContract.setTermsOfUse(new org.openmetadata.schema.entity.data.TermsOfUse()); + dpContract.setSecurity(new org.openmetadata.schema.api.data.ContractSecurity()); + dpContract.setSla(new org.openmetadata.schema.api.data.ContractSLA()); + + DataContract result = + invoke( + "inheritFromDataProductContract", + new Class[] {org.openmetadata.schema.EntityInterface.class, DataContract.class}, + entity, + dpContract); + + assertNotNull(result); + assertEquals(entityRef, result.getEntity()); + assertNull(result.getQualityExpectations()); + assertNull(result.getSchema()); + assertNull(result.getTestSuite()); + assertNull(result.getLatestResult()); + assertEquals(EntityStatus.DRAFT, result.getEntityStatus()); + assertTrue(result.getInherited()); + assertTrue(result.getTermsOfUse().getInherited()); + assertTrue(result.getSecurity().getInherited()); + assertTrue(result.getSla().getInherited()); + assertTrue(result.getSemantics().get(0).getInherited()); + } + + // --- mergeContracts --- + + @Test + void testMergeContracts_inheritsTermsOfUse() throws Exception { + DataContract entityContract = new DataContract(); + entityContract.setId(UUID.randomUUID()); + entityContract.setName("entity-contract"); + + DataContract dpContract = new DataContract(); + dpContract.setTermsOfUse(new org.openmetadata.schema.entity.data.TermsOfUse()); + dpContract.setSecurity(new org.openmetadata.schema.api.data.ContractSecurity()); + dpContract.setSla(new org.openmetadata.schema.api.data.ContractSLA()); + + DataContract result = + invoke( + "mergeContracts", + new Class[] {DataContract.class, DataContract.class}, + entityContract, + dpContract); + + assertNotNull(result.getTermsOfUse()); + assertTrue(result.getTermsOfUse().getInherited()); + assertNotNull(result.getSecurity()); + assertTrue(result.getSecurity().getInherited()); + assertNotNull(result.getSla()); + assertTrue(result.getSla().getInherited()); + } + + @Test + void testMergeContracts_entityFieldsPreserved() throws Exception { + DataContract entityContract = new DataContract(); + entityContract.setId(UUID.randomUUID()); + entityContract.setName("entity-contract"); + entityContract.setTermsOfUse(new org.openmetadata.schema.entity.data.TermsOfUse()); + entityContract.setSecurity(new org.openmetadata.schema.api.data.ContractSecurity()); + + DataContract dpContract = new DataContract(); + dpContract.setTermsOfUse(new org.openmetadata.schema.entity.data.TermsOfUse()); + dpContract.setSecurity(new org.openmetadata.schema.api.data.ContractSecurity()); + + DataContract result = + invoke( + "mergeContracts", + new Class[] {DataContract.class, DataContract.class}, + entityContract, + dpContract); + + // Entity's own terms/security should be preserved (not marked as inherited) + assertNotNull(result.getTermsOfUse()); + assertNull(result.getTermsOfUse().getInherited()); + } + + @Test + void testMergeContracts_semanticsDeduplication() throws Exception { + DataContract entityContract = new DataContract(); + entityContract.setId(UUID.randomUUID()); + entityContract.setName("entity-contract"); + entityContract.setSemantics(List.of(new SemanticsRule().withName("shared-rule"))); + + DataContract dpContract = new DataContract(); + dpContract.setSemantics( + List.of( + new SemanticsRule().withName("shared-rule"), + new SemanticsRule().withName("dp-only-rule"))); + + DataContract result = + invoke( + "mergeContracts", + new Class[] {DataContract.class, DataContract.class}, + entityContract, + dpContract); + + assertEquals(2, result.getSemantics().size()); + SemanticsRule inherited = + result.getSemantics().stream() + .filter(r -> "dp-only-rule".equals(r.getName())) + .findFirst() + .orElse(null); + assertNotNull(inherited); + assertTrue(inherited.getInherited()); + } + + // --- Helper methods --- + + private static DataContract contractWithSchema(String... columnNames) { + DataContract contract = new DataContract(); + List columns = new ArrayList<>(); + for (String name : columnNames) { + columns.add(column(name, null)); + } + contract.setSchema(columns); + return contract; + } + + private static Column column(String name, ColumnDataType dataType) { + Column col = new Column(); + col.setName(name); + col.setDataType(dataType); + return col; + } + + private static org.openmetadata.schema.type.Field field(String name) { + org.openmetadata.schema.type.Field f = new org.openmetadata.schema.type.Field(); + f.setName(name); + return f; + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SearchListFilterTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SearchListFilterTest.java index 32ded21d37e6..571bdb3de2d3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SearchListFilterTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SearchListFilterTest.java @@ -387,6 +387,46 @@ void testResolutionStatusCondition_withTestCaseFqn() { "Expected testCaseFqn filter but got: " + actual); } + @Test + void testTestCaseResultConditionWithDataContractId() { + SearchListFilter searchListFilter = new SearchListFilter(); + searchListFilter.addQueryParam("dataContractId", "abc-123-def"); + + String actual = searchListFilter.getCondition(Entity.TEST_CASE_RESULT); + + assertTrue( + actual.contains("{\"term\": {\"dataContract.id\": \"abc-123-def\"}}"), + "Expected dataContract.id filter but got: " + actual); + } + + @Test + void testTestCaseResultConditionWithDataContractIdAndOtherFilters() { + SearchListFilter searchListFilter = new SearchListFilter(); + searchListFilter.addQueryParam("dataContractId", "contract-uuid"); + searchListFilter.addQueryParam("testCaseStatus", "Failed"); + searchListFilter.addQueryParam("startTimestamp", "100"); + searchListFilter.addQueryParam("endTimestamp", "200"); + + String actual = searchListFilter.getCondition(Entity.TEST_CASE_RESULT); + + assertTrue(actual.contains("{\"term\": {\"dataContract.id\": \"contract-uuid\"}}")); + assertTrue(actual.contains("{\"term\": {\"testCaseStatus\": \"Failed\"}}")); + assertTrue(actual.contains("{\"range\": {\"timestamp\": {\"gte\": 100}}}")); + assertTrue(actual.contains("{\"range\": {\"timestamp\": {\"lte\": 200}}}")); + } + + @Test + void testTestCaseResultConditionDataContractIdEscapesDoubleQuotes() { + SearchListFilter searchListFilter = new SearchListFilter(); + searchListFilter.addQueryParam("dataContractId", "id-with-\"quotes\""); + + String actual = searchListFilter.getCondition(Entity.TEST_CASE_RESULT); + + assertTrue( + actual.contains("id-with-\\\"quotes\\\""), + "Expected escaped quotes in dataContract.id filter but got: " + actual); + } + private JsonNode parse(String json) { return JsonUtils.readTree(json); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v1130/MigrationUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v1130/MigrationUtilTest.java new file mode 100644 index 000000000000..e9a9b9a5257a --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v1130/MigrationUtilTest.java @@ -0,0 +1,241 @@ +package org.openmetadata.service.migration.utils.v1130; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.data.DataContract; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.CollectionDAO; + +class MigrationUtilTest { + private CollectionDAO collectionDAO; + private CollectionDAO.DataContractDAO dataContractDAO; + private CollectionDAO.TestCaseDAO testCaseDAO; + + @BeforeEach + void setUp() { + collectionDAO = mock(CollectionDAO.class); + dataContractDAO = mock(CollectionDAO.DataContractDAO.class); + testCaseDAO = mock(CollectionDAO.TestCaseDAO.class); + when(collectionDAO.dataContractDAO()).thenReturn(dataContractDAO); + when(collectionDAO.testCaseDAO()).thenReturn(testCaseDAO); + } + + @Test + void testMigrateTestCaseDataContractReferences_noDataContracts() { + when(dataContractDAO.listAfterWithOffset(anyInt(), anyInt())) + .thenReturn(Collections.emptyList()); + + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); + + verify(testCaseDAO, never()).findEntityById(any()); + verify(testCaseDAO, never()).update(any(TestCase.class)); + } + + @Test + void testMigrateTestCaseDataContractReferences_contractWithNoQualityExpectations() { + DataContract contract = new DataContract(); + contract.setId(UUID.randomUUID()); + contract.setFullyQualifiedName("test.contract"); + + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(0))) + .thenReturn(List.of(JsonUtils.pojoToJson(contract))); + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(1000))) + .thenReturn(Collections.emptyList()); + + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); + + verify(testCaseDAO, never()).findEntityById(any()); + } + + @Test + void testMigrateTestCaseDataContractReferences_updatesTestCase() { + UUID contractId = UUID.randomUUID(); + UUID testCaseId = UUID.randomUUID(); + + DataContract contract = new DataContract(); + contract.setId(contractId); + contract.setName("my-contract"); + contract.setFullyQualifiedName("test.contract"); + contract.setQualityExpectations( + List.of(new EntityReference().withId(testCaseId).withType(Entity.TEST_CASE))); + + TestCase testCase = new TestCase(); + testCase.setId(testCaseId); + testCase.setFullyQualifiedName("test.case.fqn"); + + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(0))) + .thenReturn(List.of(JsonUtils.pojoToJson(contract))); + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(1000))) + .thenReturn(Collections.emptyList()); + when(testCaseDAO.findEntityById(testCaseId)).thenReturn(testCase); + + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); + + verify(testCaseDAO).update(any(TestCase.class)); + assertEquals(contractId, testCase.getDataContract().getId()); + assertEquals(Entity.DATA_CONTRACT, testCase.getDataContract().getType()); + } + + @Test + void testMigrateTestCaseDataContractReferences_skipsAlreadySet() { + UUID contractId = UUID.randomUUID(); + UUID testCaseId = UUID.randomUUID(); + + DataContract contract = new DataContract(); + contract.setId(contractId); + contract.setName("my-contract"); + contract.setFullyQualifiedName("test.contract"); + contract.setQualityExpectations( + List.of(new EntityReference().withId(testCaseId).withType(Entity.TEST_CASE))); + + TestCase testCase = new TestCase(); + testCase.setId(testCaseId); + testCase.setFullyQualifiedName("test.case.fqn"); + testCase.setDataContract( + new EntityReference().withId(contractId).withType(Entity.DATA_CONTRACT)); + + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(0))) + .thenReturn(List.of(JsonUtils.pojoToJson(contract))); + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(1000))) + .thenReturn(Collections.emptyList()); + when(testCaseDAO.findEntityById(testCaseId)).thenReturn(testCase); + + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); + + verify(testCaseDAO, never()).update(any(TestCase.class)); + } + + @Test + void testMigrateTestCaseDataContractReferences_handlesTestCaseNotFound() { + UUID testCaseId = UUID.randomUUID(); + + DataContract contract = new DataContract(); + contract.setId(UUID.randomUUID()); + contract.setName("my-contract"); + contract.setFullyQualifiedName("test.contract"); + contract.setQualityExpectations( + List.of(new EntityReference().withId(testCaseId).withType(Entity.TEST_CASE))); + + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(0))) + .thenReturn(List.of(JsonUtils.pojoToJson(contract))); + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(1000))) + .thenReturn(Collections.emptyList()); + when(testCaseDAO.findEntityById(testCaseId)) + .thenThrow(new EntityNotFoundException("not found")); + + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); + + verify(testCaseDAO, never()).update(any(TestCase.class)); + } + + @Test + void testMigrateTestCaseDataContractReferences_batchProcessing() { + UUID contractId1 = UUID.randomUUID(); + UUID contractId2 = UUID.randomUUID(); + UUID testCaseId1 = UUID.randomUUID(); + UUID testCaseId2 = UUID.randomUUID(); + + DataContract contract1 = new DataContract(); + contract1.setId(contractId1); + contract1.setName("contract1"); + contract1.setFullyQualifiedName("test.contract1"); + contract1.setQualityExpectations( + List.of(new EntityReference().withId(testCaseId1).withType(Entity.TEST_CASE))); + + DataContract contract2 = new DataContract(); + contract2.setId(contractId2); + contract2.setName("contract2"); + contract2.setFullyQualifiedName("test.contract2"); + contract2.setQualityExpectations( + List.of(new EntityReference().withId(testCaseId2).withType(Entity.TEST_CASE))); + + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(0))) + .thenReturn(List.of(JsonUtils.pojoToJson(contract1), JsonUtils.pojoToJson(contract2))); + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(1000))) + .thenReturn(Collections.emptyList()); + + TestCase testCase1 = new TestCase(); + testCase1.setId(testCaseId1); + testCase1.setFullyQualifiedName("test.case1"); + + TestCase testCase2 = new TestCase(); + testCase2.setId(testCaseId2); + testCase2.setFullyQualifiedName("test.case2"); + + when(testCaseDAO.findEntityById(testCaseId1)).thenReturn(testCase1); + when(testCaseDAO.findEntityById(testCaseId2)).thenReturn(testCase2); + + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); + + verify(testCaseDAO, times(2)).update(any(TestCase.class)); + } + + @Test + void testMigrateTestCaseDataContractReferences_criticalFailure() { + when(dataContractDAO.listAfterWithOffset(anyInt(), anyInt())) + .thenThrow(new RuntimeException("DB connection failed")); + + assertThrows( + RuntimeException.class, + () -> MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO)); + } + + @Test + void testMigrateTestCaseDataContractReferences_multipleTestCasesPerContract() { + UUID contractId = UUID.randomUUID(); + UUID testCaseId1 = UUID.randomUUID(); + UUID testCaseId2 = UUID.randomUUID(); + UUID testCaseId3 = UUID.randomUUID(); + + DataContract contract = new DataContract(); + contract.setId(contractId); + contract.setName("contract"); + contract.setFullyQualifiedName("test.contract"); + contract.setQualityExpectations( + List.of( + new EntityReference().withId(testCaseId1).withType(Entity.TEST_CASE), + new EntityReference().withId(testCaseId2).withType(Entity.TEST_CASE), + new EntityReference().withId(testCaseId3).withType(Entity.TEST_CASE))); + + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(0))) + .thenReturn(List.of(JsonUtils.pojoToJson(contract))); + when(dataContractDAO.listAfterWithOffset(anyInt(), eq(1000))) + .thenReturn(Collections.emptyList()); + + TestCase tc1 = new TestCase(); + tc1.setId(testCaseId1); + tc1.setFullyQualifiedName("tc1"); + + TestCase tc2 = new TestCase(); + tc2.setId(testCaseId2); + tc2.setFullyQualifiedName("tc2"); + tc2.setDataContract(new EntityReference().withId(contractId)); + + when(testCaseDAO.findEntityById(testCaseId1)).thenReturn(tc1); + when(testCaseDAO.findEntityById(testCaseId2)).thenReturn(tc2); + when(testCaseDAO.findEntityById(testCaseId3)) + .thenThrow(new EntityNotFoundException("not found")); + + MigrationUtil.migrateTestCaseDataContractReferences(collectionDAO); + + verify(testCaseDAO, times(1)).update(any(TestCase.class)); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseResultIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseResultIndexTest.java new file mode 100644 index 000000000000..5e3ce35a42b6 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseResultIndexTest.java @@ -0,0 +1,102 @@ +package org.openmetadata.service.search.indexes; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.search.SearchIndexUtils; + +class TestCaseResultIndexTest { + + @Test + void testDataContractKeyRemovedFromTestCaseMap() { + TestCase testCase = new TestCase(); + testCase.setId(UUID.randomUUID()); + testCase.setName("test-case"); + testCase.setDataContract( + new EntityReference().withId(UUID.randomUUID()).withType("dataContract").withName("dc")); + + Map testCaseMap = JsonUtils.getMap(testCase); + assertTrue(testCaseMap.containsKey("dataContract")); + + Set keysToRemove = + Set.of("testSuites", "testSuite", "testCaseResult", "testDefinition", "dataContract"); + testCaseMap.keySet().removeAll(keysToRemove); + + assertFalse(testCaseMap.containsKey("dataContract")); + assertFalse(testCaseMap.containsKey("testSuites")); + assertFalse(testCaseMap.containsKey("testDefinition")); + } + + @Test + void testDataContractDenormalization() { + EntityReference dcRef = + new EntityReference() + .withId(UUID.randomUUID()) + .withType("dataContract") + .withName("my-contract"); + + Map esDoc = new HashMap<>(); + if (dcRef != null) { + esDoc.put("dataContract", JsonUtils.getMap(dcRef)); + } + + assertNotNull(esDoc.get("dataContract")); + Map dcMap = (Map) esDoc.get("dataContract"); + assertEquals("my-contract", dcMap.get("name")); + assertEquals("dataContract", dcMap.get("type")); + } + + @Test + void testDataContractNullHandling() { + TestCase testCase = new TestCase(); + testCase.setId(UUID.randomUUID()); + testCase.setName("test-case-no-contract"); + + Map esDoc = new HashMap<>(); + if (testCase.getDataContract() != null) { + esDoc.put("dataContract", JsonUtils.getMap(testCase.getDataContract())); + } + + assertNull(esDoc.get("dataContract")); + } + + @Test + void testExcludeFieldsRemoval() { + Set excludeFields = + Set.of("changeDescription", "failedRowsSample", "incrementalChangeDescription"); + + Map testCase = new HashMap<>(); + testCase.put("changeDescription", "should-be-removed"); + testCase.put("failedRowsSample", "should-be-removed"); + testCase.put("incrementalChangeDescription", "should-be-removed"); + testCase.put("name", "keep-this"); + testCase.put("description", "keep-this-too"); + + SearchIndexUtils.removeNonIndexableFields(testCase, excludeFields); + + assertFalse(testCase.containsKey("changeDescription")); + assertFalse(testCase.containsKey("failedRowsSample")); + assertFalse(testCase.containsKey("incrementalChangeDescription")); + assertTrue(testCase.containsKey("name")); + assertTrue(testCase.containsKey("description")); + } + + @Test + void testTimestampDenormalization() { + long timestamp = System.currentTimeMillis(); + Map esDoc = new HashMap<>(); + esDoc.put("@timestamp", timestamp); + assertEquals(timestamp, esDoc.get("@timestamp")); + } +} From 237887447b114852e4da65d4625c74c750a2450d Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 30 Mar 2026 12:31:41 +0200 Subject: [PATCH 33/38] Fix division-by-zero in validateDQ qualityScore when total is 0 Co-Authored-By: Claude Opus 4.6 --- .../service/jdbi3/DataContractRepository.java | 4 +- .../jdbi3/DataContractRepositoryTest.java | 68 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index e64904e650d1..fb08663220f4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -1358,7 +1358,9 @@ private QualityValidation validateDQ(TestSuite testSuite, QualityValidation exis .withPassed(existingValidation.getPassed() + (testSummary.size() - failedTests.size())); existingValidation.withQualityScore( - (existingValidation.getPassed() / (double) existingValidation.getTotal()) * 100); + existingValidation.getTotal() > 0 + ? (existingValidation.getPassed() / (double) existingValidation.getTotal()) * 100 + : 0.0); return existingValidation; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java index c80b6a691711..2ef1d52a0460 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java @@ -25,7 +25,9 @@ import org.openmetadata.schema.entity.datacontract.QualityValidation; import org.openmetadata.schema.entity.datacontract.SchemaValidation; import org.openmetadata.schema.entity.datacontract.SemanticsValidation; +import org.openmetadata.schema.tests.ResultSummary; import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.TestCaseResult; import org.openmetadata.schema.tests.type.TestCaseStatus; import org.openmetadata.schema.type.Column; @@ -562,6 +564,72 @@ void testGetExistingTestResults_emptyList() throws Exception { assertNotNull(result); } + // --- validateDQ (division-by-zero guard) --- + + @Test + void testValidateDQ_zeroTotal_noArithmeticException() throws Exception { + TestSuite testSuite = new TestSuite(); + testSuite.setTests(List.of(new EntityReference().withFullyQualifiedName("test1"))); + testSuite.setTestCaseResultSummary( + List.of(new ResultSummary().withTestCaseName("test1").withStatus(TestCaseStatus.Success))); + + QualityValidation existing = new QualityValidation().withTotal(0).withPassed(0).withFailed(0); + + QualityValidation result = + invoke( + "validateDQ", + new Class[] {TestSuite.class, QualityValidation.class}, + testSuite, + existing); + + assertEquals(0.0, result.getQualityScore()); + assertFalse(Double.isNaN(result.getQualityScore())); + assertFalse(Double.isInfinite(result.getQualityScore())); + } + + @Test + void testValidateDQ_withTotal_calculatesScore() throws Exception { + TestSuite testSuite = new TestSuite(); + testSuite.setTests( + List.of( + new EntityReference().withFullyQualifiedName("test1"), + new EntityReference().withFullyQualifiedName("test2"))); + testSuite.setTestCaseResultSummary( + List.of( + new ResultSummary().withTestCaseName("test1").withStatus(TestCaseStatus.Success), + new ResultSummary().withTestCaseName("test2").withStatus(TestCaseStatus.Failed))); + + QualityValidation existing = + new QualityValidation().withTotal(2).withPassed(0).withFailed(0).withQualityScore(0.0); + + QualityValidation result = + invoke( + "validateDQ", + new Class[] {TestSuite.class, QualityValidation.class}, + testSuite, + existing); + + assertEquals(1, result.getPassed()); + assertEquals(1, result.getFailed()); + assertEquals(50.0, result.getQualityScore()); + } + + @Test + void testValidateDQ_nullExistingValidation() throws Exception { + TestSuite testSuite = new TestSuite(); + testSuite.setTests(List.of()); + testSuite.setTestCaseResultSummary(null); + + QualityValidation result = + invoke( + "validateDQ", + new Class[] {TestSuite.class, QualityValidation.class}, + testSuite, + (QualityValidation) null); + + assertNotNull(result); + } + // --- compileResult --- @Test From 4a53f09cd1a4de544da430b5f2b90133ca260547 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 1 Apr 2026 12:16:35 +0200 Subject: [PATCH 34/38] Fix DataContractRepositoryTest: update renamed method and stub mock The method isSupportedEntityType was renamed to isEntityTypeSupported and made public, causing NoSuchMethod errors and mock interception failures. Co-Authored-By: Claude Opus 4.6 --- .../service/jdbi3/DataContractRepositoryTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java index 2ef1d52a0460..82a031e22544 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/DataContractRepositoryTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.mock; @@ -45,6 +46,7 @@ class DataContractRepositoryTest { @BeforeAll static void setup() { repository = mock(DataContractRepository.class); + doCallRealMethod().when(repository).isEntityTypeSupported(anyString()); } @SuppressWarnings("unchecked") @@ -333,7 +335,7 @@ void testIsSupportedEntityType_unsupported() throws Exception { } private boolean invokeIsSupportedEntityType(String entityType) throws Exception { - return invoke("isSupportedEntityType", new Class[] {String.class}, entityType); + return invoke("isEntityTypeSupported", new Class[] {String.class}, entityType); } // --- supportsSchemaValidation --- From 107ec183e024bf0335d53116e0b744583424b941 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 1 Apr 2026 15:03:18 +0200 Subject: [PATCH 35/38] Fix Copilot review issues: entityLink, NPE, compile status, dangling refs - Use contract.entity.fullyQualifiedName instead of route FQN for test case entityLink query (fixes wrong test cases for non-table contracts) - Remove invalid FQN fallback using contract route FQN - Add null-safety in validateDQ for QualityValidation fields - Call compileResult when only testsWithoutResults exist so schema/semantics failures are surfaced while DQ is pending - Add bulk DAO method to remove dataContract refs by contract ID, preventing dangling references on contract deletion Co-Authored-By: Claude Opus 4.6 --- .../service/jdbi3/CollectionDAO.java | 10 ++++ .../service/jdbi3/DataContractRepository.java | 48 +++++++++---------- .../ContractQualityCard.component.tsx | 15 +++--- .../ContractQualityCard.test.tsx | 12 ++--- .../ui/src/mocks/DataContract.mock.ts | 1 + 5 files changed, 44 insertions(+), 42 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 037a83df2643..187f4d526eb1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -7340,6 +7340,16 @@ void updateTestCaseDataContract( connectionType = POSTGRES) void removeTestCaseDataContractForSpecificContract( @Bind("id") String id, @Bind("dataContractId") String dataContractId); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE test_case SET json = JSON_REMOVE(json, '$.dataContract') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.dataContract.id')) = :dataContractId", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE test_case SET json = json - 'dataContract' WHERE json->'dataContract'->>'id' = :dataContractId", + connectionType = POSTGRES) + void removeAllTestCaseDataContractReferences(@Bind("dataContractId") String dataContractId); } interface WebAnalyticEventDAO extends EntityDAO { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index e0d1b7929c5e..c22bbef85573 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -1073,6 +1073,9 @@ public RestUtil.PutResponse validateContract(DataContract da !nullOrEmpty(testsWithoutResults) ? ContractExecutionStatus.Running : ContractExecutionStatus.Success); + } else if (result.getContractExecutionStatus() != ContractExecutionStatus.Aborted) { + // DQ triggered but no results yet — still surface schema/semantics failures + compileResult(result, ContractExecutionStatus.Running); } } else { compileResult(result, ContractExecutionStatus.Success); @@ -1345,10 +1348,11 @@ private SemanticsValidation validateSemantics(DataContract dataContract) { private QualityValidation validateDQ(TestSuite testSuite, QualityValidation existingValidation) { if (existingValidation == null) { - existingValidation = new QualityValidation(); + existingValidation = + new QualityValidation().withTotal(0).withPassed(0).withFailed(0).withQualityScore(0.0); } if (nullOrEmpty(testSuite.getTestCaseResultSummary())) { - return existingValidation; // return the existing result without updates + return existingValidation; } List currentTests = @@ -1361,14 +1365,16 @@ private QualityValidation validateDQ(TestSuite testSuite, QualityValidation exis List failedTests = testSummary.stream().filter(test -> FAILED_DQ_STATUSES.contains(test.getStatus())).toList(); + int priorFailed = existingValidation.getFailed() != null ? existingValidation.getFailed() : 0; + int priorPassed = existingValidation.getPassed() != null ? existingValidation.getPassed() : 0; + int total = existingValidation.getTotal() != null ? existingValidation.getTotal() : 0; + existingValidation - .withFailed(existingValidation.getFailed() + failedTests.size()) - .withPassed(existingValidation.getPassed() + (testSummary.size() - failedTests.size())); + .withFailed(priorFailed + failedTests.size()) + .withPassed(priorPassed + (testSummary.size() - failedTests.size())); existingValidation.withQualityScore( - existingValidation.getTotal() > 0 - ? (existingValidation.getPassed() / (double) existingValidation.getTotal()) * 100 - : 0.0); + total > 0 ? (existingValidation.getPassed() / (double) total) * 100 : 0.0); return existingValidation; } @@ -1898,24 +1904,16 @@ private void updateTestCasesWithDataContract(DataContract dataContract) { } private void removeDataContractFromTestCases(DataContract dataContract) { - if (nullOrEmpty(dataContract.getQualityExpectations())) { - return; - } - - for (EntityReference testCaseRef : dataContract.getQualityExpectations()) { - try { - daoCollection - .testCaseDAO() - .removeTestCaseDataContractForSpecificContract( - testCaseRef.getId().toString(), dataContract.getId().toString()); - - LOG.debug("Removed dataContract reference from test case {}", testCaseRef.getId()); - } catch (Exception e) { - LOG.warn( - "Failed to remove dataContract reference from test case {}: {}", - testCaseRef.getId(), - e.getMessage()); - } + try { + daoCollection + .testCaseDAO() + .removeAllTestCaseDataContractReferences(dataContract.getId().toString()); + LOG.debug("Removed dataContract references for contract {}", dataContract.getId()); + } catch (Exception e) { + LOG.warn( + "Failed to remove dataContract references for contract {}: {}", + dataContract.getId(), + e.getMessage()); } } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx index 1b7d59b9fd33..de568cb02e75 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.component.tsx @@ -22,7 +22,6 @@ import { DataContract } from '../../../generated/entity/data/dataContract'; import { DataContractResult } from '../../../generated/entity/datacontract/dataContractResult'; import { TestCase, TestCaseStatus } from '../../../generated/tests/testCase'; import { Include } from '../../../generated/type/include'; -import { useFqn } from '../../../hooks/useFqn'; import { getListTestCaseBySearch } from '../../../rest/testAPI'; import { getContractStatusType } from '../../../utils/DataContract/DataContractUtils'; import { getTestCaseDetailPagePath } from '../../../utils/RouterUtils'; @@ -38,19 +37,20 @@ const ContractQualityCard: React.FC<{ latestContractResults?: DataContractResult; }> = ({ contract, contractStatus, latestContractResults }) => { const { t } = useTranslation(); - const { fqn } = useFqn(); const [isTestCaseLoading, setIsTestCaseLoading] = useState(false); const [testCase, setTestCase] = useState([]); + const entityFqn = contract.entity?.fullyQualifiedName; + const fetchTestCases = useCallback(async () => { - if (!fqn) { + if (!entityFqn) { return; } setIsTestCaseLoading(true); try { const { data } = await getListTestCaseBySearch({ ...DEFAULT_SORT_ORDER, - entityLink: generateEntityLink(fqn), + entityLink: generateEntityLink(entityFqn), includeAllTests: true, limit: ES_MAX_PAGE_SIZE, include: Include.NonDeleted, @@ -66,7 +66,7 @@ const ContractQualityCard: React.FC<{ } finally { setIsTestCaseLoading(false); } - }, [fqn]); + }, [entityFqn]); const { showTestCaseSummaryChart, @@ -126,8 +126,7 @@ const ContractQualityCard: React.FC<{ return { id: item.id, name: item.name, - fullyQualifiedName: - item.fullyQualifiedName ?? (fqn ? `${fqn}.${item.name}` : item.name), + fullyQualifiedName: item.fullyQualifiedName ?? item.name, testCaseStatus: matchedTestCase?.testCaseStatus ?? matchedTestCase?.testCaseResult?.testCaseStatus ?? @@ -136,7 +135,7 @@ const ContractQualityCard: React.FC<{ }); return mergedData ?? []; - }, [contract, testCase, fqn]); + }, [contract, testCase]); useEffect(() => { fetchTestCases(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx index 0dad06e5e986..7e2e27f44df3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx @@ -22,12 +22,6 @@ import { getListTestCaseBySearch } from '../../../rest/testAPI'; import { showErrorToast } from '../../../utils/ToastUtils'; import ContractQualityCard from './ContractQualityCard.component'; -jest.mock('../../../hooks/useFqn', () => ({ - useFqn: jest.fn(() => ({ - fqn: 'fqn', - })), -})); - jest.mock('../../../rest/testAPI', () => ({ getListTestCaseBySearch: jest.fn(), })); @@ -281,15 +275,15 @@ describe('ContractQualityCard', () => { expect(links[0]).toHaveAttribute( 'href', - '/test-case/fqn.CLV Must be Positive' + '/test-case/CLV Must be Positive' ); expect(links[1]).toHaveAttribute( 'href', - '/test-case/fqn.Customer ID To Be Unique' + '/test-case/Customer ID To Be Unique' ); expect(links[2]).toHaveAttribute( 'href', - '/test-case/fqn.Table Row Count To Equal' + '/test-case/Table Row Count To Equal' ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/mocks/DataContract.mock.ts b/openmetadata-ui/src/main/resources/ui/src/mocks/DataContract.mock.ts index 342b6454f0bf..b5735ba8628c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/mocks/DataContract.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/mocks/DataContract.mock.ts @@ -32,6 +32,7 @@ export const MOCK_DATA_CONTRACT = { entity: { id: 'ee9d44a0-815d-4ac9-8422-4f9d02ddf04d', type: 'table', + fullyQualifiedName: 'redshift prod.dev.dbt_jaffle.customers', href: 'https://demo.getcollate.io/v1/tables/ee9d44a0-815d-4ac9-8422-4f9d02ddf04d', }, testSuite: { From f99d18e61e160dfcf80de5c78019224544d1ea02 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 22 Apr 2026 14:09:56 +0000 Subject: [PATCH 36/38] Update generated TypeScript types --- .../src/main/resources/ui/src/generated/tests/testCase.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts index 1fe8012464f8..afcddecd3a5f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts @@ -231,16 +231,13 @@ export interface FieldChange { /** * Data Contract that this test case is associated with. * - * List of data products this test case is part of. When not set, the test case inherits the - * data products from the table it belongs to. - * * This schema defines the EntityReference type used for referencing an entity. * EntityReference is used for capturing relationships from one entity to another. For * example, a table has an attribute called database of type EntityReference that captures * the relationship of a table `belongs to a` database. * - * Domains the test case belongs to. When not set, the test case inherits the domain from - * the table it belongs to. + * List of data products this test case is part of. When not set, the test case inherits the + * data products from the table it belongs to. * * This schema defines the EntityReferenceList type used for referencing an entity. * EntityReference is used for capturing relationships from one entity to another. For From 6787688243e7f7440c8408f403b33c3d4ce428c9 Mon Sep 17 00:00:00 2001 From: shrabantipaul-collate Date: Thu, 23 Apr 2026 10:28:24 +0530 Subject: [PATCH 37/38] Update openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../DataContract/ContractDetailTab/ContractDetail.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx index d4474cdfa8f4..e6c9d947b0c7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx @@ -749,7 +749,7 @@ describe('ContractDetail', () => { expect(screen.queryByText('ContractQualityCard')).toBeInTheDocument(); }); - it("should not display test cases when contract doesn't have quality expectations", async () => { + it('should not display test cases when contract does not have quality expectations', async () => { render( Date: Thu, 23 Apr 2026 13:09:01 +0530 Subject: [PATCH 38/38] fix jest tests --- .../ContractQualityCard/ContractQualityCard.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx index 7e2e27f44df3..cf0bef887b9b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractQualityCard/ContractQualityCard.test.tsx @@ -184,7 +184,7 @@ describe('ContractQualityCard', () => { expect(getListTestCaseBySearch).toHaveBeenCalledWith( expect.objectContaining({ - entityLink: '<#E::table::fqn>', + entityLink: '<#E::table::redshift prod.dev.dbt_jaffle.customers>', include: 'non-deleted', includeAllTests: true, limit: 10000,