Skip to content

Commit 355eca3

Browse files
yan-3005yan-3005
authored
cherry-pick(1.12.7): fix(csv): correct entityType in recursive import extension validation + row-count accounting (#27593) (#27669)
Cherry-pick of 4930b10 to 1.12.7. Key changes: - CsvUtil.addExtension: filter blank/empty extension tokens on export - EntityCsv: rowEntityType field for per-row entity type override in extension validation - EntityCsv: header row excluded from numberOfRowsProcessed/Passed counts - EntityCsv: countedFailureRecords dedup to count per-row failures once - EntityCsv: skip empty-value extension tokens instead of failing - DatabaseServiceRepository/DatabaseRepository/DatabaseSchemaRepository: set rowEntityType per row - BulkEntityImportPage: only short-circuit to upload on aborted or failure+processed=0 - GlossaryImportExport.spec.ts: fix version-history row counts (3->2) for header exclusion - BulkEditEntity/BulkImport/TestCaseImportExport E2E: update row counts for header exclusion - GlossaryResourceIT/TestCaseResourceIT: update expected counts for header exclusion - DatabaseServiceResourceIT: add recursive import custom property extension IT test - EntityCsvTest/CsvUtilTest: update assertSummary counts and blank extension filter assertion
1 parent 1305407 commit 355eca3

17 files changed

Lines changed: 511 additions & 95 deletions

File tree

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -979,4 +979,259 @@ private String addDomainsToTableRow(String csvLine, String newDomains) {
979979
}
980980
return String.join(",", parts);
981981
}
982+
983+
@Test
984+
void test_recursiveImportCustomPropertyExtension(TestNamespace ns)
985+
throws IOException, InterruptedException {
986+
String propName = ns.prefix("potato");
987+
String serverUrl = SdkClients.getServerUrl();
988+
String token = SdkClients.getAdminToken();
989+
com.fasterxml.jackson.databind.ObjectMapper mapper =
990+
new com.fasterxml.jackson.databind.ObjectMapper();
991+
HttpClient client = HttpClient.newHttpClient();
992+
993+
HttpRequest getStringTypeReq =
994+
HttpRequest.newBuilder()
995+
.uri(URI.create(serverUrl + "/v1/metadata/types/name/string"))
996+
.header("Authorization", "Bearer " + token)
997+
.GET()
998+
.build();
999+
HttpResponse<String> stringTypeResp =
1000+
client.send(getStringTypeReq, HttpResponse.BodyHandlers.ofString());
1001+
assertEquals(200, stringTypeResp.statusCode(), "Should fetch string type");
1002+
1003+
HttpRequest getTableTypeReq =
1004+
HttpRequest.newBuilder()
1005+
.uri(URI.create(serverUrl + "/v1/metadata/types/name/table"))
1006+
.header("Authorization", "Bearer " + token)
1007+
.GET()
1008+
.build();
1009+
HttpResponse<String> tableTypeResp =
1010+
client.send(getTableTypeReq, HttpResponse.BodyHandlers.ofString());
1011+
assertEquals(200, tableTypeResp.statusCode(), "Should fetch table type");
1012+
1013+
com.fasterxml.jackson.databind.JsonNode stringTypeNode = mapper.readTree(stringTypeResp.body());
1014+
com.fasterxml.jackson.databind.JsonNode tableTypeNode = mapper.readTree(tableTypeResp.body());
1015+
String tableTypeId = tableTypeNode.get("id").asText();
1016+
1017+
java.util.Map<String, Object> propertyTypeRef =
1018+
java.util.Map.of(
1019+
"id", stringTypeNode.get("id").asText(),
1020+
"type", "type",
1021+
"name", stringTypeNode.get("name").asText(),
1022+
"fullyQualifiedName", stringTypeNode.get("fullyQualifiedName").asText());
1023+
String customPropertyBody =
1024+
mapper.writeValueAsString(
1025+
java.util.Map.of(
1026+
"name",
1027+
propName,
1028+
"description",
1029+
"Test extension property for recursive import",
1030+
"propertyType",
1031+
propertyTypeRef));
1032+
1033+
HttpRequest registerPropReq =
1034+
HttpRequest.newBuilder()
1035+
.uri(URI.create(serverUrl + "/v1/metadata/types/" + tableTypeId))
1036+
.header("Authorization", "Bearer " + token)
1037+
.header("Content-Type", "application/json")
1038+
.PUT(HttpRequest.BodyPublishers.ofString(customPropertyBody))
1039+
.build();
1040+
HttpResponse<String> registerResp =
1041+
client.send(registerPropReq, HttpResponse.BodyHandlers.ofString());
1042+
assertEquals(200, registerResp.statusCode(), "Should register custom property on table type");
1043+
1044+
try {
1045+
DatabaseService service =
1046+
createEntity(createMinimalRequest(ns).withName(ns.prefix("ext_svc")));
1047+
Database database =
1048+
SdkClients.adminClient()
1049+
.databases()
1050+
.create(
1051+
new CreateDatabase()
1052+
.withName(ns.prefix("ext_db"))
1053+
.withService(service.getFullyQualifiedName()));
1054+
DatabaseSchema schema =
1055+
SdkClients.adminClient()
1056+
.databaseSchemas()
1057+
.create(
1058+
new CreateDatabaseSchema()
1059+
.withName(ns.prefix("ext_schema"))
1060+
.withDatabase(database.getFullyQualifiedName()));
1061+
1062+
String tableName = ns.prefix("ext_tbl");
1063+
String tableFqn = schema.getFullyQualifiedName() + "." + tableName;
1064+
1065+
String validCsv =
1066+
buildRecursiveCsv(
1067+
database, schema, tableName, tableFqn, "", propName + ":s3://bucket/file.csv");
1068+
CsvImportResult validResult =
1069+
importCsvRecursive(service.getFullyQualifiedName(), validCsv, true);
1070+
assertEquals(ApiStatus.SUCCESS, validResult.getStatus(), validResult.getImportResultsCsv());
1071+
assertEquals(0, validResult.getNumberOfRowsFailed());
1072+
assertEquals(3, validResult.getNumberOfRowsProcessed());
1073+
assertEquals(3, validResult.getNumberOfRowsPassed());
1074+
1075+
String badExtCsv =
1076+
buildRecursiveCsv(
1077+
database, schema, tableName, tableFqn, "", "unknown_prop_xyz_test:somevalue");
1078+
CsvImportResult badResult =
1079+
importCsvRecursive(service.getFullyQualifiedName(), badExtCsv, true);
1080+
assertEquals(ApiStatus.PARTIAL_SUCCESS, badResult.getStatus());
1081+
assertEquals(1, badResult.getNumberOfRowsFailed());
1082+
assertEquals(3, badResult.getNumberOfRowsProcessed());
1083+
assertEquals(2, badResult.getNumberOfRowsPassed());
1084+
1085+
String dedupCsv =
1086+
buildRecursiveCsv(
1087+
database,
1088+
schema,
1089+
tableName,
1090+
tableFqn,
1091+
"invalidownerformat",
1092+
"unknown_prop_xyz_test:somevalue");
1093+
CsvImportResult dedupResult =
1094+
importCsvRecursive(service.getFullyQualifiedName(), dedupCsv, true);
1095+
assertEquals(
1096+
1,
1097+
dedupResult.getNumberOfRowsFailed(),
1098+
"Multi-field failure on one row must count as 1 failed row");
1099+
1100+
} finally {
1101+
removeCustomPropertyFromType(tableTypeId, propName, token);
1102+
}
1103+
}
1104+
1105+
private String buildRecursiveCsv(
1106+
Database database,
1107+
DatabaseSchema schema,
1108+
String tableName,
1109+
String tableFqn,
1110+
String tableOwner,
1111+
String tableExtension) {
1112+
String header =
1113+
"name*,displayName,description,owner,tags,glossaryTerms,tiers,certification,"
1114+
+ "retentionPeriod,sourceUrl,domains,extension,entityType*,fullyQualifiedName,"
1115+
+ "column.dataTypeDisplay,column.dataType,column.arrayDataType,column.dataLength,"
1116+
+ "storedProcedure.code,storedProcedure.language";
1117+
String dbRow =
1118+
csvRow(
1119+
database.getName(),
1120+
"",
1121+
"",
1122+
"",
1123+
"",
1124+
"",
1125+
"",
1126+
"",
1127+
"",
1128+
"",
1129+
"",
1130+
"",
1131+
"database",
1132+
database.getFullyQualifiedName(),
1133+
"",
1134+
"",
1135+
"",
1136+
"",
1137+
"",
1138+
"");
1139+
String schemaRow =
1140+
csvRow(
1141+
schema.getName(),
1142+
"",
1143+
"",
1144+
"",
1145+
"",
1146+
"",
1147+
"",
1148+
"",
1149+
"",
1150+
"",
1151+
"",
1152+
"",
1153+
"databaseSchema",
1154+
schema.getFullyQualifiedName(),
1155+
"",
1156+
"",
1157+
"",
1158+
"",
1159+
"",
1160+
"");
1161+
String tableRow =
1162+
csvRow(
1163+
tableName,
1164+
"",
1165+
"",
1166+
tableOwner,
1167+
"",
1168+
"",
1169+
"",
1170+
"",
1171+
"",
1172+
"",
1173+
"",
1174+
tableExtension,
1175+
"table",
1176+
tableFqn,
1177+
"",
1178+
"",
1179+
"",
1180+
"",
1181+
"",
1182+
"");
1183+
return header + "\n" + dbRow + "\n" + schemaRow + "\n" + tableRow + "\n";
1184+
}
1185+
1186+
private void removeCustomPropertyFromType(String typeId, String propName, String token)
1187+
throws IOException, InterruptedException {
1188+
com.fasterxml.jackson.databind.ObjectMapper localMapper =
1189+
new com.fasterxml.jackson.databind.ObjectMapper();
1190+
HttpClient client = HttpClient.newHttpClient();
1191+
String baseUrl = SdkClients.getServerUrl();
1192+
String getUrl = baseUrl + "/v1/metadata/types/" + typeId + "?fields=customProperties";
1193+
HttpRequest getReq =
1194+
HttpRequest.newBuilder()
1195+
.uri(URI.create(getUrl))
1196+
.header("Authorization", "Bearer " + token)
1197+
.GET()
1198+
.build();
1199+
HttpResponse<String> getResp = client.send(getReq, HttpResponse.BodyHandlers.ofString());
1200+
if (getResp.statusCode() != 200) {
1201+
return;
1202+
}
1203+
com.fasterxml.jackson.databind.JsonNode typeNode = localMapper.readTree(getResp.body());
1204+
com.fasterxml.jackson.databind.JsonNode customProps = typeNode.get("customProperties");
1205+
if (customProps == null || !customProps.isArray()) {
1206+
return;
1207+
}
1208+
for (int i = 0; i < customProps.size(); i++) {
1209+
if (propName.equals(customProps.get(i).path("name").asText())) {
1210+
String patchBody = "[{\"op\":\"remove\",\"path\":\"/customProperties/" + i + "\"}]";
1211+
HttpRequest patchReq =
1212+
HttpRequest.newBuilder()
1213+
.uri(URI.create(baseUrl + "/v1/metadata/types/" + typeId))
1214+
.header("Authorization", "Bearer " + token)
1215+
.header("Content-Type", "application/json-patch+json")
1216+
.method("PATCH", HttpRequest.BodyPublishers.ofString(patchBody))
1217+
.build();
1218+
client.send(patchReq, HttpResponse.BodyHandlers.ofString());
1219+
break;
1220+
}
1221+
}
1222+
}
1223+
1224+
private String csvRow(String... fields) {
1225+
StringBuilder sb = new StringBuilder();
1226+
for (int i = 0; i < fields.length; i++) {
1227+
if (i > 0) sb.append(",");
1228+
String field = fields[i];
1229+
if (field.contains(",") || field.contains("\"") || field.contains("\n")) {
1230+
sb.append('"').append(field.replace("\"", "\"\"")).append('"');
1231+
} else {
1232+
sb.append(field);
1233+
}
1234+
}
1235+
return sb.toString();
1236+
}
9821237
}

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryResourceIT.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -912,11 +912,8 @@ void test_bulkImportGlossaryTermsIncrementsVersion(TestNamespace ns) {
912912
importResult = JsonUtils.readValue(result, CsvImportResult.class);
913913
assertNotNull(importResult, "Should parse CsvImportResult from response");
914914
assertEquals(ApiStatus.SUCCESS, importResult.getStatus(), "Import should succeed");
915-
// numberOfRowsProcessed = header row (1) + 3 data rows = 4
916-
assertEquals(
917-
4, importResult.getNumberOfRowsProcessed(), "Should process 4 rows (header + 3 data)");
918-
assertEquals(
919-
4, importResult.getNumberOfRowsPassed(), "All 4 rows should pass (header + 3 data)");
915+
assertEquals(3, importResult.getNumberOfRowsProcessed(), "Should process 3 data rows");
916+
assertEquals(3, importResult.getNumberOfRowsPassed(), "All 3 data rows should pass");
920917
assertEquals(0, importResult.getNumberOfRowsFailed(), "No rows should fail");
921918
assertFalse(importResult.getDryRun(), "Should not be a dry run");
922919
} catch (Exception e) {

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2838,13 +2838,13 @@ void test_importCsvWithWildcardName_multipleTablesSucceeds(TestNamespace ns) {
28382838
// Dry run with name="*" should succeed
28392839
CsvImportResult dryRunResult = importCsvWithWildcard(client, csvData, true);
28402840
assertEquals(ApiStatus.SUCCESS, dryRunResult.getStatus());
2841-
assertEquals(3, dryRunResult.getNumberOfRowsProcessed());
2841+
assertEquals(2, dryRunResult.getNumberOfRowsProcessed());
28422842

28432843
// Actual import with name="*" — previously failed because
28442844
// processChangeEventForBulkImport would call getByName("*")
28452845
CsvImportResult result = importCsvWithWildcard(client, csvData, false);
28462846
assertEquals(ApiStatus.SUCCESS, result.getStatus());
2847-
assertEquals(3, result.getNumberOfRowsProcessed());
2847+
assertEquals(2, result.getNumberOfRowsProcessed());
28482848

28492849
// Verify test cases created on different tables
28502850
TestCase tc1 =
@@ -2905,7 +2905,7 @@ void test_importCsvWithWildcardName_explicitTestSuiteTracked(TestNamespace ns) {
29052905

29062906
CsvImportResult result = importCsvWithWildcard(client, csvData, false);
29072907
assertEquals(ApiStatus.SUCCESS, result.getStatus());
2908-
assertEquals(2, result.getNumberOfRowsProcessed());
2908+
assertEquals(1, result.getNumberOfRowsProcessed());
29092909

29102910
TestCase imported =
29112911
client.testCases().getByName(table.getFullyQualifiedName() + "." + testName, "testSuite");
@@ -2966,7 +2966,7 @@ void test_importCsvWithWildcardName_dryRunDoesNotCreateEntities(TestNamespace ns
29662966

29672967
CsvImportResult dryRunResult = importCsvWithWildcard(client, csvData, true);
29682968
assertEquals(ApiStatus.SUCCESS, dryRunResult.getStatus());
2969-
assertEquals(2, dryRunResult.getNumberOfRowsProcessed());
2969+
assertEquals(1, dryRunResult.getNumberOfRowsProcessed());
29702970

29712971
// Entity should NOT exist after dry run
29722972
String expectedFqn = table.getFullyQualifiedName() + "." + testName;

‎openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -353,12 +353,12 @@ public static List<String> addExtension(List<String> csvRecord, Object extension
353353

354354
String extensionString =
355355
extensionMap.entrySet().stream()
356+
.map(entry -> Map.entry(entry.getKey(), formatValue(entry.getValue())))
357+
.filter(entry -> !entry.getValue().isBlank())
356358
.map(
357-
entry -> {
358-
String key = entry.getKey();
359-
Object value = entry.getValue();
360-
return CsvUtil.quoteCsvField(key + ENTITY_TYPE_SEPARATOR + formatValue(value));
361-
})
359+
entry ->
360+
CsvUtil.quoteCsvField(
361+
entry.getKey() + ENTITY_TYPE_SEPARATOR + entry.getValue()))
362362
.collect(Collectors.joining(FIELD_SEPARATOR));
363363

364364
csvRecord.add(extensionString);

0 commit comments

Comments
 (0)