Skip to content

Commit c679793

Browse files
Rohit0301edg956
Rohit0301
and
edg956
authored
feat: Enhanced the TagsPage to support auto-classification configurations (#24774)
* feat: Enhanced the TagsPage to support auto-classification configurations * fix: fixed the failing e2e and unit tests * fix: fixed the code smells * fix: minor UI fix * fix: moved translation changes in collate * fix: fixed the tag form name validation issue * fix: minor naming changes * fix: minor-fix * fix: Addressed PR comments * fix: Fixed the domain and owner selction issue, added missing e2e test * fix: unskip the e2e test * fix: Addressed PR comments * fix: Addressed PR comments * Fix: Updated the tag style modal * feat: Implemented UI for Automated tag feedbacks * fix: rendering recognizer feedback component * fix: Added feedback render function in TagClassBase * fix: removed the un-used translations * fix: fixed the labelType * fix: removed the un-used keys * fix: Addressed PR comments and fixed code smells * fix: Addressed PR comments * fix: fixed the tag report popup implementation * fix: fixed the failing unit test and icon render issue * Change definition of auto applied in feedback creation * fix: fixed report popupUI * fix: fixed the handleIngestionRetry method * fix: minor fix * Support table columns in feedback endpoint * fix: Fixed the failing tags e2e * fix: Addressed the copilot comments * fix: Addressed PR comments * fix: fixed e2e failing tests * fix: refactor tags form * fix: minor fix * fix: fixed code smell * fix: added view only user initializer util * fix: minor fixes * fix: minor UI fixes * added testid in mui autocomplete component * fix: code refactoring * fix: replace antd tooltip with mui * fix: added index file in resuable components * fix: created index file for IconColor Component * fix: fixed the test headings * fix: fixed the translation function use * fix: fixed the code smells * fix: added regex * fix: fixed the drawer z-index * fix: minor fixes * fix: code refactoring * fix: fixed icon for recognizer tag * fix: minor fix * fix: Addressed PR comments * fix: unit test * fix: addressed comments * Remove wildcard imports --------- Co-authored-by: Eugenio Doñaque <eugenio.donaque@getcollate.io>
1 parent f0ecc2f commit c679793

67 files changed

Lines changed: 3693 additions & 763 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎openmetadata-service/src/main/java/org/openmetadata/service/Entity.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,8 @@ public final class Entity {
144144

145145
public static final String FIELD_RELATED_TERMS = "relatedTerms";
146146

147+
public static final String FIELD_COLUMNS = "columns";
148+
147149
//
148150
// Service entities
149151
//

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/RecognizerFeedbackRepository.java‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.openmetadata.service.jdbi3;
22

3+
import static org.openmetadata.service.Entity.TABLE;
34
import static org.openmetadata.service.Entity.TAG;
45

56
import java.util.ArrayList;
@@ -8,6 +9,8 @@
89
import java.util.stream.Collectors;
910
import lombok.extern.slf4j.Slf4j;
1011
import org.openmetadata.schema.entity.classification.Tag;
12+
import org.openmetadata.schema.entity.data.Table;
13+
import org.openmetadata.schema.type.Column;
1114
import org.openmetadata.schema.type.Recognizer;
1215
import org.openmetadata.schema.type.RecognizerException;
1316
import org.openmetadata.schema.type.RecognizerFeedback;
@@ -145,7 +148,21 @@ private void validateTagIsAutoApplied(String entityLink, String tagFQN) {
145148

146149
List<TagLabel> tagsToCheck = null;
147150

148-
if (arrayFieldName != null && fieldName != null) {
151+
if (Entity.TABLE.equals(entityType) && Entity.FIELD_COLUMNS.equals(fieldName)) {
152+
TableRepository tableRepository = (TableRepository) Entity.getEntityRepository(TABLE);
153+
List<Column> results =
154+
tableRepository
155+
.getTableColumnsByFQN(
156+
entity.getFullyQualifiedName(), Integer.MAX_VALUE, 0, "tags", null, null, null)
157+
.getData();
158+
159+
for (Column column : results) {
160+
if (column.getName().equals(arrayFieldName)) {
161+
tagsToCheck = column.getTags();
162+
break;
163+
}
164+
}
165+
} else if (arrayFieldName != null && fieldName != null) {
149166
String entityJson = JsonUtils.pojoToJson(entity);
150167
com.fasterxml.jackson.databind.JsonNode rootNode = JsonUtils.readTree(entityJson);
151168

@@ -176,7 +193,7 @@ private void validateTagIsAutoApplied(String entityLink, String tagFQN) {
176193
.anyMatch(
177194
tag ->
178195
tag.getTagFQN().equals(tagFQN)
179-
&& tag.getLabelType() == TagLabel.LabelType.AUTOMATED);
196+
&& tag.getLabelType() == TagLabel.LabelType.GENERATED);
180197

181198
if (!isAutoApplied) {
182199
throw new IllegalArgumentException(
@@ -218,8 +235,38 @@ private void removeTagFromEntity(String entityLink, String tagFQN, String update
218235

219236
boolean entityModified = false;
220237

221-
if (arrayFieldName != null) {
222-
// Tag is on a nested field (columns, schemaFields, requestSchema, responseSchema, etc.)
238+
if (Entity.TABLE.equals(entityType) && Entity.FIELD_COLUMNS.equals(fieldName)) {
239+
TableRepository tableRepository = (TableRepository) Entity.getEntityRepository(TABLE);
240+
List<Column> results =
241+
tableRepository
242+
.getTableColumnsByFQN(
243+
entity.getFullyQualifiedName(), Integer.MAX_VALUE, 0, "tags", null, null, null)
244+
.getData();
245+
246+
originalEntity =
247+
((Table) originalEntity)
248+
.withColumns(
249+
results.stream()
250+
.map(c -> JsonUtils.readValue(JsonUtils.pojoToJson(c), c.getClass()))
251+
.collect(Collectors.toList()));
252+
253+
for (Column column : results) {
254+
if (column.getName().equals(arrayFieldName)) {
255+
entityModified =
256+
column
257+
.getTags()
258+
.removeIf(
259+
tag ->
260+
tag.getTagFQN().equals(tagFQN)
261+
&& tag.getLabelType() == TagLabel.LabelType.GENERATED);
262+
break;
263+
}
264+
}
265+
266+
entity = ((Table) entity).withColumns(results);
267+
268+
} else if (arrayFieldName != null) {
269+
// Tag is on a nested field (schemaFields, requestSchema, responseSchema, etc.)
223270
// We need to handle this through JSON manipulation since we don't know the specific
224271
// structure
225272

@@ -283,7 +330,7 @@ private void removeTagFromEntity(String entityLink, String tagFQN, String update
283330
.removeIf(
284331
tag ->
285332
tag.getTagFQN().equals(tagFQN)
286-
&& tag.getLabelType() == TagLabel.LabelType.AUTOMATED);
333+
&& tag.getLabelType() == TagLabel.LabelType.GENERATED);
287334
}
288335
}
289336

‎openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2514,15 +2514,15 @@ void test_tagUpdateOptimization_LargeScale(TestInfo test) throws HttpResponseExc
25142514
assertTagsContain(updated.getTags(), additionalTags);
25152515
}
25162516

2517-
private void assertTagsContain(List<TagLabel> tags, List<TagLabel> expectedTags) {
2517+
protected void assertTagsContain(List<TagLabel> tags, List<TagLabel> expectedTags) {
25182518
for (TagLabel expected : expectedTags) {
25192519
assertTrue(
25202520
tags.stream().anyMatch(tag -> tag.getTagFQN().equals(expected.getTagFQN())),
25212521
"Tags should contain: " + expected.getTagFQN());
25222522
}
25232523
}
25242524

2525-
private void assertTagsDoNotContain(List<TagLabel> tags, List<TagLabel> unexpectedTags) {
2525+
protected void assertTagsDoNotContain(List<TagLabel> tags, List<TagLabel> unexpectedTags) {
25262526
for (TagLabel unexpected : unexpectedTags) {
25272527
assertFalse(
25282528
tags.stream().anyMatch(tag -> tag.getTagFQN().equals(unexpected.getTagFQN())),
@@ -2540,7 +2540,7 @@ void test_recognizerFeedback_autoAppliedTags(TestInfo test) throws HttpResponseE
25402540
TagLabel autoAppliedTag =
25412541
new TagLabel()
25422542
.withTagFQN("PII.Sensitive")
2543-
.withLabelType(TagLabel.LabelType.AUTOMATED)
2543+
.withLabelType(TagLabel.LabelType.GENERATED)
25442544
.withState(TagLabel.State.SUGGESTED)
25452545
.withSource(TagLabel.TagSource.CLASSIFICATION);
25462546

@@ -2586,7 +2586,7 @@ void test_recognizerFeedback_exceptionList(TestInfo test) throws HttpResponseExc
25862586

25872587
// Create entity with auto-applied tag
25882588
TagLabel autoTag =
2589-
new TagLabel().withTagFQN("PII.Sensitive").withLabelType(TagLabel.LabelType.AUTOMATED);
2589+
new TagLabel().withTagFQN("PII.Sensitive").withLabelType(TagLabel.LabelType.GENERATED);
25902590

25912591
CreateEntity create = createRequest(getEntityName(test));
25922592
create.setTags(listOf(autoTag));
@@ -2627,7 +2627,7 @@ void test_recognizerFeedback_multipleEntities(TestInfo test) throws HttpResponse
26272627
// Create multiple entities with same auto-applied tag
26282628
List<T> entities = new ArrayList<>();
26292629
TagLabel autoTag =
2630-
new TagLabel().withTagFQN("PII.Sensitive").withLabelType(TagLabel.LabelType.AUTOMATED);
2630+
new TagLabel().withTagFQN("PII.Sensitive").withLabelType(TagLabel.LabelType.GENERATED);
26312631

26322632
for (int i = 0; i < 3; i++) {
26332633
CreateEntity create = createRequest(getEntityName(test) + i);
@@ -2694,7 +2694,7 @@ void test_recognizerFeedback_invalidFeedback(TestInfo test) throws HttpResponseE
26942694
"Feedback can only be submitted for auto-applied tags");
26952695
}
26962696

2697-
private RecognizerFeedback submitRecognizerFeedback(
2697+
protected RecognizerFeedback submitRecognizerFeedback(
26982698
RecognizerFeedback feedback, Map<String, String> authHeaders) throws HttpResponseException {
26992699
WebTarget target = getResource("tags/name/" + feedback.getTagFQN() + "/feedback");
27002700
return TestUtils.post(target, feedback, RecognizerFeedback.class, authHeaders);

‎openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import static jakarta.ws.rs.core.Response.Status.OK;
2121
import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED;
2222
import static java.lang.String.format;
23+
import static java.util.Collections.emptyList;
2324
import static java.util.Collections.singletonList;
2425
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
2526
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -175,6 +176,7 @@
175176
import org.openmetadata.schema.type.MetadataOperation;
176177
import org.openmetadata.schema.type.PartitionColumnDetails;
177178
import org.openmetadata.schema.type.PartitionIntervalTypes;
179+
import org.openmetadata.schema.type.RecognizerFeedback;
178180
import org.openmetadata.schema.type.SystemProfile;
179181
import org.openmetadata.schema.type.TableConstraint;
180182
import org.openmetadata.schema.type.TableConstraint.ConstraintType;
@@ -5991,4 +5993,69 @@ void test_bulkCreateOrUpdate_generatesChangeEvents(TestInfo test) throws IOExcep
59915993
+ " change events for bulk updated tables, but found "
59925994
+ bulkUpdatedEventCount);
59935995
}
5996+
5997+
private String getEntityLink(Table table, Column column) {
5998+
// Build entity link in the format: <#E::entityType::fqn>
5999+
if (column == null)
6000+
return String.format("<#E::%s::%s>", entityType, table.getFullyQualifiedName());
6001+
return String.format(
6002+
"<#E::%s::%s::%s::%s>",
6003+
entityType, table.getFullyQualifiedName(), Entity.FIELD_COLUMNS, column.getName());
6004+
}
6005+
6006+
@Test
6007+
void test_recognizerFeedback_autoAppliedTagsOnColumns(TestInfo test)
6008+
throws HttpResponseException {
6009+
if (!supportsTags) {
6010+
return; // Skip if entity doesn't support tags
6011+
}
6012+
6013+
// Create an entity with auto-applied tags (simulating recognizer output)
6014+
TagLabel autoAppliedTag =
6015+
new TagLabel()
6016+
.withTagFQN("PII.Sensitive")
6017+
.withLabelType(TagLabel.LabelType.GENERATED)
6018+
.withState(TagLabel.State.SUGGESTED)
6019+
.withSource(TagLabel.TagSource.CLASSIFICATION);
6020+
6021+
TagLabel manualTag =
6022+
new TagLabel()
6023+
.withTagFQN("Tier.Tier1")
6024+
.withLabelType(TagLabel.LabelType.MANUAL)
6025+
.withState(TagLabel.State.CONFIRMED);
6026+
Column testColumn =
6027+
getColumn("test_column", BIGINT, USER_ADDRESS_TAG_LABEL)
6028+
.withTags(listOf(autoAppliedTag, manualTag));
6029+
CreateTable create =
6030+
createRequest(getEntityName(test))
6031+
.withColumns(listOf(testColumn))
6032+
.withTableConstraints(emptyList());
6033+
6034+
Table entity = createEntity(create, ADMIN_AUTH_HEADERS);
6035+
6036+
// Submit feedback for false positive on auto-applied tag
6037+
RecognizerFeedback feedback =
6038+
new RecognizerFeedback()
6039+
.withEntityLink(getEntityLink(entity, testColumn))
6040+
.withTagFQN("PII.Sensitive")
6041+
.withFeedbackType(RecognizerFeedback.FeedbackType.FALSE_POSITIVE)
6042+
.withUserReason(RecognizerFeedback.UserReason.NOT_SENSITIVE_DATA)
6043+
.withUserComments("This field contains product IDs, not personal information");
6044+
6045+
// Submit feedback via API
6046+
RecognizerFeedback submittedFeedback = submitRecognizerFeedback(feedback, ADMIN_AUTH_HEADERS);
6047+
assertNotNull(submittedFeedback.getId());
6048+
6049+
// Verify the auto-applied tag is removed after feedback processing
6050+
TableRepository tableRepository = (TableRepository) Entity.getEntityRepository(TABLE);
6051+
List<Column> results =
6052+
tableRepository
6053+
.getTableColumnsByFQN(
6054+
entity.getFullyQualifiedName(), Integer.MAX_VALUE, 0, "tags", null, null, null)
6055+
.getData();
6056+
6057+
assertEquals(1, results.size());
6058+
assertTagsDoNotContain(results.getFirst().getTags(), listOf(autoAppliedTag));
6059+
assertTagsContain(results.getFirst().getTags(), listOf(manualTag));
6060+
}
59946061
}

‎openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tag.spec.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,11 @@ test.describe('Tag Page with Admin Roles', () => {
167167

168168
await expect(adminPage.getByRole('dialog')).toBeVisible();
169169

170-
await adminPage.getByTestId('color-color-input').fill('#6366f1');
170+
await adminPage.getByTestId('icon-picker-btn').click();
171+
await adminPage.getByRole('button', { name: `Select icon Cube01` }).click();
172+
await adminPage
173+
.getByRole('button', { name: 'Select color #F14C75' })
174+
.click();
171175

172176
const updateColor = adminPage.waitForResponse(`/api/v1/tags/*`);
173177
await adminPage.locator('button[type="submit"]').click();
@@ -250,11 +254,7 @@ test.describe('Tag Page with Admin Roles', () => {
250254

251255
await adminPage.click('[data-testid="add-new-tag-button"]');
252256

253-
await adminPage.waitForSelector('.ant-modal-content', {
254-
state: 'visible',
255-
});
256-
257-
await expect(adminPage.locator('.ant-modal-content')).toBeVisible();
257+
await expect(adminPage.getByTestId('tags-form')).toBeVisible();
258258

259259
await validateForm(adminPage);
260260

‎openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ const NEW_TAG = {
4343
displayName: `PlaywrightTag-${uuid()}`,
4444
renamedName: `PlaywrightTag-${uuid()}`,
4545
description: 'This is the PlaywrightTag',
46-
color: '#FF5733',
47-
icon: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAF8AAACFCAMAAAAKN9SOAAAAA1BMVEXmGSCqexgYAAAAI0lEQVRoge3BMQEAAADCoPVPbQwfoAAAAAAAAAAAAAAAAHgaMeAAAUWJHZ4AAAAASUVORK5CYII=',
46+
color: '#F14C75',
47+
icon: 'Cube01',
4848
};
4949
const tagFqn = `${NEW_CLASSIFICATION.name}.${NEW_TAG.name}`;
5050

@@ -261,11 +261,8 @@ test('Classification Page', async ({ page }) => {
261261
await redirectToHomePage(page);
262262
await classification.visitPage(page);
263263
await page.click('[data-testid="add-classification"]');
264-
await page.waitForSelector('.ant-modal-content', {
265-
state: 'visible',
266-
});
267264

268-
await expect(page.locator('.ant-modal-content')).toBeVisible();
265+
await expect(page.getByTestId('tags-form')).toBeVisible();
269266

270267
await validateForm(page);
271268

@@ -300,19 +297,20 @@ test('Classification Page', async ({ page }) => {
300297

301298
await page.click('[data-testid="add-new-tag-button"]');
302299

303-
await page.waitForSelector('.ant-modal-content', {
304-
state: 'visible',
305-
});
306-
307-
await expect(page.locator('.ant-modal-content')).toBeVisible();
300+
await expect(page.getByTestId('tags-form')).toBeVisible();
308301

309302
await validateForm(page);
310303

311304
await page.fill('[data-testid="name"]', NEW_TAG.name);
312305
await page.fill('[data-testid="displayName"]', NEW_TAG.displayName);
313306
await page.locator(descriptionBox).fill(NEW_TAG.description);
314-
await page.fill('[data-testid="icon-url"]', NEW_TAG.icon);
315-
await page.fill('[data-testid="tags_color-color-input"]', NEW_TAG.color);
307+
await page.getByTestId('icon-picker-btn').click();
308+
await page
309+
.getByRole('button', { name: `Select icon ${NEW_TAG.icon}` })
310+
.click();
311+
await page
312+
.getByRole('button', { name: `Select color ${NEW_TAG.color}` })
313+
.click();
316314

317315
const createTagResponse = page.waitForResponse('api/v1/tags');
318316
await submitForm(page);

‎openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ServiceBaseClass.ts‎

Lines changed: 38 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -311,27 +311,11 @@ class ServiceBaseClass {
311311
);
312312
}
313313

314-
handleIngestionRetry = async (ingestionType = 'metadata', page: Page) => {
315-
const { apiContext } = await getApiContext(page);
316-
317-
// Need to wait before start polling as Ingestion is taking time to reflect state on their db
318-
// Queued status are not stored in DB. cc: @ulixius9
319-
await page.waitForTimeout(2000);
320-
321-
const response = await apiContext
322-
.get(
323-
`/api/v1/services/ingestionPipelines?fields=pipelineStatuses&service=${
324-
this.serviceName
325-
}&pipelineType=${ingestionType}&serviceType=${getServiceCategoryFromService(
326-
this.category
327-
)}`
328-
)
329-
.then((res) => res.json());
330-
331-
const workflowData = response.data.filter(
332-
(d: { pipelineType: string }) => d.pipelineType === ingestionType
333-
)[0];
334-
314+
executeIngestionRetrySteps = async (
315+
page: Page,
316+
workflowData: { fullyQualifiedName: string; name: string },
317+
ingestionType: string
318+
) => {
335319
const oneHourBefore = Date.now() - 86400000;
336320
let consecutiveErrors = 0;
337321

@@ -400,6 +384,39 @@ class ServiceBaseClass {
400384
).toContainText('Success');
401385
};
402386

387+
handleIngestionRetryWithWorkflow = async (
388+
page: Page,
389+
workflowDetails: { fullyQualifiedName: string; name: string },
390+
ingestionType = 'metadata'
391+
) => {
392+
await page.waitForTimeout(2000);
393+
await this.executeIngestionRetrySteps(page, workflowDetails, ingestionType);
394+
};
395+
396+
handleIngestionRetry = async (ingestionType = 'metadata', page: Page) => {
397+
const { apiContext } = await getApiContext(page);
398+
399+
// Need to wait before start polling as Ingestion is taking time to reflect state on their db
400+
// Queued status are not stored in DB. cc: @ulixius9
401+
await page.waitForTimeout(2000);
402+
403+
const response = await apiContext
404+
.get(
405+
`/api/v1/services/ingestionPipelines?fields=pipelineStatuses&service=${
406+
this.serviceName
407+
}&pipelineType=${ingestionType}&serviceType=${getServiceCategoryFromService(
408+
this.category
409+
)}`
410+
)
411+
.then((res) => res.json());
412+
413+
const workflowData = response.data.find(
414+
(d: { pipelineType: string }) => d.pipelineType === ingestionType
415+
);
416+
417+
await this.executeIngestionRetrySteps(page, workflowData, ingestionType);
418+
};
419+
403420
async updateService(page: Page) {
404421
await this.updateDescriptionForIngestedTables(page);
405422
}

0 commit comments

Comments
 (0)