Lineage Improvements (#24919)

* Lineage Improvements
1. Add path preservation
2. Add Column Filter

* Remove impl doc

* Add Lineage Strategies for efficient loading of graphs

* Update getByEntityCount

* Update generated TypeScript types

* Add Builder

* Fix Build Issue

* Make NodOpProgressTracker have empty constructor

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: mohityadav766

openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java

@@ -228,7 +228,18 @@ public SearchLineageResult searchLineage(
       @Parameter(description = "Size field to limit the no.of results returned, defaults to 10")
           @DefaultValue("1000")
           @QueryParam("size")
-          int size)
+          int size,
+      @Parameter(
+              description =
+                  "Column-level lineage filter. Supports filtering by column names, tags, or glossary terms (e.g., 'columnName:customer_id', 'tag:PII', 'glossary:BusinessTerm')")
+          @QueryParam("columnFilter")
+          String columnFilter,
+      @Parameter(
+              description =
+                  "When true, preserves all nodes in the path to filtered results. When false, only returns nodes matching the filter. Default is true.")
+          @QueryParam("preservePaths")
+          @DefaultValue("true")
+          Boolean preservePaths)
       throws IOException {
     return Entity.getSearchRepository()
         .searchLineage(
@@ -241,7 +252,9 @@ public SearchLineageResult searchLineage(
                 .withIsConnectedVia(isConnectedVia(entityType))
                 .withLayerFrom(from)
                 .withLayerSize(size)
-                .withIncludeSourceFields(getRequiredLineageFields(includeSourceFields)));
+                .withIncludeSourceFields(getRequiredLineageFields(includeSourceFields))
+                .withColumnFilter(columnFilter)
+                .withPreservePaths(preservePaths));
   }
 
   @GET
@@ -329,7 +342,18 @@ public SearchLineageResult searchLineageWithDirection(
       @Parameter(description = "Size field to limit the no.of results returned, defaults to 10")
           @DefaultValue("1000")
           @QueryParam("size")
-          int size)
+          int size,
+      @Parameter(
+              description =
+                  "Column-level lineage filter. Supports filtering by column names, tags, or glossary terms (e.g., 'columnName:customer_id', 'tag:PII', 'glossary:BusinessTerm')")
+          @QueryParam("columnFilter")
+          String columnFilter,
+      @Parameter(
+              description =
+                  "When true, preserves all nodes in the path to filtered results. When false, only returns nodes matching the filter. Default is true.")
+          @QueryParam("preservePaths")
+          @DefaultValue("true")
+          Boolean preservePaths)
       throws IOException {
     return Entity.getSearchRepository()
         .searchLineageWithDirection(
@@ -343,7 +367,9 @@ public SearchLineageResult searchLineageWithDirection(
                 .withDirection(direction)
                 .withLayerFrom(from)
                 .withLayerSize(size)
-                .withIncludeSourceFields(getRequiredLineageFields(includeSourceFields)));
+                .withIncludeSourceFields(getRequiredLineageFields(includeSourceFields))
+                .withColumnFilter(columnFilter)
+                .withPreservePaths(preservePaths));
   }
 
   @GET

openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java

@@ -223,7 +223,23 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app
                   new LineageSettings()
                       .withDownstreamDepth(2)
                       .withUpstreamDepth(2)
-                      .withLineageLayer(LineageLayer.ENTITY_LINEAGE));
+                      .withLineageLayer(LineageLayer.ENTITY_LINEAGE)
+                      .withGraphPerformanceConfig(
+                          new org.openmetadata.schema.api.lineage.GraphPerformanceConfig()
+                              .withSmallGraphThreshold(5000)
+                              .withMediumGraphThreshold(50000)
+                              .withMaxInMemoryNodes(100000)
+                              .withSmallGraphBatchSize(10000)
+                              .withMediumGraphBatchSize(5000)
+                              .withLargeGraphBatchSize(1000)
+                              .withStreamingBatchSize(500)
+                              .withEnableCaching(true)
+                              .withCacheTTLSeconds(300)
+                              .withMaxCachedGraphs(100)
+                              .withEnableProgressTracking(false)
+                              .withProgressReportInterval(1000)
+                              .withUseScrollForLargeGraphs(true)
+                              .withScrollTimeoutMinutes(5)));
       Entity.getSystemRepository().createNewSetting(setting);
     }