Merge branch 'main' into fix-logout-twice

Author: chirag-madlani

.github/copilot-instructions.md

@@ -566,7 +566,7 @@ make unit_ingestion_dev_env  # For Python changes
 
 ## UI Pull Request Review Guidelines
 
-**IMPORTANT: When reviewing UI pull requests, you MUST follow the comprehensive guidelines in [/openmetadata-ui/UI_PR_REVIEW_GUIDELINES.md](../openmetadata-ui/UI_PR_REVIEW_GUIDELINES.md)**
+**IMPORTANT: When reviewing UI pull requests, you MUST follow the comprehensive guidelines in [/openmetadata-ui/UI_PR_REVIEW_GUIDELINES.md](../openmetadata-ui/UI_PR_REVIEW_GUIDELINES.md) and [/openmetadata-ui/src/main/resources/ui/playwright/PLAYWRIGHT_DEVELOPER_HANDBOOK.md](../openmetadata-ui/src/main/resources/ui/playwright/PLAYWRIGHT_DEVELOPER_HANDBOOK.md)**
 
 ### Critical UI Standards to Enforce
 

bootstrap/sql/migrations/native/1.11.4/mysql/schemaChanges.sql

@@ -0,0 +1,2 @@
+-- DO NOT RENAME OR DELETE THIS FILE
+-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1114

bootstrap/sql/migrations/native/1.11.4/postgres/schemaChanges.sql

@@ -0,0 +1,2 @@
+-- DO NOT RENAME OR DELETE THIS FILE
+-- Migrations are applied from Java code based on the version in the file name inside folder openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1114

conf/openmetadata.yaml

@@ -291,6 +291,7 @@ authenticationConfiguration:
   jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
   jwtPrincipalClaimsMapping: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
   enableSelfSignup : ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
+  enableAutoRedirect: ${AUTHENTICATION_ENABLE_AUTO_REDIRECT:-false}
   # Force secure flag on session cookies even when not using HTTPS directly.
   # Enable this when running behind a proxy/load balancer that handles SSL termination.
   # Default: false (secure flag only set when HTTPS is detected)

ingestion/src/metadata/ingestion/models/custom_pydantic.py

@@ -140,7 +140,7 @@ def model_dump(
         *,
         mask_secrets: bool = False,
         warnings: Union[bool, Literal["none", "warn", "error"]] = "none",
-        **kwargs,
+        **kwargs: Any,
     ) -> Dict[str, Any]:
         if mask_secrets:
             context = kwargs.pop("context", None) or {}

ingestion/src/metadata/ingestion/ometa/mixins/domain_mixin.py

@@ -126,3 +126,91 @@ def _handle_data_product_assets(
         payload = AssetsRequest(assets=assets)
 
         return self.client.put(path, payload.model_dump_json())
+
+    # Input Ports methods
+
+    def add_input_ports_to_data_product(
+        self, name: str, ports: List[EntityReference]
+    ) -> Dict:
+        """
+        Add input ports to a data product
+
+        Args:
+            name: Name of the data product
+            ports: List of entity references to add as input ports
+
+        Returns:
+            API response as a dictionary
+        """
+        return self._handle_data_product_ports(name, ports, "inputPorts", "add")
+
+    def remove_input_ports_from_data_product(
+        self, name: str, ports: List[EntityReference]
+    ) -> Dict:
+        """
+        Remove input ports from a data product
+
+        Args:
+            name: Name of the data product
+            ports: List of entity references to remove from input ports
+
+        Returns:
+            API response as a dictionary
+        """
+        return self._handle_data_product_ports(name, ports, "inputPorts", "remove")
+
+    # Output Ports methods
+
+    def add_output_ports_to_data_product(
+        self, name: str, ports: List[EntityReference]
+    ) -> Dict:
+        """
+        Add output ports to a data product
+
+        Args:
+            name: Name of the data product
+            ports: List of entity references to add as output ports
+
+        Returns:
+            API response as a dictionary
+        """
+        return self._handle_data_product_ports(name, ports, "outputPorts", "add")
+
+    def remove_output_ports_from_data_product(
+        self, name: str, ports: List[EntityReference]
+    ) -> Dict:
+        """
+        Remove output ports from a data product
+
+        Args:
+            name: Name of the data product
+            ports: List of entity references to remove from output ports
+
+        Returns:
+            API response as a dictionary
+        """
+        return self._handle_data_product_ports(name, ports, "outputPorts", "remove")
+
+    def _handle_data_product_ports(
+        self,
+        name: str,
+        ports: List[EntityReference],
+        port_type: str,
+        operation: str,
+    ) -> Dict:
+        """
+        Handle adding or removing ports from a data product
+
+        Args:
+            name: Name of the data product
+            ports: List of entity references to add/remove as ports
+            port_type: Type of port ("inputPorts" or "outputPorts")
+            operation: Operation to perform ("add" or "remove")
+
+        Returns:
+            API response as a dictionary
+        """
+        path = f"/dataProducts/{name}/{port_type}/{operation}"
+        payload = AssetsRequest(assets=ports)
+
+        return self.client.put(path, payload.model_dump_json())

ingestion/src/metadata/ingestion/source/database/redshift/connection.py

@@ -78,6 +78,15 @@ def test_connection(
         )
     )
 
+    def test_partition_details(engine_: Engine):
+        """Check if we have the right permissions to get partition details"""
+        with engine_.connect() as conn:
+            res = conn.execute(REDSHIFT_TEST_PARTITION_DETAILS).fetchone()
+            if not all(res):
+                raise SourceConnectionException(
+                    f"We don't have the right permissions to get partition details - {res}"
+                )
+
     def test_get_queries_permissions(engine_: Engine):
         """Check if we have the right permissions to list queries"""
         with engine_.connect() as conn:
@@ -96,9 +105,7 @@ def test_get_queries_permissions(engine_: Engine):
         "GetDatabases": partial(
             test_query, statement=REDSHIFT_GET_DATABASE_NAMES, engine=engine
         ),
-        "GetPartitionTableDetails": partial(
-            test_query, statement=REDSHIFT_TEST_PARTITION_DETAILS, engine=engine
-        ),
+        "GetPartitionTableDetails": partial(test_partition_details, engine),
     }
 
     result = test_connection_steps(

ingestion/src/metadata/ingestion/source/database/redshift/queries.py

@@ -238,7 +238,10 @@
 """
 
 
-REDSHIFT_TEST_PARTITION_DETAILS = "select * from SVV_TABLE_INFO limit 1"
+REDSHIFT_TEST_PARTITION_DETAILS = """
+SELECT
+    has_table_privilege('SVV_TABLE_INFO', 'SELECT') as can_access_svv_table_info
+"""
 
 REDSHIFT_GET_ALL_CONSTRAINTS = """
 select 

ingestion/src/metadata/pii/algorithms/presidio_utils.py

@@ -13,6 +13,7 @@
 """
 import inspect
 import logging
+from functools import cache
 from typing import Any, Callable, Iterable, List, Optional, Set, Type, Union, cast
 
 import spacy
@@ -43,6 +44,7 @@
 logger = pii_logger()
 
 
+@cache
 def load_nlp_engine(
     model_name: str = SPACY_EN_MODEL,
     supported_language: str = SUPPORTED_LANG,

ingestion/src/metadata/pii/tag_processor.py

@@ -12,6 +12,7 @@
     TagSource,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.pii.algorithms.presidio_utils import load_nlp_engine
 from metadata.pii.algorithms.tag_scoring import ScoreTagsForColumnService
 from metadata.pii.base_processor import AutoClassificationProcessor
 from metadata.pii.classification_manager import (
@@ -68,7 +69,9 @@ def __init__(
 
         # Service that runs analyzers
         if score_tags_for_column is None:
-            score_tags_for_column = ScoreTagsForColumnService()
+            score_tags_for_column = ScoreTagsForColumnService(
+                nlp_engine=load_nlp_engine()
+            )
         self.score_tags_for_column = score_tags_for_column
 
         logger.info(