Merge branch 'main' into feature/22644-iceberg-gcp-support

mohitjeswani01 · web-flow · commit 949389aec074 · 2026-04-30T15:20:34.000+05:30
diff --git a/ingestion/setup.py b/ingestion/setup.py
@@ -169,6 +169,10 @@
     "setuptools>=78.1.1,<81",  # <81 required: pkg_resources removed in setuptools 81+
     "shapely",
     "collate-data-diff>=0.11.9",
+    # Floor on dbt-extractor (transitive via collate-data-diff -> dbt-core).
+    # Pre-0.5 versions ship no cp310-manylinux_2_17_aarch64 wheel, forcing a
+    # Rust/Cargo source build on ARM runners. 0.5+ uses cp38-abi3 wheels.
+    "dbt-extractor>=0.5.0",
     "jaraco.functools<4.2.0",  # above 4.2 breaks the build
     "jaraco.context==6.0.1",
     # TODO: Remove one once we have updated datadiff version
@@ -401,7 +405,7 @@
     "mypy-boto3-glue",
     "nox",
     "pre-commit",
-    "basedpyright~=1.39.0",
+    "basedpyright==1.39.3",
     # For publishing
     "twine",
     "build",
diff --git a/ingestion/src/metadata/automations/runner.py b/ingestion/src/metadata/automations/runner.py
@@ -58,7 +58,7 @@ def _(
 
         host_port_type = type(request.connection.config.hostPort)
         docker_host_port_str = host_port_str.replace("localhost", "host.docker.internal")
-        request.connection.config.hostPort = host_port_type(docker_host_port_str)
+        request.connection.config.hostPort = host_port_type(docker_host_port_str)  # pyright: ignore[reportAttributeAccessIssue]
 
         _ = _test_connection(metadata, request.connection.config, automation_workflow)
 
diff --git a/ingestion/src/metadata/ingestion/source/api/rest/metadata.py b/ingestion/src/metadata/ingestion/source/api/rest/metadata.py
@@ -210,15 +210,15 @@ def _prepare_endpoint_data(self, path, method_type, info, collection) -> Optiona
 
     def _get_fallback_url(self) -> Optional[AnyUrl]:  # noqa: UP045
         """Return openAPISchemaURL if available, otherwise None."""
-        schema_conn = self.config.serviceConnection.root.config.openAPISchemaConnection
+        schema_conn = self.config.serviceConnection.root.config.openAPISchemaConnection  # pyright: ignore[reportAttributeAccessIssue]
         if isinstance(schema_conn, OpenAPISchemaURL):
             return schema_conn.openAPISchemaURL
         return None
 
     def _generate_collection_url(self, collection_name: str) -> Optional[AnyUrl]:  # noqa: UP045
         """generate collection url"""
         try:
-            base_url = self.config.serviceConnection.root.config.docURL
+            base_url = self.config.serviceConnection.root.config.docURL  # pyright: ignore[reportAttributeAccessIssue]
             if not base_url:
                 logger.debug(f"Could not generate collection url for {collection_name} because docURL is not present")
                 return self._get_fallback_url()
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/mode/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/mode/metadata.py
@@ -62,8 +62,8 @@ def __init__(
         metadata: OpenMetadata,
     ):
         super().__init__(config, metadata)
-        self.workspace_name = config.serviceConnection.root.config.workspaceName
-        self.filter_query_param = config.serviceConnection.root.config.filterQueryParam
+        self.workspace_name = config.serviceConnection.root.config.workspaceName  # pyright: ignore[reportAttributeAccessIssue]
+        self.filter_query_param = config.serviceConnection.root.config.filterQueryParam  # pyright: ignore[reportAttributeAccessIssue]
         self.data_sources = self.client.get_all_data_sources(self.workspace_name)
 
     @classmethod
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/metadata.py
@@ -91,7 +91,7 @@ def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
         super().__init__(config, metadata)
         self.aws_account_id = self.service_connection.awsAccountId
         self.dashboard_url = None
-        self.aws_region = self.config.serviceConnection.root.config.awsConfig.awsRegion
+        self.aws_region = self.config.serviceConnection.root.config.awsConfig.awsRegion  # pyright: ignore[reportAttributeAccessIssue]
         self.default_args = {
             "AwsAccountId": self.aws_account_id,
             "MaxResults": QUICKSIGHT_MAX_RESULTS,
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py
@@ -1117,6 +1117,6 @@ def get_base_url(self) -> str:
         """
         Get the proxy url for the tableau server
         """
-        if self.config.serviceConnection.root.config.proxyURL:
-            return str(self.config.serviceConnection.root.config.proxyURL)
+        if self.config.serviceConnection.root.config.proxyURL:  # pyright: ignore[reportAttributeAccessIssue]
+            return str(self.config.serviceConnection.root.config.proxyURL)  # pyright: ignore[reportAttributeAccessIssue]
         return str(self.config.serviceConnection.root.config.hostPort)
diff --git a/ingestion/src/metadata/ingestion/source/database/azuresql/metadata.py b/ingestion/src/metadata/ingestion/source/database/azuresql/metadata.py
@@ -75,8 +75,8 @@ def get_database_names_raw(self) -> Iterable[str]:
         yield from self._execute_database_query(AZURE_SQL_GET_DATABASES)
 
     def get_database_names(self) -> Iterable[str]:
-        if not self.config.serviceConnection.root.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.root.config.database
+        if not self.config.serviceConnection.root.config.ingestAllDatabases:  # pyright: ignore[reportAttributeAccessIssue]
+            configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
             self.set_inspector(database_name=configured_db)
             yield configured_db
         else:
diff --git a/ingestion/src/metadata/ingestion/source/database/cockroach/metadata.py b/ingestion/src/metadata/ingestion/source/database/cockroach/metadata.py
@@ -186,8 +186,8 @@ def get_database_names_raw(self) -> Iterable[str]:
         yield from self._execute_database_query(COCKROACH_GET_DB_NAMES)
 
     def get_database_names(self) -> Iterable[str]:
-        if not self.config.serviceConnection.root.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.root.config.database
+        if not self.config.serviceConnection.root.config.ingestAllDatabases:  # pyright: ignore[reportAttributeAccessIssue]
+            configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
             self.set_inspector(database_name=configured_db)
             self.set_schema_description_map()
             yield configured_db
diff --git a/ingestion/src/metadata/ingestion/source/database/greenplum/metadata.py b/ingestion/src/metadata/ingestion/source/database/greenplum/metadata.py
@@ -128,8 +128,8 @@ def get_database_names_raw(self) -> Iterable[str]:
         yield from self._execute_database_query(GREENPLUM_GET_DB_NAMES)
 
     def get_database_names(self) -> Iterable[str]:
-        if not self.config.serviceConnection.root.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.root.config.database
+        if not self.config.serviceConnection.root.config.ingestAllDatabases:  # pyright: ignore[reportAttributeAccessIssue]
+            configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
             self.set_inspector(database_name=configured_db)
             yield configured_db
         else:
diff --git a/ingestion/src/metadata/ingestion/source/database/microsoftfabric/metadata.py b/ingestion/src/metadata/ingestion/source/database/microsoftfabric/metadata.py
@@ -123,8 +123,8 @@ def get_database_names(self) -> Iterable[str]:
 
         In Microsoft Fabric, each Warehouse and Lakehouse appears as a database.
         """
-        if not self.config.serviceConnection.root.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.root.config.database
+        if not self.config.serviceConnection.root.config.ingestAllDatabases:  # pyright: ignore[reportAttributeAccessIssue]
+            configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
             self.set_inspector(database_name=configured_db)
             yield configured_db
         else:
diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/metadata.py b/ingestion/src/metadata/ingestion/source/database/mssql/metadata.py
@@ -190,8 +190,8 @@ def get_database_names_raw(self) -> Iterable[str]:
         yield from self._execute_database_query(MSSQL_GET_DATABASE)
 
     def get_database_names(self) -> Iterable[str]:
-        if not self.config.serviceConnection.root.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.root.config.database
+        if not self.config.serviceConnection.root.config.ingestAllDatabases:  # pyright: ignore[reportAttributeAccessIssue]
+            configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
             self.set_schema_description_map()
             self.set_database_description_map()
             self.set_stored_procedure_description_map()
diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py
@@ -174,8 +174,8 @@ def get_database_names_raw(self) -> Iterable[str]:
         yield from self._execute_database_query(POSTGRES_GET_DB_NAMES)
 
     def get_database_names(self) -> Iterable[str]:
-        if not self.config.serviceConnection.root.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.root.config.database
+        if not self.config.serviceConnection.root.config.ingestAllDatabases:  # pyright: ignore[reportAttributeAccessIssue]
+            configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
             self.set_inspector(database_name=configured_db)
             self.set_schema_description_map()
             yield configured_db
diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/query_parser.py b/ingestion/src/metadata/ingestion/source/database/postgres/query_parser.py
@@ -80,7 +80,7 @@ def get_table_query(self) -> Iterable[TableQuery]:
             if self.config.sourceConfig.config.queryLogFilePath:
                 yield from super().yield_table_queries_from_logs()
             else:
-                database = self.config.serviceConnection.root.config.database
+                database = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
                 if database:
                     self.engine: Engine = get_connection(self.service_connection)
                     yield from self.process_table_query()
@@ -90,7 +90,7 @@ def get_table_query(self) -> Iterable[TableQuery]:
                     for res in results:
                         row = list(res)
                         logger.info(f"Ingesting from database: {row[0]}")
-                        self.config.serviceConnection.root.config.database = row[0]
+                        self.config.serviceConnection.root.config.database = row[0]  # pyright: ignore[reportAttributeAccessIssue]
                         self.engine = get_connection(self.service_connection)
                         yield from self.process_table_query()
 
diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py
@@ -279,8 +279,8 @@ def set_external_location_map(self, database_name: str) -> None:
         self.external_location_map = {(database_name, row.schemaname, row.tablename): row.location for row in results}
 
     def get_database_names(self) -> Iterable[str]:
-        if not self.config.serviceConnection.root.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.root.config.database
+        if not self.config.serviceConnection.root.config.ingestAllDatabases:  # pyright: ignore[reportAttributeAccessIssue]
+            configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
             self._set_incremental_table_processor(configured_db)
             self.set_external_location_map(configured_db)
             yield configured_db
diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py
@@ -391,7 +391,7 @@ def get_database_names_raw(self) -> Iterable[str]:
             yield row[1]
 
     def get_database_names(self) -> Iterable[str]:
-        configured_db = self.config.serviceConnection.root.config.database
+        configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
         if configured_db:
             self.set_inspector(configured_db)
             self.set_session_query_tag()
diff --git a/ingestion/src/metadata/ingestion/source/database/timescale/query_parser.py b/ingestion/src/metadata/ingestion/source/database/timescale/query_parser.py
@@ -80,7 +80,7 @@ def get_table_query(self) -> Iterable[TableQuery]:
             if self.config.sourceConfig.config.queryLogFilePath:
                 yield from super().yield_table_queries_from_logs()
             else:
-                database = self.config.serviceConnection.root.config.database
+                database = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
                 if database:
                     self.engine: Engine = get_connection(self.service_connection)
                     yield from self.process_table_query()
@@ -90,7 +90,7 @@ def get_table_query(self) -> Iterable[TableQuery]:
                     for res in results:
                         row = list(res)
                         logger.info(f"Ingesting from database: {row[0]}")
-                        self.config.serviceConnection.root.config.database = row[0]
+                        self.config.serviceConnection.root.config.database = row[0]  # pyright: ignore[reportAttributeAccessIssue]
                         self.engine = get_connection(self.service_connection)
                         yield from self.process_table_query()
 
diff --git a/ingestion/src/metadata/ingestion/source/database/vertica/metadata.py b/ingestion/src/metadata/ingestion/source/database/vertica/metadata.py
@@ -290,7 +290,7 @@ def get_database_names_raw(self) -> Iterable[str]:
         yield from self._execute_database_query(VERTICA_LIST_DATABASES)
 
     def get_database_names(self) -> Iterable[str]:
-        configured_db = self.config.serviceConnection.root.config.database
+        configured_db = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
         if configured_db:
             self.set_inspector(database_name=configured_db)
             self.set_schema_description_map()
diff --git a/ingestion/src/metadata/ingestion/source/database/vertica/query_parser.py b/ingestion/src/metadata/ingestion/source/database/vertica/query_parser.py
@@ -55,7 +55,7 @@ def create(cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str
         return cls(config, metadata)
 
     def get_table_query(self) -> Iterable[TableQuery]:
-        database = self.config.serviceConnection.root.config.database
+        database = self.config.serviceConnection.root.config.database  # pyright: ignore[reportAttributeAccessIssue]
         if database:
             yield from super().get_table_query()
         else:
@@ -64,6 +64,6 @@ def get_table_query(self) -> Iterable[TableQuery]:
             for res in results:
                 row = list(res)
                 logger.info(f"Ingesting from database: {row[0]}")
-                self.config.serviceConnection.root.config.database = row[0]
+                self.config.serviceConnection.root.config.database = row[0]  # pyright: ignore[reportAttributeAccessIssue]
                 self.engine = get_connection(self.service_connection)
                 yield from super().get_table_query()
diff --git a/ingestion/src/metadata/ingestion/source/security/security_service.py b/ingestion/src/metadata/ingestion/source/security/security_service.py
@@ -92,7 +92,7 @@ def __init__(
         config: WorkflowSource,
         metadata: OpenMetadata,
     ):
-        config.serviceConnection.root.config.hostPort = clean_uri(config.serviceConnection.root.config.hostPort)
+        config.serviceConnection.root.config.hostPort = clean_uri(config.serviceConnection.root.config.hostPort)  # pyright: ignore[reportAttributeAccessIssue]
         super().__init__()
         self.config = config
         self.metadata = metadata
diff --git a/ingestion/src/metadata/profiler/orm/functions/median.py b/ingestion/src/metadata/profiler/orm/functions/median.py
@@ -192,33 +192,30 @@ def _(elements, compiler, **kwargs):  # pylint: disable=unused-argument
         FROM (
             SELECT
                 {col},
-                ROW_NUMBER() OVER () AS row_num
-            FROM
-                `{table}` AS median_inner,
-                (SELECT @counter := COUNT(*)
-                 FROM `{table}` AS median_count
-                 WHERE median_count.{dimension_col} = `{table}`.{dimension_col}) t_count
+                ROW_NUMBER() OVER (ORDER BY {col}) AS row_num,
+                COUNT(*) OVER () AS total_count
+            FROM `{table}` AS median_inner
             WHERE median_inner.{dimension_col} = `{table}`.{dimension_col}
-            ORDER BY {col}
             ) temp
-        WHERE temp.row_num = ROUND({percentile} * @counter)
+        WHERE temp.row_num = ROUND({percentile} * temp.total_count)
         )
         """.format(col=col, table=table, percentile=percentile, dimension_col=dimension_col)  # noqa: UP032
     else:  # noqa: RET505
-        # NON-CORRELATED MODE: Original behavior (profiler)
+        # NON-CORRELATED MODE: window-function-based count to avoid
+        # user-variable side-effect ordering (MySQL doesn't guarantee
+        # when `SELECT @v := COUNT(*)` inside a derived table is
+        # evaluated relative to the outer WHERE).
         return """
         (SELECT
             {col}
         FROM (
             SELECT
                 {col},
-                ROW_NUMBER() OVER () AS row_num
-            FROM
-                `{table}`,
-                (SELECT @counter := COUNT(*) FROM `{table}`) t_count
-            ORDER BY {col}
+                ROW_NUMBER() OVER (ORDER BY {col}) AS row_num,
+                COUNT(*) OVER () AS total_count
+            FROM `{table}`
             ) temp
-        WHERE temp.row_num = ROUND({percentile} * @counter)
+        WHERE temp.row_num = ROUND({percentile} * temp.total_count)
         )
         """.format(col=col, table=table, percentile=percentile)  # noqa: UP032
 
diff --git a/ingestion/src/metadata/profiler/source/database/mssql/profiler_source.py b/ingestion/src/metadata/profiler/source/database/mssql/profiler_source.py
@@ -65,7 +65,7 @@ def set_is_disconnect(self, config: OpenMetadataWorkflowConfig):
         # pylint: disable=import-outside-toplevel
 
         # Get the configured scheme from the source connection
-        scheme = config.source.serviceConnection.root.config.scheme
+        scheme = config.source.serviceConnection.root.config.scheme  # pyright: ignore[reportAttributeAccessIssue]
 
         # Set the appropriate is_disconnect method based on the scheme
         if scheme == MssqlScheme.mssql_pytds:
diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py
@@ -515,7 +515,7 @@ def wrapper(*args, **kwargs):
 
                 host_port_type = type(config.serviceConnection.root.config.hostPort)
                 docker_host_port_str = host_port_str.replace("localhost", "host.docker.internal")
-                config.serviceConnection.root.config.hostPort = host_port_type(docker_host_port_str)
+                config.serviceConnection.root.config.hostPort = host_port_type(docker_host_port_str)  # pyright: ignore[reportAttributeAccessIssue]
                 func(*args, **kwargs)
 
         return wrapper
diff --git a/ingestion/src/metadata/workflow/ingestion.py b/ingestion/src/metadata/workflow/ingestion.py
@@ -181,7 +181,7 @@ def validate(self, profiler_config_class: Inject[Type[ProfilerProcessorConfig]]
             )
 
         try:
-            if not self.config.source.serviceConnection.root.config.supportsProfiler:
+            if not self.config.source.serviceConnection.root.config.supportsProfiler:  # pyright: ignore[reportAttributeAccessIssue]
                 raise AttributeError()  # noqa: TRY301
         except AttributeError:
             if profiler_config_class.model_validate(self.config.processor.model_dump().get("config")).ignoreValidation:
@@ -197,7 +197,7 @@ def import_source_class(self) -> Type[Source]:  # noqa: UP006
         source_type = self.config.source.type.lower()
         try:
             return (
-                import_from_module(self.config.source.serviceConnection.root.config.sourcePythonClass)
+                import_from_module(self.config.source.serviceConnection.root.config.sourcePythonClass)  # pyright: ignore[reportAttributeAccessIssue]
                 if source_type.startswith(CUSTOM_CONNECTOR_PREFIX)
                 else import_source_class(service_type=self.service_type, source_type=source_type)
             )
diff --git a/ingestion/tests/integration/profiler/test_median_mysql.py b/ingestion/tests/integration/profiler/test_median_mysql.py
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.test.ts
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/JSONLogicSearchClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/JSONLogicSearchClassBase.ts
