Skip to content

Commit dac16c8

Browse files
Gabe Pescoclaude
authored andcommitted
feat(databricks): add force_shared_connection and guard connector internals
On connector 4.6+ with the kernel extra, Reyden / Real-Time warehouses open connections on the kernel backend, which ignores credentials_provider and runs its own U2M browser flow per connection. force_shared_connection runs every query over one connection (and restores the pre-shared-login behaviour) as an escape hatch for that. The capability check now imports every connector internal the shared login uses and checks each accepts the arguments passed to it, instead of probing for one module. If a connector upgrade changes them, U2M OAuth falls back to a shared connection with a warning rather than failing to connect. Checked against connector 4.2.6, 4.3.0 and 4.9.0. shared_connection and the connection kwargs now both use _uses_shared_u2m_login, so they cannot disagree. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Gabe Pesco <PescoG@medinsight.milliman.com>
1 parent 6400bf9 commit dac16c8

4 files changed

Lines changed: 98 additions & 18 deletions

File tree

‎docs/integrations/engines/databricks.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,7 @@ The only relevant SQLMesh configuration parameter is the optional `catalog` para
270270
| `force_databricks_connect` | When running locally, force the use of Databricks Connect for all model operations (so don't use SQL Connector for SQL models) | bool | N |
271271
| `disable_databricks_connect` | When running locally, disable the use of Databricks Connect for all model operations (so use SQL Connector for all models) | bool | N |
272272
| `disable_spark_session` | Do not use SparkSession if it is available (like when running in a notebook). | bool | N |
273+
| `force_shared_connection` | SQL Connector Only: Run all queries over a single connection instead of one per thread. With U2M OAuth, use this if the connector opens connections on its kernel backend (Reyden / Real-Time warehouses), which runs a separate browser login for each connection.| bool | N |
273274

274275
### Query tags
275276

‎sqlmesh/core/config/connection.py‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,7 @@ class DatabricksConnectionConfig(ConnectionConfig):
823823
force_databricks_connect: bool = False
824824
disable_databricks_connect: bool = False
825825
disable_spark_session: bool = False
826+
force_shared_connection: bool = False
826827

827828
concurrent_tasks: int = 1
828829
register_comments: bool = True
@@ -839,11 +840,23 @@ class DatabricksConnectionConfig(ConnectionConfig):
839840
@computed_field # type: ignore[prop-decorator]
840841
@property
841842
def shared_connection(self) -> bool:
842-
"""U2M OAuth connections share one login, so each thread can have its own connection. Connectors too old
843-
to support that fall back to sharing a single connection, so only one browser flow is run."""
843+
"""U2M OAuth connections share one login, so each thread can have its own connection. When the login can't
844+
be shared, or `force_shared_connection` is set, a single connection is shared so only one browser flow runs."""
845+
return self.force_shared_connection or (
846+
self._uses_u2m_oauth and not self._uses_shared_u2m_login
847+
)
848+
849+
@property
850+
def _uses_shared_u2m_login(self) -> bool:
851+
"""Whether every connection is given the same U2M OAuth login, letting each thread have its own connection."""
844852
from sqlmesh.utils.databricks import supports_shared_u2m_login
845853

846-
return self._uses_u2m_oauth and not supports_shared_u2m_login()
854+
return bool(
855+
self._uses_u2m_oauth
856+
and self.server_hostname
857+
and not self.force_shared_connection
858+
and supports_shared_u2m_login()
859+
)
847860

848861
@property
849862
def _uses_u2m_oauth(self) -> bool:
@@ -1001,14 +1014,14 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]:
10011014
# ref: https://docs.databricks.com/en/dev-tools/python-sql-connector.html#oauth-user-to-machine-u2m-authentication
10021015
# All connections share one login so that each thread can have its own connection without
10031016
# running its own browser flow. `credentials_provider` takes precedence over `auth_type` on the
1004-
# connector's Thrift and SEA backends. `auth_type` is still passed because the connector's kernel
1005-
# backend rejects a custom `credentials_provider` and runs its own U2M flow from `auth_type`.
1006-
from sqlmesh.utils.databricks import (
1007-
get_shared_u2m_credentials_provider,
1008-
supports_shared_u2m_login,
1009-
)
1010-
1011-
if self.server_hostname and supports_shared_u2m_login():
1017+
# connector's Thrift and SEA backends. `auth_type` is still passed for the connector's kernel
1018+
# backend, which ignores `credentials_provider` and runs its own U2M flow for each connection
1019+
# (`force_shared_connection` avoids that by only opening one connection).
1020+
from sqlmesh.utils.databricks import get_shared_u2m_credentials_provider
1021+
1022+
if self._uses_shared_u2m_login:
1023+
if t.TYPE_CHECKING:
1024+
assert self.server_hostname is not None
10121025
conn_kwargs["credentials_provider"] = get_shared_u2m_credentials_provider(
10131026
self.server_hostname, self.auth_type
10141027
)

‎sqlmesh/utils/databricks.py‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
from __future__ import annotations
22

3-
import importlib.util
3+
import inspect
4+
import logging
45
import re
56
import typing as t
67
from functools import lru_cache
78
from threading import Lock
89

10+
logger = logging.getLogger(__name__)
11+
912
HeaderFactory = t.Callable[[], t.Dict[str, str]]
1013

1114

@@ -105,11 +108,39 @@ def _create_auth_provider(self) -> t.Any:
105108

106109
@lru_cache(maxsize=1)
107110
def supports_shared_u2m_login() -> bool:
108-
"""Whether the installed connector has the internals `SharedU2MCredentialsProvider` builds on (4.1.0+)."""
111+
"""Whether the connector internals `SharedU2MCredentialsProvider` builds on accept the arguments it passes them.
112+
113+
These internals aren't public, so a connector upgrade could change them. If one has, U2M OAuth falls back to
114+
sharing a single connection rather than failing to connect.
115+
"""
116+
try:
117+
from databricks.sql import __version__
118+
except ImportError:
119+
# The connector isn't installed, so there are no connections to share a login across
120+
return False
121+
109122
try:
110-
return importlib.util.find_spec("databricks.sql.common.unified_http_client") is not None
111-
except ModuleNotFoundError:
123+
from databricks.sql.auth.auth import get_python_sql_connector_auth_provider
124+
from databricks.sql.common.unified_http_client import UnifiedHttpClient
125+
from databricks.sql.exc import RequestError # noqa: F401
126+
from databricks.sql.utils import build_client_context
127+
128+
inspect.signature(build_client_context).bind(
129+
"hostname", __version__, user_agent_entry="sqlmesh"
130+
)
131+
inspect.signature(UnifiedHttpClient).bind(None)
132+
inspect.signature(get_python_sql_connector_auth_provider).bind(
133+
"hostname", http_client=None, auth_type="databricks-oauth"
134+
)
135+
except Exception as e:
136+
logger.warning(
137+
"Databricks SQL connector %s is not compatible with SQLMesh's shared U2M OAuth login (%s). U2M OAuth "
138+
"connections will share a single connection, so queries will run one at a time.",
139+
__version__,
140+
e,
141+
)
112142
return False
143+
return True
113144

114145

115146
_shared_u2m_credentials_providers: t.Dict[t.Tuple[str, str], SharedU2MCredentialsProvider] = {}

‎tests/core/test_connection_config.py‎

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import base64
2+
import logging
23
import re
34
import typing as t
45
from unittest.mock import MagicMock, patch
@@ -1472,14 +1473,18 @@ def test_databricks__u2m_oauth__connection_pool(make_config):
14721473
)
14731474

14741475

1475-
def test_databricks__u2m_oauth__old_connector_shares_connection(make_config):
1476-
"""Connectors without the internals the shared login builds on fall back to one shared connection."""
1476+
def test_databricks__u2m_oauth__incompatible_connector_shares_connection(make_config, caplog):
1477+
"""If a connector upgrade changes the internals the shared login builds on, U2M OAuth falls back to one shared
1478+
connection and logs a warning rather than failing to connect."""
14771479
from sqlmesh.utils import databricks
14781480
from sqlmesh.utils.connection_pool import ThreadLocalSharedConnectionPool
14791481

14801482
databricks.supports_shared_u2m_login.cache_clear()
14811483
try:
1482-
with patch("importlib.util.find_spec", return_value=None):
1484+
with (
1485+
patch("databricks.sql.utils.build_client_context", lambda server_hostname: None),
1486+
caplog.at_level(logging.WARNING, logger="sqlmesh.utils.databricks"),
1487+
):
14831488
config = make_config(
14841489
type="databricks",
14851490
server_hostname="dbc-test.cloud.databricks.com",
@@ -1496,10 +1501,40 @@ def test_databricks__u2m_oauth__old_connector_shares_connection(make_config):
14961501
connection_kwargs = config._connection_factory_with_kwargs.keywords
14971502
assert connection_kwargs["auth_type"] == "databricks-oauth"
14981503
assert "credentials_provider" not in connection_kwargs
1504+
1505+
assert "not compatible with SQLMesh's shared U2M OAuth login" in caplog.text
14991506
finally:
15001507
databricks.supports_shared_u2m_login.cache_clear()
15011508

15021509

1510+
@pytest.mark.parametrize(
1511+
"auth_kwargs",
1512+
[
1513+
{"auth_type": "databricks-oauth"},
1514+
{"access_token": "foo"},
1515+
],
1516+
)
1517+
def test_databricks__force_shared_connection(make_config, auth_kwargs):
1518+
"""`force_shared_connection` runs every query over one connection, for connector backends that can't share a
1519+
U2M OAuth login (the kernel backend runs its own browser flow for each connection)."""
1520+
from sqlmesh.utils.connection_pool import ThreadLocalSharedConnectionPool
1521+
1522+
config = make_config(
1523+
type="databricks",
1524+
server_hostname="dbc-test.cloud.databricks.com",
1525+
http_path="sql/test/foo",
1526+
concurrent_tasks=4,
1527+
force_shared_connection=True,
1528+
**auth_kwargs,
1529+
)
1530+
assert isinstance(config, DatabricksConnectionConfig)
1531+
assert config.shared_connection is True
1532+
1533+
adapter = config.create_engine_adapter()
1534+
assert isinstance(adapter._connection_pool, ThreadLocalSharedConnectionPool)
1535+
assert "credentials_provider" not in config._connection_factory_with_kwargs.keywords
1536+
1537+
15031538
@patch.object(DatabricksConnectionConfig, "_connection_factory_with_kwargs")
15041539
def test_databricks__m2m_oauth__connection_pool(mock_connection_factory_with_kwargs, make_config):
15051540
from sqlmesh.utils.connection_pool import ThreadLocalConnectionPool

0 commit comments

Comments
 (0)