Skip to content

Commit 5f84992

Browse files
committed
fix(clickhouse): isolate cleanup catalog state
Signed-off-by: mday-io <mdaytn@gmail.com>
1 parent 90a0b45 commit 5f84992

2 files changed

Lines changed: 334 additions & 11 deletions

File tree

‎sqlmesh/core/context.py‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3066,17 +3066,21 @@ def _cleanup_environments(
30663066
f"Environment '{name}' is not expired or does not exist. Nothing to clean up."
30673067
)
30683068

3069-
if expired_environments_summaries:
3070-
self._ensure_virtual_catalog_injection()
3071-
30723069
for expired_env_summary in expired_environments_summaries:
30733070
expired_env = self.state_reader.get_environment(expired_env_summary.name)
30743071

30753072
if expired_env:
3073+
cleanup_default_adapter, cleanup_engine_adapters, failure = (
3074+
self._cleanup_adapters_for_environment(expired_env)
3075+
)
3076+
if failure:
3077+
logger.warning(failure)
3078+
failures.append(failure)
3079+
continue
30763080
failures.extend(
30773081
cleanup_expired_views(
3078-
default_adapter=self.engine_adapter,
3079-
engine_adapters=self.engine_adapters,
3082+
default_adapter=cleanup_default_adapter,
3083+
engine_adapters=cleanup_engine_adapters,
30803084
environments=[expired_env],
30813085
console=self.console,
30823086
)
@@ -3088,6 +3092,62 @@ def _cleanup_environments(
30883092
self.state_sync.delete_expired_environments(current_ts=current_ts, name=name)
30893093
return failures
30903094

3095+
def _cleanup_adapters_for_environment(
3096+
self, environment: Environment
3097+
) -> t.Tuple[EngineAdapter, t.Dict[str, EngineAdapter], t.Optional[str]]:
3098+
"""Create cleanup-scoped adapters for an expired environment.
3099+
3100+
Persisted catalog-qualified view names indicate that virtual catalog injection was active,
3101+
so cleanup can clone only the selected adapters with the historical catalog and leave the
3102+
context's adapters unchanged.
3103+
"""
3104+
engine_adapters = self.engine_adapters
3105+
default_adapter = self.engine_adapter
3106+
catalogs_by_gateway: t.Dict[str, t.Set[str]] = collections.defaultdict(set)
3107+
3108+
for snapshot in environment.snapshots:
3109+
if not snapshot.is_model or snapshot.is_symbolic:
3110+
continue
3111+
3112+
gateway = (
3113+
snapshot.model_gateway
3114+
if environment.gateway_managed and snapshot.model_gateway in engine_adapters
3115+
else self.selected_gateway
3116+
)
3117+
adapter = engine_adapters.get(gateway, default_adapter)
3118+
catalog = snapshot.qualified_view_name.catalog_for_environment(
3119+
environment.naming_info, dialect=adapter.dialect
3120+
)
3121+
if catalog and adapter.supports_virtual_catalog() is True:
3122+
catalogs_by_gateway[gateway].add(catalog)
3123+
3124+
for gateway, catalogs in catalogs_by_gateway.items():
3125+
if len(catalogs) > 1:
3126+
catalogs_description = ", ".join(f"'{catalog}'" for catalog in sorted(catalogs))
3127+
return (
3128+
default_adapter,
3129+
engine_adapters,
3130+
(
3131+
f"Failed to clean up expired environment '{environment.name}': gateway "
3132+
f"'{gateway}' references multiple virtual catalogs: {catalogs_description}"
3133+
),
3134+
)
3135+
3136+
cleanup_engine_adapters = engine_adapters.copy()
3137+
cleanup_default_adapter = default_adapter
3138+
for gateway, catalogs in catalogs_by_gateway.items():
3139+
cleanup_adapter = engine_adapters.get(gateway, default_adapter).with_settings()
3140+
cleanup_adapter.inject_virtual_catalog(gateway)
3141+
# inject_virtual_catalog() may initialize adapter-specific state in addition to
3142+
# _default_catalog. Override only the cleanup clone with the catalog persisted in the
3143+
# expired environment so historical names pass SINGLE_CATALOG_ONLY validation.
3144+
cleanup_adapter._default_catalog = next(iter(catalogs))
3145+
cleanup_engine_adapters[gateway] = cleanup_adapter
3146+
if gateway == self.selected_gateway:
3147+
cleanup_default_adapter = cleanup_adapter
3148+
3149+
return cleanup_default_adapter, cleanup_engine_adapters, None
3150+
30913151
def _try_connection(self, connection_name: str, validator: t.Callable[[], None]) -> None:
30923152
connection_name = connection_name.capitalize()
30933153
try:

0 commit comments

Comments
 (0)