Skip to content

Commit a596f5d

Browse files
fix(snowflake): change the clustering key before dropping a clustered column
Snowflake rejects DROP COLUMN while the column still belongs to the clustering key, so the clustering change has to come first. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 966e34d commit a596f5d

6 files changed

Lines changed: 670 additions & 11 deletions

File tree

‎sqlmesh/core/engine_adapter/mixins.py‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -383,23 +383,41 @@ def get_alter_operations(
383383
self.get_data_objects(target_table_schema, {target_table.name}), 0
384384
)
385385

386+
clustering_operation: t.Optional[TableAlterClusterByOperation] = None
386387
if current_table_info and target_table_info:
387388
if target_table_info.is_clustered:
388389
if target_table_info.clustering_key and (
389390
current_table_info.clustering_key != target_table_info.clustering_key
390391
):
391-
operations.append(
392-
TableAlterChangeClusterKeyOperation(
393-
target_table=current_table,
394-
clustering_key=target_table_info.clustering_key,
395-
dialect=self.dialect,
396-
)
392+
clustering_operation = TableAlterChangeClusterKeyOperation(
393+
target_table=current_table,
394+
clustering_key=target_table_info.clustering_key,
395+
dialect=self.dialect,
397396
)
398397
elif current_table_info.is_clustered:
399-
operations.append(TableAlterDropClusterKeyOperation(target_table=current_table))
398+
clustering_operation = TableAlterDropClusterKeyOperation(target_table=current_table)
399+
400+
if clustering_operation:
401+
operations.insert(
402+
self._clustering_alter_position(operations, current_table_info),
403+
clustering_operation,
404+
)
400405

401406
return operations
402407

408+
def _clustering_alter_position(
409+
self,
410+
operations: t.List[TableAlterOperation],
411+
current_table_info: t.Any,
412+
) -> int:
413+
"""Where to place a clustering change among the column alters.
414+
415+
Other engines keep the historical order: column changes first, then the
416+
clustering change. Snowflake overrides this so a key that still names a
417+
dropped column is removed before that column drop.
418+
"""
419+
return len(operations)
420+
403421

404422
def logical_merge(
405423
engine_adapter: EngineAdapter,

‎sqlmesh/core/engine_adapter/snowflake.py‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,16 @@
1616
ClusteredByMixin,
1717
RowDiffMixin,
1818
GrantsFromInfoSchemaMixin,
19+
TableAlterChangeClusterKeyOperation,
1920
)
21+
from sqlmesh.core.schema_diff import TableAlterDropColumnOperation, TableAlterOperation
2022
from sqlmesh.core.engine_adapter.shared import (
2123
CatalogSupport,
2224
DataObject,
2325
DataObjectType,
2426
SourceQuery,
2527
set_catalog,
2628
)
27-
from sqlmesh.core.schema_diff import TableAlterOperation
2829
from sqlmesh.utils import optional_import, get_source_columns_to_types
2930
from sqlmesh.utils.errors import SQLMeshError
3031
from sqlmesh.utils.pandas import columns_to_types_from_dtypes
@@ -692,6 +693,46 @@ def clone_table(
692693
**kwargs,
693694
)
694695

696+
def _clustering_alter_position(
697+
self,
698+
operations: t.List[TableAlterOperation],
699+
current_table_info: t.Any,
700+
) -> int:
701+
"""Place a clustering change before the first drop of a column that key still names.
702+
703+
Snowflake rejects DROP COLUMN while the column belongs to the clustering key.
704+
Column names from columns() are plain strings. A key references one when
705+
normalize_identifiers gives both the same name.
706+
"""
707+
clustering_key = getattr(current_table_info, "clustering_key", None)
708+
if not clustering_key:
709+
return len(operations)
710+
711+
referenced = self._clustering_key_column_names(clustering_key)
712+
for index, operation in enumerate(operations):
713+
if (
714+
not isinstance(operation, TableAlterDropColumnOperation)
715+
or not operation.column_parts
716+
):
717+
continue
718+
column_name = normalize_identifiers(
719+
exp.to_identifier(operation.column_parts[0].name), dialect=self.dialect
720+
).name
721+
if column_name in referenced:
722+
return index
723+
return len(operations)
724+
725+
def _clustering_key_column_names(self, clustering_key: str) -> t.Set[str]:
726+
expressions = TableAlterChangeClusterKeyOperation(
727+
target_table=exp.to_table("_"),
728+
clustering_key=clustering_key,
729+
dialect=self.dialect,
730+
).cluster_key_expressions
731+
return {
732+
normalize_identifiers(expression, dialect=self.dialect).name
733+
for expression in expressions
734+
}
735+
695736
def alter_table(
696737
self,
697738
alter_expressions: t.Union[t.List[exp.Alter], t.List[TableAlterOperation]],

‎tests/core/engine_adapter/test_bigquery.py‎

Lines changed: 108 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@
1313
import sqlmesh.core.dialect as d
1414
from sqlmesh.core.engine_adapter import BigQueryEngineAdapter
1515
from sqlmesh.core.engine_adapter.bigquery import select_partitions_expr
16-
from sqlmesh.core.engine_adapter.shared import DataObjectType
16+
from sqlmesh.core.engine_adapter.mixins import TableAlterDropClusterKeyOperation
17+
from sqlmesh.core.engine_adapter.shared import DataObject, DataObjectType
18+
from sqlmesh.core.schema_diff import TableAlterDropColumnOperation
1719
from sqlmesh.core.node import IntervalUnit
1820
from sqlmesh.utils import AttributeDict
1921
from sqlmesh.utils.errors import SQLMeshError
@@ -1380,3 +1382,108 @@ def test_sync_grants_config_no_schema(
13801382

13811383
with pytest.raises(ValueError, match="Table test_table does not have a schema \\(dataset\\)"):
13821384
adapter.sync_grants_config(relation, new_grants_config)
1385+
1386+
1387+
def _mock_bq_data_objects(
1388+
mocker: MockerFixture,
1389+
adapter: BigQueryEngineAdapter,
1390+
current_obj: DataObject,
1391+
target_obj: DataObject,
1392+
) -> None:
1393+
"""Patch _get_data_objects on *adapter* to return the supplied DataObjects."""
1394+
1395+
def _side_effect(
1396+
schema_name: t.Any, object_names: t.Optional[t.Set[str]] = None
1397+
) -> t.List[DataObject]:
1398+
results = []
1399+
for obj in [current_obj, target_obj]:
1400+
if object_names is None or obj.name in object_names:
1401+
results.append(obj)
1402+
return results
1403+
1404+
mocker.patch.object(adapter, "_get_data_objects", side_effect=_side_effect)
1405+
1406+
1407+
def test_bigquery_clustering_column_drop_before_clustering_change(
1408+
make_mocked_engine_adapter: t.Callable, mocker: MockerFixture
1409+
) -> None:
1410+
"""BigQuery keeps the historical order: DROP COLUMN before clustering key change.
1411+
1412+
Dropping a column that belongs to the current clustering key must NOT trigger
1413+
a reorder. BigQuery must emit DROP COLUMN first, then DROP CLUSTERING KEY.
1414+
"""
1415+
adapter = make_mocked_engine_adapter(BigQueryEngineAdapter, patch_get_data_objects=False)
1416+
1417+
current_columns = {"tenant_id": "INT64", "partition_month": "INT64"}
1418+
target_columns = {"tenant_id": "INT64"}
1419+
adapter.columns = lambda table_name, **kwargs: {
1420+
k: exp.DataType.build(v, dialect="bigquery")
1421+
for k, v in (
1422+
current_columns if table_name == "project.dataset.current_table" else target_columns
1423+
).items()
1424+
}
1425+
1426+
_mock_bq_data_objects(
1427+
mocker,
1428+
adapter,
1429+
current_obj=DataObject(
1430+
catalog="project",
1431+
schema="dataset",
1432+
name="current_table",
1433+
type=DataObjectType.TABLE,
1434+
clustering_key="(partition_month)",
1435+
),
1436+
target_obj=DataObject(
1437+
catalog="project",
1438+
schema="dataset",
1439+
name="target_table",
1440+
type=DataObjectType.TABLE,
1441+
clustering_key=None,
1442+
),
1443+
)
1444+
1445+
operations = adapter.get_alter_operations(
1446+
"project.dataset.current_table", "project.dataset.target_table"
1447+
)
1448+
1449+
assert len(operations) == 2
1450+
assert isinstance(operations[0], TableAlterDropColumnOperation)
1451+
assert isinstance(operations[1], TableAlterDropClusterKeyOperation)
1452+
1453+
1454+
def test_bigquery_no_column_change_clustering_drop_only(
1455+
make_mocked_engine_adapter: t.Callable, mocker: MockerFixture
1456+
) -> None:
1457+
"""BigQuery: same columns, only clustering key removed — one DROP CLUSTERING KEY."""
1458+
adapter = make_mocked_engine_adapter(BigQueryEngineAdapter, patch_get_data_objects=False)
1459+
1460+
columns = {"tenant_id": "INT64", "partition_month": "INT64"}
1461+
adapter.columns = lambda table_name, **kwargs: {
1462+
k: exp.DataType.build(v, dialect="bigquery") for k, v in columns.items()
1463+
}
1464+
1465+
_mock_bq_data_objects(
1466+
mocker,
1467+
adapter,
1468+
current_obj=DataObject(
1469+
catalog="project",
1470+
schema="dataset",
1471+
name="current_table",
1472+
type=DataObjectType.TABLE,
1473+
clustering_key="(partition_month)",
1474+
),
1475+
target_obj=DataObject(
1476+
catalog="project",
1477+
schema="dataset",
1478+
name="target_table",
1479+
type=DataObjectType.TABLE,
1480+
clustering_key=None,
1481+
),
1482+
)
1483+
1484+
operations = adapter.get_alter_operations(
1485+
"project.dataset.current_table", "project.dataset.target_table"
1486+
)
1487+
1488+
assert len(operations) == 1
1489+
assert isinstance(operations[0], TableAlterDropClusterKeyOperation)

‎tests/core/engine_adapter/test_redshift.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import sqlmesh.core.dialect as d
1212
from sqlmesh.core.engine_adapter import RedshiftEngineAdapter
1313
from sqlmesh.core.engine_adapter.shared import DataObject, DataObjectType
14+
from sqlmesh.core.schema_diff import TableAlterDropColumnOperation
1415
from sqlmesh.core.model import load_sql_based_model
1516
from sqlmesh.core.model.definition import SqlModel
1617
from sqlmesh.utils.errors import SQLMeshError
@@ -912,3 +913,46 @@ def test_merge_logical(
912913
'INSERT INTO "target" ("ID", "ts") SELECT "ID", "ts" FROM (SELECT "ID" AS "ID", "ts" AS "ts", ROW_NUMBER() OVER (PARTITION BY "ID" ORDER BY "ID") AS _row_number FROM "__temp_test_abcdefgh") AS _t WHERE _row_number = 1',
913914
'DROP TABLE IF EXISTS "__temp_test_abcdefgh"',
914915
]
916+
917+
918+
def test_redshift_no_clustering_operations_on_column_drop(
919+
make_mocked_engine_adapter: t.Callable,
920+
) -> None:
921+
"""Dropping a column on Redshift never emits clustering statements."""
922+
adapter = make_mocked_engine_adapter(RedshiftEngineAdapter)
923+
adapter.cursor.fetchall.return_value = []
924+
925+
current_columns = {"TENANT_ID": "INT", "PARTITION_MONTH": "INT"}
926+
target_columns = {"TENANT_ID": "INT"}
927+
adapter.columns = lambda table_name, **kwargs: {
928+
k: exp.DataType.build(v)
929+
for k, v in (current_columns if "current" in str(table_name) else target_columns).items()
930+
}
931+
932+
operations = adapter.get_alter_operations("current_table", "target_table")
933+
934+
assert len(operations) == 1
935+
assert all(isinstance(op, TableAlterDropColumnOperation) for op in operations)
936+
937+
938+
def test_redshift_multiple_column_drops_no_clustering_operations(
939+
make_mocked_engine_adapter: t.Callable,
940+
) -> None:
941+
"""Dropping multiple columns on Redshift preserves the schema-diff order.
942+
943+
No clustering operations must be injected between or after the column drops.
944+
"""
945+
adapter = make_mocked_engine_adapter(RedshiftEngineAdapter)
946+
adapter.cursor.fetchall.return_value = []
947+
948+
current_columns = {"EXTRA": "INT", "PARTITION_MONTH": "INT", "TENANT_ID": "INT"}
949+
target_columns = {"TENANT_ID": "INT"}
950+
adapter.columns = lambda table_name, **kwargs: {
951+
k: exp.DataType.build(v)
952+
for k, v in (current_columns if "current" in str(table_name) else target_columns).items()
953+
}
954+
955+
operations = adapter.get_alter_operations("current_table", "target_table")
956+
957+
assert len(operations) == 2
958+
assert all(isinstance(op, TableAlterDropColumnOperation) for op in operations)

0 commit comments

Comments
 (0)