Skip to content

Commit a2d027a

Browse files
committed
fix(clickhouse): make partition ALTER actions deterministic
Signed-off-by: mday-io <mdaytn@gmail.com>
1 parent e3f3b7a commit a2d027a

2 files changed

Lines changed: 15 additions & 62 deletions

File tree

‎sqlmesh/core/engine_adapter/clickhouse.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,7 @@ def _build_alter_partition_exp(
446446
) -> exp.Alter:
447447
alter_expr = exp.Alter(this=target_table, kind="TABLE")
448448

449-
for partition in partitions_to_replace:
449+
for partition in sorted(partitions_to_replace):
450450
alter_expr.append(
451451
"actions",
452452
exp.ReplacePartition(
@@ -457,7 +457,7 @@ def _build_alter_partition_exp(
457457
),
458458
)
459459

460-
for partition in partitions_to_drop:
460+
for partition in sorted(partitions_to_drop):
461461
alter_expr.append(
462462
"actions",
463463
exp.DropPartition(

‎tests/core/engine_adapter/test_clickhouse.py‎

Lines changed: 13 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1170,29 +1170,16 @@ def test_insert_overwrite_by_condition_where_partitioned(
11701170
),
11711171
)
11721172

1173-
sql_calls = to_sql_calls(adapter)
1174-
1175-
assert len(sql_calls) == 7
1176-
assert sql_calls[:5] == [
1173+
assert to_sql_calls(adapter) == [
11771174
'CREATE TABLE "__temp_target_abcd" AS "__temp_existing_records_abcd"',
11781175
"""INSERT INTO "__temp_target_abcd" ("id", "ds") SELECT "id", "ds" FROM (SELECT * FROM "__temp_new_records_abcd") AS "_subquery" WHERE "ds" BETWEEN '2024-02-15' AND '2024-04-30'""",
11791176
'''CREATE TABLE IF NOT EXISTS "__temp_affected_partitions_abcd" ENGINE=MergeTree ORDER BY () AS SELECT DISTINCT "partition_id" FROM (SELECT "_partition_id" AS "partition_id" FROM "__temp_existing_records_abcd" WHERE "ds" BETWEEN '2024-02-15' AND '2024-04-30' UNION DISTINCT SELECT "_partition_id" AS "partition_id" FROM "__temp_target_abcd") AS "_affected_partitions"''',
11801177
"""INSERT INTO "__temp_target_abcd" SELECT "id", "ds" FROM "__temp_existing_records_abcd" WHERE NOT ("ds" BETWEEN '2024-02-15' AND '2024-04-30') AND "_partition_id" IN (SELECT "partition_id" FROM "__temp_affected_partitions_abcd")""",
11811178
'DROP TABLE IF EXISTS "__temp_affected_partitions_abcd"',
1179+
"""ALTER TABLE "__temp_existing_records_abcd" REPLACE PARTITION ID '1' FROM "__temp_target_abcd", REPLACE PARTITION ID '2' FROM "__temp_target_abcd", REPLACE PARTITION ID '4' FROM "__temp_target_abcd", DROP PARTITION ID '3'""",
1180+
'DROP TABLE IF EXISTS "__temp_target_abcd"',
11821181
]
11831182

1184-
alter_prefix = 'ALTER TABLE "__temp_existing_records_abcd" '
1185-
assert sql_calls[5].startswith(alter_prefix)
1186-
assert sorted(sql_calls[5].removeprefix(alter_prefix).split(", ")) == sorted(
1187-
[
1188-
"REPLACE PARTITION ID '1' FROM \"__temp_target_abcd\"",
1189-
"REPLACE PARTITION ID '2' FROM \"__temp_target_abcd\"",
1190-
"REPLACE PARTITION ID '4' FROM \"__temp_target_abcd\"",
1191-
"DROP PARTITION ID '3'",
1192-
]
1193-
)
1194-
assert sql_calls[6] == 'DROP TABLE IF EXISTS "__temp_target_abcd"'
1195-
11961183

11971184
def test_insert_overwrite_by_condition_by_key(
11981185
make_mocked_engine_adapter: t.Callable, mocker: MockerFixture, make_temp_table_name: t.Callable
@@ -1301,45 +1288,22 @@ def test_insert_overwrite_by_condition_by_key_partitioned(
13011288
dynamic_key_unique=False,
13021289
)
13031290

1304-
sql_calls = to_sql_calls(adapter)
1305-
1306-
assert len(sql_calls) == 14
1307-
assert sql_calls[:4] == [
1291+
assert to_sql_calls(adapter) == [
13081292
'CREATE TABLE "__temp_target_abcd" AS "__temp_existing_records_abcd"',
13091293
'INSERT INTO "__temp_target_abcd" ("id", "ds") SELECT "id", "ds" FROM (SELECT DISTINCT ON ("id") * FROM "__temp_new_records_abcd") AS "_subquery"',
13101294
'CREATE TABLE IF NOT EXISTS "__temp_affected_partitions_abcd" ENGINE=MergeTree ORDER BY () AS SELECT DISTINCT "partition_id" FROM (SELECT "_partition_id" AS "partition_id" FROM "__temp_existing_records_abcd" WHERE "id" IN (SELECT "id" FROM "__temp_target_abcd") UNION DISTINCT SELECT "_partition_id" AS "partition_id" FROM "__temp_target_abcd") AS "_affected_partitions"',
13111295
'INSERT INTO "__temp_target_abcd" SELECT "id", "ds" FROM "__temp_existing_records_abcd" WHERE NOT ("id" IN (SELECT "id" FROM "__temp_target_abcd")) AND "_partition_id" IN (SELECT "partition_id" FROM "__temp_affected_partitions_abcd")',
1312-
]
1313-
1314-
assert sql_calls[4] == 'DROP TABLE IF EXISTS "__temp_affected_partitions_abcd"'
1315-
alter_prefix = 'ALTER TABLE "__temp_existing_records_abcd" '
1316-
assert sql_calls[5].startswith(alter_prefix)
1317-
assert sorted(sql_calls[5].removeprefix(alter_prefix).split(", ")) == sorted(
1318-
[
1319-
"REPLACE PARTITION ID '1' FROM \"__temp_target_abcd\"",
1320-
"REPLACE PARTITION ID '2' FROM \"__temp_target_abcd\"",
1321-
"REPLACE PARTITION ID '4' FROM \"__temp_target_abcd\"",
1322-
"DROP PARTITION ID '3'",
1323-
]
1324-
)
1325-
assert sql_calls[6] == 'DROP TABLE IF EXISTS "__temp_target_abcd"'
1326-
assert sql_calls[7:11] == [
1296+
'DROP TABLE IF EXISTS "__temp_affected_partitions_abcd"',
1297+
"""ALTER TABLE "__temp_existing_records_abcd" REPLACE PARTITION ID '1' FROM "__temp_target_abcd", REPLACE PARTITION ID '2' FROM "__temp_target_abcd", REPLACE PARTITION ID '4' FROM "__temp_target_abcd", DROP PARTITION ID '3'""",
1298+
'DROP TABLE IF EXISTS "__temp_target_abcd"',
13271299
'CREATE TABLE "__temp_target_abcd" AS "__temp_existing_records_abcd"',
13281300
'INSERT INTO "__temp_target_abcd" ("id", "ds") SELECT "id", "ds" FROM (SELECT * FROM "__temp_new_records_abcd") AS "_subquery"',
13291301
'CREATE TABLE IF NOT EXISTS "__temp_affected_partitions_abcd" ENGINE=MergeTree ORDER BY () AS SELECT DISTINCT "partition_id" FROM (SELECT "_partition_id" AS "partition_id" FROM "__temp_existing_records_abcd" WHERE "id" IN (SELECT DISTINCT "id" FROM "__temp_target_abcd") UNION DISTINCT SELECT "_partition_id" AS "partition_id" FROM "__temp_target_abcd") AS "_affected_partitions"',
13301302
'INSERT INTO "__temp_target_abcd" SELECT "id", "ds" FROM "__temp_existing_records_abcd" WHERE NOT ("id" IN (SELECT DISTINCT "id" FROM "__temp_target_abcd")) AND "_partition_id" IN (SELECT "partition_id" FROM "__temp_affected_partitions_abcd")',
1303+
'DROP TABLE IF EXISTS "__temp_affected_partitions_abcd"',
1304+
"""ALTER TABLE "__temp_existing_records_abcd" REPLACE PARTITION ID '1' FROM "__temp_target_abcd", REPLACE PARTITION ID '2' FROM "__temp_target_abcd", REPLACE PARTITION ID '4' FROM "__temp_target_abcd", DROP PARTITION ID '3'""",
1305+
'DROP TABLE IF EXISTS "__temp_target_abcd"',
13311306
]
1332-
assert sql_calls[11] == 'DROP TABLE IF EXISTS "__temp_affected_partitions_abcd"'
1333-
assert sql_calls[12].startswith(alter_prefix)
1334-
assert sorted(sql_calls[12].removeprefix(alter_prefix).split(", ")) == sorted(
1335-
[
1336-
"REPLACE PARTITION ID '1' FROM \"__temp_target_abcd\"",
1337-
"REPLACE PARTITION ID '2' FROM \"__temp_target_abcd\"",
1338-
"REPLACE PARTITION ID '4' FROM \"__temp_target_abcd\"",
1339-
"DROP PARTITION ID '3'",
1340-
]
1341-
)
1342-
assert sql_calls[13] == 'DROP TABLE IF EXISTS "__temp_target_abcd"'
13431307

13441308

13451309
def test_insert_overwrite_by_condition_inc_by_partition(
@@ -1375,23 +1339,12 @@ def test_insert_overwrite_by_condition_inc_by_partition(
13751339
keep_existing_partition_rows=False,
13761340
)
13771341

1378-
sql_calls = to_sql_calls(adapter)
1379-
1380-
assert len(sql_calls) == 4
1381-
assert sql_calls[:2] == [
1342+
assert to_sql_calls(adapter) == [
13821343
'CREATE TABLE "__temp_target_abcd" AS "__temp_existing_records_abcd"',
13831344
'INSERT INTO "__temp_target_abcd" ("id", "ds") SELECT "id", "ds" FROM (SELECT * FROM "__temp_new_records_abcd") AS "_subquery"',
1345+
"""ALTER TABLE "__temp_existing_records_abcd" REPLACE PARTITION ID '1' FROM "__temp_target_abcd", REPLACE PARTITION ID '2' FROM "__temp_target_abcd", REPLACE PARTITION ID '4' FROM "__temp_target_abcd\"""",
1346+
'DROP TABLE IF EXISTS "__temp_target_abcd"',
13841347
]
1385-
alter_prefix = 'ALTER TABLE "__temp_existing_records_abcd" '
1386-
assert sql_calls[2].startswith(alter_prefix)
1387-
assert sorted(sql_calls[2].removeprefix(alter_prefix).split(", ")) == sorted(
1388-
[
1389-
"REPLACE PARTITION ID '1' FROM \"__temp_target_abcd\"",
1390-
"REPLACE PARTITION ID '2' FROM \"__temp_target_abcd\"",
1391-
"REPLACE PARTITION ID '4' FROM \"__temp_target_abcd\"",
1392-
]
1393-
)
1394-
assert sql_calls[3] == 'DROP TABLE IF EXISTS "__temp_target_abcd"'
13951348

13961349

13971350
def test_to_time_column():

0 commit comments

Comments
 (0)