Skip to content

Commit 7bc1fa3

Browse files
committed
Merge branch 'main' into feature/DRM/indirect-non-breaking-always-uses-prod-ledger
2 parents 960603b + f0ed98e commit 7bc1fa3

6 files changed

Lines changed: 76 additions & 28 deletions

File tree

‎.github/workflows/pr.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,10 @@ jobs:
150150
. ./venv/Scripts/activate
151151
python.exe -m pip install --upgrade pip
152152
make install-dev
153+
- name: Install DuckLake extension
154+
run: |
155+
. ./venv/Scripts/activate
156+
python -c "import duckdb; duckdb.connect().install_extension('ducklake')"
153157
- name: Run fast unit tests
154158
run: |
155159
. ./venv/Scripts/activate

‎sqlmesh/core/metric/definition.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,10 @@ def is_measure(node: exp.Expr) -> bool:
199199
return is_measure(parent)
200200
if isinstance(parent, (exp.Binary, exp.Paren, exp.Distinct)):
201201
return is_measure(parent)
202+
# AGG(x ORDER BY y) is parsed as AGG(Order(this=x, expressions=[Ordered(y)])).
203+
# Only the Order's `this` can be the measure, not the ORDER BY expressions.
204+
if isinstance(parent, exp.Order) and node.arg_key == "this":
205+
return is_measure(parent)
202206
return False
203207

204208
for node in expression.walk():

‎tests/core/engine_adapter/test_duckdb.py‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -126,17 +126,25 @@ def test_drop_catalog_motherduck(make_mocked_engine_adapter: t.Callable) -> None
126126
assert to_sql_calls(adapter) == ['DROP DATABASE IF EXISTS "foo" CASCADE']
127127

128128

129-
def test_ducklake_partitioning(adapter: EngineAdapter, duck_conn, tmp_path):
129+
@pytest.fixture
130+
def ducklake_conn(duck_conn):
131+
# Don't call install_extension here: on Windows it fails with "Access is denied"
132+
# once an earlier test in this process has loaded DuckLake. LOAD auto-installs it
133+
# when missing, and Windows CI pre-installs it before pytest.
134+
duck_conn.load_extension("ducklake")
135+
return duck_conn
136+
137+
138+
@pytest.mark.xdist_group("ducklake_extension")
139+
def test_ducklake_partitioning(adapter: EngineAdapter, ducklake_conn, tmp_path):
130140
catalog = "a_ducklake_db"
131141

132-
duck_conn.install_extension("ducklake")
133-
duck_conn.load_extension("ducklake")
134-
duck_conn.execute(
142+
ducklake_conn.execute(
135143
f"ATTACH 'ducklake:{tmp_path}/{catalog}.ducklake' AS {catalog} (DATA_PATH '{tmp_path}');"
136144
)
137145

138146
# no partitions on catalog creation
139-
partition_info = duck_conn.execute(
147+
partition_info = ducklake_conn.execute(
140148
f"SELECT * FROM __ducklake_metadata_{catalog}.main.ducklake_partition_info"
141149
).fetchdf()
142150
assert partition_info.empty
@@ -150,31 +158,30 @@ def test_ducklake_partitioning(adapter: EngineAdapter, duck_conn, tmp_path):
150158
)
151159

152160
# 1 partition after table creation
153-
partition_info = duck_conn.execute(
161+
partition_info = ducklake_conn.execute(
154162
f"SELECT * FROM __ducklake_metadata_{catalog}.main.ducklake_partition_info"
155163
).fetchdf()
156164
assert partition_info.shape[0] == 1
157165

158166

159-
def test_drop_table_ducklake_no_cascade(adapter: EngineAdapter, duck_conn, tmp_path):
167+
@pytest.mark.xdist_group("ducklake_extension")
168+
def test_drop_table_ducklake_no_cascade(adapter: EngineAdapter, ducklake_conn, tmp_path):
160169
# DuckLake does not implement DROP TABLE/VIEW ... CASCADE, so the adapter must
161170
# omit CASCADE for objects in a DuckLake catalog while keeping it for native catalogs.
162171
catalog = "a_ducklake_db"
163172

164-
duck_conn.install_extension("ducklake")
165-
duck_conn.load_extension("ducklake")
166-
duck_conn.execute(
173+
ducklake_conn.execute(
167174
f"ATTACH 'ducklake:{tmp_path}/{catalog}.ducklake' AS {catalog} (DATA_PATH '{tmp_path}');"
168175
)
169176

170-
duck_conn.execute(f"CREATE SCHEMA {catalog}.phys")
171-
duck_conn.execute(f"CREATE SCHEMA {catalog}.virt")
172-
duck_conn.execute(f"CREATE TABLE {catalog}.phys.t (i INTEGER)")
173-
duck_conn.execute(f"CREATE VIEW {catalog}.virt.v AS SELECT * FROM {catalog}.phys.t")
177+
ducklake_conn.execute(f"CREATE SCHEMA {catalog}.phys")
178+
ducklake_conn.execute(f"CREATE SCHEMA {catalog}.virt")
179+
ducklake_conn.execute(f"CREATE TABLE {catalog}.phys.t (i INTEGER)")
180+
ducklake_conn.execute(f"CREATE VIEW {catalog}.virt.v AS SELECT * FROM {catalog}.phys.t")
174181

175182
# native catalog, cascade is passed through
176-
duck_conn.execute("CREATE TABLE memory.main.native_t (i INTEGER)")
177-
duck_conn.execute("CREATE VIEW memory.main.native_v AS SELECT * FROM memory.main.native_t")
183+
ducklake_conn.execute("CREATE TABLE memory.main.native_t (i INTEGER)")
184+
ducklake_conn.execute("CREATE VIEW memory.main.native_v AS SELECT * FROM memory.main.native_t")
178185

179186
adapter.drop_table(f"{catalog}.phys.t", cascade=True)
180187
adapter.drop_view(f"{catalog}.virt.v", cascade=True)
@@ -187,19 +194,18 @@ def test_drop_table_ducklake_no_cascade(adapter: EngineAdapter, duck_conn, tmp_p
187194
assert not adapter.table_exists("memory.main.native_v")
188195

189196

197+
@pytest.mark.xdist_group("ducklake_extension")
190198
def test_drop_table_ducklake_no_cascade_mixed_case_alias(
191-
adapter: EngineAdapter, duck_conn, tmp_path
199+
adapter: EngineAdapter, ducklake_conn, tmp_path
192200
):
193201
# Catalog aliases keep their case in duckdb_databases(), but SQLMesh normalizes
194202
# table names to lowercase, so the catalog type lookup must be case-insensitive.
195-
duck_conn.install_extension("ducklake")
196-
duck_conn.load_extension("ducklake")
197-
duck_conn.execute(
203+
ducklake_conn.execute(
198204
f"ATTACH 'ducklake:{tmp_path}/my_lake.ducklake' AS \"MyLake\" (DATA_PATH '{tmp_path}');"
199205
)
200-
duck_conn.execute("CREATE SCHEMA MyLake.phys")
201-
duck_conn.execute("CREATE TABLE MyLake.phys.t (i INTEGER)")
202-
duck_conn.execute("CREATE VIEW MyLake.phys.v AS SELECT * FROM MyLake.phys.t")
206+
ducklake_conn.execute("CREATE SCHEMA MyLake.phys")
207+
ducklake_conn.execute("CREATE TABLE MyLake.phys.t (i INTEGER)")
208+
ducklake_conn.execute("CREATE VIEW MyLake.phys.v AS SELECT * FROM MyLake.phys.t")
203209

204210
adapter.drop_table("mylake.phys.t", cascade=True)
205211
adapter.drop_view("mylake.phys.v", cascade=True)

‎tests/core/linter/test_helpers.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
from sqlmesh.core.model import SqlModel
88

99

10-
def test_get_position_of_model_block():
11-
context = Context(paths=["examples/sushi"])
10+
def test_get_position_of_model_block(copy_to_temp_path):
11+
context = Context(paths=copy_to_temp_path("examples/sushi"))
1212

1313
sql_models = [
1414
model
@@ -40,8 +40,8 @@ def test_get_position_of_model_block():
4040
assert read_range.endswith(";")
4141

4242

43-
def test_get_range_of_a_key_in_model_block_testing_on_sushi():
44-
context = Context(paths=["examples/sushi"])
43+
def test_get_range_of_a_key_in_model_block_testing_on_sushi(copy_to_temp_path):
44+
context = Context(paths=copy_to_temp_path("examples/sushi"))
4545

4646
sql_models = [
4747
model

‎tests/core/metric/test_metric.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,3 +154,11 @@ def test_get_measure_and_dim_tables():
154154
"a",
155155
("b",),
156156
)
157+
assert _get_measure_and_dim_tables(d.parse_one("ARRAY_AGG(a.x ORDER BY a.x)")) == ("a", ())
158+
assert _get_measure_and_dim_tables(d.parse_one("ARRAY_AGG(a.x ORDER BY b.y)")) == ("a", ("b",))
159+
assert _get_measure_and_dim_tables(
160+
d.parse_one("ARRAY_AGG(IF(c.z = 'dim', a.x, NULL) ORDER BY b.y)")
161+
) == ("a", ("b", "c"))
162+
# A column that only appears in the ORDER BY clause cannot be the measure.
163+
with pytest.raises(ConfigError, match="Could not infer a measures table"):
164+
_get_measure_and_dim_tables(d.parse_one("ARRAY_AGG(1 ORDER BY b.y)"))

‎tests/core/metric/test_rewriter.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
from sqlglot import parse_one
22

3-
from sqlmesh.core.metric import rewrite
3+
from sqlmesh.core import dialect as d
4+
from sqlmesh.core.metric import expand_metrics, load_metric_ddl, rewrite
45
from sqlmesh.core.reference import ReferenceGraph
6+
from sqlmesh.utils import UniqueKeyDict
57

68

79
def test_rewrite(sushi_context_pre_scheduling, assert_exp_eq):
@@ -222,3 +224,27 @@ def test_rewrite(sushi_context_pre_scheduling, assert_exp_eq):
222224
# dialect=context.config.dialect,
223225
# ),
224226
# )
227+
228+
229+
def test_rewrite_order_by_metric(sushi_context_pre_scheduling):
230+
context = sushi_context_pre_scheduling
231+
graph = ReferenceGraph(context.models.values())
232+
meta = load_metric_ddl(
233+
d.parse_one(
234+
"METRIC (name sorted_amounts, expression ARRAY_AGG(sushi.orders.amount ORDER BY sushi.orders.amount))",
235+
dialect=context.config.dialect,
236+
),
237+
dialect=context.config.dialect,
238+
)
239+
metrics = expand_metrics(UniqueKeyDict("metas", {meta.name: meta}))
240+
241+
query = rewrite(
242+
"SELECT c.customer_id, METRIC(sorted_amounts) FROM sushi.customers AS c GROUP BY c.customer_id",
243+
graph=graph,
244+
metrics=metrics,
245+
)
246+
sql = query.sql()
247+
# Stop before the generated NULLS LAST so the assertion doesn't depend on null ordering.
248+
assert "ARRAY_AGG(sushi__orders.amount ORDER BY sushi__orders.amount" in sql
249+
assert "sushi.orders AS sushi__orders" in sql
250+
parse_one(sql, dialect=context.config.dialect)

0 commit comments

Comments
 (0)