Skip to content

Commit 1c7d73a

Browse files
anxkhnEhabEasee
authored andcommitted
fix(dlt): use timestamp macros for incremental model time filter (#5867)
Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> Signed-off-by: EhabEasee <ehab.elbadrawi@easee.com>
1 parent 79b9d1c commit 1c7d73a

3 files changed

Lines changed: 26 additions & 5 deletions

File tree

‎sqlmesh/integrations/dlt.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ def generate_incremental_model(
194194
FROM
195195
{from_clause}
196196
WHERE
197-
{time_column} BETWEEN @start_ds AND @end_ds
197+
{time_column} BETWEEN @start_ts AND @end_ts
198198
"""
199199

200200

‎tests/cli/test_cli.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -921,7 +921,7 @@ def test_dlt_filesystem_pipeline(tmp_path):
921921
FROM
922922
filesystem_pipeline_dataset.equipment as c
923923
WHERE
924-
TO_TIMESTAMP(CAST(c._dlt_load_id AS DOUBLE)) BETWEEN @start_ds AND @end_ds
924+
TO_TIMESTAMP(CAST(c._dlt_load_id AS DOUBLE)) BETWEEN @start_ts AND @end_ts
925925
"""
926926

927927
with open(equipment_model_path) as file:
@@ -1058,7 +1058,7 @@ def test_dlt_pipeline(runner, tmp_path):
10581058
FROM
10591059
sushi_dataset.sushi_types as c
10601060
WHERE
1061-
TO_TIMESTAMP(CAST(c._dlt_load_id AS DOUBLE)) BETWEEN @start_ds AND @end_ds
1061+
TO_TIMESTAMP(CAST(c._dlt_load_id AS DOUBLE)) BETWEEN @start_ts AND @end_ts
10621062
"""
10631063

10641064
dlt_sushi_types_model_path = tmp_path / "models/incremental_sushi_types.sql"
@@ -1089,7 +1089,7 @@ def test_dlt_pipeline(runner, tmp_path):
10891089
FROM
10901090
sushi_dataset._dlt_loads as c
10911091
WHERE
1092-
TO_TIMESTAMP(CAST(c.load_id AS DOUBLE)) BETWEEN @start_ds AND @end_ds
1092+
TO_TIMESTAMP(CAST(c.load_id AS DOUBLE)) BETWEEN @start_ts AND @end_ts
10931093
"""
10941094

10951095
with open(dlt_loads_model_path) as file:
@@ -1116,7 +1116,7 @@ def test_dlt_pipeline(runner, tmp_path):
11161116
ON
11171117
c._dlt_parent_id = p._dlt_id
11181118
WHERE
1119-
TO_TIMESTAMP(CAST(p._dlt_load_id AS DOUBLE)) BETWEEN @start_ds AND @end_ds
1119+
TO_TIMESTAMP(CAST(p._dlt_load_id AS DOUBLE)) BETWEEN @start_ts AND @end_ts
11201120
"""
11211121

11221122
with open(dlt_sushi_fillings_model_path) as file:

‎tests/integrations/test_dlt.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
from sqlmesh.integrations.dlt import generate_incremental_model
2+
3+
4+
def test_generate_incremental_model_filters_on_timestamp_macros() -> None:
5+
# The DLT-generated model's time column is a timestamp
6+
# (TO_TIMESTAMP(...)). It must therefore be filtered with the inclusive
7+
# timestamp macros @start_ts/@end_ts, not the categorical date macros
8+
# @start_ds/@end_ds, which both render midnight and exclude any rows past
9+
# 00:00:00 on a single-day run.
10+
model = generate_incremental_model(
11+
"dataset_sqlmesh.incremental_equipment",
12+
" CAST(c.item_id AS BIGINT) AS item_id",
13+
"",
14+
"dataset.equipment",
15+
"duckdb",
16+
"c._dlt_load_id",
17+
)
18+
19+
assert "BETWEEN @start_ts AND @end_ts" in model
20+
assert "@start_ds" not in model
21+
assert "@end_ds" not in model

0 commit comments

Comments
 (0)