Skip to content

Commit 7b576f1

Browse files
committed
perf: reuse table mapping during snapshot promotion
Signed-off-by: perjmi <perjmi@yahoo.com>
1 parent d69c262 commit 7b576f1

4 files changed

Lines changed: 181 additions & 9 deletions

File tree

‎sqlmesh/core/renderer.py‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ def _render(
149149
table_mapping: t.Optional[t.Dict[str, str]] = None,
150150
deployability_index: t.Optional[DeployabilityIndex] = None,
151151
runtime_stage: RuntimeStage = RuntimeStage.LOADING,
152+
render_table_mapping: t.Optional[TableMapping] = None,
152153
**kwargs: t.Any,
153154
) -> t.List[t.Optional[exp.Expr]]:
154155
"""Renders a expression, expanding macros with provided kwargs
@@ -161,6 +162,7 @@ def _render(
161162
table_mapping: Table mapping of physical locations. Takes precedence over snapshot mappings.
162163
deployability_index: Determines snapshots that are deployable in the context of this evaluation.
163164
runtime_stage: Indicates the current runtime stage, for example if we're still loading the project, etc.
165+
render_table_mapping: Shared snapshot mapping and overrides prepared for this promotion.
164166
kwargs: Additional kwargs to pass to the renderer.
165167
166168
Returns:
@@ -232,6 +234,7 @@ def _resolve_table(table: str | exp.Table) -> str:
232234
end=end,
233235
execution_time=execution_time,
234236
runtime_stage=runtime_stage,
237+
render_table_mapping=render_table_mapping,
235238
),
236239
snapshots=snapshots,
237240
default_catalog=self._default_catalog,
@@ -430,6 +433,7 @@ def _resolve_tables(
430433
table_mapping: t.Optional[t.Dict[str, str]] = None,
431434
expand: t.Iterable[str] = tuple(),
432435
deployability_index: t.Optional[DeployabilityIndex] = None,
436+
render_table_mapping: t.Optional[TableMapping] = None,
433437
**kwargs: t.Any,
434438
) -> E:
435439
if not snapshots and not table_mapping and not expand:
@@ -444,10 +448,14 @@ def _resolve_tables(
444448

445449
snapshots = snapshots or {}
446450
table_mapping = table_mapping or {}
447-
mapping = {
448-
**self._to_table_mapping(snapshots.values(), deployability_index),
449-
**table_mapping,
450-
}
451+
mapping: t.Dict[str, str] = (
452+
render_table_mapping
453+
if render_table_mapping is not None
454+
else {
455+
**self._to_table_mapping(snapshots.values(), deployability_index),
456+
**table_mapping,
457+
}
458+
)
451459
expand = set(expand) | {
452460
name for name, snapshot in snapshots.items() if snapshot.is_embedded
453461
}
@@ -477,6 +485,7 @@ def _expand(node: exp.Expr) -> exp.Expr:
477485
table_mapping=table_mapping,
478486
expand=expand,
479487
deployability_index=deployability_index,
488+
render_table_mapping=render_table_mapping,
480489
**kwargs,
481490
)
482491
if nested_query is not None:
@@ -489,6 +498,20 @@ def _expand(node: exp.Expr) -> exp.Expr:
489498

490499
expression = expression.transform(_expand, copy=False) # type: ignore
491500

501+
if isinstance(mapping, TableMapping):
502+
# Normalize the shared mapping once, then replace only tables in this expression.
503+
normalized_keys = mapping.normalized_keys(self._dialect)
504+
mapping = {
505+
key: mapping[key]
506+
for table in expression.find_all(exp.Table)
507+
if (
508+
key := normalized_keys.get(
509+
exp.normalize_table_name(table, dialect=self._dialect)
510+
)
511+
)
512+
is not None
513+
}
514+
492515
if mapping:
493516
expression = exp.replace_tables(
494517
expression, mapping, dialect=self._dialect, copy=False

‎sqlmesh/core/snapshot/evaluator.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
from sqlmesh.core.engine_adapter.shared import InsertOverwriteStrategy, DataObjectType, DataObject
4242
from sqlmesh.core.model.meta import GrantsTargetLayer
4343
from sqlmesh.core.macros import RuntimeStage
44+
from sqlmesh.core.renderer import TableMapping
4445
from sqlmesh.core.model import (
4546
AuditResult,
4647
IncrementalUnmanagedKind,
@@ -286,6 +287,8 @@ def promote(
286287
on_complete: A callback to call on each successfully promoted snapshot.
287288
"""
288289

290+
from sqlmesh.core.snapshot import to_table_mapping
291+
289292
tables_by_gateway: t.Dict[t.Union[str, None], t.List[exp.Table]] = defaultdict(list)
290293
for snapshot in target_snapshots:
291294
if snapshot.is_model and not snapshot.is_symbolic:
@@ -314,6 +317,16 @@ def promote(
314317
deployability_index = deployability_index or DeployabilityIndex.all_deployable()
315318
# Renderers look snapshots up by model name, not by SnapshotId.
316319
snapshots_by_name = {s.name: s for s in (snapshots or {}).values()}
320+
# Share the physical mapping and overrides across all promotion renders and workers.
321+
render_table_mapping = TableMapping(
322+
{
323+
**to_table_mapping(snapshots_by_name.values(), deployability_index),
324+
**(table_mapping or {}),
325+
}
326+
)
327+
for snapshot in target_snapshots:
328+
if snapshot.is_model:
329+
render_table_mapping.normalized_keys(snapshot.model.dialect)
317330
with self.concurrent_context():
318331
concurrent_apply_to_snapshots(
319332
target_snapshots,
@@ -324,6 +337,7 @@ def promote(
324337
execution_time=execution_time,
325338
snapshots=snapshots_by_name,
326339
table_mapping=table_mapping,
340+
render_table_mapping=render_table_mapping,
327341
environment_naming_info=environment_naming_info,
328342
deployability_index=deployability_index, # type: ignore
329343
on_complete=on_complete,
@@ -1264,6 +1278,7 @@ def _promote_snapshot(
12641278
execution_time: t.Optional[TimeLike] = None,
12651279
snapshots: t.Optional[t.Dict[str, Snapshot]] = None,
12661280
table_mapping: t.Optional[t.Dict[str, str]] = None,
1281+
render_table_mapping: t.Optional[TableMapping] = None,
12671282
) -> None:
12681283
if not snapshot.is_model:
12691284
return
@@ -1298,11 +1313,16 @@ def _promote_snapshot(
12981313
environment=environment_naming_info.name,
12991314
snapshots=snapshots,
13001315
snapshot=snapshot,
1316+
render_table_mapping=render_table_mapping,
13011317
**render_kwargs,
13021318
)
13031319

13041320
adapter.execute(
1305-
snapshot.model.render_on_virtual_update(snapshots=snapshots, **render_kwargs)
1321+
snapshot.model.render_on_virtual_update(
1322+
snapshots=snapshots,
1323+
render_table_mapping=render_table_mapping,
1324+
**render_kwargs,
1325+
)
13061326
)
13071327

13081328
if on_complete is not None:
@@ -2016,6 +2036,7 @@ def promote(
20162036
snapshots=kwargs.get("snapshots"),
20172037
deployability_index=kwargs.get("deployability_index"),
20182038
table_mapping=kwargs.get("table_mapping"),
2039+
render_table_mapping=kwargs.get("render_table_mapping"),
20192040
runtime_stage=kwargs.get("runtime_stage"),
20202041
)
20212042
self.adapter.create_view(

‎tests/core/test_model.py‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from sqlglot.schema import MappingSchema
1616
from sqlmesh.cli.project_init import init_example_project, ProjectTemplate
1717
from sqlmesh.core.environment import EnvironmentNamingInfo
18-
from sqlmesh.core.renderer import TableMapping
18+
from sqlmesh.core.renderer import ExpressionRenderer, TableMapping
1919
from sqlmesh.core.model.kind import TimeColumn, ModelKindName, SeedKind
2020

2121
from sqlmesh import CustomMaterialization, CustomKind
@@ -9900,7 +9900,10 @@ def resolve_named(evaluator, name):
99009900
)
99019901

99029902

9903-
def test_resolve_tables_expand_reveals_table_after_find_check(make_snapshot: t.Callable):
9903+
@pytest.mark.parametrize("use_cached_mapping", [False, True])
9904+
def test_resolve_tables_expand_reveals_table_after_find_check(
9905+
make_snapshot: t.Callable, use_cached_mapping: bool
9906+
):
99049907
"""Embedded-model expansion (`expand=`) runs as an `expression.transform` *before* the new
99059908
`expression.find(exp.Table)` short-circuit in `_resolve_tables`, so a table reference that
99069909
only exists *after* inlining an embedded model's query must still be seen by `find()` and
@@ -9909,6 +9912,8 @@ def test_resolve_tables_expand_reveals_table_after_find_check(make_snapshot: t.C
99099912
must still resolve to its physical table name, not be silently skipped because it wasn't
99109913
present at the time `_resolve_tables` was first called."""
99119914

9915+
from sqlmesh.core.snapshot import to_table_mapping
9916+
99129917
grandparent = load_sql_based_model(d.parse("MODEL (name grandparent); SELECT 1 AS c"))
99139918
grandparent_snapshot = make_snapshot(grandparent)
99149919
grandparent_snapshot.categorize_as(SnapshotChangeCategory.BREAKING)
@@ -9922,7 +9927,12 @@ def test_resolve_tables_expand_reveals_table_after_find_check(make_snapshot: t.C
99229927
child = load_sql_based_model(d.parse("MODEL (name child); SELECT c FROM mid;"))
99239928

99249929
snapshots = {'"grandparent"': grandparent_snapshot, '"mid"': mid_snapshot}
9925-
query = child.render_query(snapshots=snapshots)
9930+
query = child.render_query(
9931+
snapshots=snapshots,
9932+
render_table_mapping=(
9933+
TableMapping(to_table_mapping(snapshots.values(), None)) if use_cached_mapping else None
9934+
),
9935+
)
99269936
assert query is not None
99279937
rendered_sql = query.sql()
99289938

@@ -10071,6 +10081,28 @@ def items(self):
1007110081
assert ItemsCountingDict.items_call_count == 0
1007210082

1007310083

10084+
@pytest.mark.parametrize(
10085+
"dialect",
10086+
[
10087+
"duckdb,normalization_strategy=case_insensitive",
10088+
"snowflake,normalization_strategy=uppercase",
10089+
],
10090+
)
10091+
def test_cached_table_mapping_preserves_key_order(dialect):
10092+
# Updating an existing raw key must not move it after a later dialect-equivalent key.
10093+
table_mapping = {'"DB"."T"': "early_table", "DB.T": "last_table"}
10094+
table_mapping.update({'"DB"."T"': "override_table"})
10095+
renderer = ExpressionRenderer(parse_one("SELECT * FROM DB.T", dialect=dialect), dialect, [])
10096+
10097+
uncached = renderer.render(table_mapping=table_mapping)
10098+
cached = renderer.render(
10099+
table_mapping=table_mapping, render_table_mapping=TableMapping(table_mapping)
10100+
)
10101+
10102+
assert cached == uncached
10103+
assert exp.table_name(cached[0].find(exp.Table)) == "last_table"
10104+
10105+
1007410106
def test_table_mapping_normalized_keys():
1007510107
table_mapping = TableMapping({'"db"."a"': "view_a", "db.A": "view_a_upper"})
1007610108

@@ -10414,7 +10446,8 @@ def dummy_model_entry(evaluator: MacroEvaluator) -> exp.Select:
1041410446
assert len(context.engine_adapters) == 2
1041510447

1041610448

10417-
def test_model_on_virtual_update(make_snapshot: t.Callable):
10449+
@pytest.mark.parametrize("use_cached_mapping", [False, True])
10450+
def test_model_on_virtual_update(make_snapshot: t.Callable, use_cached_mapping: bool):
1041810451
# Macro to test resolution within virtual statement
1041910452
@macro()
1042010453
def resolve_parent_name(evaluator, name):
@@ -10495,6 +10528,7 @@ def resolve_parent_name(evaluator, name):
1049510528
rendered_on_virtual_update = model.render_on_virtual_update(
1049610529
snapshots=snapshots,
1049710530
table_mapping=table_mapping,
10531+
render_table_mapping=TableMapping(table_mapping) if use_cached_mapping else None,
1049810532
this_model=model_snapshot.qualified_view_name.table_for_environment(
1049910533
environment_naming_info, dialect=dialect
1050010534
),
@@ -10526,6 +10560,7 @@ def resolve_parent_name(evaluator, name):
1052610560
rendered_parent_on_virtual_update = parent.render_on_virtual_update(
1052710561
snapshots=snapshots,
1052810562
table_mapping=table_mapping,
10563+
render_table_mapping=TableMapping(table_mapping) if use_cached_mapping else None,
1052910564
this_model=parent_snapshot.qualified_view_name.table_for_environment(
1053010565
environment_naming_info, dialect=dialect
1053110566
),

‎tests/core/test_snapshot_evaluator.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
SnapshotTableCleanupTask,
5959
)
6060
from sqlmesh.core.snapshot.definition import to_view_mapping
61+
from sqlmesh.core import snapshot as snapshot_module
6162
from sqlmesh.core.snapshot.evaluator import (
6263
CustomMaterialization,
6364
EngineManagedStrategy,
@@ -5766,3 +5767,95 @@ def test_promote_resolves_this_model_with_single_mapping_entry(
57665767
assert spy.call_count == 20
57675768
for call in spy.call_args_list:
57685769
assert len(call.args[1]) == 1
5770+
5771+
5772+
@pytest.mark.parametrize("ddl_concurrent_tasks", [1, 4])
5773+
@pytest.mark.parametrize("use_view_mapping", [False, True])
5774+
def test_promote_reuses_table_mapping(
5775+
mocker: MockerFixture, make_snapshot, ddl_concurrent_tasks, use_view_mapping
5776+
):
5777+
@macro()
5778+
def resolve_upstream_query(evaluator):
5779+
return evaluator.resolve_tables(
5780+
exp.select("COUNT(*)").from_("test_schema.model_0").subquery()
5781+
)
5782+
5783+
adapter_mock = mocker.patch("sqlmesh.core.engine_adapter.EngineAdapter")
5784+
adapter_mock.dialect = "duckdb"
5785+
adapter_mock.with_settings.return_value = adapter_mock
5786+
evaluator = SnapshotEvaluator(adapter_mock, ddl_concurrent_tasks=ddl_concurrent_tasks)
5787+
5788+
snapshots = {}
5789+
for i in range(10):
5790+
model = load_sql_based_model(
5791+
d.parse(
5792+
f"""
5793+
MODEL (
5794+
name test_schema.model_{i},
5795+
kind FULL,
5796+
virtual_properties (
5797+
upstream_count = (SELECT COUNT(*) FROM test_schema.model_0),
5798+
macro_count = @resolve_upstream_query()
5799+
),
5800+
);
5801+
SELECT 1 AS a;
5802+
ON_VIRTUAL_UPDATE_BEGIN;
5803+
GRANT SELECT ON VIEW test_schema.model_0 TO ROLE owner;
5804+
ON_VIRTUAL_UPDATE_END;
5805+
"""
5806+
)
5807+
)
5808+
snapshot = make_snapshot(model)
5809+
snapshot.categorize_as(SnapshotChangeCategory.BREAKING)
5810+
snapshots[snapshot.snapshot_id] = snapshot
5811+
5812+
upstream = next(iter(snapshots.values()))
5813+
upstream.dev_version_ = "preview"
5814+
# Representative snapshots use the production table even when they are not deployable.
5815+
deployability_index = DeployabilityIndex(
5816+
indexed_ids=frozenset(), representative_shared_version_ids={upstream.snapshot_id}
5817+
)
5818+
mapping_spy = mocker.spy(snapshot_module, "to_table_mapping")
5819+
replacement_spy = mocker.spy(exp, "replace_tables")
5820+
5821+
for environment_name in ("first", "second"):
5822+
environment_naming_info = EnvironmentNamingInfo(name=environment_name)
5823+
view_mapping = (
5824+
to_view_mapping(snapshots.values(), environment_naming_info)
5825+
if use_view_mapping
5826+
else None
5827+
)
5828+
expected_table = (
5829+
view_mapping[upstream.name]
5830+
if view_mapping is not None
5831+
else upstream.table_name(deployability_index.is_representative(upstream))
5832+
)
5833+
mapping_spy.reset_mock()
5834+
replacement_spy.reset_mock()
5835+
adapter_mock.create_view.reset_mock()
5836+
adapter_mock.execute.reset_mock()
5837+
5838+
evaluator.promote(
5839+
list(snapshots.values()),
5840+
environment_naming_info,
5841+
snapshots=snapshots,
5842+
table_mapping=view_mapping,
5843+
deployability_index=deployability_index,
5844+
)
5845+
5846+
full_mapping_calls = [
5847+
call for call in mapping_spy.call_args_list if len(call.args[0]) == len(snapshots)
5848+
]
5849+
assert len(full_mapping_calls) == 1
5850+
assert all(len(call.args[1]) <= 1 for call in replacement_spy.call_args_list)
5851+
assert adapter_mock.create_view.call_count == len(snapshots)
5852+
for call in adapter_mock.create_view.call_args_list:
5853+
for query in call.kwargs["view_properties"].values():
5854+
assert exp.table_name(query.find(exp.Table)) == expected_table
5855+
for call in adapter_mock.execute.call_args_list:
5856+
for statement in call.args[0]:
5857+
assert exp.table_name(statement.find(exp.Table)) == expected_table
5858+
5859+
# A later promotion must see changed versions, deployability and environment overrides.
5860+
upstream.version = "updated"
5861+
deployability_index = DeployabilityIndex.none_deployable()

0 commit comments

Comments
 (0)