Skip to content

Commit fb98872

Browse files
fix(run): clarify unfinalized environment waits
An unfinalized environment can outlive a failed plan. Report the stored state without asserting that the plan process is active, and explain the configured wait and recovery path while retaining the guard. Generated-by: OpenAI Codex Signed-off-by: bosprimigenious <bosprimigenious@foxmail.com>
1 parent c4dbac4 commit fb98872

3 files changed

Lines changed: 93 additions & 4 deletions

File tree

‎docs/reference/configuration.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ Configuration for the `sqlmesh run` command. Please note that this is only appli
112112
| `environment_check_interval` | The number of seconds to wait between attempts to check the target environment for readiness (Default: 30 seconds) | int | N |
113113
| `environment_check_max_wait` | The maximum number of seconds to wait for the target environment to be ready (Default: 6 hours) | int | N |
114114

115+
`sqlmesh run` checks the target environment's persisted finalization state. This check does not establish whether a plan process is active. A failed promotion can leave the environment unfinalized, so `run` continues to wait. If the plan failed, inspect its logs, resolve the errors, and successfully reapply it with `sqlmesh plan <environment>` before retrying the run.
116+
115117
## Format
116118

117119
Formatting settings for the `sqlmesh format` command and UI.

‎sqlmesh/core/context.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -920,13 +920,18 @@ def _block_until_finalized() -> str:
920920
if environment_state.finalized_ts:
921921
return environment_state.plan_id
922922
self.console.log_warning(
923-
f"Environment '{environment}' is being updated by plan '{environment_state.plan_id}'. "
924-
f"Retrying in {self.config.run.environment_check_interval} seconds..."
923+
f"Environment '{environment}' is not finalized after plan '{environment_state.plan_id}'. "
924+
"The plan may still be running or may have failed. "
925+
"Waiting for environment finalization, "
926+
f"retrying in {self.config.run.environment_check_interval} seconds "
927+
f"(configured maximum wait: {self.config.run.environment_check_max_wait} seconds)."
925928
)
926929
time.sleep(self.config.run.environment_check_interval)
927930
raise SQLMeshError(
928-
f"Exceeded the maximum wait time for environment '{environment}' to be ready. "
929-
"This means that the environment either failed to update or the update is taking longer than expected. "
931+
f"Exceeded the maximum wait time for environment '{environment}' to finalize "
932+
f"(configured maximum wait: {self.config.run.environment_check_max_wait} seconds). "
933+
"The plan may still be running or may have failed. Inspect the plan logs and, if it failed, "
934+
f"resolve its errors and reapply it with `sqlmesh plan {environment}` to finalize the environment. "
930935
"See https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#run to adjust the timeout settings."
931936
)
932937

‎tests/core/integration/test_run.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,108 @@
11
from __future__ import annotations
22

33
import typing as t
4+
from pathlib import Path
45
import pytest
56
import time_machine
67
from pytest_mock.plugin import MockerFixture
78

89
from sqlmesh.core import constants as c
910
from sqlmesh.core import dialect as d
1011
from sqlmesh.core.config.categorizer import CategorizerConfig
12+
from sqlmesh.core.config import (
13+
Config,
14+
DuckDBConnectionConfig,
15+
GatewayConfig,
16+
ModelDefaultsConfig,
17+
RunConfig,
18+
)
19+
from sqlmesh.core.context import Context
1120
from sqlmesh.core.model import (
1221
SqlModel,
1322
PythonModel,
1423
load_sql_based_model,
1524
)
1625
from sqlmesh.utils.date import to_timestamp
26+
from sqlmesh.utils.concurrency import NodeExecutionFailedError
27+
from sqlmesh.utils.errors import SQLMeshError
1728

1829
if t.TYPE_CHECKING:
1930
pass
2031

2132
pytestmark = pytest.mark.slow
2233

2334

35+
def test_run_waits_for_finalization_after_failed_plan(tmp_path: Path, mocker: MockerFixture):
36+
model_path = tmp_path / "models" / "example.sql"
37+
model_path.parent.mkdir()
38+
original_model = "MODEL (name repro.example, kind FULL); SELECT 1::INT AS id;"
39+
updated_model = original_model.replace("SELECT 1", "SELECT 2")
40+
model_path.write_text(original_model)
41+
context = Context(
42+
paths=tmp_path,
43+
config=Config(
44+
gateways={
45+
"local": GatewayConfig(
46+
connection=DuckDBConnectionConfig(database=str(tmp_path / "repro.duckdb"))
47+
)
48+
},
49+
default_gateway="local",
50+
model_defaults=ModelDefaultsConfig(dialect="duckdb"),
51+
disable_anonymized_analytics=True,
52+
run=RunConfig(environment_check_interval=1, environment_check_max_wait=2),
53+
),
54+
)
55+
try:
56+
context.plan("prod", auto_apply=True, no_prompts=True)
57+
initial_environment = context.state_sync.get_environment("prod")
58+
assert initial_environment is not None and initial_environment.finalized_ts
59+
60+
model_path.write_text(
61+
updated_model
62+
+ "\nON_VIRTUAL_UPDATE_BEGIN;"
63+
+ "\nSELECT CAST('intentional_promotion_failure' AS INTEGER);"
64+
+ "\nON_VIRTUAL_UPDATE_END;"
65+
)
66+
context.load()
67+
with pytest.raises(NodeExecutionFailedError):
68+
context.plan("prod", auto_apply=True, no_prompts=True)
69+
70+
failed_environment = context.state_sync.get_environment("prod")
71+
assert failed_environment is not None
72+
assert failed_environment.finalized_ts is None
73+
run = mocker.spy(context, "_run")
74+
warnings = mocker.spy(context.console, "log_warning")
75+
with pytest.raises(SQLMeshError, match="Exceeded the maximum wait time") as exc_info:
76+
context.run("prod", skip_janitor=True)
77+
78+
run.assert_not_called()
79+
assert context.state_sync.get_environment("prod") == failed_environment
80+
assert warnings.call_count == 2
81+
82+
model_path.write_text(updated_model)
83+
context.load()
84+
context.plan("prod", auto_apply=True, no_prompts=True)
85+
recovered_environment = context.state_sync.get_environment("prod")
86+
assert recovered_environment is not None and recovered_environment.finalized_ts
87+
assert not context.run("prod", skip_janitor=True).is_failure
88+
run.assert_called_once()
89+
90+
warning = warnings.call_args_list[0].args[0]
91+
assert failed_environment.plan_id in warning
92+
assert "finaliz" in warning
93+
assert "1 seconds" in warning
94+
assert "2 seconds" in warning
95+
assert "failed" in warning
96+
assert "is being updated" not in warning
97+
assert "finaliz" in str(exc_info.value)
98+
assert "sqlmesh plan prod" in str(exc_info.value)
99+
assert "https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#run" in str(
100+
exc_info.value
101+
)
102+
finally:
103+
context.close()
104+
105+
24106
@time_machine.travel("2023-01-08 15:00:00 UTC")
25107
def test_run_with_select_models(
26108
init_and_plan_context: t.Callable,

0 commit comments

Comments
 (0)