diff --git a/integration_tests/tests/test_metric_stability.py b/integration_tests/tests/test_metric_stability.py index 1fe162313..1b98d5c47 100644 --- a/integration_tests/tests/test_metric_stability.py +++ b/integration_tests/tests/test_metric_stability.py @@ -12,6 +12,26 @@ OTHER_VALUE_COLUMN = "other_amount" DBT_TEST_NAME = "elementary.metric_stability" +# metric_value goes through dbt's type_float, which is 4 bytes on these. A value +# around 45 then carries about 1e-5 of storage error, orders of magnitude above +# the tolerance the boundary tests probe, so a move sitting exactly on a +# threshold is not decidable there and the assertion would be meaningless. +SINGLE_PRECISION_TARGETS = [ + "clickhouse", + "duckdb", + "spark", + "fabricspark", + "databricks_catalog", + "athena", + "trino", + "dremio", +] + +# dbt-dremio seeds a fractional column as a bare `decimal`, which Dremio reads +# at scale 0, so 45.30 and 45.20 both arrive as 45 and the move under test never +# exists. Tests whose data cannot be expressed as integers skip it. +FRACTIONAL_SEED_TARGETS = ["dremio"] + BASE_AMOUNT = 100 OTHER_BASE_AMOUNT = 500 DAYS_OF_HISTORY = 6 @@ -32,8 +52,8 @@ def _rows( - restatements: Optional[Dict[int, int]] = None, - other_restatements: Optional[Dict[int, int]] = None, + restatements: Optional[Dict[int, float]] = None, + other_restatements: Optional[Dict[int, float]] = None, ) -> List[Dict[str, Any]]: """One row per day, midday so it lands unambiguously inside a daily bucket.""" restatements = restatements or {} @@ -63,6 +83,16 @@ def _run(dbt_project: DbtProject, test_id: str, data, **overrides) -> str: return result["status"] +def _run_result(dbt_project: DbtProject, test_id: str, data, **overrides) -> dict: + return dbt_project.test( + test_id, + DBT_TEST_NAME, + {**BASE_ARGS, **overrides}, + data=data, + test_vars={"enable_elementary_test_materialization": True}, + ) + + def _bucket_values( dbt_project: DbtProject, test_id: str, column_name: str = VALUE_COLUMN ) -> Dict[Any, List[float]]: @@ -306,6 +336,365 @@ def test_metric_stability_ignores_measurements_taken_while_settling( assert _run(dbt_project, test_id, _rows(), **args) == "pass" +# A ratio sits on a fixed scale, so the move that matters is the absolute one. +# 45.30% dropping to 45.10% is 20 basis points, which is what max_change_absolute +# expresses directly. +BASE_MARGIN = 45.30 +RATIO_ARGS: Dict[str, Any] = {"metrics": ["average"]} + + +@pytest.mark.skip_targets(FRACTIONAL_SEED_TARGETS) +def test_metric_stability_absolute_threshold_on_a_ratio( + test_id: str, dbt_project: DbtProject +): + """An absolute threshold means the same move at every level of the ratio.""" + args = {**RATIO_ARGS, "max_change_absolute": 0.20} + + assert ( + _run(dbt_project, test_id, _rows({d: BASE_MARGIN for d in range(1, 7)}), **args) + == "pass" + ) + assert ( + _run(dbt_project, test_id, _rows({d: BASE_MARGIN for d in range(1, 7)}), **args) + == "pass" + ) + + # 10bp, inside the tolerance. + rows = {d: BASE_MARGIN for d in range(1, 7)} + rows[SETTLED_DAYS_AGO] = 45.20 + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # A further 30bp, outside it. + rows[SETTLED_DAYS_AGO] = 44.90 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +def test_metric_stability_both_thresholds_must_be_exceeded( + test_id: str, dbt_project: DbtProject +): + """With both set the absolute one is a floor under the relative one. + + A small absolute move on a small base looks large in relative terms, which + is exactly the noise an absolute floor is meant to suppress. + + The data is integral because a column of 5.00 is written as 5.0, which agate + normalizes to zero decimals: the seed column becomes INT64 on BigQuery, + integer on Athena and bigint on Dremio, and the load fails before any + assertion runs. Postgres, DuckDB and ClickHouse coerce on insert and hide it. + """ + args = {**RATIO_ARGS, "max_change_percent": 1.0, "max_change_absolute": 20} + low = 500 + + assert ( + _run(dbt_project, test_id, _rows({d: low for d in range(1, 7)}), **args) + == "pass" + ) + assert ( + _run(dbt_project, test_id, _rows({d: low for d in range(1, 7)}), **args) + == "pass" + ) + + # 10 is 2% of a base of 500, so the relative threshold alone would fire. It + # is under the absolute floor, so together they do not. + rows = {d: low for d in range(1, 7)} + rows[SETTLED_DAYS_AGO] = 510 + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # 50 clears the floor and is still well over 1%, so both agree. + rows[SETTLED_DAYS_AGO] = 560 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +def test_metric_stability_percent_still_applies_when_both_set( + test_id: str, dbt_project: DbtProject +): + """The relative leg has to keep biting when an absolute one is present. + + Every other both-thresholds case is decided by the absolute leg, so dropping + max_change_percent from that branch would not have been caught. This move + clears the absolute threshold and stays under the relative one, so it can + only pass if both are still being applied. + """ + args = {**RATIO_ARGS, "max_change_percent": 1.0, "max_change_absolute": 20} + base = 4530 + rows = {d: base for d in range(1, 7)} + + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # 30 absolute (over the floor of 20) but 0.66% relative (under 1%). + rows[SETTLED_DAYS_AGO] = 4560 + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # 70 absolute and 1.5% relative, so both agree. + rows[SETTLED_DAYS_AGO] = 4630 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +def test_metric_stability_absolute_tolerates_zero_baseline( + test_id: str, dbt_project: DbtProject +): + """Relative change is undefined at zero; absolute change is not. + + With only a relative threshold any move away from exactly zero has to fail, + because the division has no meaning there. An absolute threshold is well + defined at zero, so a metric legitimately sitting at 0 is held to the same + tolerance as everything else. + """ + # first_check pins the baseline at the original zero. Under the default + # last_check the second move would be measured against 2, not 0, and the + # zero baseline would never be the thing deciding: an implementation that + # never fires at zero would still pass. + args = { + **RATIO_ARGS, + "max_change_absolute": 5, + "change_since": ["first_check"], + } + rows = {d: 0 for d in range(1, 7)} + + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # A move away from zero inside the tolerance. A relative threshold would + # have had to fail this. + rows[SETTLED_DAYS_AGO] = 2 + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # And the threshold still bites beyond it, still measured against zero. + rows[SETTLED_DAYS_AGO] = 10 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +def test_metric_stability_both_detect_zero_baseline( + test_id: str, dbt_project: DbtProject +): + """At a zero baseline the absolute threshold has to decide on its own. + + Relative change has no meaning at zero, so a relative conjunct of the form + "baseline != 0 and ..." is false there. Combined with the absolute one under + AND it would suppress every zero-baseline restatement, so adding an absolute + threshold as a noise floor would silently switch that detection off. + """ + args = {**RATIO_ARGS, "max_change_percent": 10, "max_change_absolute": 5} + rows = {d: 0 for d in range(1, 7)} + + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + rows[SETTLED_DAYS_AGO] = 1000 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +@pytest.mark.skip_targets(FRACTIONAL_SEED_TARGETS) +def test_metric_stability_unset_threshold_still_fails_any_change( + test_id: str, dbt_project: DbtProject +): + """An unconfigured threshold means zero, not "some small number". + + max_change_percent defaults to none internally so that unset is + distinguishable from an explicit zero. The other tests restate by a factor + of two, so they would still pass if unset silently resolved to anything + under 100%. This pins a change far below any plausible accidental default. + """ + assert _run(dbt_project, test_id, _rows()) == "pass" + assert _run(dbt_project, test_id, _rows()) == "pass" + + # 0.1% of the base amount, which must still fail with nothing configured. + assert _run(dbt_project, test_id, _rows({SETTLED_DAYS_AGO: 100.1})) == "fail" + + +@pytest.mark.skip_targets(FRACTIONAL_SEED_TARGETS) +def test_metric_stability_absolute_on_both_baselines( + test_id: str, dbt_project: DbtProject +): + """Each baseline gets its own threshold pair, OR'd together.""" + args = { + **RATIO_ARGS, + "change_since": ["last_check", "first_check"], + "max_change_absolute": 0.20, + } + rows = {d: BASE_MARGIN for d in range(1, 7)} + + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # 15bp per step: under the tolerance against the previous measurement, but + # first_check sees the total move from where the bucket started. + rows[SETTLED_DAYS_AGO] = 45.15 + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + rows[SETTLED_DAYS_AGO] = 45.00 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +@pytest.mark.parametrize( + "overrides,expected_argument", + [ + pytest.param({"max_change_absolute": -1}, "max_change_absolute", id="abs_neg"), + pytest.param({"max_change_percent": -1}, "max_change_percent", id="pct_neg"), + pytest.param( + {"max_change_absolute": True}, "max_change_absolute", id="abs_bool" + ), + pytest.param({"max_change_percent": True}, "max_change_percent", id="pct_bool"), + pytest.param({"max_change_absolute": "x"}, "max_change_absolute", id="abs_str"), + pytest.param({"max_change_percent": "x"}, "max_change_percent", id="pct_str"), + pytest.param({"max_change_percent": None}, "max_change_percent", id="pct_null"), + pytest.param( + {"max_change_absolute": float("inf")}, + "max_change_absolute", + id="abs_inf", + marks=pytest.mark.skip_for_dbt_fusion, + ), + pytest.param( + {"max_change_absolute": float("nan")}, + "max_change_absolute", + id="abs_nan", + marks=pytest.mark.skip_for_dbt_fusion, + ), + ], +) +def test_metric_stability_rejects_bad_thresholds( + test_id: str, + dbt_project: DbtProject, + overrides: Dict[str, Any], + expected_argument: str, +): + """Every threshold value that cannot mean what it says must raise. + + The description is asserted on, not just the status. Several of these would + error anyway without any validation: a string raises inside the arithmetic, + and True renders as a threshold of 1. Only an error naming the argument + shows that the check, rather than an incidental crash, rejected it. + + Booleans matter because bool subclasses int in python, so Jinja's `number` + test accepts True. inf and nan matter because they are numbers, are not + negative and clear every bound, and then render as a bare inf / nan, which + ClickHouse reads as a literal that makes every comparison false, so the test + would never fail again. + + inf and nan skip dbt-fusion, which does not carry either through to the + macro: both arrive as null, which for an optional argument means "unset", + so the validation under test is never reached. + + A string is rejected rather than parsed. dbt reads YAML 1.1, where an + exponent without a decimal point is text, so `2e-3` arrives here as a + string; the message says to write it with a point rather than telling the + user to unquote something they never quoted. + """ + result = _run_result(dbt_project, test_id, _rows(), **overrides) + + assert result["status"] == "error" + assert expected_argument in (result.get("test_results_description") or "") + + +@pytest.mark.skip_targets(SINGLE_PRECISION_TARGETS) +def test_metric_stability_absolute_bites_on_large_metrics( + test_id: str, dbt_project: DbtProject +): + """A noise floor must not quietly override an explicit absolute threshold. + + The predicate used to AND a fixed 1e-9 percent floor onto every comparison, + absolute-only ones included. On a daily sum of 2.5e11 a move of 2 is 8e-10 + percent, under that floor, so `max_change_absolute: 1` never reported it + however far the value moved in absolute terms. + + The base carries a fraction so the seed column stays floating point: whole + numbers this large are inferred as integer and overflow a 32-bit int column. + """ + args = {"max_change_absolute": 1} + base = 250000000000.5 + rows = {d: base for d in range(1, 7)} + + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # Twice the threshold, and far below the old relative floor. + rows[SETTLED_DAYS_AGO] = base + 2 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +@pytest.mark.skip_targets(SINGLE_PRECISION_TARGETS) +def test_metric_stability_absolute_boundary_on_large_values( + test_id: str, dbt_project: DbtProject +): + """The boundary stays symmetric as the metric grows. + + Representation error grows with the operands, so a slack of a fixed size + stops absorbing it. At a bucket of 12345678.30 the two directions of a 0.20 + move compute 0.2000000011175871 and 0.19999999925494194, so the move was + reported going down and not going up: the same one-direction-only behaviour + the smaller boundary test guards against, returning about a million times + the threshold. + """ + args = {**RATIO_ARGS, "max_change_absolute": 0.20} + base = 12345678.30 + rows = {d: base for d in range(1, 7)} + + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # Exactly 0.20 down, the direction that rounds against us here. + rows[SETTLED_DAYS_AGO] = 12345678.10 + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # 0.70 further, comfortably past the threshold. + rows[SETTLED_DAYS_AGO] = 12345678.80 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +@pytest.mark.skip_targets(SINGLE_PRECISION_TARGETS) +def test_metric_stability_absolute_float_boundary( + test_id: str, dbt_project: DbtProject +): + """A move of exactly the threshold must not fire, in either direction. + + Decimal thresholds have no exact float representation, so two moves that are + both 0.20 in decimal land either side of it: 45.30 -> 45.10 comes out as + 0.19999999999999574 and 45.30 -> 45.50 as 0.20000000000000284. Without a + tolerance the second fires and the first does not, which is the customer's + own 20bp example behaving differently depending on which way it moved. + """ + args = {**RATIO_ARGS, "max_change_absolute": 0.20} + rows = {d: BASE_MARGIN for d in range(1, 7)} + + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # Exactly 20bp up, the direction that rounds against us. + rows[SETTLED_DAYS_AGO] = 45.50 + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # 30bp further, comfortably past the threshold. + rows[SETTLED_DAYS_AGO] = 45.80 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + +@pytest.mark.skip_targets(SINGLE_PRECISION_TARGETS) +def test_metric_stability_absolute_boundary_on_unit_scale( + test_id: str, dbt_project: DbtProject +): + """The same move must behave the same however the metric is scaled. + + 20bp is 0.20 on a 0-100 scale and 0.0020 on a 0-1 scale. Rendering the + threshold with a fixed ten decimals kept the tolerance on the first and + dropped it on the second, so 0.1000 -> 0.0980 (a diff of + 0.0020000000000000018) fired while the identical move on the other scale + did not. + """ + args = {**RATIO_ARGS, "max_change_absolute": 0.0020} + rows = {d: 0.1000 for d in range(1, 7)} + + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # Exactly 20bp down, the direction that rounds against us. + rows[SETTLED_DAYS_AGO] = 0.0980 + assert _run(dbt_project, test_id, _rows(rows), **args) == "pass" + + # 30bp further, comfortably past the threshold. + rows[SETTLED_DAYS_AGO] = 0.0950 + assert _run(dbt_project, test_id, _rows(rows), **args) == "fail" + + def test_metric_stability_ignores_unsettled_buckets( test_id: str, dbt_project: DbtProject ): diff --git a/macros/edr/data_monitoring/monitors_query/metric_stability_query.sql b/macros/edr/data_monitoring/monitors_query/metric_stability_query.sql index 6b31c6049..6596474b2 100644 --- a/macros/edr/data_monitoring/monitors_query/metric_stability_query.sql +++ b/macros/edr/data_monitoring/monitors_query/metric_stability_query.sql @@ -9,6 +9,7 @@ days_back, min_bucket_age, max_change_percent=0, + max_change_absolute=none, change_since=["last_check"], column_names=none, data_monitoring_metrics_table=none, @@ -52,10 +53,58 @@ and updated_at >= {{ settled_at_expr }} {% endset %} - {# Suppress floating-point aggregation noise; zero baselines are handled separately. #} - {%- set change_percent_noise_floor = 0.000000001 %} - {%- set change_threshold = "%.10f" | format( - [max_change_percent, change_percent_noise_floor] | max + {# A relative threshold suits magnitudes, whose scale varies by orders of + magnitude between models. It does not suit ratios: a margin already sits + on a fixed scale, so the move that transfers is the absolute one. And + because the relative form divides by the baseline, its sensitivity is + recomputed per bucket, so a setting meaning 20bp at a 45% margin means + about 9bp at 20% and 27bp at 60%. max_change_absolute expresses the move + directly. + + Both legs are written as `moved > threshold * scale`, with no division, + so neither needs a zero-baseline escape: at a zero baseline the relative + leg reduces to `moved > 0`, which is satisfied by any change rather than + falsified by it, and the absolute leg decides on its own. The earlier + form built a separate predicate per branch, which is how the zero + baseline was suppressed once already. #} + {# Two coefficients, both scaled by the operands, because the errors they + cover scale with the operands too. + + noise_floor stands in for an absolute threshold when none was given, so + that re-aggregating the same rows in a different order does not read as + a restatement. It is the old 1e-9 percent floor written as a fraction, + so the default behaviour is unchanged. It is deliberately NOT added on + top of an explicit max_change_absolute: setting one states the + resolution wanted, and folding the floor into it dropped real moves on + large metrics (a move of 2 on a daily sum of 2.5e11, with + max_change_absolute: 1, went unreported). + + comparison_tolerance absorbs the representation error in the comparison + itself. It is about 4 ulps of a double, which is the smallest value that + still covers the boundary cases below with room to spare (measured: 30x + the worst observed excess, over magnitudes from 0.1 to 1e12). Sizing it + more loosely masks real moves on large metrics: at 1e-13 a baseline of + 1e12 carries a tolerance of 0.2, which hides a move of 0.01 under an + absolute threshold of 0.001. A decimal threshold has no exact binary form, so 45.30 -> 45.10 + is 0.19999999999999574 while 45.50 -> 45.30 is 0.20000000000000284, and + without it the same 20bp move is reported in one direction only. A fixed + slack cannot do this, because that error grows with the operands: at a + bucket of 12345678.30 it reaches 1e-9, five orders of magnitude above a + slack sized for values around 45. #} + {%- set noise_floor = 0.00000000001 %} + {%- set comparison_tolerance = 0.000000000000001 %} + + {#- Thresholds render through `string`, python's shortest round-tripping + form, so the literal in the SQL is the double that was configured. + Fixed decimals are not: at "%.10f" a tolerance survived on a 0-100 + scale and vanished on a 0-1 scale, so the same move behaved differently + depending on how the metric happened to be scaled. -#} + {%- set percent_fraction = (max_change_percent / 100.0) | string %} + {%- set tolerance_coefficient = comparison_tolerance | string %} + {%- set absolute_literal = ( + max_change_absolute | string + if max_change_absolute is not none + else none ) %} {%- set exceeds_conditions = [] %} @@ -64,12 +113,18 @@ {%- set baseline_column = baseline ~ "_value" %} {%- if baseline_column not in baseline_columns %} {%- do baseline_columns.append(baseline_column) %} + {%- set moved = "abs(measured_value - " ~ baseline_column ~ ")" %} + {%- set scale = "(abs(measured_value) + abs(" ~ baseline_column ~ "))" %} + {%- set tolerance = tolerance_coefficient ~ " * " ~ scale %} + {%- set absolute_leg = ( + absolute_literal + if absolute_literal is not none + else (noise_floor | string) ~ " * " ~ scale + ) %} {% set exceeds_condition %} - ({{ baseline_column }} is not null and ( - ({{ baseline_column }} = 0 and measured_value != 0) - or ({{ baseline_column }} != 0 and - {{ elementary.metric_stability_change_percent(baseline_column) }} > {{ change_threshold }}) - )) + ({{ baseline_column }} is not null + and {{ moved }} > {{ percent_fraction }} * abs({{ baseline_column }}) + {{ tolerance }} + and {{ moved }} > {{ absolute_leg }} + {{ tolerance }}) {% endset %} {% do exceeds_conditions.append(exceeds_condition) %} {%- endif %} @@ -194,12 +249,14 @@ first_check_value as initial_value, case when is_current = 1 then measured_value - last_check_value end as change_since_last_check, case when is_current = 1 then measured_value - first_check_value end as change_since_first_check, + {#- nullif inside the helper already yields NULL at a zero or + missing baseline, so is_current is the only guard left. -#} case - when is_current = 1 and last_check_value is not null and last_check_value != 0 + when is_current = 1 then {{ elementary.metric_stability_change_percent("last_check_value") }} end as change_percent_since_last_check, case - when is_current = 1 and first_check_value is not null and first_check_value != 0 + when is_current = 1 then {{ elementary.metric_stability_change_percent("first_check_value") }} end as change_percent_since_first_check from latest_measurement @@ -212,7 +269,13 @@ {# Relative change from a baseline column, in percentage points. Shared by the WHERE predicate and the reported columns so the two cannot drift apart. + + nullif keeps this NULL rather than raising at a zero baseline. SQL does not + guarantee that a `= 0` guard around the call is evaluated first, so on an + engine that evaluates both sides of an OR the division would raise. #} {% macro metric_stability_change_percent(baseline_column) -%} - abs(measured_value - {{ baseline_column }}) / abs({{ baseline_column }}) * 100.0 + abs(measured_value - {{ baseline_column }}) + / nullif(abs({{ baseline_column }}), 0) + * 100.0 {%- endmacro %} diff --git a/macros/edr/tests/test_metric_stability.sql b/macros/edr/tests/test_metric_stability.sql index 5eab81833..c355ba6e6 100644 --- a/macros/edr/tests/test_metric_stability.sql +++ b/macros/edr/tests/test_metric_stability.sql @@ -1,4 +1,25 @@ {# Detect restatements of settled bucket metrics within the observation window. + + Thresholds: max_change_percent is relative, which suits magnitudes whose + scale varies between models. max_change_absolute is the raw move, which + suits ratios that already sit on a fixed scale (20 basis points is 0.20 on a + 0-100 scale). Setting both fails only when both are exceeded, so the + absolute one acts as a noise floor under the relative one. At a zero + baseline, where relative change has no meaning, the absolute threshold + decides on its own. + + max_change_absolute is one number for every column, metric and dimension the + test covers, so it only means something when they share a scale. A test over + a 0-100 margin and a 0-1 rate holds both to the same raw move, and a test + over `average` and `null_count` holds a ratio and a row count to it. Use one + test per scale, or max_change_percent, which is scale-free. + + Both thresholds are bounded by how precisely the warehouse stores a metric. + metric_value goes through dbt's type_float, which is 4 bytes on DuckDB, + ClickHouse, Spark, Databricks and Dremio. There a sum of 12345678.30 + restated to 12345678.00 stores as the same value twice, and no threshold, + relative or absolute, can see a move that was never recorded. + Configuration, baseline behavior, and limits: https://docs.elementary-data.com/data-tests/metric-stability #} {% test metric_stability( @@ -9,6 +30,7 @@ min_bucket_age, change_since=["last_check"], max_change_percent=0, + max_change_absolute=none, time_bucket=none, where_expression=none, days_back=none, @@ -33,9 +55,12 @@ min_bucket_age, change_since, max_change_percent, + max_change_absolute, days_back, backfill_days, ) %} + {% set max_change_percent = arguments.max_change_percent %} + {% set max_change_absolute = arguments.max_change_absolute %} {% set model_relation = arguments.model_relation %} {% set model_graph_node = arguments.model_graph_node %} {% set timestamp_column = arguments.timestamp_column %} @@ -96,6 +121,7 @@ days_back=days_back, min_bucket_age=min_bucket_age, max_change_percent=max_change_percent, + max_change_absolute=max_change_absolute, change_since=change_since, column_names=collected.columns, measurement_windows=collected.windows, @@ -362,6 +388,7 @@ min_bucket_age, change_since, max_change_percent, + max_change_absolute, days_back, backfill_days ) %} @@ -391,22 +418,72 @@ }} {%- endif %} - {%- if max_change_percent is not number %} - {{ - exceptions.raise_compiler_error( - "max_change_percent must be a number, got '" - ~ max_change_percent - ~ "'. Write it unquoted, e.g. max_change_percent: 25." - ) - }} - {%- endif %} - {%- if max_change_percent < 0 %} - {{ - exceptions.raise_compiler_error( - "max_change_percent must be non-negative." - ) - }} - {%- endif %} + {#- max_change_percent defaults to 0, so a none here can only come from an + explicit `max_change_percent: null`. master rejected that, and it must + not quietly become "report any change at all". -#} + {%- set thresholds = {} %} + {%- for arg_name, arg_value, example, optional in [ + ("max_change_percent", max_change_percent, "25", false), + ("max_change_absolute", max_change_absolute, "0.20", true), + ] %} + {%- if arg_value is none %} + {%- if not optional %} + {{ + exceptions.raise_compiler_error( + arg_name + ~ " cannot be null. Omit it to keep the default of 0," + ~ " or give a number." + ) + }} + {%- endif %} + {%- do thresholds.update({arg_name: none}) %} + {%- else %} + {#- bool subclasses int in python, so `is number` alone lets + `max_change_percent: true` through as a threshold of 1. + + The message names the decimal point because dbt reads YAML 1.1, + where an exponent written without one is text rather than a + number: `max_change_absolute: 2e-3`, a natural way to write 20bp + on a 0-1 scale, arrives here as the string '2e-3'. Telling the + user to unquote something they never quoted is not actionable. -#} + {%- if arg_value is boolean or arg_value is not number %} + {{ + exceptions.raise_compiler_error( + arg_name + ~ " must be a number, got '" + ~ arg_value + ~ "'. Write it with a decimal point, e.g. " + ~ arg_name + ~ ": " + ~ example + ~ ". An exponent without one, such as 2e-3, is read as" + ~ " text rather than a number." + ) + }} + {%- endif %} + {#- inf and nan are numbers, are not negative and clear every bound, + so without this they render as a bare inf / nan: most warehouses + read that as an unknown column and fail at runtime, and + ClickHouse reads it as a literal that makes every comparison + false, so the test would simply never fail again. -#} + {%- if arg_value - arg_value != 0 %} + {{ + exceptions.raise_compiler_error( + arg_name ~ " must be finite, got '" ~ arg_value ~ "'." + ) + }} + {%- endif %} + {%- if arg_value < 0 %} + {{ + exceptions.raise_compiler_error( + arg_name ~ " must be non-negative, got '" ~ arg_value ~ "'." + ) + }} + {%- endif %} + {%- do thresholds.update({arg_name: arg_value}) %} + {%- endif %} + {%- endfor %} + {%- for arg_name, arg_value in [ ("days_back", days_back), ("backfill_days", backfill_days), @@ -553,6 +630,8 @@ "columns": column_definitions, "metrics": metrics, "change_since": change_since, + "max_change_percent": thresholds["max_change_percent"], + "max_change_absolute": thresholds["max_change_absolute"], "model_relation": model_relation, "model_graph_node": model_graph_node, "timestamp_column": timestamp_column,