Skip to content

Commit 7f719e3

Browse files
fix: make kl_divergence and chi_square audits run on DuckDB and Trino
kl_divergence failed on every engine: - table_a_with_q joined USING (@source_column) against a CTE that only has @target_column, so different columns failed ("missing from right side of join") - with the same column on both sides, num_rows became ambiguous - the final SELECT grouped by the column values, so the divergence was compared per value instead of summed over the distribution - single-argument LOG() is log10 on DuckDB and does not exist on Trino Rename the value column to x in both distributions, join them on it, sum (p - q) * LN(p / q) over the values present in both (the same terms the old query kept), and compare the single result with the threshold. chi_square referenced the chi_square select alias in HAVING, which Trino rejects ("Column 'chi_square' cannot be resolved"). Compute the statistic in a CTE and filter it with WHERE. kl_divergence also used HAVING on its alias; the rewrite filters with WHERE. Signed-off-by: shinichitazawa <shinichitazawa.aws@gmail.com>
1 parent d69c262 commit 7f719e3

2 files changed

Lines changed: 36 additions & 57 deletions

File tree

‎sqlmesh/core/audit/builtin.py‎

Lines changed: 22 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -532,84 +532,44 @@ def create_non_blocking_copy(audit: Audit) -> Audit:
532532
WITH
533533
table_a AS (
534534
SELECT
535-
@source_column,
535+
@source_column AS x,
536536
COUNT(*) AS num_rows
537537
FROM @this_model
538538
WHERE @condition
539539
GROUP BY @source_column
540540
),
541541
table_b AS (
542542
SELECT
543-
@target_column,
543+
@target_column AS x,
544544
COUNT(*) AS num_rows
545545
FROM @this_model
546546
WHERE @condition
547547
GROUP BY @target_column
548548
),
549549
table_a_with_p AS (
550550
SELECT
551-
@source_column,
552-
num_rows,
551+
x,
553552
num_rows / SUM(num_rows) OVER () AS p
554553
FROM table_a
555554
),
556555
table_b_with_q AS (
557556
SELECT
558-
@target_column,
559-
num_rows,
557+
x,
560558
num_rows / SUM(num_rows) OVER () AS q
561559
FROM table_b
562560
),
563-
table_a_with_q AS (
561+
-- Values that appear in only one column are skipped, as their terms are undefined
562+
divergence AS (
564563
SELECT
565-
@source_column,
566-
num_rows,
567-
p,
568-
COALESCE(q, 0) AS q
564+
SUM((p - q) * LN(p / q)) AS kl_divergence
569565
FROM table_a_with_p
570-
LEFT JOIN table_b_with_q USING (@source_column)
571-
),
572-
table_b_with_p AS (
573-
SELECT
574-
@target_column,
575-
num_rows,
576-
q,
577-
COALESCE(p, 0) AS p
578-
FROM table_b_with_q
579-
LEFT JOIN table_a_with_p USING (@target_column)
580-
),
581-
table_a_with_kl AS (
582-
SELECT
583-
@source_column,
584-
num_rows,
585-
p,
586-
q,
587-
p * LOG(p / NULLIF(q, 0)) AS kl
588-
FROM table_a_with_q
589-
),
590-
table_b_with_kl AS (
591-
SELECT
592-
@target_column,
593-
num_rows,
594-
p,
595-
q,
596-
q * LOG(q / NULLIF(p, 0)) AS kl
597-
FROM table_b_with_p
598-
),
599-
unioned AS (
600-
SELECT *
601-
FROM table_a_with_kl
602-
UNION ALL
603-
SELECT *
604-
FROM table_b_with_kl
566+
INNER JOIN table_b_with_q
567+
ON table_a_with_p.x = table_b_with_q.x
605568
)
606569
SELECT
607-
@source_column,
608-
@target_column,
609-
SUM(kl) AS kl_divergence
610-
FROM unioned
611-
GROUP BY @source_column, @target_column
612-
HAVING kl_divergence > @threshold
570+
kl_divergence
571+
FROM divergence
572+
WHERE kl_divergence > @threshold
613573
""",
614574
)
615575

@@ -646,17 +606,23 @@ def create_non_blocking_copy(audit: Audit) -> Audit:
646606
(SELECT COUNT(*) FROM samples) as g_t -- g_t is the grand total
647607
FROM samples AS r
648608
GROUP BY x_a, x_b
609+
),
610+
statistic AS (
611+
SELECT
612+
(@c - 1) * (@r - 1) as degrees_of_freedom,
613+
SUM((observed - @E) * (observed - @E) / @E) as chi_square
614+
FROM contingency_table
649615
)
650616
651617
SELECT
652-
(@c - 1) * (@r - 1) as degrees_of_freedom,
653-
SUM((observed - @E) * (observed - @E) / @E) as chi_square
654-
FROM contingency_table
618+
degrees_of_freedom,
619+
chi_square
620+
FROM statistic
655621
-- H0: the two variables are independent
656622
-- H1: the two variables are dependent
657623
-- if chi_square > critical_value, reject H0
658624
-- if chi_square <= critical_value, fail to reject H0
659-
HAVING NOT @IF(@dependent, chi_square > @critical_value, chi_square <= @critical_value)
625+
WHERE NOT @IF(@dependent, chi_square > @critical_value, chi_square <= @critical_value)
660626
""",
661627
)
662628

‎tests/core/test_audit.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -651,7 +651,20 @@ def test_chi_square_audit(model: Model):
651651
)
652652
assert (
653653
rendered_query.sql()
654-
== """WITH "samples" AS (SELECT "a" AS "x_a", "b" AS "x_b" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE (NOT "a" IS NULL AND NOT "b" IS NULL) AND TRUE), "contingency_table" AS (SELECT "x_a", "x_b", COUNT(*) AS "observed", (SELECT COUNT(*) FROM "samples" AS "t" WHERE "r"."x_a" = "t"."x_a") AS "tot_a", (SELECT COUNT(*) FROM "samples" AS "t" WHERE "r"."x_b" = "t"."x_b") AS "tot_b", (SELECT COUNT(*) FROM "samples" AS "samples") AS "g_t" /* g_t is the grand total */ FROM "samples" AS "r" GROUP BY "x_a", "x_b") SELECT ((SELECT COUNT(DISTINCT "x_a") FROM "contingency_table" AS "contingency_table") - 1) * ((SELECT COUNT(DISTINCT "x_b") FROM "contingency_table" AS "contingency_table") - 1) AS "degrees_of_freedom", SUM(("observed" - ("tot_a" * "tot_b" / "g_t")) * ("observed" - ("tot_a" * "tot_b" / "g_t")) / ("tot_a" * "tot_b" / "g_t")) AS "chi_square" FROM "contingency_table" AS "contingency_table" /* H0: the two variables are independent */ /* H1: the two variables are dependent */ /* if chi_square > critical_value, reject H0 */ /* if chi_square <= critical_value, fail to reject H0 */ HAVING NOT "chi_square" > 9.48773"""
654+
== """WITH "samples" AS (SELECT "a" AS "x_a", "b" AS "x_b" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE (NOT "a" IS NULL AND NOT "b" IS NULL) AND TRUE), "contingency_table" AS (SELECT "x_a", "x_b", COUNT(*) AS "observed", (SELECT COUNT(*) FROM "samples" AS "t" WHERE "r"."x_a" = "t"."x_a") AS "tot_a", (SELECT COUNT(*) FROM "samples" AS "t" WHERE "r"."x_b" = "t"."x_b") AS "tot_b", (SELECT COUNT(*) FROM "samples" AS "samples") AS "g_t" /* g_t is the grand total */ FROM "samples" AS "r" GROUP BY "x_a", "x_b"), "statistic" AS (SELECT ((SELECT COUNT(DISTINCT "x_a") FROM "contingency_table" AS "contingency_table") - 1) * ((SELECT COUNT(DISTINCT "x_b") FROM "contingency_table" AS "contingency_table") - 1) AS "degrees_of_freedom", SUM(("observed" - ("tot_a" * "tot_b" / "g_t")) * ("observed" - ("tot_a" * "tot_b" / "g_t")) / ("tot_a" * "tot_b" / "g_t")) AS "chi_square" FROM "contingency_table" AS "contingency_table") SELECT "degrees_of_freedom" AS "degrees_of_freedom", "chi_square" AS "chi_square" FROM "statistic" AS "statistic" /* H0: the two variables are independent */ /* H1: the two variables are dependent */ /* if chi_square > critical_value, reject H0 */ /* if chi_square <= critical_value, fail to reject H0 */ WHERE NOT "chi_square" > 9.48773"""
655+
)
656+
657+
658+
def test_kl_divergence_audit(model: Model):
659+
rendered_query = model.render_audit_query(
660+
builtin.kl_divergence_audit,
661+
source_column=exp.to_column("a"),
662+
target_column=exp.to_column("b"),
663+
threshold=exp.convert(0.1),
664+
)
665+
assert (
666+
rendered_query.sql()
667+
== """WITH "table_a" AS (SELECT "a" AS "x", COUNT(*) AS "num_rows" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_0" WHERE TRUE GROUP BY "a"), "table_b" AS (SELECT "b" AS "x", COUNT(*) AS "num_rows" FROM (SELECT * FROM "db"."test_model" AS "test_model" WHERE "ds" BETWEEN '1970-01-01' AND '1970-01-01') AS "_1" WHERE TRUE GROUP BY "b"), "table_a_with_p" AS (SELECT "x", "num_rows" / SUM("num_rows") OVER () AS "p" FROM "table_a" AS "table_a"), "table_b_with_q" AS (SELECT "x", "num_rows" / SUM("num_rows") OVER () AS "q" FROM "table_b" AS "table_b"), "divergence" /* Values that appear in only one column are skipped, as their terms are undefined */ AS (SELECT SUM(("p" - "q") * LN("p" / "q")) AS "kl_divergence" FROM "table_a_with_p" AS "table_a_with_p" INNER JOIN "table_b_with_q" AS "table_b_with_q" ON "table_a_with_p"."x" = "table_b_with_q"."x") SELECT "kl_divergence" AS "kl_divergence" FROM "divergence" AS "divergence" WHERE "kl_divergence" > 0.1"""
655668
)
656669

657670

0 commit comments

Comments
 (0)