diff --git a/carousel/diff-diff-dml-carousel.pdf b/carousel/diff-diff-dml-carousel.pdf new file mode 100644 index 00000000..28057302 Binary files /dev/null and b/carousel/diff-diff-dml-carousel.pdf differ diff --git a/carousel/generate_dml_carousel.py b/carousel/generate_dml_carousel.py new file mode 100644 index 00000000..1ead3f15 --- /dev/null +++ b/carousel/generate_dml_carousel.py @@ -0,0 +1,1332 @@ +#!/usr/bin/env python3 +"""Generate LinkedIn carousel PDF for the DMLDiD launch (v3.11.0). + +Mirrors the POSTER-MODE architecture of ``generate_lwdid_carousel.py`` +(magazine sidebar with progress tick, light gradient background, split-color +logo, footer wordmark, dark slides, soft card shadows; poster type floors: +body >= 16pt, captions >= 13pt, headlines 32-54pt, code >= 15pt; one idea + +one visual + roughly 25 words per slide). + +Palette: "Circuit" - graphite/slate structure with a single SIGNAL GREEN +accent reserved for the ML/curved-truth motif (the deck's visual thesis: +the relationship the fixed-form model cannot bend to). The misspecified +linear fit is always warm gray. Two dark slides: the ridge twist (slide 4) +and the code slide (slide 9). + +Narrative spine (ML-FORWARD hook - user decision 2026-08-29: lead with the +capability, then earn it with the confidently-wrong receipt): + +1. Cover -- "Plug any ML model into your DiD." Curve-vs-line + motif (illustrative); subtitle names DMLDiD + the + 3.11 release (version derived from pyproject, never + hardcoded). +2. The bet -- covariate adjustment is a modeling bet: classic DiD + estimators fix the nuisance forms (logit + linear + OLS); when assignment or trends bend in X, the bet + loses silently. +3. The receipt -- simulated staggered panel, truth known by + construction (said on-slide): linear reads 2.5909 + (reported SE 0.0743) against a true 2.2388 - almost + five standard errors off, behind a narrow nominal + interval (nominal-inference caveat on-slide). +4. Twist (DARK) -- ridge won't save you: 2.5898. A penalty on the wrong + functional form is still the wrong functional form. +5. The paper -- Chang (2020), The Econometrics Journal: two ideas - + Neyman-orthogonal scores + cross-fitting. + ATTRIBUTED PARAPHRASE ONLY, no quote marks: the + paper review (docs/methodology/papers/ + chang-2020-review.md) is pinned to the arXiv v3 + layout and the published PDF has not been + cross-checked word-for-word, so this deck carries NO + verbatim pull quote (unlike the CiC deck, whose + quote was verified against its published PDF). +6. The math -- the Case 1 score annotated term-by-term (matplotlib + mathtext, big type), plus the two one-liners: + orthogonality (nuisance error enters at second + order) and cross-fitting (no unit scored by a + learner that saw it). +7. The payoff -- the four-learner chart: linear/ridge miss high, + sieve 2.2804 and a hand-rolled PolynomialRidge + 2.2818 recover the truth line. The double-robustness + beat rides ON the slide: the propensity model is + misspecified in every arm; the flexible OUTCOME + model alone rescues the point estimate. +8. When to use it -- the honest decision split (mirrors + docs/choosing_estimator.rst): nonlinear / + high-dimensional covariates -> DMLDiD; a plausible + logit + linear spec -> CallawaySantAnna (fewer + moving parts). Closing band (user emphasis, + 2026-08-29): four learners BUILT IN with no extra + installs, or bring your own - scikit-learn + REGRESSORS already fit the duck-typed + fit()/predict() outcome contract, and propensity + CLASSIFIERS additionally need predict_proba() + (both contracts stated on-slide). +9. Code (DARK) -- the tutorial's own fit (sieve, n_folds=5, seed=42), + >= 15pt, plus the bring-your-own swap rendered as a + real sklearn import + constructor call. +10. Production -- feature grid (post-fit staggered surface: event + study / group / HonestDiD / sup-t bands; survey + designs + clustering with PSU-cohesive folds; + repeated cross-sections; per-cell cross-fit + diagnostics) + validation strip (Chang Case 1 score + anchored to DoubleML at machine precision; committed + spike). +11. CTA -- pip install; tutorial 32 mentioned ONCE, here. + +Claim discipline (verified against the committed executed notebook +``docs/tutorials/32_dml_did.ipynb`` and ``docs/methodology/REGISTRY.md``): + +- EVERY estimate on the deck is a committed, seed-locked tutorial-32 value + from a SIMULATED example with truth known by construction - labeled + on-slide wherever a truth value appears. notebook <-> deck sync is + pinned by ``tests/test_dml_carousel_claims.py`` (parses this module's + constants via ``ast`` and locates each on the committed notebook + surface). The 30-seed robustness sweep from prototyping is deliberately + NOT on the deck (it is not on the committed notebook surface). +- "Almost five standard errors" mirrors the tutorial's own wording and is + a DISTANCE in reported-SE units; the deck never presents the simulated + SEs as valid coverage. The rate-condition caveat is SLIDE-LOCAL wherever + reported uncertainty appears (review round 1): slide 3 labels the value + "reported SE" / "narrow nominal interval" and carries the + nominal-not-theory-backed caption, slide 4 says "nominal precision" with + its own qualifier line, and the payoff slide keeps the + illustrative-inference caption. +- The math slide labels the displayed expression as the UNCENTERED summand + ``s`` (its cell mean is the ATT; the centered eq. 3.1 score is + ``psi = s - ATT`` - said in the caption), and scopes orthogonality to + the LEARNED nuisances g and l only (the treated share p carries the + Theorem 2 augmented-score variance correction, per the REGISTRY + chang_panel_score note) - review round 1 correction. +- The repeated-cross-section card states Chang Assumption 2.3's + same-target-population requirement ON the card ("fresh samples from the + SAME target population each wave") - review round 1 correction. +- The COVER motif is stylized art (a curved relationship vs a straight + fit), labeled "illustrative" on-slide; it plots no tutorial data. +- CallawaySantAnna is named twice, neutrally and honestly: slide 2 (the + fixed-form nuisances it and classic DR estimators use - stated as a + design choice, not a defect) and slide 8 (the choosing_estimator + decision rule RECOMMENDING it when a parametric spec is plausible). No + "only"/"first" absolutes anywhere (claims test bans them). No external + competitor is named. +- The math slide renders the ``chang_panel_score`` summand exactly as the + REGISTRY documents it (Equation 3.1 as implemented); g-hat is the + conditional propensity, p-hat the unconditional treated share (the + REGISTRY notation split). +- The DoubleML anchor is scoped ON-SLIDE to the Case 1 score (committed + spike ``benchmarks/doubleml/chang_case1_parity.py``, ATT diff 4.4e-16 + per the REGISTRY DR-score families note) - never an end-to-end + staggered-parity claim. +- Survey/cluster support and the staggered ATT(g,t) construction are + documented library extensions of Chang (REGISTRY DMLDiD Notes); the + production slide says "documented extension" rather than attributing + them to the paper. +- The learner-flexibility claims are library-surface facts, not + comparisons: the four built-in learners live in ``diff_diff/_learners.py`` + on the library's own numpy/scipy solvers (no additional dependency - the + claims test asserts the module imports no external ML package), and + scikit-learn compatibility is the documented duck-typed protocol + (regressor fit()/predict(); classifier fit()/predict_proba()) + (``docs/api/dml_did.rst`` names scikit-learn; sklearn is an ecosystem + library the estimator interoperates with, never a compared-against + competitor). The sklearn import on the code slide is illustrative of the + contract; the notebook's own custom-learner demo is a numpy-only object + (the notebooks CI environment deliberately excludes sklearn). +- Tutorial 32 is referenced ONCE, on the CTA slide (house precedent). + +Run with:: + + python carousel/generate_dml_carousel.py + +Produces ``carousel/diff-diff-dml-carousel.pdf``. Generation requires +``fpdf2``, ``Pillow``, and ``matplotlib`` (carousel-only dependencies, not +part of the library's install extras). +""" + +import os +import re +import tempfile +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 +from fpdf import FPDF # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +# Page dimensions (4:5 portrait -- same as the other decks) +WIDTH = 270 # mm +HEIGHT = 337.5 # mm + +# Version label derived from pyproject.toml (regex, not tomllib: py>=3.9). +_PYPROJECT = Path(__file__).parent.parent / "pyproject.toml" +_m = re.search(r'^version\s*=\s*"([^"]+)"', _PYPROJECT.read_text(encoding="utf-8"), re.MULTILINE) +if _m is None: + raise RuntimeError(f"could not parse project version from {_PYPROJECT}") +VERSION_LABEL = "v" + _m.group(1) +RELEASE_LABEL = ".".join(_m.group(1).split(".")[:2]) + +TOTAL_SLIDES = 11 + +# ------------------------------------------------------------------------- +# "Circuit" palette -- graphite/slate structure, SIGNAL GREEN = the ML / +# curved-truth motif. The wrong linear fit is always warm gray. +# ------------------------------------------------------------------------- +GRAPHITE = (30, 41, 59) # #1e293b primary accent / structure +SLATE = (51, 65, 85) # #334155 +SIGNAL = (22, 163, 74) # #16a34a THE CURVED TRUTH / ML (chart grammar) +SIGNAL_BRIGHT = (74, 222, 128) # #4ade80 signal on the dark slides +SIGNAL_TINT = (240, 253, 244) # #f0fdf4 gradient start +MINT = (187, 247, 208) # #bbf7d0 light fills / dark-slide text pop +WRONG_GRAY = (120, 113, 108) # #78716c the misspecified fit (warm gray) +SHADOW = (203, 213, 225) # #cbd5e1 soft card shadow + +# Text + structural (shared with the other decks for legibility) +NAVY = (15, 23, 42) # #0f172a primary text; dark-slide gradient start +GRAY = (100, 116, 139) # #64748b +LIGHT_GRAY = (148, 163, 184) # #94a3b8 +WHITE = (255, 255, 255) +DEEP_GREEN = (20, 46, 34) # #142e22 dark-slide gradient end +PANEL_NAVY = (30, 41, 59) # #1e293b code panel fill +GREEN_CODE = (74, 222, 128) # #4ade80 code string/number literals +SLATE_CODE = (148, 163, 184) # #94a3b8 + +GRAPHITE_HEX = "#1e293b" +SIGNAL_HEX = "#16a34a" +SIGNAL_BRIGHT_HEX = "#4ade80" +MINT_HEX = "#bbf7d0" +WRONG_GRAY_HEX = "#78716c" +NAVY_HEX = "#0f172a" +GRAY_HEX = "#64748b" +LIGHT_GRAY_HEX = "#94a3b8" + +# ------------------------------------------------------------------------- +# Seed-locked numbers from the committed executed tutorial +# (docs/tutorials/32_dml_did.ipynb). Every constant below is located on the +# committed notebook surface by tests/test_dml_carousel_claims.py - never +# edit one without re-running it. The example is SIMULATED with truth known +# by construction (stated on every slide that shows a truth value). +# ------------------------------------------------------------------------- + +TRUTH = 2.2388 # DGP-implied overall ATT (the fit's own aggregation weights) +LINEAR = (2.5909, 0.0743) # outcome_learner="linear": ATT, SE +RIDGE = (2.5898, 0.0736) # outcome_learner="ridge" +SIEVE = (2.2804, 0.0438) # outcome_learner="sieve" +POLY = (2.2818, 0.0438) # hand-rolled duck-typed PolynomialRidge object + +N_UNITS, N_PERIODS = 600, 6 +N_NEVER = 354 # never-treated units in the simulated panel +COHORTS = (4, 5) + +# Survey lane provenance (RCS + SurveyDesign fit) +SURVEY_DF, SURVEY_PSU, SURVEY_STRATA = 16, 20, 4 + +# Tutorial fit config rendered on the code slide (claims test pins these +# against the notebook's code cells) +CODE_LEARNER, CODE_FOLDS, CODE_SEED = "sieve", 5, 42 + +# DoubleML anchor (REGISTRY DR-score families note; committed spike +# benchmarks/doubleml/chang_case1_parity.py) - Case 1 SCORE parity only. +DOUBLEML_ATT_DIFF = "4.4e-16" + + +class DMLCarouselPDF(FPDF): + def __init__(self): + super().__init__(orientation="P", unit="mm", format=(WIDTH, HEIGHT)) + self.set_auto_page_break(False) + self._temp_files = [] + + def cleanup(self): + for f in self._temp_files: + try: + os.unlink(f) + except OSError: + pass + + # ----------------------------------------------------------------- + # Magazine vertical sidebar -- GRAPHITE bar, SIGNAL progress tick. + # ----------------------------------------------------------------- + def _draw_vertical_sidebar(self, slide_number, total=TOTAL_SLIDES, dark=False): + bar_x = 14 + bar_y_top = 45 + bar_y_bottom = 275 + self.set_draw_color(*(LIGHT_GRAY if dark else GRAPHITE)) + self.set_line_width(0.6) + self.line(bar_x, bar_y_top, bar_x, bar_y_bottom) + + ratio = (slide_number - 1) / (total - 1) if total > 1 else 0.0 + tick_y = bar_y_top + ratio * (bar_y_bottom - bar_y_top) + self.set_draw_color(*(SIGNAL_BRIGHT if dark else SIGNAL)) + self.set_line_width(1.2) + self.line(bar_x - 4, tick_y, bar_x + 7, tick_y) + + # ----------------------------------------------------------------- + # Backgrounds + footer + # ----------------------------------------------------------------- + def light_gradient_background(self): + """Green tint #f0fdf4 fading to white.""" + steps = 50 + r0, g0, b0 = SIGNAL_TINT + for i in range(steps): + ratio = i / steps + self.set_fill_color( + int(r0 + (255 - r0) * ratio), + int(g0 + (255 - g0) * ratio), + int(b0 + (255 - b0) * ratio), + ) + y = i * HEIGHT / steps + self.rect(0, y, WIDTH, HEIGHT / steps + 1, "F") + + def dark_gradient_background(self): + """Near-black navy #0f172a fading to deep green #142e22.""" + steps = 50 + r0, g0, b0 = NAVY + r1, g1, b1 = DEEP_GREEN + for i in range(steps): + ratio = i / steps + self.set_fill_color( + int(r0 + (r1 - r0) * ratio), + int(g0 + (g1 - g0) * ratio), + int(b0 + (b1 - b0) * ratio), + ) + y = i * HEIGHT / steps + self.rect(0, y, WIDTH, HEIGHT / steps + 1, "F") + + def add_footer(self, dark=False): + """Centered split-color ``diff-diff vX.Y.Z`` wordmark.""" + self.set_font("Helvetica", "B", 12) + dd_text = "diff-diff " + v_text = VERSION_LABEL + dd_w = self.get_string_width(dd_text) + v_w = self.get_string_width(v_text) + start_x = (WIDTH - dd_w - v_w) / 2 + + self.set_xy(start_x, HEIGHT - 18) + self.set_text_color(*(LIGHT_GRAY if dark else GRAY)) + self.cell(dd_w, 10, dd_text) + self.set_text_color(*(SIGNAL_BRIGHT if dark else SIGNAL)) + self.cell(v_w, 10, v_text) + + # ----------------------------------------------------------------- + # Text helpers + # ----------------------------------------------------------------- + def centered_text(self, y, text, size=28, bold=True, color=NAVY, italic=False): + self.set_xy(0, y) + style = ("B" if bold else "") + ("I" if italic else "") + self.set_font("Helvetica", style, size) + self.set_text_color(*color) + self.cell(WIDTH, size * 0.5, text, align="C") + + def _kicker(self, y, text, color=SIGNAL): + """Editorial section label: letter-spaced caps with flanking rules.""" + spaced = " ".join(text.upper()) + self.set_font("Helvetica", "B", 14) + tw = self.get_string_width(spaced) + mid_y = y + 3 + rule = 20 + gap = 8 + self.set_draw_color(*color) + self.set_line_width(0.7) + self.line(WIDTH / 2 - tw / 2 - gap - rule, mid_y, WIDTH / 2 - tw / 2 - gap, mid_y) + self.line(WIDTH / 2 + tw / 2 + gap, mid_y, WIDTH / 2 + tw / 2 + gap + rule, mid_y) + self.set_xy(0, y) + self.set_text_color(*color) + self.cell(WIDTH, 6, spaced, align="C") + + def draw_split_logo(self, y, size=18): + """Split-color diff-diff logo with SIGNAL middle dash.""" + self.set_xy(0, y) + self.set_font("Helvetica", "B", size) + self.set_text_color(*NAVY) + self.cell(WIDTH / 2 - 5, 10, "diff", align="R") + self.set_text_color(*SIGNAL) + self.cell(10, 10, "-", align="C") + self.set_text_color(*NAVY) + self.cell(WIDTH / 2 - 5, 10, "diff", align="L") + + # ----------------------------------------------------------------- + # Shadowed card helper (poster mode: used sparingly, big type inside) + # ----------------------------------------------------------------- + def _shadow_rect(self, x, y, w, h): + self.set_fill_color(*SHADOW) + self.rect(x + 1.4, y + 1.4, w, h, "F") + + def _stat_card(self, x, y, w, h, headline, sub_lines, accent, headline_size=40): + """One shadowed stat card: big number + caption lines (>= 14pt).""" + self._shadow_rect(x, y, w, h) + self.set_fill_color(*WHITE) + self.set_draw_color(220, 220, 220) + self.set_line_width(0.5) + self.rect(x, y, w, h, "DF") + self.set_fill_color(*accent) + self.rect(x, y, w, 3.2, "F") + + self.set_xy(x, y + 13) + self.set_font("Helvetica", "B", headline_size) + self.set_text_color(*accent) + self.cell(w, 16, headline, align="C") + + ly = y + 36 + for line, emphasize in sub_lines: + self.set_xy(x + 6, ly) + self.set_font("Helvetica", "B" if emphasize else "", 14) + self.set_text_color(*(NAVY if emphasize else GRAY)) + self.cell(w - 12, 8, line, align="C") + ly += 10.5 + + # ----------------------------------------------------------------- + # Figure helpers (matplotlib -> PNG -> fpdf image) + # ----------------------------------------------------------------- + def _save_fig(self, fig, dpi=200, transparent=False, facecolor="white"): + fd, path = tempfile.mkstemp(suffix=".png") + os.close(fd) + self._temp_files.append(path) + try: + fig.savefig( + path, + dpi=dpi, + bbox_inches="tight", + pad_inches=0.1, + transparent=transparent, + facecolor=None if transparent else facecolor, + ) + finally: + plt.close(fig) + with PILImage.open(path) as img: + pw, ph = img.size + return path, pw, ph + + # ----------------------------------------------------------------- + # Cover motif -- a curved covariate relationship (signal green) that + # a straight gray fit cannot bend to. Stylized art, not a data claim + # (it plots no tutorial data) - labeled "illustrative" on-slide. + # ----------------------------------------------------------------- + def _render_cover_motif(self): + rng = np.random.default_rng(2020) + x = np.linspace(-2.6, 2.6, 140) + truth = 1.6 * (x**2 - 1.0) * 0.55 + 0.6 + + fig, ax = plt.subplots(figsize=(10, 3.7)) + fig.patch.set_alpha(0) + ax.set_facecolor("none") + + xs = rng.uniform(-2.6, 2.6, 170) + ys = 1.6 * (xs**2 - 1.0) * 0.55 + 0.6 + rng.normal(0, 0.42, xs.size) + ax.plot(xs, ys, "o", color=GRAPHITE_HEX, markersize=5.5, alpha=0.30) + + coef = np.polyfit(xs, ys, 1) + ax.plot( + x, + np.polyval(coef, x), + color=WRONG_GRAY_HEX, + linewidth=3.2, + linestyle=(0, (6, 3)), + ) + ax.plot(x, truth, color=SIGNAL_HEX, linewidth=4.2, zorder=5) + + ax.text( + 2.68, + float(truth[-1]) - 0.15, + "what an ML learner\ncan fit", + fontsize=14, + color=SIGNAL_HEX, + fontweight="bold", + va="center", + ) + ax.text( + 2.68, + float(np.polyval(coef, 2.6)) - 1.05, + "what a linear\nmodel fits", + fontsize=13.5, + color=WRONG_GRAY_HEX, + fontweight="bold", + va="center", + ) + ax.text( + 3.9, + -1.62, + "illustrative", + fontsize=15, + color=LIGHT_GRAY_HEX, + style="italic", + ha="right", + ) + + ax.set_xlim(-2.8, 4.0) + ax.set_ylim(-1.75, 3.3) + ax.set_xticks([]) + ax.set_yticks([]) + for spine in ax.spines.values(): + spine.set_visible(False) + fig.tight_layout(pad=0.2) + return self._save_fig(fig, transparent=True) + + # ----------------------------------------------------------------- + # Math slide equation (matplotlib mathtext -> crisp big type) + # ----------------------------------------------------------------- + def _render_score_equation(self, dark=False): + """Equation composed from measured fragments, with the second + fraction built MANUALLY (numerator text, drawn bar, denominator + text) so the annotation arrows can anchor to center-placed text + objects instead of mathtext bounding boxes - mathtext bboxes carry + asymmetric slop (worst around \\left[...\\right]), which pushed + every arrow visibly right of its target in the first two cuts.""" + fg = "white" if dark else NAVY_HEX + accent = SIGNAL_BRIGHT_HEX if dark else SIGNAL_HEX + gray = LIGHT_GRAY_HEX if dark else GRAY_HEX + + fig, ax = plt.subplots(figsize=(10.8, 4.1)) + fig.patch.set_alpha(0) + ax.set_facecolor("none") + ax.set_xlim(0, 10) + ax.set_ylim(0, 10) + ax.axis("off") + + y_eq = 6.1 + frags = [ + ("prefix", r"$s \,=$"), + ("bracket", r"$\left[\,D \;-\; \hat g(X)\,\frac{1-D}{1-\hat g(X)}\,\right]$"), + ("dot", r"$\cdot$"), + ] + texts = {} + cursor = 0.0 + lead_gap = {"prefix": 0.0, "bracket": -0.12, "dot": 0.18} + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + inv = ax.transData.inverted() + boxes = {} + for name, tex in frags: + t = ax.text( + cursor + lead_gap[name], y_eq, tex, fontsize=30, color=fg, ha="left", va="center" + ) + fig.canvas.draw() + bb = t.get_window_extent(renderer=renderer) + (x0, y0), (x1, y1) = inv.transform([(bb.x0, bb.y0), (bb.x1, bb.y1)]) + texts[name] = t + boxes[name] = [x0, y0, x1, y1] + cursor = x1 + + # Manual fraction: numerator / bar / p-hat, all centered on one x + # so the arrows can anchor EXACTLY (ha="center" text at a known x). + num_t = ax.text( + cursor + 0.35, + y_eq + 0.28, + r"$\Delta Y \;-\; \hat\ell(X)$", + fontsize=25, + color=fg, + ha="left", + va="bottom", + ) + fig.canvas.draw() + bb = num_t.get_window_extent(renderer=renderer) + (nx0, _ny0), (nx1, _ny1) = inv.transform([(bb.x0, bb.y0), (bb.x1, bb.y1)]) + frac_cx = (nx0 + nx1) / 2 + bar_pad = 0.12 + bar = ax.plot( + [nx0 - bar_pad, nx1 + bar_pad], + [y_eq + 0.08, y_eq + 0.08], + color=fg, + linewidth=2.0, + solid_capstyle="butt", + )[0] + den_t = ax.text( + frac_cx, y_eq - 0.16, r"$\hat p$", fontsize=25, color=fg, ha="center", va="top" + ) + frac_parts = [num_t, bar, den_t] + + # Recenter the assembled equation on x = 5. + span_mid = (boxes["prefix"][0] + (nx1 + bar_pad)) / 2 + shift = 5.0 - span_mid + for name, _ in frags: + x_old, y_old = texts[name].get_position() + texts[name].set_position((x_old + shift, y_old)) + boxes[name][0] += shift + boxes[name][2] += shift + for artist in frac_parts: + if artist is bar: + xs = bar.get_xdata() + bar.set_xdata([xs[0] + shift, xs[1] + shift]) + else: + x_old, y_old = artist.get_position() + artist.set_position((x_old + shift, y_old)) + frac_cx += shift + nx0 += shift + nx1 += shift + + # Bracket arrow center: the bbox RIGHT edge carries \right]-related + # slop, so use the reliable left edge and the dot fragment's left + # edge (which visually abuts the closing bracket) as the span. + bracket_cx = (boxes["bracket"][0] + boxes["dot"][0] - lead_gap["dot"]) / 2 + bracket_bottom = boxes["bracket"][1] + + pad = 0.45 + arrow_kw = dict(arrowstyle="->", color=accent, lw=1.8) + # propensity weight -> the bracket, from below + ax.annotate( + "", + xy=(bracket_cx, bracket_bottom - 0.15), + xytext=(bracket_cx, bracket_bottom - pad - 0.9), + arrowprops=arrow_kw, + ) + ax.text( + bracket_cx, + bracket_bottom - pad - 1.05, + "propensity weight\n(treated vs reweighted controls)", + fontsize=13.5, + color=gray, + ha="center", + va="top", + ) + # outcome-model residual -> the numerator, from above + num_top = y_eq + 1.22 + ax.annotate( + "", + xy=(frac_cx, num_top + 0.1), + xytext=(frac_cx, num_top + pad + 0.85), + arrowprops=arrow_kw, + ) + ax.text( + frac_cx, + num_top + pad + 1.0, + "outcome-model residual\n(what the trend model missed)", + fontsize=13.5, + color=gray, + ha="center", + va="bottom", + ) + # treated share -> p-hat, from below (den_t is ha="center" at + # frac_cx, so this arrow hits the glyph center exactly) + den_bottom = y_eq - 1.28 + ax.annotate( + "", + xy=(frac_cx, den_bottom + 0.1), + xytext=(frac_cx, den_bottom - pad - 0.75), + arrowprops=arrow_kw, + ) + ax.text( + frac_cx, + den_bottom - pad - 0.9, + "treated share\n(normalizer)", + fontsize=13.5, + color=gray, + ha="center", + va="top", + ) + + ax.text( + 5.0, + 0.15, + r"$\hat g(X)$: propensity learner " + r"$\hat\ell(X)$: outcome learner " + r"$\hat p$: treated share", + fontsize=14.5, + color=gray, + ha="center", + ) + fig.tight_layout(pad=0.2) + return self._save_fig(fig, transparent=True) + + # ----------------------------------------------------------------- + # Payoff chart -- the four learner estimates vs the truth line. + # ONLY committed tutorial values (point estimates + 1.96*SE whiskers, + # labeled illustrative-inference on-slide). + # ----------------------------------------------------------------- + def _render_payoff_chart(self): + fig, ax = plt.subplots(figsize=(10.2, 4.1)) + + learners = [ + ("linear", LINEAR, WRONG_GRAY_HEX), + ("ridge", RIDGE, WRONG_GRAY_HEX), + ("sieve", SIEVE, SIGNAL_HEX), + ("custom\nPolynomialRidge", POLY, SIGNAL_HEX), + ] + xs = np.arange(len(learners)) + ax.axhline(TRUTH, color=GRAPHITE_HEX, linewidth=2.2, linestyle=(0, (5, 3))) + ax.text( + 3.52, + TRUTH - 0.028, + f"true effect {TRUTH:.4f}\n(simulated - known\nby construction)", + fontsize=13.5, + color=GRAPHITE_HEX, + fontweight="bold", + va="top", + ) + + for i, (name, (att, se), color) in enumerate(learners): + ax.errorbar( + [i], + [att], + yerr=[1.96 * se], + fmt="o", + color=color, + markersize=14, + capsize=7, + linewidth=2.8, + capthick=2.4, + ) + ax.annotate( + f"{att:.4f}", + xy=(i, att), + xytext=(i + 0.09, att + 0.035), + fontsize=15, + fontweight="bold", + color=color, + ) + + ax.set_xticks(xs) + ax.set_xticklabels([n for n, _, _ in learners], fontsize=14.5, color=NAVY_HEX) + ax.set_ylabel("estimated overall ATT", fontsize=14, color=NAVY_HEX) + ax.tick_params(axis="y", labelsize=13, colors=GRAY_HEX) + ax.set_xlim(-0.5, 4.6) + ax.set_ylim(2.1, 2.82) + for s in ("top", "right"): + ax.spines[s].set_visible(False) + for s in ("bottom", "left"): + ax.spines[s].set_color(LIGHT_GRAY_HEX) + fig.tight_layout(pad=0.3) + return self._save_fig(fig, facecolor="white") + + # ----------------------------------------------------------------- + # Code-block helper (dark panel with token-colored lines) + # ----------------------------------------------------------------- + def _add_code_block( + self, x, y, w, token_lines, font_size=15, line_height=12.5, fill=PANEL_NAVY + ): + n_lines = len(token_lines) + total_h = n_lines * line_height + 26 + + self.set_fill_color(*fill) + self.rect(x, y, w, total_h, "F") + + self.set_font("Courier", "", font_size) + char_w = self.get_string_width("M") + + pad_x = 14 + pad_y = 13 + for i, tokens in enumerate(token_lines): + cx = x + pad_x + cy = y + pad_y + i * line_height + for text, color in tokens: + if not text: + continue + self.set_xy(cx, cy) + self.set_text_color(*color) + self.cell(char_w * len(text), 10, text) + cx += char_w * len(text) + return total_h + + # ----------------------------------------------------------------- + # Slides + # ----------------------------------------------------------------- + def slide_01_cover(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(1) + + self.draw_split_logo(26, size=34) + + self.centered_text(60, "Plug any ML model", size=52) + self.centered_text(90, "into your DiD.", size=52, color=SIGNAL) + + motif_path, _pw, _ph = self._render_cover_motif() + motif_w = 224 + self.image(motif_path, (WIDTH - motif_w) / 2, 122, motif_w) + + self.centered_text( + 212, + f"New in diff-diff {RELEASE_LABEL}: the DMLDiD estimator.", + size=22, + color=GRAPHITE, + ) + self.centered_text( + 230, + "Double/debiased machine learning DiD (Chang 2020).", + size=19, + bold=False, + ) + self.add_footer() + + def slide_02_bet(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(2) + + self._kicker(42, "Why You'd Want This") + + self.centered_text(78, "Covariate adjustment", size=44) + self.centered_text(104, "is a modeling bet.", size=44, color=GRAPHITE) + + self.set_xy(28, 138) + self.set_font("Helvetica", "", 20) + self.set_text_color(*NAVY) + self.multi_cell( + WIDTH - 56, + 11, + "Classic doubly-robust DiD (CallawaySantAnna and friends) fixes the " + "nuisance models up front: a logit for treatment, a linear " + "regression for trends.", + align="C", + ) + + self.set_draw_color(*SIGNAL) + self.set_line_width(1.4) + self.line(WIDTH / 2 - 30, 186, WIDTH / 2 + 30, 186) + + self.set_xy(28, 200) + self.set_font("Helvetica", "B", 22) + self.set_text_color(*GRAPHITE) + self.multi_cell( + WIDTH - 56, + 12, + "When treatment or trends bend in X, that bet loses - and nothing " + "in the output tells you.", + align="C", + ) + self.add_footer() + + def slide_03_receipt(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(3) + + self._kicker(36, "The Receipt") + + self.centered_text(62, "Confidently wrong.", size=48) + + card_w, card_h = 108, 78 + gap = 14 + left_x = (WIDTH - 2 * card_w - gap) / 2 + y0 = 100 + self._stat_card( + left_x, + y0, + card_w, + card_h, + f"{LINEAR[0]:.4f}", + [ + (f"reported SE {LINEAR[1]:.4f}", True), + ("linear outcome model -", False), + ("a narrow nominal interval", False), + ], + WRONG_GRAY, + ) + self._stat_card( + left_x + card_w + gap, + y0, + card_w, + card_h, + f"{TRUTH:.4f}", + [ + ("the true effect", True), + ("simulated staggered panel -", False), + ("truth known by construction", False), + ], + SIGNAL, + ) + + self.centered_text(198, "Almost five standard errors off.", size=30, color=GRAPHITE) + + self.set_xy(30, 222) + self.set_font("Helvetica", "", 18) + self.set_text_color(*GRAY) + self.multi_cell( + WIDTH - 60, + 9.5, + f"{N_UNITS} units, {N_PERIODS} periods, cohorts treated at t = {COHORTS[0]} " + f"and {COHORTS[1]}, {N_NEVER} never treated. Assignment and trends both " + "bend in X - the linear model cannot. (SEs here are nominal, not " + "theory-backed: the DGP deliberately breaks the rate conditions.)", + align="C", + ) + self.add_footer() + + def slide_04_twist(self): + self.add_page() + self.dark_gradient_background() + self._draw_vertical_sidebar(4, dark=True) + + self._kicker(40, "It Gets Worse", color=SIGNAL_BRIGHT) + + self.centered_text(84, "Ridge won't save you.", size=48, color=WHITE) + + self.centered_text(136, f"{RIDGE[0]:.4f}", size=64, color=LIGHT_GRAY) + self.centered_text( + 166, + f"reported SE {RIDGE[1]:.4f} - same miss, same nominal precision", + size=19, + bold=False, + color=LIGHT_GRAY, + ) + + self.set_xy(30, 204) + self.set_font("Helvetica", "B", 24) + self.set_text_color(*MINT) + self.multi_cell( + WIDTH - 60, + 13, + "A penalty on the wrong functional form\nis still the wrong functional form.", + align="C", + ) + + self.set_xy(30, 250) + self.set_font("Helvetica", "", 16) + self.set_text_color(*LIGHT_GRAY) + self.multi_cell( + WIDTH - 60, + 8.5, + "Regularization tunes a model family. It cannot leave the family. " + "(Reported SEs are nominal - illustrative under this DGP.)", + align="C", + ) + self.add_footer(dark=True) + + def slide_05_paper(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(5) + + self._kicker(38, "The Paper") + + self.centered_text(66, "Chang (2020):", size=40) + self.centered_text(92, "stop betting. Learn them.", size=40, color=SIGNAL) + + y0 = 128 + for num, head, body in [ + ( + "1", + "Neyman-orthogonal scores", + "built so small nuisance errors hit the estimate only at second order", + ), + ( + "2", + "Cross-fitting", + "nuisances learned on K-1 folds, applied to the held-out fold - " + "flexible learners without overfitting bias", + ), + ]: + self.set_fill_color(*SIGNAL) + self.set_xy(34, y0) + self.set_font("Helvetica", "B", 26) + self.set_text_color(*SIGNAL) + self.cell(14, 12, num) + self.set_xy(52, y0) + self.set_font("Helvetica", "B", 23) + self.set_text_color(*NAVY) + self.cell(WIDTH - 86, 12, head) + self.set_xy(52, y0 + 14) + self.set_font("Helvetica", "", 17) + self.set_text_color(*GRAY) + self.multi_cell(WIDTH - 92, 9, body) + y0 += 52 + + self.set_xy(30, 244) + self.set_font("Helvetica", "I", 15) + self.set_text_color(*GRAY) + self.multi_cell( + WIDTH - 60, + 8, + "Double/debiased machine learning for difference-in-differences models " + "- The Econometrics Journal, 23(2).", + align="C", + ) + self.add_footer() + + def slide_06_math(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(6) + + self._kicker(34, "The Math") + + self.centered_text(58, "One orthogonal score per cell.", size=36) + + eq_path, _pw, _ph = self._render_score_equation() + eq_w = 232 + self.image(eq_path, (WIDTH - eq_w) / 2, 84, eq_w) + + y0 = 182 + for head, body in [ + ( + "Orthogonal.", + "The score's derivative in the LEARNED nuisances g and l is " + "zero at the truth - learner error is second-order. (The " + "treated share p gets its own variance correction.)", + ), + ( + "Cross-fit.", + "No unit is scored by a learner that trained on it - " + "flexibility without self-overfitting.", + ), + ]: + self.set_xy(34, y0) + self.set_font("Helvetica", "B", 20) + self.set_text_color(*SIGNAL) + head_w = self.get_string_width(head) + 4 + self.cell(head_w, 10, head) + self.set_xy(34 + head_w, y0) + self.set_font("Helvetica", "", 17) + self.set_text_color(*NAVY) + self.multi_cell(WIDTH - 68 - head_w, 9.4, body) + y0 += 34 + + self.set_xy(30, 256) + self.set_font("Helvetica", "I", 13.5) + self.set_text_color(*GRAY) + self.multi_cell( + WIDTH - 60, + 7, + "Chang eq. 3.1 (panel case) as implemented: the mean of the summand " + "s over a (cohort, period) cell is that cell's ATT; the centered " + "score is psi = s - ATT.", + align="C", + ) + self.add_footer() + + def slide_07_payoff(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(7) + + self._kicker(34, "The Payoff") + + self.centered_text(58, "Same data. Same propensity.", size=32) + self.centered_text(78, "Flexible outcome model: truth.", size=32, color=SIGNAL) + + chart_path, _pw, _ph = self._render_payoff_chart() + chart_w = 226 + self.image(chart_path, (WIDTH - chart_w) / 2, 98, chart_w) + + self.set_xy(28, 232) + self.set_font("Helvetica", "", 16.5) + self.set_text_color(*NAVY) + self.multi_cell( + WIDTH - 56, + 9, + "That is double robustness doing real work: the propensity model is " + "misspecified in every arm - a flexible outcome model alone rescues " + "the point estimate.", + align="C", + ) + self.set_xy(28, 262) + self.set_font("Helvetica", "I", 13) + self.set_text_color(*GRAY) + self.multi_cell( + WIDTH - 56, + 7, + "Seed-locked simulated example; whiskers are 1.96 SE, illustrative " + "under the deliberately misspecified propensity.", + align="C", + ) + self.add_footer() + + def slide_08_when(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(8) + + self._kicker(36, "When To Reach For It") + + col_w = 106 + gap = 16 + left_x = (WIDTH - 2 * col_w - gap) / 2 + y0 = 66 + col_h = 118 + + self._shadow_rect(left_x, y0, col_w, col_h) + self.set_fill_color(*WHITE) + self.set_draw_color(220, 220, 220) + self.rect(left_x, y0, col_w, col_h, "DF") + self.set_fill_color(*SIGNAL) + self.rect(left_x, y0, col_w, 3.2, "F") + self.set_xy(left_x, y0 + 10) + self.set_font("Helvetica", "B", 24) + self.set_text_color(*SIGNAL) + self.cell(col_w, 12, "DMLDiD", align="C") + self.set_xy(left_x + 8, y0 + 28) + self.set_font("Helvetica", "", 16) + self.set_text_color(*NAVY) + self.multi_cell( + col_w - 16, + 9, + "Covariates with nonlinear or high-dimensional relationships to " + "treatment or trends.", + align="C", + ) + self.set_xy(left_x + 8, y0 + 82) + self.set_font("Helvetica", "B", 15) + self.set_text_color(*GRAPHITE) + self.multi_cell( + col_w - 16, + 8.5, + "Bring your own learners:\nregressors fit() / predict(),\nclassifiers predict_proba().", + align="C", + ) + + rx = left_x + col_w + gap + self._shadow_rect(rx, y0, col_w, col_h) + self.set_fill_color(*WHITE) + self.set_draw_color(220, 220, 220) + self.rect(rx, y0, col_w, col_h, "DF") + self.set_fill_color(*GRAPHITE) + self.rect(rx, y0, col_w, 3.2, "F") + self.set_xy(rx, y0 + 10) + self.set_font("Helvetica", "B", 24) + self.set_text_color(*GRAPHITE) + self.cell(col_w, 12, "CallawaySantAnna", align="C") + self.set_xy(rx + 8, y0 + 28) + self.set_font("Helvetica", "", 16) + self.set_text_color(*NAVY) + self.multi_cell( + col_w - 16, + 9, + "A logit + linear spec is plausible for your covariates.", + align="C", + ) + self.set_xy(rx + 8, y0 + 82) + self.set_font("Helvetica", "B", 15) + self.set_text_color(*GRAPHITE) + self.multi_cell(col_w - 16, 8.5, "Fewer moving parts.\nStill doubly robust.", align="C") + + self.set_xy(28, y0 + col_h + 20) + self.set_font("Helvetica", "", 18) + self.set_text_color(*NAVY) + self.multi_cell( + WIDTH - 56, + 10, + "Same staggered surface either way: ATT(g,t) cells, event studies, " + "group effects, HonestDiD.", + align="C", + ) + + band_y = y0 + col_h + 52 + self.set_fill_color(*GRAPHITE) + self.rect(28, band_y, WIDTH - 56, 40, "F") + self.set_xy(34, band_y + 6) + self.set_font("Helvetica", "B", 17) + self.set_text_color(*SIGNAL_BRIGHT) + self.cell( + WIDTH - 68, + 9, + "Four learners built in: linear, ridge, sieve, logit - no extra installs.", + align="C", + ) + self.set_xy(34, band_y + 19) + self.set_font("Helvetica", "B", 16) + self.set_text_color(*WHITE) + self.multi_cell( + WIDTH - 68, + 8.5, + "Or bring your own: scikit-learn regressors already fit the fit() /\n" + "predict() contract - classifiers add predict_proba().", + align="C", + ) + self.add_footer() + + def slide_09_code(self): + self.add_page() + self.dark_gradient_background() + self._draw_vertical_sidebar(9, dark=True) + + self._kicker(34, "The Code", color=SIGNAL_BRIGHT) + + self.centered_text(58, "Three lines. Any learner.", size=40, color=WHITE) + + W_ = WHITE + G_ = GREEN_CODE + S_ = SLATE_CODE + code = [ + [("est = DMLDiD(", W_)], + [(" outcome_learner=", W_), (f'"{CODE_LEARNER}"', G_), (",", W_)], + [ + (" n_folds=", W_), + (str(CODE_FOLDS), G_), + (", seed=", W_), + (str(CODE_SEED), G_), + (",", W_), + ], + [(" base_period=", W_), ('"universal"', G_), (",", W_)], + [(")", W_)], + [ + ("res = est.fit(df, outcome=", W_), + ('"y"', G_), + (", unit=", W_), + ('"unit"', G_), + (", time=", W_), + ('"time"', G_), + (",", W_), + ], + [ + (" first_treat=", W_), + ('"first_treat"', G_), + (", covariates=[", W_), + ('"x1"', G_), + (", ", W_), + ('"x2"', G_), + ("])", W_), + ], + [("res.aggregate(", W_), ('"event_study"', G_), (")", W_)], + [("", W_)], + [("# or bring your own - sklearn fits the contract:", S_)], + [("from sklearn.ensemble import GradientBoostingRegressor", W_)], + [("DMLDiD(outcome_learner=GradientBoostingRegressor())", W_)], + ] + block_w = 226 + self._add_code_block( + (WIDTH - block_w) / 2, 88, block_w, code, font_size=15, line_height=11.8 + ) + + self.set_xy(30, 252) + self.set_font("Helvetica", "", 16) + self.set_text_color(*LIGHT_GRAY) + self.multi_cell( + WIDTH - 60, + 8.5, + "seed= pins the fold draws - the shown call reproduces the shown numbers exactly.", + align="C", + ) + self.add_footer(dark=True) + + def slide_10_production(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(10) + + self._kicker(34, "Built For Production") + + self.centered_text(56, "The full staggered toolkit.", size=36) + + features = [ + ( + "Post-fit aggregation", + "event studies, group effects,\nHonestDiD, uniform sup-t bands", + ), + ( + "Survey data", + f"designs + clustering: PSU-cohesive\nfolds; design df: {SURVEY_PSU} PSUs\n" + f"- {SURVEY_STRATA} strata = {SURVEY_DF}", + ), + ( + "Repeated cross-sections", + "panel=False runs Chang's Case 2:\nfresh samples from the SAME\ntarget population each wave", + ), + ( + "Cross-fit diagnostics", + "per-cell fold losses, overlap,\ntrimming - inspect every nuisance", + ), + ] + card_w, card_h = 108, 62 + gap_x, gap_y = 14, 12 + x0 = (WIDTH - 2 * card_w - gap_x) / 2 + y0 = 82 + for i, (head, body) in enumerate(features): + cx = x0 + (i % 2) * (card_w + gap_x) + cy = y0 + (i // 2) * (card_h + gap_y) + self._shadow_rect(cx, cy, card_w, card_h) + self.set_fill_color(*WHITE) + self.set_draw_color(220, 220, 220) + self.rect(cx, cy, card_w, card_h, "DF") + self.set_fill_color(*SIGNAL) + self.rect(cx, cy, card_w, 3, "F") + self.set_xy(cx + 6, cy + 9) + self.set_font("Helvetica", "B", 17) + self.set_text_color(*GRAPHITE) + self.cell(card_w - 12, 9, head, align="C") + self.set_xy(cx + 6, cy + 22) + self.set_font("Helvetica", "", 13.5) + self.set_text_color(*GRAY) + self.multi_cell(card_w - 12, 7.5, body, align="C") + + strip_y = y0 + 2 * card_h + gap_y + 16 + self.set_fill_color(*GRAPHITE) + self.rect(28, strip_y, WIDTH - 56, 30, "F") + self.set_xy(34, strip_y + 5) + self.set_font("Helvetica", "B", 15.5) + self.set_text_color(*SIGNAL_BRIGHT) + self.cell(WIDTH - 68, 8, f"Chang Case 1 score vs DoubleML: ATT diff {DOUBLEML_ATT_DIFF}") + self.set_xy(34, strip_y + 15) + self.set_font("Helvetica", "", 13) + self.set_text_color(*LIGHT_GRAY) + self.cell( + WIDTH - 68, + 7, + "committed parity spike; staggered cells + survey lanes are documented " + "extensions of the paper", + ) + self.add_footer() + + def slide_11_cta(self): + self.add_page() + self.light_gradient_background() + self._draw_vertical_sidebar(11) + + self.draw_split_logo(38, size=34) + + self.centered_text(80, "Your covariates aren't linear.", size=38) + self.centered_text(106, "Your DiD can keep up now.", size=38, color=SIGNAL) + + chip_w = 150 + chip_x = (WIDTH - chip_w) / 2 + self.set_fill_color(*PANEL_NAVY) + self.rect(chip_x, 142, chip_w, 22, "F") + self.set_xy(chip_x, 148) + self.set_font("Courier", "B", 19) + self.set_text_color(*GREEN_CODE) + self.cell(chip_w, 10, "pip install diff-diff", align="C") + + self.set_xy(30, 184) + self.set_font("Helvetica", "", 17) + self.set_text_color(*NAVY) + self.multi_cell( + WIDTH - 60, + 9, + "Tutorial 32 reproduces every number on this deck - learner " + "comparison, HonestDiD, and the survey lane included:", + align="C", + ) + self.centered_text( + 218, + "diff-diff.readthedocs.io/en/latest/tutorials/32_dml_did.html", + size=14, + color=SLATE, + ) + + self.centered_text(242, "github.com/igerber/diff-diff", size=17) + + self.set_xy(0, 264) + self.set_font("Helvetica", "I", 13) + self.set_text_color(*GRAY) + self.cell( + WIDTH, + 6, + "Chang (2020), The Econometrics Journal 23(2) - doi:10.1093/ectj/utaa001.", + align="C", + ) + self.add_footer() + + +def main(): + pdf = DMLCarouselPDF() + try: + pdf.slide_01_cover() + pdf.slide_02_bet() + pdf.slide_03_receipt() + pdf.slide_04_twist() + pdf.slide_05_paper() + pdf.slide_06_math() + pdf.slide_07_payoff() + pdf.slide_08_when() + pdf.slide_09_code() + pdf.slide_10_production() + pdf.slide_11_cta() + + output_path = Path(__file__).parent / "diff-diff-dml-carousel.pdf" + pdf.output(str(output_path)) + print(f"PDF saved to: {output_path}") + finally: + pdf.cleanup() + + +if __name__ == "__main__": + main() diff --git a/tests/test_dml_carousel_claims.py b/tests/test_dml_carousel_claims.py new file mode 100644 index 00000000..fb2200e6 --- /dev/null +++ b/tests/test_dml_carousel_claims.py @@ -0,0 +1,485 @@ +"""Claim-sync guard for the DMLDiD launch carousel. + +The deck (``carousel/generate_dml_carousel.py``) hardcodes committed, +seed-locked numbers from the executed tutorial-32 notebook as module +constants. The synchronization chain is: + +- library <-> notebook: tutorial 32 is numbers-locked and executed against + the committed library (``tests/test_t32_dml_did_drift.py`` re-derives the + estimates and hash-pins the code cells); +- notebook <-> carousel: THIS file parses the deck's constants (via ``ast`` + - the generator imports ``fpdf2``, a carousel-only dependency deliberately + absent from CI) and locates each on the COMMITTED NOTEBOOK SURFACE, not + restated here. If the tutorial is ever retuned and the carousel is + forgotten, the notebook surface changes and this file fails. +- REGISTRY <-> carousel: the DoubleML parity strip renders the diff the + REGISTRY DR-score families note records for the committed spike. + +Skips cleanly when ``carousel/`` or ``docs/`` is absent (the +isolated-install CI jobs copy only ``tests/``). +""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path + +import pytest + +from tests._tutorial_drift import notebook_markdown, notebook_output_text + +GENERATOR = Path(__file__).resolve().parents[1] / "carousel" / "generate_dml_carousel.py" +REGISTRY = Path(__file__).resolve().parents[1] / "docs" / "methodology" / "REGISTRY.md" +NB = "docs/tutorials/32_dml_did.ipynb" + +TOTAL_SLIDES = 11 + +# Constant values the code slide renders via f-strings / str() - resolved +# here so the snippet reconstruction stays a pure-AST operation. +DECK_LITERALS = {"CODE_LEARNER": "sieve", "CODE_FOLDS": 5, "CODE_SEED": 42} + + +@pytest.fixture(scope="module") +def deck_constants(): + if not GENERATOR.exists(): + pytest.skip(f"{GENERATOR} not available in this CI environment.") + tree = ast.parse(GENERATOR.read_text(encoding="utf-8")) + consts = {} + for node in tree.body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + try: + consts[target.id] = ast.literal_eval(node.value) + except (ValueError, TypeError): + pass + elif isinstance(target, ast.Tuple) and all( + isinstance(el, ast.Name) for el in target.elts + ): + try: + values = ast.literal_eval(node.value) + except (ValueError, TypeError): + continue + if isinstance(values, tuple) and len(values) == len(target.elts): + for el, val in zip(target.elts, values): + consts[el.id] = val + return consts + + +@pytest.fixture(scope="module") +def visible_strings(): + """Slide copy: every string literal in the generator EXCEPT docstrings. + + f-strings contribute their constant fragments (``ast.Constant`` nodes + inside ``ast.JoinedStr``), so phrase pins must not span a ``{...}`` + substitution. + """ + if not GENERATOR.exists(): + pytest.skip(f"{GENERATOR} not available in this CI environment.") + tree = ast.parse(GENERATOR.read_text(encoding="utf-8")) + docstring_nodes = set() + for node in ast.walk(tree): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + body = node.body + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + docstring_nodes.add(id(body[0].value)) + return " ".join( + node.value + for node in ast.walk(tree) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and id(node) not in docstring_nodes + ) + + +# Chart helpers attributed to the slide that places them. +_SLIDE_HELPERS = { + "slide_01_cover": ("_render_cover_motif",), + "slide_06_math": ("_render_score_equation",), + "slide_07_payoff": ("_render_payoff_chart",), +} + + +def _per_slide(collect): + if not GENERATOR.exists(): + pytest.skip(f"{GENERATOR} not available in this CI environment.") + tree = ast.parse(GENERATOR.read_text(encoding="utf-8")) + per_func = {} + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + per_func[node.name] = collect(node) + slides = {} + for name, value in per_func.items(): + if name.startswith("slide_"): + for helper in _SLIDE_HELPERS.get(name, ()): + value = value + per_func.get(helper, type(value)()) + slides[name] = value + assert ( + len(slides) == TOTAL_SLIDES + ), f"expected {TOTAL_SLIDES} slide_* methods, found {sorted(slides)}" + return slides + + +@pytest.fixture(scope="module") +def slide_strings(): + """Per-slide visible strings (a qualifier must live ON the slide that + makes the claim, not merely somewhere in the module).""" + + def collect(node): + return " ".join( + n.value + for n in ast.walk(node) + if isinstance(n, ast.Constant) and isinstance(n.value, str) + ) + + return _per_slide(collect) + + +@pytest.fixture(scope="module") +def slide_constant_refs(): + """Per-slide referenced module-level names (each pinned statistical + constant must actually be RENDERED by its intended slide - a hardcoded + literal bypassing the constant would otherwise pass).""" + + def collect(node): + return [n.id for n in ast.walk(node) if isinstance(n, ast.Name)] + + return _per_slide(collect) + + +@pytest.fixture(scope="module") +def nb_output(): + return notebook_output_text(NB) + + +def _notebook_code(rel_path: str) -> str: + nb_path = Path(__file__).resolve().parents[1] / rel_path + if not nb_path.exists(): + pytest.skip(f"Notebook not found at {nb_path}.") + return "\n".join( + "".join(c["source"]) if isinstance(c["source"], list) else c["source"] + for c in json.loads(nb_path.read_text())["cells"] + if c["cell_type"] == "code" + ) + + +class TestEstimatesMatchNotebook: + """Every estimate on the deck is on the committed tutorial-32 surface.""" + + def test_truth_and_sieve_lines(self, deck_constants, nb_output): + c = deck_constants + att, se = c["SIEVE"] + for needle in ( + f"DGP-implied overall ATT: {c['TRUTH']:.4f}", + f"DMLDiD (sieve) estimate: {att:.4f} +/- {se:.4f}", + ): + assert needle in nb_output, f"deck value not on tutorial-32 surface: {needle!r}" + + def test_learner_table_values(self, deck_constants, nb_output): + # The notebook renders the comparison as a pandas table; pin each + # deck (ATT, SE) pair to ITS OWN ROW (label + ATT + SE in order on + # one line), so swapped labels or a mislabeled constant cannot pass + # on mere presence of the numbers (review round 1 P2). + import re + + rows = { + "LINEAR": "linear", + "RIDGE": "ridge", + "SIEVE": "sieve", + "POLY": r"PolynomialRidge \(custom\)", + } + for key, label in rows.items(): + att, se = deck_constants[key] + pattern = rf"{label}\s+{att:.4f}\s+{se:.4f}" + assert re.search( + pattern, nb_output + ), f"{key}: no notebook table row matches {pattern!r}" + + def test_dgp_shape_values(self, deck_constants, nb_output): + c = deck_constants + md = notebook_markdown(NB) + assert f"{c['N_NEVER']} never-treated units" in md + # cohort sizes render in the DGP cell's value_counts output + assert str(c["N_NEVER"]) in nb_output + code = _notebook_code(NB) + assert f"n_units, periods = {c['N_UNITS']}," in code + # exact cohort-construction expression (review round: a bare + # str(g)-in-code fallback was vacuous - single digits occur + # everywhere) + assert f"np.where(early, {c['COHORTS'][0]}, {c['COHORTS'][1]})" in code + + def test_survey_provenance_line(self, deck_constants, nb_output): + c = deck_constants + needle = ( + f"design df: {c['SURVEY_DF']} ({c['SURVEY_PSU']} PSUs - " + f"{c['SURVEY_STRATA']} strata)" + ) + assert needle in nb_output, f"deck value not on tutorial-32 surface: {needle!r}" + + def test_five_se_claim_is_arithmetic(self, deck_constants): + # "Almost five standard errors off" must follow from the pinned + # values (a distance in reported-SE units, mirroring the tutorial's + # own wording - never presented as valid coverage). + c = deck_constants + att, se = c["LINEAR"] + ratio = abs(att - c["TRUTH"]) / se + assert 4.0 < ratio < 5.0, ratio + + def test_recovery_claim_is_arithmetic(self, deck_constants): + # The payoff headline ("Flexible outcome model: truth.") rests on + # sieve/custom landing within ~1 reported SE of the DGP truth. + c = deck_constants + for key in ("SIEVE", "POLY"): + att, se = c[key] + assert abs(att - c["TRUTH"]) / se < 1.05, key + + +class TestRegistrySync: + def test_doubleml_diff_matches_registry(self, deck_constants): + if not REGISTRY.exists(): + pytest.skip("REGISTRY.md not available in this CI environment.") + text = REGISTRY.read_text(encoding="utf-8") + assert ( + f"ATT diff {deck_constants['DOUBLEML_ATT_DIFF']}" in text + ), "the deck's DoubleML parity figure is not the REGISTRY's" + assert "chang_case1_parity.py" in text + + +class TestGuardrailPhrases: + def test_qualifiers_are_slide_local(self, slide_strings): + expectations = { + # the cover motif is stylized art, labeled on-slide + "slide_01_cover": ("illustrative",), + # the receipt states simulation + known truth WITH the numbers, + # labels uncertainty as reported/nominal, and carries the + # rate-condition caveat locally (review round 1 P1) + "slide_03_receipt": ( + "truth known by construction", + "simulated", + "reported SE ", + "narrow nominal interval", + "nominal, not ", + ), + # the twist repeats the nominal qualifier locally + "slide_04_twist": ("nominal precision", "nominal - illustrative"), + # the math slide scopes orthogonality to the learned nuisances + # and distinguishes the summand from the centered score + "slide_06_math": ( + "LEARNED nuisances g and l", + "psi = s - ATT", + "own variance correction", + ), + # the payoff carries the DR beat + the illustrative-inference + # qualifier (the tutorial's rate-condition caveat) + simulation + "slide_07_payoff": ( + "misspecified in every arm", + "illustrative", + "known\nby construction", + "Seed-locked simulated example", + ), + # the parity strip scopes the DoubleML anchor to the committed + # Case 1 spike and labels staggered/survey as extensions + "slide_10_production": ( + "committed parity spike", + "documented extensions of the paper", + "Chang Case 1 score", + # A2.3: fresh samples must come from the SAME target + # population (review round 1 P1) + "SAME\ntarget population each wave", + ), + } + for slide, phrases in expectations.items(): + for phrase in phrases: + assert phrase in slide_strings[slide], ( + f"qualifier {phrase!r} is not on {slide} - it must ride " + "with the claim it scopes" + ) + + def test_statistical_constants_reach_their_slides(self, slide_constant_refs): + expectations = { + "slide_03_receipt": ("LINEAR", "TRUTH", "N_UNITS", "N_PERIODS", "N_NEVER", "COHORTS"), + "slide_04_twist": ("RIDGE",), + "slide_07_payoff": ("LINEAR", "RIDGE", "SIEVE", "POLY", "TRUTH"), + "slide_09_code": ("CODE_LEARNER", "CODE_FOLDS", "CODE_SEED"), + "slide_10_production": ( + "SURVEY_PSU", + "SURVEY_STRATA", + "SURVEY_DF", + "DOUBLEML_ATT_DIFF", + ), + } + for slide, names in expectations.items(): + for name in names: + assert name in slide_constant_refs[slide], ( + f"constant {name} is not rendered by {slide} - a " + "hardcoded literal may have replaced the synced value" + ) + + def test_no_absolutes_or_competitor_claims(self, visible_strings): + # CS is named neutrally/honestly (slide 8 RECOMMENDS it for the + # parametric case); DoubleML appears only as the parity anchor. No + # superlatives, no external-competitor comparisons, no coverage + # overclaims for the deliberately misspecified demo. + lowered = visible_strings.lower() + for banned in ( + "the only ", + "first and only", + "no other estimator", + "nothing else", + "better than", + "state of the art", + "state-of-the-art", + "game changer", + "revolutionary", + "econml", + "geolift", + "causalimpact", + "csdid", + "unbiased", # the deck shows recovery, never claims unbiasedness + "valid coverage", + "guarantees", + "in each nuisance", # orthogonality covers g and l, not p + "confident ci", + "same confidence", + ): + assert banned not in lowered, f"banned absolute/competitive claim: {banned!r}" + + def test_no_verbatim_pull_quote(self, slide_strings): + # The Chang paper review is pinned to the arXiv layout and the + # published PDF has not been cross-checked word-for-word, so the + # paper slide must carry NO quotation-marked pull quote (module + # docstring records the decision). + paper = slide_strings["slide_05_paper"] + for ch in ('"', "“", "”"): + assert ch not in paper, "the paper slide must not carry a quoted pull quote" + + def test_cs_is_a_real_exported_estimator(self): + init = Path(__file__).resolve().parents[1] / "diff_diff" / "__init__.py" + if not init.exists(): + pytest.skip("diff_diff/__init__.py not available in this CI environment.") + assert "CallawaySantAnna" in init.read_text(encoding="utf-8") + + def test_code_slide_fragments_match_notebook_invocation(self, deck_constants, visible_strings): + # Slide 9's fit is the tutorial's own configuration; the displayed + # learner/folds/seed render from the deck constants, so + # reconstructing the fragments pins deck <-> notebook. + code = _notebook_code(NB) + c = deck_constants + for fragment in ( + 'base_period="universal"', + f'outcome_learner="{c["CODE_LEARNER"]}"', + f"n_folds={c['CODE_FOLDS']},", + f"seed={c['CODE_SEED']},", + 'covariates=["x1", "x2"]', + 'aggregate("event_study")', + ): + assert fragment in code, f"tutorial 32 lost the displayed invocation: {fragment!r}" + assert "outcome_learner=" in visible_strings + # the bring-your-own swap line renders a real sklearn constructor + assert "GradientBoostingRegressor()" in visible_strings + + def test_builtin_learner_claim_is_dependency_free(self): + # Slide 8 says the four built-in learners need "no extra installs": + # diff_diff/_learners.py must import no external ML package. + learners = Path(__file__).resolve().parents[1] / "diff_diff" / "_learners.py" + if not learners.exists(): + pytest.skip("diff_diff/_learners.py not available in this CI environment.") + tree = ast.parse(learners.read_text(encoding="utf-8")) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(a.name.split(".")[0] for a in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module.split(".")[0]) + assert not imported & {"sklearn", "statsmodels", "xgboost", "lightgbm", "torch"}, imported + + def test_sklearn_contract_claim_is_documented(self): + # Slide 8/9 say scikit-learn estimators fit the duck-typed + # contract - the API docs must actually document that claim. + rst = Path(__file__).resolve().parents[1] / "docs" / "api" / "dml_did.rst" + if not rst.exists(): + pytest.skip("docs/api/dml_did.rst not available in this CI environment.") + assert "scikit-learn" in rst.read_text(encoding="utf-8") + + def test_module_docstring_states_both_contracts(self): + # The review-round P1: the guard below excludes docstrings, so the + # module docstring itself must also carry the classifier contract + # (predict_proba), never the bare fit()/predict() shorthand. + if not GENERATOR.exists(): + pytest.skip(f"{GENERATOR} not available in this CI environment.") + tree = ast.parse(GENERATOR.read_text(encoding="utf-8")) + doc = ast.get_docstring(tree) or "" + assert "predict_proba()" in doc + assert "CLASSIFIERS" in doc or "classifier" in doc.lower() + + def test_flexibility_band_is_slide_local(self, slide_strings): + # The user-emphasized flexibility beats (2026-08-29) live on their + # slides: built-ins + no-extra-installs on slide 8, the sklearn + # contract line on the code slide. + assert "no extra installs" in slide_strings["slide_08_when"] + assert "scikit-learn regressors already fit the" in slide_strings["slide_08_when"] + # the propensity contract is predict_proba, not predict - both + # contracts must be stated (review round 2 P2) + assert "predict_proba()" in slide_strings["slide_08_when"] + assert "classifiers" in slide_strings["slide_08_when"] + assert "sklearn fits the contract" in slide_strings["slide_09_code"] + + def test_code_slide_snippet_is_valid_python(self): + # Review round 3 P2: the displayed fit call was invalid Python + # (`fit(df, outcome="y", ..., covariates=...)` - positional + # ellipsis after keyword arguments) under a caption claiming the + # shown call reproduces the shown numbers. Reconstruct the snippet + # from the slide's token lines and require it to parse. + if not GENERATOR.exists(): + pytest.skip(f"{GENERATOR} not available in this CI environment.") + tree = ast.parse(GENERATOR.read_text(encoding="utf-8")) + code_list = None + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "slide_09_code": + for stmt in ast.walk(node): + if ( + isinstance(stmt, ast.Assign) + and isinstance(stmt.targets[0], ast.Name) + and stmt.targets[0].id == "code" + ): + code_list = stmt.value + assert code_list is not None, "slide_09_code lost its `code` token-line list" + + def _frag(nodeval): + # token text: plain Constant or an f-string of Constants/Names + if isinstance(nodeval, ast.Constant): + return str(nodeval.value) + if isinstance(nodeval, ast.JoinedStr): + out = [] + for v in nodeval.values: + if isinstance(v, ast.Constant): + out.append(str(v.value)) + elif isinstance(v, ast.FormattedValue) and isinstance(v.value, ast.Name): + out.append(str(DECK_LITERALS[v.value.id])) + return "".join(out) + if isinstance(nodeval, ast.Call): # str(CONST) + arg = nodeval.args[0] + if isinstance(arg, ast.Name): + return str(DECK_LITERALS[arg.id]) + raise AssertionError(f"unrecognized token node: {ast.dump(nodeval)[:80]}") + + lines = [] + for line in code_list.elts: + assert isinstance(line, ast.List) + lines.append("".join(_frag(tok.elts[0]) for tok in line.elts)) + snippet = "\n".join(lines) + ast.parse(snippet) # must be syntactically valid Python + assert 'base_period="universal"' in snippet # the tutorial fit's config + assert 'unit="unit"' in snippet and 'first_treat="first_treat"' in snippet + + def test_tutorial_referenced_once_on_cta(self, visible_strings): + assert visible_strings.count("Tutorial 32") == 1 + assert "tutorials/32_dml_did.html" in visible_strings + nb_path = Path(__file__).resolve().parents[1] / NB + assert nb_path.exists(), "CTA points to a tutorial that does not exist"