diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d142f6d..7291fa4 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -59,7 +59,7 @@ gunicorn app:app -b 0.0.0.0:5000 # production Before suggesting non-trivial changes, consult these: -- **[docs/plans/business_line_reorg.md](../docs/plans/business_line_reorg.md)** — the 2026-09 business-line reorg (provider seam + canonical schema, ticker-only Parameters bar, readiness prefetch). Batches **B1–B9 landed**; §10 lists the deferred follow-ups. ADRs [0011](../docs/decisions/0011-pluggable-data-provider-seam.md) / [0012](../docs/decisions/0012-parameter-ownership-and-prefetch.md) are **Accepted** — don't re-litigate. +- **[docs/plans/business_line_reorg.md](../docs/plans/business_line_reorg.md)** — the 2026-09 business-line reorg (provider seam + canonical schema, ticker-only Parameters bar, readiness prefetch). Batches **B1–B10 landed**; §10 lists the two remaining deferred follow-ups (risk-free-rate global setting, ADR 0011 `symbol` column). ADRs [0011](../docs/decisions/0011-pluggable-data-provider-seam.md) / [0012](../docs/decisions/0012-parameter-ownership-and-prefetch.md) are **Accepted** — don't re-litigate. - **[docs/constraints.md](../docs/constraints.md)** — external/historical constraints (yfinance limits, SQLite choice, single-machine assumption, intentional "magic numbers"). Read this before flagging anything as tech debt. - **[docs/glossary.md](../docs/glossary.md)** — domain terms (IV vs HV, Greeks, regime, anomaly flags). Read this before assuming a term means what you think it means. - **[docs/decisions/](../docs/decisions/)** — Architecture Decision Records. Each ADR explains the context, options considered, and accepted trade-offs for a major design choice. diff --git a/CLAUDE.md b/CLAUDE.md index 169b86d..179969f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,10 +12,11 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co > **Business-line reorg (2026-09) — [ADR 0011](docs/decisions/0011-pluggable-data-provider-seam.md) > (provider seam + canonical schema) + [0012](docs/decisions/0012-parameter-ownership-and-prefetch.md) > (ticker-only Parameters bar + readiness prefetch), both Accepted.** -> Batches B1–B9 **landed**; the architecture below reflects the end state. Only +> Batches B1–B10 **landed**; the architecture below reflects the end state. Two > deferred follow-ups remain — see [`docs/plans/business_line_reorg.md`](docs/plans/business_line_reorg.md) -> §10 (risk-free-rate global setting, `market_review_prices` L5 → provider seam, -> ADR 0011 `symbol` column). Do not re-litigate the Accepted ADRs. +> §10: the risk-free-rate global setting and the ADR 0011 `symbol` column +> (ticker→symbol rename, waits for a real second provider). Do not re-litigate +> the Accepted ADRs. ## Commands diff --git a/CODEBUDDY.md b/CODEBUDDY.md index 169b86d..179969f 100644 --- a/CODEBUDDY.md +++ b/CODEBUDDY.md @@ -12,10 +12,11 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co > **Business-line reorg (2026-09) — [ADR 0011](docs/decisions/0011-pluggable-data-provider-seam.md) > (provider seam + canonical schema) + [0012](docs/decisions/0012-parameter-ownership-and-prefetch.md) > (ticker-only Parameters bar + readiness prefetch), both Accepted.** -> Batches B1–B9 **landed**; the architecture below reflects the end state. Only +> Batches B1–B10 **landed**; the architecture below reflects the end state. Two > deferred follow-ups remain — see [`docs/plans/business_line_reorg.md`](docs/plans/business_line_reorg.md) -> §10 (risk-free-rate global setting, `market_review_prices` L5 → provider seam, -> ADR 0011 `symbol` column). Do not re-litigate the Accepted ADRs. +> §10: the risk-free-rate global setting and the ADR 0011 `symbol` column +> (ticker→symbol rename, waits for a real second provider). Do not re-litigate +> the Accepted ADRs. ## Commands diff --git a/data_pipeline/read/_query.py b/data_pipeline/read/_query.py index d824dc0..5969843 100644 --- a/data_pipeline/read/_query.py +++ b/data_pipeline/read/_query.py @@ -76,6 +76,72 @@ def get_cleaned_daily(ticker: str, start: dt.date | None = None, end: dt.date | return df +# DOMAIN: default lookback for the market-review close panel when the caller +# passes no explicit range. Matches the 400-day window the old +# market_review_prices ladder used (services/market_review/fetch.py). +_PANEL_LOOKBACK_DAYS = 400 + + +def get_close_panel( + symbols: list[str], + start: dt.date | None = None, + end: dt.date | None = None, +) -> pd.DataFrame: + """Wide close-price panel from ``clean_bars`` for *symbols* (ADR 0011, L5). + + Replaces the old ``market_review_prices`` acquisition path: every symbol — + the primary ticker and each benchmark — is coverage-healed through the same + ``needs_backfill`` / background ``ensure_range`` machinery the rest of the + read layer uses, then its ``close`` series is read from ``clean_bars``. + + Missing ranges are kicked once up front and then awaited **together** for a + single short grace window, so a cold panel costs ~one grace period, not one + per symbol. Whatever coverage exists at the end of that window is returned; + the background backfills keep filling and the next call sees the rest. + + Returns a date-indexed frame with one column per symbol that had any data + (input order), or an empty frame when nothing is available. + """ + start = start or (dt.date.today() - dt.timedelta(days=_PANEL_LOOKBACK_DAYS)) + end = end or dt.date.today() + init_db() + + kicked: list[str] = [] + for sym in symbols: + try: + _u.manual_update(sym, days=7) + if _bf.needs_backfill(sym, start, end): + _kick_backfill(sym, start, end) + kicked.append(sym) + else: + _bf.ensure_range(sym, start, end) + except Exception as exc: # noqa: BLE001 — one bad symbol must not sink the panel + logger.warning("close-panel coverage check failed for %s: %s", sym, exc) + + if kicked: + deadline = time.monotonic() + _BACKFILL_WAIT_SECONDS + while time.monotonic() < deadline: + if not any(_bf.needs_backfill(s, start, end) for s in kicked): + break + time.sleep(0.25) + + series: dict[str, pd.Series] = {} + for sym in symbols: + # fetch_df indexes by date when the column is present. + df = fetch_df( + "SELECT date, close FROM clean_bars WHERE ticker=? AND date>=? AND date<=? ORDER BY date", + (sym, start.isoformat(), end.isoformat()), + ) + if df.empty or "close" not in df.columns: + continue + s = df["close"].dropna() + if not s.empty: + series[sym] = s + if not series: + return pd.DataFrame() + return pd.DataFrame(series).sort_index() + + def get_processed( ticker: str, frequency: str = "D", start: dt.date | None = None, end: dt.date | None = None ) -> pd.DataFrame: diff --git a/data_pipeline/read/facade.py b/data_pipeline/read/facade.py index 657e9ea..0af81fd 100644 --- a/data_pipeline/read/facade.py +++ b/data_pipeline/read/facade.py @@ -8,8 +8,8 @@ *make data ready* is delegated to ``orchestrate``. Contracts: - ``DataService`` staticmethods — initialize, manual_update, seed_history, - has_data_for_date, ensure_range, get_cleaned_daily, get_processed, - get_processed_data, get_latest_spot. + has_data_for_date, ensure_range, get_cleaned_daily, get_close_panel, + get_processed, get_processed_data, get_latest_spot. Dependencies UPWARD: - store (db), orchestrate (backfill / update) Dependencies DOWNWARD: @@ -73,6 +73,11 @@ def ensure_range(ticker: str, start, end) -> bool: def get_cleaned_daily(ticker: str, start=None, end=None): return _q.get_cleaned_daily(ticker, start, end) + @staticmethod + def get_close_panel(symbols: list[str], start=None, end=None): + """Wide close-price panel over ``clean_bars`` (market review, ADR 0011 L5).""" + return _q.get_close_panel(symbols, start, end) + @staticmethod def get_processed(ticker: str, frequency: str = "D", start=None, end=None): return _q.get_processed(ticker, frequency, start, end) diff --git a/data_pipeline/store/db.py b/data_pipeline/store/db.py index 0ec52ae..fd5f568 100644 --- a/data_pipeline/store/db.py +++ b/data_pipeline/store/db.py @@ -210,17 +210,9 @@ def init_db(db_path: str | None = None): _create_table(cur, "raw_prices", _BARS_COLUMNS) _create_table(cur, "clean_prices", _CLEAN_BARS_COLUMNS) _create_table(cur, "processed_prices", _FEATURE_BARS_COLUMNS) - # Market review benchmark close prices - cur.execute( - """ - CREATE TABLE IF NOT EXISTS market_review_prices ( - ticker TEXT NOT NULL, - date TEXT NOT NULL, - close REAL, - PRIMARY KEY (ticker, date) - ) - """ - ) + # NOTE (batch B10): the market-review benchmark panel dropped its own + # ``market_review_prices`` table — benchmark symbols are stored in + # ``clean_bars`` like any other ticker (ADR 0011 L5). # Market regime daily log (see core.regime) cur.execute( """ @@ -308,7 +300,6 @@ def get_conn(db_path: str | None = None): "raw_prices", "clean_prices", "processed_prices", - "market_review_prices", "regime_log", "data_quality_log", "tracked_strategies", diff --git a/data_pipeline/store/repos.py b/data_pipeline/store/repos.py index 83e03f3..43658e3 100644 --- a/data_pipeline/store/repos.py +++ b/data_pipeline/store/repos.py @@ -112,45 +112,15 @@ def update_tracked_strategy_closed(position_id: int, closed_date: str, closed_va return cur.rowcount -# ── Market review (benchmark / instrument close panel) ──────────── +# ── Schema bootstrap ───────────────────────────────────────────── def ensure_schema() -> None: """Bootstrap the SQLite schema (idempotent). Safe to call before any read.""" init_db() -def fetch_market_review_latest_dates(tickers: list[str]) -> dict[str, str | None]: - """Return ``{ticker: latest date string | None}`` from ``market_review_prices``.""" - out: dict[str, str | None] = {} - with get_conn() as conn: - for t in tickers: - row = conn.execute("SELECT MAX(date) FROM market_review_prices WHERE ticker = ?", (t,)).fetchone() - out[t] = row[0] if row and row[0] else None - return out - - -def upsert_market_review_prices(rows: Iterable[tuple[str, str, float]]) -> None: - """Insert or replace ``(ticker, date, close)`` rows in ``market_review_prices``.""" - rows = list(rows) - if not rows: - return - with get_conn() as conn: - conn.executemany( - "INSERT INTO market_review_prices (ticker, date, close) " - "VALUES (?, ?, ?) ON CONFLICT(ticker, date) DO UPDATE SET close=excluded.close", - rows, - ) - conn.commit() - - -def fetch_market_review_panel(range_start: str) -> pd.DataFrame: - """Return ``market_review_prices`` rows with ``date >= range_start`` as a DataFrame.""" - with get_conn() as conn: - return pd.read_sql_query( - "SELECT ticker, date, close FROM market_review_prices WHERE date >= ? ORDER BY date", - conn, - params=(range_start,), - parse_dates=["date"], - ) +# NOTE (batch B10): the market-review benchmark panel no longer has its own +# table / ladder here — benchmark symbols flow through ``clean_bars`` like any +# other ticker and are read via ``DataService.get_close_panel`` (ADR 0011 L5). # ── Regime log ─────────────────────────────────────────────────── diff --git a/docs/architecture_review.md b/docs/architecture_review.md index 6621899..7298c10 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -80,7 +80,7 @@ Rescoped in batch B1 of [ADR 0011](decisions/0011-pluggable-data-provider-seam.m | ~~`data_pipeline/providers/yf_client.py` (391 lines, fan-in 11)~~ — **resolved 2026-09-10 (B1)** | it sat 9 lines below the 400-line god-file threshold | the option-chain section was extracted pre-emptively, as prescribed, into `providers/yf_snapshot.py`; `yf_client.py` is now a ~35-line shim. The pressure moved to `providers/yf_snapshot.py` (≈340 lines) and `providers/yfinance_provider.py` (≈290 lines) — watch them before adding endpoints | | `static/sim/black_scholes.js` + `core/options/greeks` (risk-free rate) | the same constant is hard-coded in the client simulation **and** the server-side Greeks; changing one without the other makes the two pricings diverge silently | batch B8 retired the Config tab, so there is no global-setting surface to put it in; making it a parameter (store entry + form field + backend path) is a small feature — do it before anyone edits either constant | | `data_pipeline/providers/yf_client.py` (compat shim, ADR 0011 B1) | re-exports `fetch_spot` / `fetch_option_chain` / `fetch_close_panel` / `fetch_daily_ohlcv` with their old yfinance-shaped contracts; it was a **"one release"** bridge so importers did not have to change in the B1 PR | once `grep -rn "providers.yf_client\|yf_client import" services/ data_pipeline/read/` is empty (importers moved to the canonical `providers.get_provider()` shapes), delete `yf_client.py` and its re-exports from `providers/__init__.py` in one commit | -| `services/market_review/fetch.py` → `market_review_prices` (L5 in plan §4.1) | a second acquisition path outside the provider seam: its own L1/L2/L3 close-panel ladder writes a `market_review_prices` table that `data_pipeline/orchestrate/readiness.py` does **not** plan, so the benchmark panel still lazy-fetches on the Market Review slice | fold the ladder into `providers` + a canonical `bars` read (ADR 0011's L5 exit); until then, add `market_review` benchmark tickers to `KIND_DATASETS` so the readiness pass warms them | +| ~~`services/market_review/fetch.py` → `market_review_prices` (L5 in plan §4.1)~~ — **resolved 2026-09-11 (B10)** | a second acquisition path outside the provider seam: its own L1/L2/L3 close-panel ladder wrote a `market_review_prices` table that the readiness pass did **not** plan, so the benchmark panel lazy-fetched on the Market Review slice | folded into the seam: benchmark symbols are ordinary `clean_bars` tickers now, read via `DataService.get_close_panel` (heals through `ensure_range`); `market_review_prices` + its three repo functions deleted; `services/market/readiness.py` expands the plan with the benchmark symbols so `POST /` prefetches them. `services/market_review/fetch.py` keeps only its 5-min L1 memo | | ~~`services/market/analysis/assessment.py` option overlay~~ — **retired 2026-09-11 (B9, plan §10 F5-a)** | B7 left the projection-vs-positions overlay + the sizing max-loss unfed (positions moved to the Portfolio tab) | removed: the `option_data` branches in `assessment.py`, `FormService.parse_option_data`, `MarketAnalyzer.analyze_options` / `MarketChartAssembly.analyze_options`, and `core/market/option_pnl.py` + `core/market/charts/option_pnl.py` (whole files). Assessment sizing is now debit-only; position P&L lives in the Portfolio tab | ## 3. Guardrails (how the score is kept) diff --git a/docs/decisions/0011-pluggable-data-provider-seam.md b/docs/decisions/0011-pluggable-data-provider-seam.md index ad13c74..52e5f67 100644 --- a/docs/decisions/0011-pluggable-data-provider-seam.md +++ b/docs/decisions/0011-pluggable-data-provider-seam.md @@ -151,6 +151,11 @@ writes both families) and `scripts/migrate_canonical_tables.py` backfills an exi update `docs/constraints.md` §1, `docs/l0_architecture.md`, `docs/architecture_review.md` §2, and `scripts/doc_guard.py::_ALLOWED_DEPS` in the same batches. +- Landed 2026-09-11 (batch B10): L5 closed. `services/market_review/fetch.py`'s + parallel ladder and the `market_review_prices` table are gone — the benchmark + close panel is now `DataService.get_close_panel` over `clean_bars`, so every + acquisition path runs through `providers/`. Deferred still: the `symbol` column + (ticker→symbol rename), which waits for a real second provider. ## References diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index c55d175..ce897df 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -4,12 +4,12 @@ **ADRs**: [0011](../decisions/0011-pluggable-data-provider-seam.md) (data-provider seam + canonical schema — **Accepted**), [0012](../decisions/0012-parameter-ownership-and-prefetch.md) (parameter ownership + readiness prefetch — **Accepted**) -> **Status: LANDED (2026-09-10) + acceptance review B9 (2026-09-11).** All eight -> §6 batches plus the B9 review-remediation shipped on branch -> `worktree-business-line-reorg`; the §0 ledger records what actually shipped and -> §10 the review findings. Only the deferred follow-ups in §10 remain (risk-free -> global setting, `market_review_prices` L5, ADR 0011 `symbol` column). Batches -> are individually revertible — `git revert `. +> **Status: LANDED.** All eight §6 batches, the B9 acceptance-review +> remediation, and B10 (market-review L5 → provider seam) shipped and merged to +> `main` (B1–B9 via PR #10 / `1c9f49e`; B10 on its own PR). The §0 ledger +> records what actually shipped and §10 the review findings. Only two deferred +> follow-ups in §10 remain: the risk-free-rate global setting and the ADR 0011 +> `symbol` column. Batches are individually revertible — `git revert `. --- @@ -49,7 +49,8 @@ | B6 — `ticker`-only Parameters bar | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q3 resolved (dedicated Portfolio tab); bar + collapse persisted; transitional settings group inside the bar's form until B7. Actuals in §8 | | B7 — module-scoped params | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (manifest). Backend query-arg contract + per-module allow-list; `state/*ParamsState.js`; module toolbars; bridge + hidden fields deleted; Config tab emptied (B8 decides its fate). See §8 B7 | | B8 — retire / repurpose Config tab | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q2 resolved (**deleted**). `grep tab_config` returns nothing; risk-free-rate follow-up on the watch list | -| B9 — acceptance-review remediation | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-11 | §10 review: F1 (memo `variant` key), F2 (real collapse), F3 (SVG chevron), F4 (feature_bars self-heal), F5 a+b+c (a=retire option overlay per owner; b/c=stale UI), F6 (docstrings). Watch-list: `summary.py` deleted; risk-free-rate + L5 + `symbol` column deferred. **Open**: F7c (merge to main). | +| B9 — acceptance-review remediation | ✅ landed | #10 | `main` (`1c9f49e`) · 2026-09-11 | §10 review: F1 (memo `variant` key), F2 (real collapse), F3 (SVG chevron), F4 (feature_bars self-heal), F5 a+b+c (a=retire option overlay per owner; b/c=stale UI), F6 (docstrings). Watch-list: `summary.py` deleted; risk-free-rate + L5 + `symbol` column deferred. F7c: B1–B9 (15 commits) merged to `main`. | +| B10 — market-review L5 → provider seam | ✅ landed | — | branch `b10-market-review-seam` · 2026-09-11 | `market_review_prices` table + ladder deleted; benchmark closes read from `clean_bars` via new `DataService.get_close_panel` (heals through `ensure_range`); `services/market/readiness.py` expands the readiness plan with the benchmark symbols so `POST /` prefetches them. `services/market_review/fetch.py` keeps only its 5-min L1 memo. §2 watch-list L5 row resolved. New tests: `TestGetClosePanel`, `test_market_readiness.py`. | States: `⬜ not started` → `🔨 in progress (PR #n)` → `✅ landed` → (`↩ reverted`). Keep the row order; edit the row in place. @@ -93,7 +94,7 @@ snapshot / none), two compute paths (streaming `/render/` vs. client-fired |---|---|---|---| | Parameter | — (form only) | — | — | | Summary *(dormant)* | multi-ticker aggregate | tickers | `services/market/analysis/summary.py` — fan-in 0, on the §2 watch list | -| Market Review | `clean_prices` + `market_review_prices` close panel | `ticker`, `start_time`, `end_time` | streaming `generate_market_review_slice` | +| Market Review | `clean_prices` + `market_review_prices` close panel *(B10: benchmark closes moved into `clean_bars`; `market_review_prices` deleted)* | `ticker`, `start_time`, `end_time` | streaming `generate_market_review_slice` | | Statistical Analysis | `processed_prices` | `ticker`, `parsed_start_time`, `frequency` | streaming `generate_statistical_slice` | | Assessment & Projections | `processed_prices` | `ticker`, `parsed_start_time`, `frequency`, `risk_threshold`, `rolling_window`, `side_bias`→`target_bias`, `account_size`, `max_risk_pct` | streaming `generate_assessment_slice` | | Market Regime | `regime_log` + live `^VIX` / `SPY` | `days` (30/180/365/1095) | client → `/api/regime/{current,history,backfill}` | @@ -204,6 +205,11 @@ and call `/api/*` directly. | L5 | Second acquisition path outside `data_ops` | `services/market_review/fetch.py` writes `market_review_prices` on its own ladder | fold into the provider seam | | L6 | Live option/spot data has no persistence contract | `services/options/preload.py` + in-process `_option_chain_cache` only; deliberate per ADR 0004 (no option history) | keep, but make the "live vs. stored" split explicit in the seam | +> This table is the plan-time diagnosis. Resolution: L1 → B4, L2 → B1, L3 → B5 +> (`orchestrate/`), L4 → B1/B2 (canonical schema), **L5 → B10** (benchmark closes +> now read from `clean_bars` via `DataService.get_close_panel`; +> `market_review_prices` deleted), L6 → B1 (`providers` snapshots vs. stored bars). + ### 4.2 Prefetch / readiness today - `POST /` computes nothing — `create_job` stores `form_data`, returns the skeleton. @@ -318,8 +324,11 @@ providers→ (leaf: only utils + the external SDK) `fetch_data_context(...)` (a thin thing in `services/market/`, allowed to call `read/`) that produces a pure `DataContext`, and `core` keeps only the data-in/data-out container. Removes both `core-purity` markers. -- L5: `services/market_review/fetch.py`'s ladder becomes a `read/` function over a - canonical `bars` table (no separate `market_review_prices` shape — see §5.3). +- L5 (**done, B10**): `services/market_review/fetch.py`'s ladder became + `DataService.get_close_panel` — a `read/` function over `clean_bars` (no + separate `market_review_prices` shape). Benchmark symbols heal through + `ensure_range` like any ticker, and `services/market/readiness.py` adds them to + the `POST /` readiness plan. ### 5.3 Canonical internal schema + provider mapping @@ -793,12 +802,51 @@ all vestigial. All removed; `arch_baseline.json` `dead_code_candidates` reset `doc_guard.py` clean; `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / **dead 0**); `audit_tags.py` 16 vs 16. +**B10 — market-review L5 → provider seam, 2026-09-11.** (ADR 0011's L5 exit; +watch-list item / F7b.) + +- **The ladder is gone.** `services/market_review/fetch.py` no longer owns an + acquisition path: `market_review_prices` (table + `fetch_market_review_latest_dates` + / `upsert_market_review_prices` / `fetch_market_review_panel`) is deleted, and + the module now calls **`DataService.get_close_panel(symbols, start, end)`** — a + new `read/_query.py` function that reads `close` from `clean_bars` per symbol + and heals coverage through the same `needs_backfill` → background `ensure_range` + machinery every other read uses. Missing symbols are kicked once up front and + awaited **together** for one short grace window (not one per symbol). The 5-min + L1 in-memory memo over the assembled `(data, returns, display)` triple stays — + it is a compute cache, not an acquisition path. +- **Benchmarks are first-class tickers now.** SPX / US10Y / Gold / … flow through + `providers.history()` → `raw_bars` → `clean_bars` → `feature_bars` like any + ticker (they already passed `is_valid_ticker_format`). The feature columns are + computed and ignored by market review, which is a small, cached, once-daily + cost — the price of not having a second schema. +- **Readiness prefetches the panel.** `data_pipeline.orchestrate` may not import + `core`, so `services/market/readiness.py::_augment_with_benchmarks` expands the + `POST /` plan with `BENCHMARKS.values()` whenever `market_review` is a requested + module. `check_and_kick` then warms them on daemon threads alongside the user's + ticker — closing the other half of F4's spirit (the benchmark panel used to + cold-fetch on the slice). +- **Docs**: `architecture_review.md` §2 watch-list L5 row → resolved; §4.1 / §5.2 + resolution notes; `db.py` + `repos.py` carry a B10 NOTE where the table was; + `scripts/build_pages_site.py` demo fixture reads `clean_bars`. +- **Tests**: `test_background_backfill.py::TestGetClosePanel` (reads seeded + `clean_bars`; one shared grace window across missing symbols; one bad symbol + does not sink the panel); `test_market_readiness.py` (benchmark expansion fires + only for `market_review`, no dup when the ticker *is* a benchmark, + `prepare_readiness` kicks them); `test_market_review.py` rewritten to stub + `get_close_panel` instead of seeding the dropped table; `test_db_errors.py` + asserts `market_review_prices` is **absent**. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → exit 0; + `pytest tests/e2e` → exit 0; `npx vitest run` unchanged; `ruff` clean; + `doc_guard.py` clean; `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / + dead 0); `audit_tags.py` 16 vs 16; `grep -rn market_review_prices` → docs only. + ### Watch-list follow-ups — 2026-09-11 assessment | Item | Call | Why | |---|---|---| | **Risk-free rate** hard-coded (`r = 0.05` default in `static/sim/{analyze,stats}.js`, `core/options/greeks/portfolio.py`, `grid.js` `r_pct=5`) → true global setting | **Defer the setting; do the cheap consolidation if wanted.** | A *user-facing* setting needs a surface — B8 deliberately deleted the Config tab, so this means re-introducing one for a number that moves ~quarterly. The silent-divergence risk is cheaply killed by one shared constant (`utils/constants.RISK_FREE_RATE` + a `static/sim/` mirror + a parity test) without any UI. Full setting = wait until someone actually wants to tweak it. | -| **`market_review_prices` (L5)** → fold into provider seam | **Defer to its own batch (B10).** | It is ADR 0011's stated L5 exit and it would let readiness prefetch the benchmark panel (closing the other half of F4's spirit), but it is a real refactor: benchmark tickers (SPY/QQQ/…) would route through `ensure_range`/`clean_bars` instead of the close-only ladder. ~1 day + tests. Not a bundle-in. | +| ~~**`market_review_prices` (L5)** → fold into provider seam~~ — **DONE 2026-09-11 (B10)** | done | benchmark closes now read from `clean_bars` via `DataService.get_close_panel` (heals through `ensure_range`); `market_review_prices` + its three repo functions deleted; `services/market/readiness.py` expands the `POST /` readiness plan with the benchmark symbols. `services/market_review/fetch.py` keeps only its 5-min L1 memo. See the B10 section below. | | **ADR 0011 `symbol` column** (ticker→symbol rename) | **Agree — stay deferred.** | Pure churn with one provider (`ticker == symbol` for yfinance). Do it *with* the second provider, when the mapping actually has two shapes to reconcile. | | ~~**`services/market/analysis/summary.py`**~~ — **DELETED 2026-09-11 (B9)** | done | `summary_data` was never set; module + `tab_summary.html` + sidebar button + `summary_pending` + `renderCorrelationHeatmap`/`corrToColor` removed; `dead_code_candidates` baseline 1 → 0. | @@ -837,5 +885,5 @@ Findings worked as **batch B9 (remediation)** under the §0 rules. | F4 | minor — readiness gap | ✅ **fixed (B9)** | `needs_backfill` probed `clean_bars` only, so a DB with clean rows but stale/missing `feature_bars` (a past `process_frequencies` failure, or clean extended without a reprocess) was never healed — Statistical/Assessment read `feature_bars`. `process_frequencies` writes D/W/ME/QE together so a *new* frequency is not the trigger; the partial-failure state is. | `backfill._feature_bars_behind` (probe frequency='D', 1:1 with clean); `needs_backfill` returns True on feature lag; `_ensure_range_impl` reprocesses (no download) on the clean-covered short-circuit; `get_processed` self-heals like `get_cleaned_daily`. Tests: `TestFeatureBarsHeal`. | | F5 | mixed | ✅ **fixed (B9)** | **(a) was a B7 regression, not dead code:** `assessment.py` read `form_data["option_data"]` (projection-vs-positions overlay + sizing max-loss); B7 stopped `POST /` carrying positions, leaving the overlay permanently unfed. **(b/c) stale UI:** `routes/core.py` GET vars + `index.html` `badge-meta` + `market_review.html` meta-chips all showed POST-time defaults after B7 (`badge` always "Monthly, Neutral"). | **(a) retired** (owner call): removed the `option_data` branches, `parse_option_data`, `analyze_options` (both), `core/market/option_pnl.py` + `charts/option_pnl.py`, the `plot_url` block — Assessment sizing is debit-only. **(b/c)** GET vars + imports removed; `badge-meta` dropped (ticker only); 3 stale `market_review` chips removed; 4 "Parameter tab" empty-state refs + `tab_simulation` placeholder fixed. | | F6 | trivial — stale docstrings | ✅ **fixed (review commit `9691776`)** | `providers/base.py` "yf_option_chain.py" → `yf_snapshot.py`; `providers/yf_client.py` listed `core/market/data_context.py` as an importer (B4 removed it); `providers/__init__.py` + `yfinance_provider.py` said `downloader.py` (it is `ingest/ohlcv.py` since B3); §0 ledger planning-row status. | done | -| F7 | housekeeping | 🔨 partial | (a) `providers/yf_client.py` "one-release" shim has no tracked removal trigger. (b) `market_review_prices` (L5) is still a parallel acquisition path outside the seam. (c) Branch is ahead of `origin/main`, unmerged. | (a)+(b) **added to `architecture_review.md` §2 watch list** (review commit); (c) push + merge still pending. | +| F7 | housekeeping | ✅ **done** | (a) `providers/yf_client.py` "one-release" shim has no tracked removal trigger. (b) `market_review_prices` (L5) is still a parallel acquisition path outside the seam. (c) Branch is ahead of `origin/main`, unmerged. | (a) **on `architecture_review.md` §2 watch list** with a grep trigger (still pending — shim importers not yet moved). (b) **resolved in B10** — L5 folded into the seam. (c) B1–B9 merged via PR #10 (`1c9f49e`, 2026-09-11). | diff --git a/scripts/build_pages_site.py b/scripts/build_pages_site.py index 9d9960c..08415b6 100644 --- a/scripts/build_pages_site.py +++ b/scripts/build_pages_site.py @@ -258,14 +258,15 @@ def _refresh_api_fixtures(ticker: str) -> None: } (FIXTURES_DIR / "regime_history.json").write_text(json.dumps(history, ensure_ascii=False), encoding="utf-8") - # -- validate_tickers: NVDA + benchmark tickers, latest closes from DB + # -- validate_tickers: NVDA + benchmark tickers, latest closes from DB. + # B10: benchmark closes now live in clean_bars alongside every other ticker. print("[snapshot] validate_tickers fixture …") conn = sqlite3.connect(REPO_ROOT / "market_data.sqlite") - tickers = [r[0] for r in conn.execute("SELECT DISTINCT ticker FROM market_review_prices")] + tickers = [r[0] for r in conn.execute("SELECT DISTINCT ticker FROM clean_bars")] results = {} for t in tickers: row = conn.execute( - "SELECT close FROM market_review_prices WHERE ticker=? ORDER BY date DESC LIMIT 1", (t,) + "SELECT close FROM clean_bars WHERE ticker=? AND close IS NOT NULL ORDER BY date DESC LIMIT 1", (t,) ).fetchone() price = round(float(row[0]), 2) if row and row[0] is not None else None results[t] = {"valid": price is not None, "price": price, "message": "demo snapshot"} diff --git a/services/market/readiness.py b/services/market/readiness.py index fabe4bd..306ac92 100644 --- a/services/market/readiness.py +++ b/services/market/readiness.py @@ -16,7 +16,8 @@ - ``prepare_readiness(tickers, modules, *, start, end) -> list[ReadinessStatus]`` - ``warm_live_snapshots(tickers) -> None`` — fire-and-forget Dependencies UPWARD: - - data_pipeline.orchestrate.readiness (plan + kick), services.options.preload + - data_pipeline.orchestrate.readiness (plan + kick), services.options.preload, + core.market_review.constants (benchmark symbol list — batch B10) Dependencies DOWNWARD: - routes/core.py """ @@ -41,6 +42,15 @@ # preload cache. LIVE_CHAIN_MODULES = frozenset({"options_chain", "payoff_ratio"}) +# WHY (batch B10): the Market Review slice renders the entered ticker *plus* a +# fixed benchmark panel (SPX / US10Y / Gold / …). Since B10 those benchmark +# symbols are ordinary ``clean_bars`` tickers, so the readiness pass can kick +# them alongside the user's ticker instead of letting the slice cold-fetch +# them. ``data_pipeline.orchestrate`` may not import ``core``, so the benchmark +# list is expanded here (a ``services`` module) rather than inside +# ``plan_datasets``. +_BENCHMARK_MODULE = "market_review" + def prepare_readiness( tickers: list[str], @@ -55,6 +65,7 @@ def prepare_readiness( ``/render/`` can tell "covered" from "still downloading". """ plan = plan_datasets(tickers, modules, start=start, end=end) + plan = _augment_with_benchmarks(plan, modules, start=start, end=end) if not plan: logger.info("readiness: no stored dataset required for modules=%s", modules) statuses = check_and_kick(plan) @@ -63,6 +74,26 @@ def prepare_readiness( return statuses +def _augment_with_benchmarks(plan, modules, *, start, end): + """Add the Market Review benchmark symbols to the plan (batch B10). + + A no-op unless ``market_review`` is one of the requested modules. Benchmark + entries the base plan already carries (a user who typed a benchmark ticker) + are not duplicated. + """ + if _BENCHMARK_MODULE not in modules: + return plan + from core.market_review.constants import BENCHMARKS + + have = {(r.ticker, r.dataset) for r in plan} + extra = [ + r + for r in plan_datasets(list(BENCHMARKS.values()), [_BENCHMARK_MODULE], start=start, end=end) + if (r.ticker, r.dataset) not in have + ] + return plan + extra + + def warm_live_snapshots(tickers: list[str]) -> None: """Warm the option-chain preload cache for ``tickers`` on daemon threads.""" for ticker in tickers: diff --git a/services/market_review/__init__.py b/services/market_review/__init__.py index 33f3938..6bdde7c 100644 --- a/services/market_review/__init__.py +++ b/services/market_review/__init__.py @@ -1,10 +1,11 @@ """Market review — I/O orchestration package. -Owns the L1/L2/L3 cache ladder (in-memory TTL → SQLite ``market_review_prices`` -→ yfinance) that produces the close-price panel, then delegates the pure -computation to ``core.market_review``. This is the layer permitted to touch -``data_pipeline`` for market review (ADR 0003 / architecture review §2 -`core-purity`). +Assembles the benchmark close-price panel (via ``DataService.get_close_panel``, +which reads ``clean_bars`` and heals coverage through ``ensure_range`` — +ADR 0011 L5) behind a 5-minute L1 in-memory cache, then delegates the pure +computation to ``core.market_review``. Batch B10 retired the standalone +``market_review_prices`` ladder; benchmark symbols now flow through the +provider seam like any other ticker. Public entry points keep the historical ``(instrument, start, end)`` signature so routes / services / tests call them the same way they called the old @@ -17,7 +18,6 @@ _fetch_market_data, _mr_cache, _mr_cache_lock, - fetch_close_panel, fetch_market_data, ) @@ -29,5 +29,4 @@ "BENCHMARKS", "_mr_cache", "_mr_cache_lock", - "fetch_close_panel", ] diff --git a/services/market_review/facade.py b/services/market_review/facade.py index c5aafce..704739a 100644 --- a/services/market_review/facade.py +++ b/services/market_review/facade.py @@ -1,10 +1,10 @@ """Market review orchestration facade. -Thin glue: fetch the close-price panel via the cache ladder, then hand it to -the pure ``core.market_review`` builders. The historical -``(instrument, start, end)`` signature is preserved so routes / services / -tests call these the same way they called the old ``core.market_review`` -functions. +Thin glue: fetch the close-price panel (L1 cache over +``DataService.get_close_panel``), then hand it to the pure +``core.market_review`` builders. The historical ``(instrument, start, end)`` +signature is preserved so routes / services / tests call these the same way +they called the old ``core.market_review`` functions. Dependencies: - core.market_review (build_review, build_timeseries) diff --git a/services/market_review/fetch.py b/services/market_review/fetch.py index bad52c8..ffc7442 100644 --- a/services/market_review/fetch.py +++ b/services/market_review/fetch.py @@ -1,20 +1,21 @@ -"""Market review data fetching — I/O owner. +"""Market review data fetching — panel assembly over the provider seam. -Domain: Market Review — Data Fetching (L1/L2/L3 cache ladder) +Domain: Market Review — Data Fetching Context: - - L1 in-memory cache (5-min TTL) - - L2 SQLite market_review_prices table - - L3 yfinance incremental download - -This module is the *only* place in ``core``/``services`` that builds the -market-review panel, so WAL pragmas and the throttle apply uniformly. It is -legal for a ``services`` module to import ``data_pipeline`` (ADR 0003 / -architecture review §2 `core-purity`). + - L1 in-memory cache (5-min TTL) over the assembled (data, returns, display) + triple. + - The close-price panel itself comes from ``DataService.get_close_panel``, + which reads ``clean_bars`` and heals coverage through the same + ``ensure_range`` machinery every other read uses (ADR 0011, L5). Batch B10 + removed the old ``market_review_prices`` ladder (its own L2 table + a + parallel yfinance download) so benchmark symbols now flow through the + provider seam like any other ticker and the submit-time readiness pass can + prefetch them. Contracts: - fetch_market_data(instrument, start_date, end_date) -> tuple[pd.DataFrame, pd.DataFrame, list] Dependencies: - - data_pipeline.providers.yf_client, data_pipeline.store.db + - data_pipeline.read.facade.DataService - core.market_review.constants (BENCHMARKS) """ @@ -28,13 +29,7 @@ import pandas as pd from core.market_review.constants import BENCHMARKS -from data_pipeline.providers.yf_client import fetch_close_panel -from data_pipeline.store.repos import ( - ensure_schema, - fetch_market_review_latest_dates, - fetch_market_review_panel, - upsert_market_review_prices, -) +from data_pipeline.read.facade import DataService logger = logging.getLogger(__name__) @@ -47,6 +42,9 @@ # Bound on distinct cache keys (each holds a full multi-ticker price panel). _MR_CACHE_MAX = 64 +# DOMAIN: default lookback when the caller passes no explicit start date. +_DEFAULT_LOOKBACK_DAYS = 400 + def fetch_market_data(instrument: str, start_date=None, end_date=None): cache_key = (instrument, str(start_date), str(end_date)) @@ -77,47 +75,16 @@ def fetch_market_data(instrument: str, start_date=None, end_date=None): display_names = [instrument] + list(BENCHMARKS.keys()) ticker_to_display = dict(zip(all_tickers, display_names, strict=False)) - ensure_schema() - today_str = dt.date.today().isoformat() range_start = ( - start_date.isoformat() - if isinstance(start_date, dt.date) - else (dt.date.today() - dt.timedelta(days=400)).isoformat() + start_date if isinstance(start_date, dt.date) else (dt.date.today() - dt.timedelta(days=_DEFAULT_LOOKBACK_DAYS)) ) + range_end = end_date if isinstance(end_date, dt.date) else dt.date.today() - latest_map = fetch_market_review_latest_dates(all_tickers) - tickers_needing_download = [t for t in all_tickers if latest_map.get(t) is None or latest_map[t] < today_str] - - if tickers_needing_download: - try: - download_start = range_start - for t in tickers_needing_download: - latest = latest_map.get(t) - if latest is None: - download_start = range_start - break - elif latest < download_start: - download_start = latest - close_data = fetch_close_panel(tickers_needing_download, start=download_start, end=today_str) - if not close_data.empty: - rows = [] - for t in tickers_needing_download: - if t in close_data.columns: - series = close_data[t].dropna() - for date_idx, val in series.items(): - rows.append((t, date_idx.strftime("%Y-%m-%d"), float(val))) - if rows: - upsert_market_review_prices(rows) - except Exception as e: - logger.warning("Market review yfinance download failed: %s", e) - - df = fetch_market_review_panel(range_start) - if df.empty: - logger.warning("No market review data in DB, falling back to yfinance") - raw = fetch_close_panel(all_tickers, period="400d") - data = raw.ffill() if raw is not None and not raw.empty else pd.DataFrame() - else: - data = df.pivot(index="date", columns="ticker", values="close").sort_index().ffill() + panel = DataService.get_close_panel(all_tickers, range_start, range_end) + # ffill across the panel: benchmarks trade on different calendars, so a US + # holiday leaves a NaN in one column that we carry forward rather than + # dropping the whole row. + data = panel.sort_index().ffill() if not panel.empty else pd.DataFrame() valid_tickers = [t for t in all_tickers if t in data.columns and data[t].notna().any()] if instrument not in valid_tickers: diff --git a/tests/test_background_backfill.py b/tests/test_background_backfill.py index 4117cb1..a9ef252 100644 --- a/tests/test_background_backfill.py +++ b/tests/test_background_backfill.py @@ -209,3 +209,103 @@ def test_ensure_range_reprocesses_without_downloading(self, monkeypatch): feat = fetch_df("SELECT frequency, COUNT(*) AS n FROM feature_bars WHERE ticker='FEATHEAL2' GROUP BY frequency") assert not feat.empty, "feature_bars must be populated after the heal" assert _bf.needs_backfill("FEATHEAL2", start, end) is False + + +class TestGetClosePanel: + """``get_close_panel`` — the ADR 0011 L5 replacement for the old + ``market_review_prices`` ladder (batch B10).""" + + @pytest.fixture(autouse=True) + def _no_incremental_update(self, monkeypatch): + # The per-symbol freshness poke is not under test here and would hit + # the network; every case exercises only the coverage/read path. + monkeypatch.setattr("data_pipeline.orchestrate.update.manual_update", lambda *a, **k: False) + + @staticmethod + def _seed_clean(ticker, start, end, base=100.0): + from data_pipeline.store.db import upsert_many + + days = pd.bdate_range(start, end) + rows = [ + ( + ticker, + d.date().isoformat(), + base, + base + 1, + base - 1, + base + i * 0.1, + base + i * 0.1, + 1_000_000, + 1, + 0, + 0, + 0, + 0, + ) + for i, d in enumerate(days) + ] + upsert_many( + "clean_bars", + [ + "ticker", + "date", + "open", + "high", + "low", + "close", + "adj_close", + "volume", + "is_trading_day", + "missing_any", + "price_jump_flag", + "vol_anom_flag", + "ohlc_inconsistent", + ], + rows, + ) + + def test_reads_close_series_per_symbol_from_clean_bars(self, monkeypatch): + init_db() + start, end = dt.date(2024, 1, 1), dt.date(2024, 6, 28) + self._seed_clean("PANELA", start, end, base=50.0) + self._seed_clean("PANELB", start, end, base=200.0) + # coverage already satisfied — the panel must not kick a backfill + monkeypatch.setattr(_bf, "needs_backfill", lambda *a, **k: False) + kicks = {"n": 0} + monkeypatch.setattr(_q, "_kick_backfill", lambda *a, **k: kicks.__setitem__("n", kicks["n"] + 1)) + + panel = _q.get_close_panel(["PANELA", "PANELB"], start, end) + + assert list(panel.columns) == ["PANELA", "PANELB"] + assert not panel.empty + assert kicks["n"] == 0 + assert panel["PANELA"].iloc[0] == 50.0 + assert panel["PANELB"].iloc[0] == 200.0 + + def test_missing_symbols_are_kicked_once_and_awaited_together(self, monkeypatch): + init_db() + start, end = dt.date(2024, 1, 1), dt.date(2024, 6, 28) + monkeypatch.setattr(_bf, "needs_backfill", lambda *a, **k: True) + kicked: list[str] = [] + monkeypatch.setattr(_q, "_kick_backfill", lambda t, s, e: kicked.append(t)) + + t0 = time.monotonic() + panel = _q.get_close_panel(["MISS1", "MISS2", "MISS3"], start, end) + elapsed = time.monotonic() - t0 + + assert kicked == ["MISS1", "MISS2", "MISS3"], "one kick per symbol" + # _fast_grace sets the wait to 0.5s; the wait is shared, not per-symbol + assert elapsed < 1.2, f"grace window was not shared across symbols ({elapsed:.2f}s)" + assert panel.empty + + def test_one_unavailable_symbol_does_not_sink_the_panel(self, monkeypatch): + init_db() + start, end = dt.date(2024, 1, 1), dt.date(2024, 6, 28) + self._seed_clean("GOODSYM", start, end, base=75.0) + monkeypatch.setattr(_bf, "needs_backfill", lambda t, *a, **k: t != "GOODSYM") + monkeypatch.setattr(_q, "_kick_backfill", lambda *a, **k: None) + + panel = _q.get_close_panel(["GOODSYM", "NODATA"], start, end) + + assert list(panel.columns) == ["GOODSYM"] + assert not panel.empty diff --git a/tests/test_db_errors.py b/tests/test_db_errors.py index 27eced1..5d60bf8 100644 --- a/tests/test_db_errors.py +++ b/tests/test_db_errors.py @@ -23,7 +23,8 @@ def test_creates_tables(self, tmp_path): assert "raw_prices" in tables assert "clean_prices" in tables assert "processed_prices" in tables - assert "market_review_prices" in tables + # B10: market_review_prices dropped — benchmarks live in clean_bars now + assert "market_review_prices" not in tables def test_idempotent(self, tmp_path): db_path = str(tmp_path / "test.sqlite") diff --git a/tests/test_market_readiness.py b/tests/test_market_readiness.py new file mode 100644 index 0000000..c1eb44b --- /dev/null +++ b/tests/test_market_readiness.py @@ -0,0 +1,58 @@ +"""Services-layer readiness: benchmark-panel prefetch (batch B10). + +``data_pipeline.orchestrate.readiness`` cannot import ``core`` (layer rule), so +the Market Review benchmark symbols are folded into the plan in +``services.market.readiness``. These tests pin that expansion and the fact that +it only fires for the ``market_review`` module. +""" + +from __future__ import annotations + +import datetime as dt + +from core.market_review.constants import BENCHMARKS +from services.market import readiness as mr + +TODAY = dt.date(2026, 9, 10) +START = TODAY - dt.timedelta(days=400) + + +def _tickers(plan): + return {r.ticker for r in plan} + + +def test_benchmarks_are_added_when_market_review_is_requested(): + base = mr.plan_datasets(["AAPL"], ["market_review"], start=START, end=TODAY) + augmented = mr._augment_with_benchmarks(base, ["market_review"], start=START, end=TODAY) + + assert _tickers(augmented) == {"AAPL", *BENCHMARKS.values()} + # every benchmark entry targets the clean_bars dataset + assert {r.dataset for r in augmented} == {"clean_bars"} + + +def test_no_benchmarks_without_market_review(): + base = mr.plan_datasets(["AAPL"], ["statistical", "assessment"], start=START, end=TODAY) + augmented = mr._augment_with_benchmarks(base, ["statistical", "assessment"], start=START, end=TODAY) + + assert augmented == base + assert _tickers(augmented) == {"AAPL"} + + +def test_a_benchmark_typed_as_the_ticker_is_not_duplicated(): + a_benchmark = next(iter(BENCHMARKS.values())) + base = mr.plan_datasets([a_benchmark], ["market_review"], start=START, end=TODAY) + augmented = mr._augment_with_benchmarks(base, ["market_review"], start=START, end=TODAY) + + pairs = [(r.ticker, r.dataset) for r in augmented] + assert pairs.count((a_benchmark, "clean_bars")) == 1 + assert _tickers(augmented) == set(BENCHMARKS.values()) + + +def test_prepare_readiness_kicks_benchmark_backfills(monkeypatch): + kicked: list[str] = [] + monkeypatch.setattr(mr, "check_and_kick", lambda plan, **k: kicked.extend(r.ticker for r in plan) or []) + + mr.prepare_readiness(["NVDA"], ["market_review"], start=START, end=TODAY) + + assert "NVDA" in kicked + assert set(BENCHMARKS.values()) <= set(kicked) diff --git a/tests/test_market_review.py b/tests/test_market_review.py index 7c9fa53..c34475b 100644 --- a/tests/test_market_review.py +++ b/tests/test_market_review.py @@ -1,4 +1,11 @@ -"""Tests for core.market_review — data processing, caching, and output format.""" +"""Tests for services.market_review — L1 cache + output format. + +Batch B10 folded the benchmark close panel into the provider seam: the ladder +no longer owns a ``market_review_prices`` table, it calls +``DataService.get_close_panel`` (which reads ``clean_bars``). These tests stub +that call and focus on the L1 cache behaviour and the shape of what +``core.market_review`` produces from a panel. +""" import datetime as dt from unittest.mock import patch @@ -6,13 +13,12 @@ import numpy as np import pandas as pd -from data_pipeline.store.db import get_conn, init_db - # ── Helpers ─────────────────────────────────────────────────────── -def _make_benchmark_prices(tickers: list[str], days: int = 60) -> pd.DataFrame: - """Generate synthetic close prices for multiple tickers.""" +def _make_close_panel(tickers: list[str], days: int = 120) -> pd.DataFrame: + """A synthetic wide close-price panel, date-indexed, one column per ticker — + the shape ``DataService.get_close_panel`` returns.""" dates = pd.bdate_range(end=dt.date.today(), periods=days) rng = np.random.default_rng(42) data = {} @@ -22,21 +28,16 @@ def _make_benchmark_prices(tickers: list[str], days: int = 60) -> pd.DataFrame: return pd.DataFrame(data, index=dates) -def _seed_market_review_prices(tickers: list[str], days: int = 60) -> None: - """Seed the market_review_prices table with synthetic data.""" - init_db() - df = _make_benchmark_prices(tickers, days) - rows = [] - for t in tickers: - for date_idx, val in df[t].items(): - rows.append((t, date_idx.strftime("%Y-%m-%d"), float(val))) - with get_conn() as conn: - conn.executemany( - "INSERT INTO market_review_prices (ticker, date, close) " - "VALUES (?, ?, ?) ON CONFLICT(ticker, date) DO UPDATE SET close=excluded.close", - rows, - ) - conn.commit() +def _patch_panel(primary: str): + """Patch ``DataService.get_close_panel`` to return a panel covering + *primary* + every benchmark.""" + from services.market_review import BENCHMARKS + + panel = _make_close_panel([primary] + list(BENCHMARKS.values())) + return patch( + "services.market_review.fetch.DataService.get_close_panel", + return_value=panel, + ) # ── Cache tests ────────────────────────────────────────────────── @@ -44,35 +45,31 @@ def _seed_market_review_prices(tickers: list[str], days: int = 60) -> None: class TestMarketReviewCache: def test_cache_hit_avoids_refetch(self, clear_mr_cache): - """After first fetch, second call should use L1 cache.""" - from services.market_review import BENCHMARKS, _fetch_market_data + """After first fetch, the second call uses the L1 cache and never + re-enters the read layer.""" + from services.market_review import _fetch_market_data - all_tickers = ["AAPL"] + list(BENCHMARKS.values()) - _seed_market_review_prices(all_tickers, days=60) clear_mr_cache() + with _patch_panel("AAPL") as mocked: + data1, _ret1, disp1 = _fetch_market_data("AAPL") + data2, _ret2, disp2 = _fetch_market_data("AAPL") - with patch("services.market_review.fetch_close_panel"): - # First call — should use DB (not yfinance since we seeded) - data1, ret1, disp1 = _fetch_market_data("AAPL") - # Second call — should hit L1 cache - data2, ret2, disp2 = _fetch_market_data("AAPL") - assert data1.shape == data2.shape - assert disp1 == disp2 + assert data1.shape == data2.shape + assert disp1 == disp2 + assert mocked.call_count == 1, "second call must be served from the L1 cache" def test_cache_returns_copy(self, clear_mr_cache): - """Cached data should be a copy — mutations don't affect cache.""" - from services.market_review import BENCHMARKS, _fetch_market_data + """Cached data is a copy — mutating a result must not corrupt the cache.""" + from services.market_review import _fetch_market_data - all_tickers = ["MSFT"] + list(BENCHMARKS.values()) - _seed_market_review_prices(all_tickers, days=60) clear_mr_cache() - - with patch("services.market_review.fetch_close_panel"): + with _patch_panel("MSFT"): data1, _, _ = _fetch_market_data("MSFT") original_shape = data1.shape data1.drop(data1.index[:10], inplace=True) data2, _, _ = _fetch_market_data("MSFT") - assert data2.shape == original_shape + + assert data2.shape == original_shape # ── market_review output format tests ──────────────────────────── @@ -80,30 +77,24 @@ def test_cache_returns_copy(self, clear_mr_cache): class TestMarketReviewOutput: def test_returns_dataframe(self, clear_mr_cache): - """market_review() returns a DataFrame with MultiIndex columns.""" - from services.market_review import BENCHMARKS, market_review + from services.market_review import market_review - all_tickers = ["GOOGL"] + list(BENCHMARKS.values()) - _seed_market_review_prices(all_tickers, days=100) clear_mr_cache() - - with patch("services.market_review.fetch_close_panel"): + with _patch_panel("GOOGL"): result = market_review("GOOGL") - assert isinstance(result, pd.DataFrame) - assert isinstance(result.columns, pd.MultiIndex) - assert len(result) > 0 - def test_result_contains_expected_assets(self, clear_mr_cache): - """Result index should contain the primary ticker and benchmark names.""" - from services.market_review import BENCHMARKS, market_review + assert isinstance(result, pd.DataFrame) + assert isinstance(result.columns, pd.MultiIndex) + assert len(result) > 0 - all_tickers = ["TSLA"] + list(BENCHMARKS.values()) - _seed_market_review_prices(all_tickers, days=100) - clear_mr_cache() + def test_result_contains_primary_ticker(self, clear_mr_cache): + from services.market_review import market_review - with patch("services.market_review.fetch_close_panel"): + clear_mr_cache() + with _patch_panel("TSLA"): result = market_review("TSLA") - assert "TSLA" in result.index + + assert "TSLA" in result.index # ── market_review_timeseries tests ─────────────────────────────── @@ -111,31 +102,25 @@ def test_result_contains_expected_assets(self, clear_mr_cache): class TestMarketReviewTimeseries: def test_returns_dict_structure(self, clear_mr_cache): - """market_review_timeseries() returns dict with expected keys.""" - from services.market_review import BENCHMARKS, market_review_timeseries + from services.market_review import market_review_timeseries - all_tickers = ["AMZN"] + list(BENCHMARKS.values()) - _seed_market_review_prices(all_tickers, days=100) clear_mr_cache() - - with patch("services.market_review.fetch_close_panel"): + with _patch_panel("AMZN"): result = market_review_timeseries("AMZN") - assert "dates" in result - assert "assets" in result - assert "instrument" in result - assert len(result["dates"]) > 0 + + assert "dates" in result + assert "assets" in result + assert "instrument" in result + assert len(result["dates"]) > 0 def test_assets_have_expected_fields(self, clear_mr_cache): - """Each asset entry should have prices, cum_return, rolling_vol.""" - from services.market_review import BENCHMARKS, market_review_timeseries + from services.market_review import market_review_timeseries - all_tickers = ["META"] + list(BENCHMARKS.values()) - _seed_market_review_prices(all_tickers, days=100) clear_mr_cache() - - with patch("services.market_review.fetch_close_panel"): + with _patch_panel("META"): result = market_review_timeseries("META") - for _asset_name, asset_data in result["assets"].items(): - assert "prices" in asset_data - assert "cum_returns" in asset_data - assert "rolling_vol" in asset_data + + for _asset_name, asset_data in result["assets"].items(): + assert "prices" in asset_data + assert "cum_returns" in asset_data + assert "rolling_vol" in asset_data