From 5b25fbcec2280f3db224bf6e26a26c5f30313a5d Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Thu, 10 Sep 2026 16:34:42 +0800 Subject: [PATCH 01/15] =?UTF-8?q?refactor(data-pipeline):=20B1=20provider?= =?UTF-8?q?=20seam=20=E6=8A=BD=E5=8F=96=EF=BC=8Cyfinance=20=E6=94=B6?= =?UTF-8?q?=E5=8F=A3=E5=88=B0=20providers/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 业务线重构计划书 §6 B1，行为不变。 - 新增 data_pipeline/providers/{base,_log,_registry,yfinance_provider,yf_snapshot}.py： MarketDataProvider 协议 + canonical schema（CanonicalBar / OptionLeg / OptionChainSnapshot）+ 名称注册表（MARKET_DATA_PROVIDER，默认 yfinance） - yf_client.py 退化为兼容 re-export；downloader.py 只保留缺口检测与 raw_prices upsert，不再 import yfinance - doc_guard 的 single-yf-exit / yfinance-throttle 作用域改为 data_pipeline/providers/ - 过 §8 决策闸 Q5：协议对照 futu 字段表 → iv 归一为小数、bid/ask 可空、 inTheMoney 不入 canonical（结论表记入 ADR 0011） - 计划书 §0 台账 B1 → landed；实际形态与两处偏差记在 §8 验收：pytest -m "not network" --ignore=tests/e2e → 459 passed / 5 skipped； doc_guard clean；arch_metrics --check ok（无需重置 baseline）；audit_tags 不变； 生产代码 import yfinance 仅 providers/ 两处；routes/ 与 services/ 零改动。 --- CLAUDE.md | 12 +- CODEBUDDY.md | 12 +- data_pipeline/downloader.py | 45 +- data_pipeline/providers/__init__.py | 44 ++ data_pipeline/providers/_log.py | 28 ++ data_pipeline/providers/_registry.py | 55 +++ data_pipeline/providers/base.py | 153 +++++++ data_pipeline/providers/yf_snapshot.py | 316 +++++++++++++ data_pipeline/providers/yfinance_provider.py | 264 +++++++++++ data_pipeline/yf_client.py | 418 ++---------------- docs/architecture_review.md | 12 +- docs/automation.md | 2 +- docs/constraints.md | 9 +- .../0011-pluggable-data-provider-seam.md | 23 + docs/l0_architecture.md | 6 +- docs/plans/business_line_reorg.md | 29 +- scripts/doc_guard.py | 41 +- tests/test_downloader_gap.py | 20 +- tests/test_provider_seam.py | 248 +++++++++++ tests/test_yf_failure_injection.py | 40 +- 20 files changed, 1307 insertions(+), 470 deletions(-) create mode 100644 data_pipeline/providers/__init__.py create mode 100644 data_pipeline/providers/_log.py create mode 100644 data_pipeline/providers/_registry.py create mode 100644 data_pipeline/providers/base.py create mode 100644 data_pipeline/providers/yf_snapshot.py create mode 100644 data_pipeline/providers/yfinance_provider.py create mode 100644 tests/test_provider_seam.py diff --git a/CLAUDE.md b/CLAUDE.md index 4939417..96c3f09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,10 +150,14 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which stops concurrent UI requests from stampeding Yahoo. `_query.py` calls `_update`/`_range` module functions directly (never the facade) to avoid an import cycle. -- **`yf_client.py`** is the **only** module allowed to call yfinance (enforced by `doc_guard` - `single-yf-exit`; exceptions registered in `docs/architecture_review.md` §2). Every call goes - through `yf_throttle()` (token bucket, 5 req/s, burst 5). **Never** pass - `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails (ADR 0005). +- **`providers/`** is the **only** package allowed to call yfinance (the chokepoint moved here from + `yf_client.py` in batch B1 of ADR 0011; enforced by `doc_guard` `single-yf-exit`, exceptions + registered in `docs/architecture_review.md` §2). It owns the mapping from the vendor's fields onto + one canonical schema (`providers/base.py`) — IV as a decimal, nullable bid/ask, no `inTheMoney`. + Every call goes through `yf_throttle()` (token bucket, 5 req/s, burst 5). `yf_client.py` is a + one-release compatibility shim over the package and `downloader.py` keeps only gap detection + + upsert. **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently + fails (ADR 0005). - **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, `busy_timeout=5000`) and does **not** close on exit. `repos.py` is the only place that builds SQL. diff --git a/CODEBUDDY.md b/CODEBUDDY.md index 4939417..96c3f09 100644 --- a/CODEBUDDY.md +++ b/CODEBUDDY.md @@ -150,10 +150,14 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which stops concurrent UI requests from stampeding Yahoo. `_query.py` calls `_update`/`_range` module functions directly (never the facade) to avoid an import cycle. -- **`yf_client.py`** is the **only** module allowed to call yfinance (enforced by `doc_guard` - `single-yf-exit`; exceptions registered in `docs/architecture_review.md` §2). Every call goes - through `yf_throttle()` (token bucket, 5 req/s, burst 5). **Never** pass - `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails (ADR 0005). +- **`providers/`** is the **only** package allowed to call yfinance (the chokepoint moved here from + `yf_client.py` in batch B1 of ADR 0011; enforced by `doc_guard` `single-yf-exit`, exceptions + registered in `docs/architecture_review.md` §2). It owns the mapping from the vendor's fields onto + one canonical schema (`providers/base.py`) — IV as a decimal, nullable bid/ask, no `inTheMoney`. + Every call goes through `yf_throttle()` (token bucket, 5 req/s, burst 5). `yf_client.py` is a + one-release compatibility shim over the package and `downloader.py` keeps only gap detection + + upsert. **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently + fails (ADR 0005). - **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, `busy_timeout=5000`) and does **not** close on exit. `repos.py` is the only place that builds SQL. diff --git a/data_pipeline/downloader.py b/data_pipeline/downloader.py index 9d86c32..be3ec96 100644 --- a/data_pipeline/downloader.py +++ b/data_pipeline/downloader.py @@ -1,4 +1,22 @@ -"""Market data downloader: fetches raw price data from external sources.""" +"""Market data downloader: business-day gap detection + raw OHLCV upsert. + +Domain: Data Pipeline — Ingest Glue +Context: + - Acquisition itself lives in ``data_pipeline/providers/``. This module keeps + only the DB-aware parts: business-day gap detection, the auto-backfill cap, + and the ``raw_prices`` upsert. Batch B1 (see + docs/plans/business_line_reorg.md §6) moved the ``yf.download`` call behind + ``providers.yfinance_provider.download_daily_frame``, so this module no + longer imports yfinance. +Contracts: + - ``upsert_raw_prices(ticker, start, end, days) -> PipelineResult`` — never + raises; degraded outcomes are reported through the ``PipelineResult``. + - ``find_missing_business_days(ticker, start, end) -> list[date]`` +Dependencies UPWARD: + - providers.yfinance_provider (download), .db (fetch_df / upsert_many) +Dependencies DOWNWARD: + - data_pipeline/data_ops (_update / _range), services/regime/ops/_bootstrap.py +""" import datetime as dt import logging @@ -6,9 +24,8 @@ from pathlib import Path import pandas as pd -import yfinance as yf # doc-guard: allow=single-yf-exit -from utils.network import yf_throttle +from data_pipeline.providers.yfinance_provider import download_daily_frame from . import PipelineResult from .db import fetch_df, upsert_many @@ -95,24 +112,16 @@ def find_missing_business_days(ticker: str, start: dt.date, end: dt.date) -> lis def _download_yf(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: - # Test tickers never hit the network. Useful for unit tests + ad-hoc - # smoke tests under rate-limit conditions; see `_load_test_fixture`. + """Download daily OHLCV for ``[start, end]`` (inclusive), fixture-aware. + + ``TEST_*`` tickers never hit the network (useful for unit tests + ad-hoc + smoke tests under rate-limit conditions; see ``_load_test_fixture``). + Everything else is delegated to the yfinance provider. + """ if ticker.startswith("TEST_"): logger.info("Loading fixture data for test ticker %s (%s..%s)", ticker, start, end) return _load_test_fixture(ticker, start, end) - # yfinance 'end' is exclusive, so pass end + 1 day to include the requested end date - yf_end = end + dt.timedelta(days=1) - yf_throttle() - df = yf.download(ticker, start=start, end=yf_end, interval="1d", progress=False, auto_adjust=False) - if df is None or df.empty: - return pd.DataFrame() - if isinstance(df.columns, pd.MultiIndex): - df.columns = df.columns.droplevel(1) - cols = ["Open", "High", "Low", "Close", "Adj Close", "Volume"] - for c in cols: - if c not in df.columns: - df[c] = pd.NA - return df[cols].rename(columns={"Adj Close": "Adj_Close"}) + return download_daily_frame(ticker, start, end) def upsert_raw_prices( diff --git a/data_pipeline/providers/__init__.py b/data_pipeline/providers/__init__.py new file mode 100644 index 0000000..396efe8 --- /dev/null +++ b/data_pipeline/providers/__init__.py @@ -0,0 +1,44 @@ +"""Data-provider seam — the only package that touches an external market-data API. + +Domain: Data Pipeline — Providers +Context: + - ADR 0011. Every external acquisition call lives under this package and is + mapped onto the canonical schema in ``providers/base.py``; processing and + serving stay provider-agnostic. + - ``yf_client.py`` is a compatibility shim over this package for one release; + ``downloader.py`` keeps only gap detection + DB upsert. +Contracts: + - ``get_provider(name=None)``: resolve a ``MarketDataProvider`` implementation. + - ``available_providers()``: registered provider names. + - Canonical shapes: ``CanonicalBar`` / ``OptionLeg`` / ``OptionChainSnapshot``. +Dependencies UPWARD: + - (none) +Dependencies DOWNWARD: + - providers/base, providers/_registry, providers/yfinance_provider, + providers/yf_snapshot +""" + +from __future__ import annotations + +from data_pipeline.providers._registry import available_providers, get_provider +from data_pipeline.providers.base import ( + CANONICAL_BAR_COLUMNS, + CANONICAL_LEG_COLUMNS, + CanonicalBar, + MarketDataProvider, + OptionChainSnapshot, + OptionLeg, + bars_to_frame, +) + +__all__ = [ + "CANONICAL_BAR_COLUMNS", + "CANONICAL_LEG_COLUMNS", + "CanonicalBar", + "MarketDataProvider", + "OptionChainSnapshot", + "OptionLeg", + "available_providers", + "bars_to_frame", + "get_provider", +] diff --git a/data_pipeline/providers/_log.py b/data_pipeline/providers/_log.py new file mode 100644 index 0000000..a89ae90 --- /dev/null +++ b/data_pipeline/providers/_log.py @@ -0,0 +1,28 @@ +"""Provider-side hook into the pipeline's failure log. + +Domain: Data Pipeline — Providers (shared) +Context: + - Every acquisition failure worth surfacing in ``/health/data`` lands in + ``data_quality_log`` (see ``data_pipeline/quality_log.py``). Both option-chain + and general yfinance provider modules need to record failures, and neither + may import the other, so the best-effort wrapper lives here. +Why the ``source`` strings still read ``yf_client.*``: + - Batch B1 moved these calls without changing the stored ``data_quality_log`` + rows; renaming the source labels is a separate, observable change and is + deliberately deferred (see docs/plans/business_line_reorg.md §6 B1). +Dependencies UPWARD: + - data_pipeline.quality_log (imported lazily — keeps package import cheap and + avoids a cycle at import time) +""" + +from __future__ import annotations + + +def _log_dq(source: str, error_class: str, message: str, *, ticker: str | None = None) -> None: + """Best-effort write to ``data_quality_log``. Never raises.""" + try: + from data_pipeline.quality_log import log_failure + + log_failure(source, error_class, message, ticker=ticker) + except Exception: # noqa: BLE001 + pass diff --git a/data_pipeline/providers/_registry.py b/data_pipeline/providers/_registry.py new file mode 100644 index 0000000..0703494 --- /dev/null +++ b/data_pipeline/providers/_registry.py @@ -0,0 +1,55 @@ +"""Provider registry: name → MarketDataProvider instance. + +Domain: Data Pipeline — Provider Selection +Context: + - ADR 0011: acquisition is selected by name, so a second vendor is one new + provider module plus one line in ``_FACTORIES`` — no caller changes. + yfinance stays the only implementation until a second provider is actually + needed (the seam is the deliverable, not the vendor). + - Selection order: explicit ``name`` argument → ``MARKET_DATA_PROVIDER`` env + var → ``DEFAULT_PROVIDER``. +Contracts: + - ``get_provider(name=None) -> MarketDataProvider`` + - ``available_providers() -> tuple[str, ...]`` +Dependencies UPWARD: + - (none) +Dependencies DOWNWARD: + - providers/base, providers/yfinance_provider +""" + +from __future__ import annotations + +import os +from collections.abc import Callable + +from data_pipeline.providers.base import MarketDataProvider +from data_pipeline.providers.yfinance_provider import YFinanceProvider + +PROVIDER_ENV_VAR = "MARKET_DATA_PROVIDER" +DEFAULT_PROVIDER = "yfinance" + +_FACTORIES: dict[str, Callable[[], MarketDataProvider]] = { + DEFAULT_PROVIDER: YFinanceProvider, +} +# WHY (cache): providers are stateless, but re-reading env/config on every fetch +# is pointless. Instances are keyed by resolved name. +_INSTANCES: dict[str, MarketDataProvider] = {} + + +def available_providers() -> tuple[str, ...]: + """Return the registered provider names, sorted.""" + return tuple(sorted(_FACTORIES)) + + +def get_provider(name: str | None = None) -> MarketDataProvider: + """Return the (cached) provider instance for ``name``. + + Raises ``ValueError`` for an unknown name rather than silently falling back + to yfinance — a typo in ``MARKET_DATA_PROVIDER`` must fail loudly. + """ + resolved = name or os.environ.get(PROVIDER_ENV_VAR) or DEFAULT_PROVIDER + if resolved not in _FACTORIES: + raise ValueError(f"unknown data provider {resolved!r}; available: {list(available_providers())}") + if resolved not in _INSTANCES: + _INSTANCES[resolved] = _FACTORIES[resolved]() + return _INSTANCES[resolved] diff --git a/data_pipeline/providers/base.py b/data_pipeline/providers/base.py new file mode 100644 index 0000000..cc94b08 --- /dev/null +++ b/data_pipeline/providers/base.py @@ -0,0 +1,153 @@ +"""Provider seam: the canonical internal schema and the acquisition protocol. + +Domain: Data Pipeline — Acquisition Seam +Context: + - ADR 0011 splits acquisition from processing/serving: a *provider* owns every + call to an external market-data API and maps that API's fields onto one + canonical internal schema, so nothing downstream of acquisition sees a + vendor-specific shape. + - yfinance is the only implementation today (ADR 0002, as amended by 0011). + This protocol is deliberately sketched against *two* field maps — yfinance + and the archived futu integration + (``archive/futu_integration/field_mapping.md``) — so it does not bake in + yfinance-isms. That comparison is the §8 Q5 decision gate for batch B1 and is + recorded in ADR 0011. +Contracts: + - ``MarketDataProvider``: the minimal acquisition surface. + - ``CANONICAL_BAR_COLUMNS`` / ``CANONICAL_LEG_COLUMNS``: canonical frame columns. + - ``CanonicalBar`` / ``OptionLeg`` / ``OptionChainSnapshot``: canonical records. +Unit conventions — INVARIANT for every provider implementation: + - ``iv`` is a **decimal** (0.2436 means 24.36 %). futu reports percent, + yfinance reports decimal; the provider normalises at its own boundary. + - ``bid`` / ``ask`` may be ``None``. futu's ``get_stock_quote`` exposes no + bid/ask without an ORDER_BOOK subscription, so "absent" must be expressible — + a provider must never invent a quote. + - ``inTheMoney`` is intentionally **not** canonical: it is derivable from + ``(strike, spot)`` and futu has no equivalent column. + - OHLCV values are plain floats; ``volume`` / ``open_interest`` are + non-negative counts. +Dependencies UPWARD: + - (none — no external SDK is imported here; implementations sit beside it) +Dependencies DOWNWARD: + - providers/yfinance_provider.py, providers/yf_option_chain.py, + providers/_registry.py +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Protocol, runtime_checkable + +import pandas as pd + +# INVARIANT: the column names (and order) of the frame ``history()`` returns, +# indexed by a tz-naive DatetimeIndex of trading days. +CANONICAL_BAR_COLUMNS: tuple[str, ...] = ("open", "high", "low", "close", "adj_close", "volume") + +# INVARIANT: the fields every canonical option leg exposes. See the unit +# conventions in the module docstring for the semantics of ``iv`` / ``bid``. +CANONICAL_LEG_COLUMNS: tuple[str, ...] = ( + "strike", + "bid", + "ask", + "last", + "iv", + "open_interest", + "volume", +) + + +@dataclass(frozen=True) +class CanonicalBar: + """One daily OHLCV bar in the canonical schema.""" + + provider: str + symbol: str + date: dt.date + open: float | None = None + high: float | None = None + low: float | None = None + close: float | None = None + adj_close: float | None = None + volume: float | None = None + + +@dataclass(frozen=True) +class OptionLeg: + """One option contract quote in the canonical schema.""" + + strike: float + bid: float | None = None + ask: float | None = None + last: float | None = None + iv: float | None = None + open_interest: float | None = None + volume: float | None = None + + +@dataclass(frozen=True) +class OptionChainSnapshot: + """A live option-chain snapshot. Never persisted — see ADR 0004.""" + + provider: str + symbol: str + spot: float | None + expiries: tuple[str, ...] = () + chain: Mapping[str, Mapping[str, tuple[OptionLeg, ...]]] = field(default_factory=dict) + + def legs(self, expiry: str, side: str) -> tuple[OptionLeg, ...]: + """Return the legs for ``expiry`` and ``side`` (``calls`` | ``puts``).""" + return tuple(self.chain.get(expiry, {}).get(side, ())) + + +@runtime_checkable +class MarketDataProvider(Protocol): + """The acquisition surface a data provider must offer. + + An implementation owns (a) every call to its external API and (b) the mapping + from that API's fields onto the canonical schema above. Callers resolve an + implementation through ``providers.get_provider()`` rather than importing a + concrete provider, so adding a vendor is a localised change (ADR 0011). + """ + + @property + def name(self) -> str: + """Stable provider id, also stored in the ``provider`` column.""" + ... + + def history(self, symbol: str, start: dt.date, end: dt.date) -> pd.DataFrame: + """Daily bars for ``[start, end]`` as a ``CANONICAL_BAR_COLUMNS`` frame.""" + ... + + def close_panel( + self, + symbols: list[str], + *, + start: dt.date | str | None = None, + end: dt.date | str | None = None, + period: str | None = None, + ) -> pd.DataFrame: + """Wide Close-price frame: index = date, columns = symbols.""" + ... + + def spot(self, symbol: str) -> float | None: + """Latest traded price for ``symbol``, or ``None`` when unavailable.""" + ... + + def option_chain(self, symbol: str) -> OptionChainSnapshot: + """Live chain snapshot (spot + expiries + canonical legs).""" + ... + + +def bars_to_frame(bars: Iterable[CanonicalBar]) -> pd.DataFrame: + """Render ``CanonicalBar`` records as a ``CANONICAL_BAR_COLUMNS`` DataFrame.""" + rows = list(bars) + if not rows: + return pd.DataFrame(columns=list(CANONICAL_BAR_COLUMNS)) + frame = pd.DataFrame( + [{col: getattr(bar, col) for col in CANONICAL_BAR_COLUMNS} for bar in rows], + index=pd.DatetimeIndex([pd.Timestamp(bar.date) for bar in rows]), + ) + return frame.sort_index() diff --git a/data_pipeline/providers/yf_snapshot.py b/data_pipeline/providers/yf_snapshot.py new file mode 100644 index 0000000..15796ef --- /dev/null +++ b/data_pipeline/providers/yf_snapshot.py @@ -0,0 +1,316 @@ +"""yfinance live-snapshot acquisition: spot price + option chain. + +Domain: Data Pipeline — yfinance Provider (Live Snapshots) +Context: + - Live snapshots are the endpoints ADR 0004 says are never persisted: the + current spot and the current option chain. They are also the largest + yfinance surface, so they live in their own module and keep + ``providers/yfinance_provider.py`` under the 400-line god-file cap — a split + pre-registered in docs/architecture_review.md §2. + - Batch B1 moved this code verbatim out of ``data_pipeline/yf_client.py`` (no + behaviour change); only the canonical mapping at the bottom is new. + - INVARIANT (keeps the import graph acyclic): this module must never import + ``providers/yfinance_provider.py``. +Contracts: + - ``fetch_spot(ticker)`` / ``fetch_spots_bulk(tickers)`` -> ``float`` / ``dict``. + - ``fetch_option_chain(ticker)`` keeps the legacy payload contract + (``ticker`` / ``spot`` / ``expiries`` / ``chain{expiry:{calls,puts}}``) for + callers that still import it through the ``data_pipeline.yf_client`` shim. + - ``to_option_chain_snapshot(payload)`` -> canonical ``OptionChainSnapshot``. +Design rules: + - CONSTRAINT: every public function calls ``yf_throttle()`` before each + yfinance call — see docs/decisions/0005-token-bucket-throttle.md. + - WHY (never raise on transient failure): callers receive ``None`` / an empty + chain and decide how to surface the error. Raising here would cascade into + unhandled 500s from several routes. +Dependencies UPWARD: + - utils.network (throttle), data_pipeline.quality_log (via providers._log) +Dependencies DOWNWARD: + - providers/base, providers/_log +""" + +from __future__ import annotations + +import logging +import math +import os +import threading +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any + +import pandas as pd +import yfinance as yf + +from data_pipeline.providers._log import _log_dq +from data_pipeline.providers.base import OptionChainSnapshot, OptionLeg +from utils.network import yf_throttle + +logger = logging.getLogger(__name__) + + +# Standard option-chain numeric columns we always coerce. +_OPT_NUMERIC_COLS = ( + "strike", + "bid", + "ask", + "lastPrice", + "impliedVolatility", + "openInterest", + "volume", +) + + +# --------------------------------------------------------------------------- +# Spot price +# --------------------------------------------------------------------------- +def fetch_spot(ticker: str) -> float | None: + """Return the current spot price for ``ticker`` or ``None`` on failure. + + Tries ``fast_info.last_price`` then ``regularMarketPrice`` then a tiny + history fallback (1d) so we always have *something* for tickers whose + fast_info is flaky. + """ + try: + yf_throttle() + tk = yf.Ticker(ticker) + fi = tk.fast_info + price = getattr(fi, "last_price", None) or getattr(fi, "regularMarketPrice", None) + if price is not None: + return float(price) + except Exception as exc: # noqa: BLE001 — yfinance raises a wide variety + logger.debug("fetch_spot fast_info failed for %s: %s", ticker, exc) + + try: + yf_throttle() + hist = yf.Ticker(ticker).history(period="1d") + if not hist.empty and "Close" in hist.columns: + return float(hist["Close"].iloc[-1]) + except Exception as exc: # noqa: BLE001 + logger.warning("fetch_spot history fallback failed for %s: %s", ticker, exc) + _log_dq("yf_client.fetch_spot", "spot_unavailable", str(exc), ticker=ticker) + return None + + +def fetch_spots_bulk(tickers: list[str]) -> dict[str, float]: + """Return ``{ticker: spot}`` for every ticker that resolved successfully. + + Sequential (one yfinance call per ticker) — the global token bucket in + ``yf_throttle`` already paces us. Failures are logged and omitted from + the result; callers must handle missing keys. + """ + out: dict[str, float] = {} + for t in tickers: + spot = fetch_spot(t) + if spot is not None: + out[t] = spot + return out + + +# --------------------------------------------------------------------------- +# Option chain +# --------------------------------------------------------------------------- +def fetch_option_chain(ticker: str) -> dict[str, Any]: + """Fetch a full option-chain snapshot. + + Returns + ------- + dict + Shape:: + + { + "ticker": str, + "spot": float | None, + "expiries": list[str], # YYYY-MM-DD strings + "chain": { + expiry_str: { + "calls": pd.DataFrame, + "puts": pd.DataFrame, + } + } + } + + Numeric columns on the DataFrames are coerced via ``pd.to_numeric``. + ``openInterest`` / ``volume`` NaNs are filled with 0. + + On total failure (no expiries returned), ``expiries`` and ``chain`` + are empty but ``spot`` may still be populated. + """ + spot = fetch_spot(ticker) + expiries: list[str] = [] + chain: dict[str, dict[str, pd.DataFrame]] = {} + + try: + yf_throttle() + tk = yf.Ticker(ticker) + expiries = list(tk.options or []) + except Exception as exc: # noqa: BLE001 + logger.warning("fetch_option_chain: failed to list expiries for %s: %s", ticker, exc) + _log_dq("yf_client.fetch_option_chain", "expiries_unavailable", str(exc), ticker=ticker) + return {"ticker": ticker, "spot": spot, "expiries": [], "chain": {}} + + # WHY: track consecutive empty responses. Yahoo's 429 / curl_cffi's + # empty-body behaviour applies to the WHOLE option-chain endpoint + # uniformly — once one expiry returns None, the rest will too. Bail + # after a few empties to avoid burning the throttle on guaranteed misses. + EMPTY_FAIL_FAST = 3 + # WHY (concurrency): option_chain HTTP latency (~2s) typically exceeds the + # 1.5s throttle gap, so overlapping HTTP across a small worker pool + # reduces total wall time even though throttle still serialises call + # *starts*. Cap at 2 by default — higher counts give diminishing returns + # and risk tripping Yahoo's burst detection. CONSTRAINT: every worker + # MUST call yf_throttle() before each yfinance call (ADR 0005). + max_workers = max(1, int(os.environ.get("YF_OPTION_CHAIN_WORKERS", "2"))) + if max_workers == 1 or len(expiries) <= 1: + return _fetch_option_chain_serial(ticker, spot, expiries, EMPTY_FAIL_FAST) + + consecutive_empty_lock = threading.Lock() + consecutive_empty = {"n": 0, "abort": False} + + def _fetch_one(exp: str): + if consecutive_empty["abort"]: + return exp, None + try: + yf_throttle() + # WHY (per-thread Ticker): yfinance.Ticker is not documented as + # thread-safe; create a fresh instance per worker to avoid + # hidden shared state in tk._options_data. + opt = yf.Ticker(ticker).option_chain(exp) + except Exception as exc: # noqa: BLE001 + logger.warning("fetch_option_chain: %s exp=%s failed: %s", ticker, exp, exc) + # Exceptions (e.g. 429s surfacing as raised errors) count toward + # the fail-fast budget too, otherwise an error-storm burns the + # throttle across every expiry instead of aborting early. + with consecutive_empty_lock: + consecutive_empty["n"] += 1 + if consecutive_empty["n"] >= EMPTY_FAIL_FAST: + consecutive_empty["abort"] = True + return exp, "error" + if opt is None or opt.calls is None or opt.puts is None: + with consecutive_empty_lock: + consecutive_empty["n"] += 1 + if consecutive_empty["n"] == 1: + logger.warning("fetch_option_chain: %s exp=%s returned no data (rate-limited?)", ticker, exp) + if consecutive_empty["n"] >= EMPTY_FAIL_FAST: + consecutive_empty["abort"] = True + return exp, None + with consecutive_empty_lock: + consecutive_empty["n"] = 0 + return exp, _coerce_chain_side_payload(opt) + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(_fetch_one, exp): exp for exp in expiries} + for future in as_completed(futures): + exp, payload = future.result() + if isinstance(payload, dict): + chain[exp] = payload + + # WHY: preserve the user-meaningful order (front-month first) from the + # original ``expiries`` list — as_completed yields in completion order. + ordered_chain = {exp: chain[exp] for exp in expiries if exp in chain} + return {"ticker": ticker, "spot": spot, "expiries": list(ordered_chain.keys()), "chain": ordered_chain} + + +def _coerce_chain_side_payload(opt: Any) -> dict[str, pd.DataFrame]: + """Coerce one ``opt.calls`` / ``opt.puts`` pair: numeric columns, OI/volume NaN→0.""" + calls = opt.calls.copy() + puts = opt.puts.copy() + for col in _OPT_NUMERIC_COLS: + if col in calls.columns: + calls[col] = pd.to_numeric(calls[col], errors="coerce") + if col in puts.columns: + puts[col] = pd.to_numeric(puts[col], errors="coerce") + for col in ("openInterest", "volume"): + if col in calls.columns: + calls[col] = calls[col].fillna(0) + if col in puts.columns: + puts[col] = puts[col].fillna(0) + return {"calls": calls, "puts": puts} + + +def _fetch_option_chain_serial( + ticker: str, spot: float | None, expiries: list[str], empty_fail_fast: int +) -> dict[str, Any]: + """Sequential fallback path used when YF_OPTION_CHAIN_WORKERS=1 or only + one expiry exists. Behaviourally identical to the pre-concurrency loop. + """ + chain: dict[str, dict[str, pd.DataFrame]] = {} + consecutive_empty = 0 + tk = yf.Ticker(ticker) + for exp in expiries: + try: + yf_throttle() + opt = tk.option_chain(exp) + if opt is None or opt.calls is None or opt.puts is None: + consecutive_empty += 1 + if consecutive_empty == 1: + logger.warning("fetch_option_chain: %s exp=%s returned no data (rate-limited?)", ticker, exp) + if consecutive_empty >= empty_fail_fast: + logger.warning( + "fetch_option_chain: %s aborting after %d empty expiries (likely rate-limited)", + ticker, + consecutive_empty, + ) + break + continue + consecutive_empty = 0 + chain[exp] = _coerce_chain_side_payload(opt) + except Exception as exc: # noqa: BLE001 + logger.warning("fetch_option_chain: %s exp=%s failed: %s", ticker, exp, exc) + continue + + return {"ticker": ticker, "spot": spot, "expiries": list(chain.keys()), "chain": chain} + + +# --------------------------------------------------------------------------- +# Canonical mapping +# --------------------------------------------------------------------------- +def _opt_float(value: Any) -> float | None: + """Coerce a yfinance cell to float, mapping NaN/inf/None to ``None``.""" + if value is None: + return None + try: + out = float(value) + except (TypeError, ValueError): + return None + return None if math.isnan(out) or math.isinf(out) else out + + +def _leg_from_row(row: pd.Series) -> OptionLeg: + """Map one yfinance chain row onto a canonical ``OptionLeg``.""" + return OptionLeg( + strike=_opt_float(row.get("strike")) or 0.0, + bid=_opt_float(row.get("bid")), + ask=_opt_float(row.get("ask")), + last=_opt_float(row.get("lastPrice")), + iv=_opt_float(row.get("impliedVolatility")), + open_interest=_opt_float(row.get("openInterest")), + volume=_opt_float(row.get("volume")), + ) + + +def to_option_chain_snapshot(payload: Mapping[str, Any], *, provider: str = "yfinance") -> OptionChainSnapshot: + """Map a legacy ``fetch_option_chain`` payload onto the canonical schema. + + INVARIANT: no ``inTheMoney`` column is carried over (derivable), and ``iv`` + stays a decimal — see ``providers/base.py`` for the unit conventions. + """ + expiries = tuple(payload.get("expiries") or ()) + raw_chain = payload.get("chain") or {} + chain: dict[str, dict[str, tuple[OptionLeg, ...]]] = {} + for expiry in expiries: + sides = raw_chain.get(expiry) + if not sides: + continue + chain[expiry] = { + side: tuple(_leg_from_row(row) for _, row in sides[side].iterrows()) + for side in ("calls", "puts") + if side in sides + } + return OptionChainSnapshot( + provider=provider, + symbol=str(payload.get("ticker") or ""), + spot=payload.get("spot"), + expiries=expiries, + chain=chain, + ) diff --git a/data_pipeline/providers/yfinance_provider.py b/data_pipeline/providers/yfinance_provider.py new file mode 100644 index 0000000..df53dec --- /dev/null +++ b/data_pipeline/providers/yfinance_provider.py @@ -0,0 +1,264 @@ +"""yfinance provider: historical bars, close panels, canonical mapping. + +Domain: Data Pipeline — yfinance Provider +Context: + - This module (with ``yf_snapshot.py``) is the **only** place in the repo that + imports ``yfinance``; batch B1 moved these calls here out of + ``data_pipeline/yf_client.py`` and ``data_pipeline/downloader.py`` without + changing behaviour. See ADR 0002 (as amended by ADR 0011) and + docs/plans/business_line_reorg.md §6. + - ``YFinanceProvider`` is the canonical seam implementation (``history()`` / + ``close_panel()`` / ``spot()`` / ``option_chain()``); it delegates the live + snapshots to ``yf_snapshot.py``. The module-level ``fetch_*`` functions keep + their original yfinance-shaped contracts so the ~11 existing importers keep + working through the ``yf_client`` shim — new code should prefer the + canonical shapes (ADR 0011). +Design rules: + - CONSTRAINT: every public function calls ``yf_throttle()`` before each + yfinance call — see docs/decisions/0005-token-bucket-throttle.md. + - WHY (no caching here): caching is the caller's concern (e.g. ``app.py`` + option-chain cache, ``DataService`` 60s freshness window). + - WHY (never raise on transient failure): callers receive an empty DataFrame + and decide how to surface the error. Raising here would cascade into + unhandled 500s from many different routes. + - INVARIANT: returns plain Python types (float / DataFrame), never + yfinance-specific objects — keeps the rest of the pipeline mockable. + - CONSTRAINT: never pass ``session=requests.Session()`` — yfinance ≥0.2.50 uses + curl_cffi and silently fails (ADR 0005 / docs/constraints.md §2). +Dependencies UPWARD: + - utils.network (throttle), data_pipeline.quality_log (via providers._log) +Dependencies DOWNWARD: + - providers/base, providers/_log, providers/yf_snapshot +""" + +from __future__ import annotations + +import datetime as dt +import logging +import time + +import pandas as pd +import yfinance as yf + +from data_pipeline.providers._log import _log_dq +from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS, OptionChainSnapshot +from data_pipeline.providers.yf_snapshot import fetch_option_chain, fetch_spot, to_option_chain_snapshot +from utils.network import yf_throttle + +logger = logging.getLogger(__name__) + + +# WHY: yfinance's Title-Case columns (as returned by ``yf.download``) mapped onto +# the canonical lowercase schema. ``Adj Close`` arrives with a space from +# ``yf.download`` and as ``Adj_Close`` from the test fixtures / upsert path. +_YF_TO_CANONICAL = { + "Open": "open", + "High": "high", + "Low": "low", + "Close": "close", + "Adj Close": "adj_close", + "Adj_Close": "adj_close", + "Volume": "volume", +} + + +# --------------------------------------------------------------------------- +# Close-only panel (used by correlation matrix and market review) +# --------------------------------------------------------------------------- +def fetch_close_panel( + tickers: list[str], + period: str | None = "90d", + *, + start: str | None = None, + end: str | None = None, + max_retries: int = 2, + retry_base_delay: float = 3.0, +) -> pd.DataFrame: + """Return a wide DataFrame of Close prices for ``tickers``. + + Either pass ``period`` (e.g. ``"90d"``, ``"400d"``) OR ``start``/``end`` + date strings — when ``start`` is provided it takes precedence. On failure + returns an empty DataFrame. One yfinance call total (yfinance natively + supports multi-ticker download). + + WHY (retry loop): Yahoo occasionally returns an empty payload on the first + call after a wake-from-sleep; one retry resolves it without escalating. + """ + if not tickers: + return pd.DataFrame() + if start is not None: + kwargs = {"start": start, "end": end} + else: + kwargs = {"period": period or "90d"} + last_err: Exception | None = None + for attempt in range(max_retries): + try: + yf_throttle() + data = yf.download(tickers, auto_adjust=False, progress=False, **kwargs) + if data is None or data.empty: + if attempt < max_retries - 1: + logger.warning( + "fetch_close_panel empty payload, retrying in %.1fs (attempt %d)", + retry_base_delay * (attempt + 1), + attempt + 1, + ) + time.sleep(retry_base_delay * (attempt + 1)) + continue + return pd.DataFrame() + if isinstance(data.columns, pd.MultiIndex): + if "Close" not in data.columns.get_level_values(0): + return pd.DataFrame() + close = data["Close"] + if isinstance(close.columns, pd.MultiIndex): + close.columns = close.columns.droplevel(1) + else: + if "Close" not in data.columns: + return pd.DataFrame() + close = data[["Close"]] + return close + except Exception as exc: # noqa: BLE001 + last_err = exc + is_rate_limit = "rate" in str(exc).lower() or "too many" in str(exc).lower() + if is_rate_limit and attempt < max_retries - 1: + time.sleep(retry_base_delay * (attempt + 1)) + continue + break + if last_err is not None: + logger.warning("fetch_close_panel failed for %s: %s", tickers, last_err) + is_rate_limit = "rate" in str(last_err).lower() or "too many" in str(last_err).lower() + _log_dq( + "yf_client.fetch_close_panel", + "rate_limited" if is_rate_limit else "download_error", + str(last_err), + ticker=",".join(tickers), + ) + return pd.DataFrame() + + +# --------------------------------------------------------------------------- +# Daily OHLCV +# --------------------------------------------------------------------------- +def fetch_daily_ohlcv( + ticker: str, + start, + end, + *, + auto_adjust: bool = False, + max_retries: int = 2, + retry_base_delay: float = 3.0, +) -> pd.DataFrame: + """Download daily OHLCV bars for ``ticker`` in ``[start, end)``. + + Returns a DataFrame indexed by Date with columns + ``[Open, High, Low, Close, Adj Close, Volume]``. Empty DataFrame on + failure. Includes simple retry loop for transient empty responses. + """ + last_err: Exception | None = None + for attempt in range(max_retries): + try: + yf_throttle() + df = yf.download( + ticker, + start=start, + end=end, + interval="1d", + progress=False, + auto_adjust=auto_adjust, + ) + if df is None or df.empty: + if attempt < max_retries - 1: + time.sleep(retry_base_delay * (attempt + 1)) + continue + return pd.DataFrame() + if isinstance(df.columns, pd.MultiIndex): + df.columns = df.columns.droplevel(1) + df.index = pd.DatetimeIndex(df.index) + return df + except Exception as exc: # noqa: BLE001 + last_err = exc + is_rate_limit = "rate" in str(exc).lower() or "too many" in str(exc).lower() + if is_rate_limit and attempt < max_retries - 1: + time.sleep(retry_base_delay * (attempt + 1)) + continue + break + if last_err is not None: + logger.warning("fetch_daily_ohlcv failed for %s: %s", ticker, last_err) + is_rate_limit = "rate" in str(last_err).lower() or "too many" in str(last_err).lower() + _log_dq( + "yf_client.fetch_daily_ohlcv", + "rate_limited" if is_rate_limit else "download_error", + str(last_err), + ticker=ticker, + ) + return pd.DataFrame() + + +def download_daily_frame(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: + """Download daily OHLCV for ``[start, end]`` (inclusive) from yfinance. + + Returns a frame with Title-Case columns plus ``Adj_Close`` — the shape + ``data_pipeline.downloader.upsert_raw_prices`` consumes. ``history()`` on + ``YFinanceProvider`` is the canonical equivalent. + """ + # yfinance 'end' is exclusive, so pass end + 1 day to include the requested end date + yf_end = end + dt.timedelta(days=1) + yf_throttle() + df = yf.download(ticker, start=start, end=yf_end, interval="1d", progress=False, auto_adjust=False) + if df is None or df.empty: + return pd.DataFrame() + if isinstance(df.columns, pd.MultiIndex): + df.columns = df.columns.droplevel(1) + cols = ["Open", "High", "Low", "Close", "Adj Close", "Volume"] + for c in cols: + if c not in df.columns: + df[c] = pd.NA + return df[cols].rename(columns={"Adj Close": "Adj_Close"}) + + +# --------------------------------------------------------------------------- +# Canonical mapping +# --------------------------------------------------------------------------- +def to_canonical_bars(frame: pd.DataFrame | None) -> pd.DataFrame: + """Map a yfinance OHLCV frame onto the canonical lower-case schema.""" + if frame is None or frame.empty: + return pd.DataFrame(columns=list(CANONICAL_BAR_COLUMNS)) + out = pd.DataFrame(index=pd.DatetimeIndex(frame.index)) + for source, target in _YF_TO_CANONICAL.items(): + if source in frame.columns: + out[target] = pd.to_numeric(frame[source], errors="coerce") + for col in CANONICAL_BAR_COLUMNS: + if col not in out.columns: + out[col] = pd.NA + return out[list(CANONICAL_BAR_COLUMNS)].sort_index() + + +# --------------------------------------------------------------------------- +# Provider implementation +# --------------------------------------------------------------------------- +class YFinanceProvider: + """yfinance implementation of ``MarketDataProvider`` (the only one today).""" + + name = "yfinance" + + def history(self, symbol: str, start: dt.date, end: dt.date) -> pd.DataFrame: + """Canonical daily bars for ``[start, end]`` (inclusive).""" + return to_canonical_bars(download_daily_frame(symbol, start, end)) + + def close_panel( + self, + symbols: list[str], + *, + start: dt.date | str | None = None, + end: dt.date | str | None = None, + period: str | None = None, + ) -> pd.DataFrame: + """Wide Close-price panel for ``symbols``.""" + return fetch_close_panel(symbols, period or "90d", start=start, end=end) + + def spot(self, symbol: str) -> float | None: + """Latest traded price for ``symbol``.""" + return fetch_spot(symbol) + + def option_chain(self, symbol: str) -> OptionChainSnapshot: + """Canonical live option-chain snapshot for ``symbol``.""" + return to_option_chain_snapshot(fetch_option_chain(symbol), provider=self.name) diff --git a/data_pipeline/yf_client.py b/data_pipeline/yf_client.py index ab8d0ea..caf65e2 100644 --- a/data_pipeline/yf_client.py +++ b/data_pipeline/yf_client.py @@ -1,398 +1,36 @@ -""" -Unified yfinance access layer. +"""Compatibility shim over the yfinance provider seam. +Domain: Data Pipeline — yfinance compatibility surface Context: -- All direct yfinance calls (`yf.Ticker`, `yf.download`, `option_chain`, - `fast_info`) for live snapshot data go through this module. OHLCV historical - bulk downloads remain in ``downloader.py`` (which has DB-aware gap detection). -- Yahoo Finance has no SLA: rate limits, transient 5xx, and silently empty - payloads are routine. See docs/constraints.md §2 and ADR 0002 / 0005. - + - Batch B1 (docs/plans/business_line_reorg.md §6) moved every yfinance call + into ``data_pipeline/providers/``. This module is kept for one release so the + existing importers (``services/``, ``core/market/data_context.py``, + ``data_pipeline/data_ops/``) do not have to change in the same PR as the + extraction. See ADR 0011. + - New code should import from ``data_pipeline.providers`` (canonical shapes) + instead of here. +Contracts: + - Re-exports the legacy function contracts unchanged: ``fetch_spot``, + ``fetch_spots_bulk``, ``fetch_option_chain``, ``fetch_close_panel``, + ``fetch_daily_ohlcv``. Design rules: -- CONSTRAINT: every public function MUST call ``yf_throttle()`` before each - yfinance call. See docs/decisions/0005-token-bucket-throttle.md. -- WHY (no caching here): caching is the caller's concern (e.g. - ``app._option_chain_cache``, ``data_service`` 60s freshness window). -- WHY (never raise on transient failure): callers receive ``None`` / empty - dict and decide how to surface the error. Raising here would cascade into - unhandled 500s from many different routes. -- INVARIANT: returns plain Python types (float / dict / DataFrame), never - yfinance-specific objects — keeps the rest of the pipeline mockable. + - CONSTRAINT: this module must not import ``yfinance``; the single exit point + is ``data_pipeline/providers/`` (enforced by doc_guard ``single-yf-exit``). +Dependencies UPWARD: + - providers/yf_snapshot (live snapshots), providers/yfinance_provider (bars) +Dependencies DOWNWARD: + - services/*, core/market/data_context.py, data_pipeline/data_ops/* """ from __future__ import annotations -import logging -import os -import threading -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Any - -import pandas as pd -import yfinance as yf - -from utils.network import yf_throttle - -logger = logging.getLogger(__name__) - - -def _log_dq(source: str, error_class: str, message: str, *, ticker: str | None = None) -> None: - """Best-effort write to ``data_quality_log``. Never raises.""" - try: - from data_pipeline.quality_log import log_failure - - log_failure(source, error_class, message, ticker=ticker) - except Exception: # noqa: BLE001 - pass - - -# Standard option-chain numeric columns we always coerce. -_OPT_NUMERIC_COLS = ( - "strike", - "bid", - "ask", - "lastPrice", - "impliedVolatility", - "openInterest", - "volume", -) - - -# --------------------------------------------------------------------------- -# Spot price -# --------------------------------------------------------------------------- -def fetch_spot(ticker: str) -> float | None: - """Return the current spot price for ``ticker`` or ``None`` on failure. - - Tries ``fast_info.last_price`` then ``regularMarketPrice`` then a tiny - history fallback (1d) so we always have *something* for tickers whose - fast_info is flaky. - """ - try: - yf_throttle() - tk = yf.Ticker(ticker) - fi = tk.fast_info - price = getattr(fi, "last_price", None) or getattr(fi, "regularMarketPrice", None) - if price is not None: - return float(price) - except Exception as exc: # noqa: BLE001 — yfinance raises a wide variety - logger.debug("fetch_spot fast_info failed for %s: %s", ticker, exc) - - try: - yf_throttle() - hist = yf.Ticker(ticker).history(period="1d") - if not hist.empty and "Close" in hist.columns: - return float(hist["Close"].iloc[-1]) - except Exception as exc: # noqa: BLE001 - logger.warning("fetch_spot history fallback failed for %s: %s", ticker, exc) - _log_dq("yf_client.fetch_spot", "spot_unavailable", str(exc), ticker=ticker) - return None - - -def fetch_spots_bulk(tickers: list[str]) -> dict[str, float]: - """Return ``{ticker: spot}`` for every ticker that resolved successfully. - - Sequential (one yfinance call per ticker) — the global token bucket in - ``yf_throttle`` already paces us. Failures are logged and omitted from - the result; callers must handle missing keys. - """ - out: dict[str, float] = {} - for t in tickers: - spot = fetch_spot(t) - if spot is not None: - out[t] = spot - return out - - -# --------------------------------------------------------------------------- -# Option chain -# --------------------------------------------------------------------------- -def fetch_option_chain(ticker: str) -> dict[str, Any]: - """Fetch a full option-chain snapshot. - - Returns - ------- - dict - Shape:: - - { - "ticker": str, - "spot": float | None, - "expiries": list[str], # YYYY-MM-DD strings - "chain": { - expiry_str: { - "calls": pd.DataFrame, - "puts": pd.DataFrame, - } - } - } - - Numeric columns on the DataFrames are coerced via ``pd.to_numeric``. - ``openInterest`` / ``volume`` NaNs are filled with 0. - - On total failure (no expiries returned), ``expiries`` and ``chain`` - are empty but ``spot`` may still be populated. - """ - spot = fetch_spot(ticker) - expiries: list[str] = [] - chain: dict[str, dict[str, pd.DataFrame]] = {} - - try: - yf_throttle() - tk = yf.Ticker(ticker) - expiries = list(tk.options or []) - except Exception as exc: # noqa: BLE001 - logger.warning("fetch_option_chain: failed to list expiries for %s: %s", ticker, exc) - _log_dq("yf_client.fetch_option_chain", "expiries_unavailable", str(exc), ticker=ticker) - return {"ticker": ticker, "spot": spot, "expiries": [], "chain": {}} - - # WHY: track consecutive empty responses. Yahoo's 429 / curl_cffi's - # empty-body behaviour applies to the WHOLE option-chain endpoint - # uniformly — once one expiry returns None, the rest will too. Bail - # after a few empties to avoid burning the throttle on guaranteed misses. - EMPTY_FAIL_FAST = 3 - # WHY (concurrency): option_chain HTTP latency (~2s) typically exceeds the - # 1.5s throttle gap, so overlapping HTTP across a small worker pool - # reduces total wall time even though throttle still serialises call - # *starts*. Cap at 2 by default — higher counts give diminishing returns - # and risk tripping Yahoo's burst detection. CONSTRAINT: every worker - # MUST call yf_throttle() before each yfinance call (ADR 0005). - max_workers = max(1, int(os.environ.get("YF_OPTION_CHAIN_WORKERS", "2"))) - if max_workers == 1 or len(expiries) <= 1: - return _fetch_option_chain_serial(ticker, spot, expiries, EMPTY_FAIL_FAST) - - consecutive_empty_lock = threading.Lock() - consecutive_empty = {"n": 0, "abort": False} - - def _fetch_one(exp: str): - if consecutive_empty["abort"]: - return exp, None - try: - yf_throttle() - # WHY (per-thread Ticker): yfinance.Ticker is not documented as - # thread-safe; create a fresh instance per worker to avoid - # hidden shared state in tk._options_data. - opt = yf.Ticker(ticker).option_chain(exp) - except Exception as exc: # noqa: BLE001 - logger.warning("fetch_option_chain: %s exp=%s failed: %s", ticker, exp, exc) - # Exceptions (e.g. 429s surfacing as raised errors) count toward - # the fail-fast budget too, otherwise an error-storm burns the - # throttle across every expiry instead of aborting early. - with consecutive_empty_lock: - consecutive_empty["n"] += 1 - if consecutive_empty["n"] >= EMPTY_FAIL_FAST: - consecutive_empty["abort"] = True - return exp, "error" - if opt is None or opt.calls is None or opt.puts is None: - with consecutive_empty_lock: - consecutive_empty["n"] += 1 - if consecutive_empty["n"] == 1: - logger.warning("fetch_option_chain: %s exp=%s returned no data (rate-limited?)", ticker, exp) - if consecutive_empty["n"] >= EMPTY_FAIL_FAST: - consecutive_empty["abort"] = True - return exp, None - with consecutive_empty_lock: - consecutive_empty["n"] = 0 - calls = opt.calls.copy() - puts = opt.puts.copy() - for col in _OPT_NUMERIC_COLS: - if col in calls.columns: - calls[col] = pd.to_numeric(calls[col], errors="coerce") - if col in puts.columns: - puts[col] = pd.to_numeric(puts[col], errors="coerce") - for col in ("openInterest", "volume"): - if col in calls.columns: - calls[col] = calls[col].fillna(0) - if col in puts.columns: - puts[col] = puts[col].fillna(0) - return exp, {"calls": calls, "puts": puts} - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = {executor.submit(_fetch_one, exp): exp for exp in expiries} - for future in as_completed(futures): - exp, payload = future.result() - if isinstance(payload, dict): - chain[exp] = payload - - # WHY: preserve the user-meaningful order (front-month first) from the - # original ``expiries`` list — as_completed yields in completion order. - ordered_chain = {exp: chain[exp] for exp in expiries if exp in chain} - return {"ticker": ticker, "spot": spot, "expiries": list(ordered_chain.keys()), "chain": ordered_chain} - - -def _fetch_option_chain_serial( - ticker: str, spot: float | None, expiries: list[str], empty_fail_fast: int -) -> dict[str, Any]: - """Sequential fallback path used when YF_OPTION_CHAIN_WORKERS=1 or only - one expiry exists. Behaviourally identical to the pre-concurrency loop. - """ - chain: dict[str, dict[str, pd.DataFrame]] = {} - consecutive_empty = 0 - tk = yf.Ticker(ticker) - for exp in expiries: - try: - yf_throttle() - opt = tk.option_chain(exp) - if opt is None or opt.calls is None or opt.puts is None: - consecutive_empty += 1 - if consecutive_empty == 1: - logger.warning("fetch_option_chain: %s exp=%s returned no data (rate-limited?)", ticker, exp) - if consecutive_empty >= empty_fail_fast: - logger.warning( - "fetch_option_chain: %s aborting after %d empty expiries (likely rate-limited)", - ticker, - consecutive_empty, - ) - break - continue - consecutive_empty = 0 - calls = opt.calls.copy() - puts = opt.puts.copy() - for col in _OPT_NUMERIC_COLS: - if col in calls.columns: - calls[col] = pd.to_numeric(calls[col], errors="coerce") - if col in puts.columns: - puts[col] = pd.to_numeric(puts[col], errors="coerce") - for col in ("openInterest", "volume"): - if col in calls.columns: - calls[col] = calls[col].fillna(0) - if col in puts.columns: - puts[col] = puts[col].fillna(0) - chain[exp] = {"calls": calls, "puts": puts} - except Exception as exc: # noqa: BLE001 - logger.warning("fetch_option_chain: %s exp=%s failed: %s", ticker, exp, exc) - continue - - return {"ticker": ticker, "spot": spot, "expiries": list(chain.keys()), "chain": chain} - - -# --------------------------------------------------------------------------- -# Close-only panel (used by correlation matrix and market review) -# --------------------------------------------------------------------------- -def fetch_close_panel( - tickers: list[str], - period: str | None = "90d", - *, - start: str | None = None, - end: str | None = None, - max_retries: int = 2, - retry_base_delay: float = 3.0, -) -> pd.DataFrame: - """Return a wide DataFrame of Close prices for ``tickers``. - - Either pass ``period`` (e.g. ``"90d"``, ``"400d"``) OR ``start``/``end`` - date strings — when ``start`` is provided it takes precedence. On failure - returns an empty DataFrame. One yfinance call total (yfinance natively - supports multi-ticker download). - - WHY (retry loop): Yahoo occasionally returns an empty payload on the first - call after a wake-from-sleep; one retry resolves it without escalating. - """ - if not tickers: - return pd.DataFrame() - if start is not None: - kwargs = {"start": start, "end": end} - else: - kwargs = {"period": period or "90d"} - last_err: Exception | None = None - for attempt in range(max_retries): - try: - yf_throttle() - data = yf.download(tickers, auto_adjust=False, progress=False, **kwargs) - if data is None or data.empty: - if attempt < max_retries - 1: - logger.warning( - "fetch_close_panel empty payload, retrying in %.1fs (attempt %d)", - retry_base_delay * (attempt + 1), - attempt + 1, - ) - time.sleep(retry_base_delay * (attempt + 1)) - continue - return pd.DataFrame() - if isinstance(data.columns, pd.MultiIndex): - if "Close" not in data.columns.get_level_values(0): - return pd.DataFrame() - close = data["Close"] - if isinstance(close.columns, pd.MultiIndex): - close.columns = close.columns.droplevel(1) - else: - if "Close" not in data.columns: - return pd.DataFrame() - close = data[["Close"]] - return close - except Exception as exc: # noqa: BLE001 - last_err = exc - is_rate_limit = "rate" in str(exc).lower() or "too many" in str(exc).lower() - if is_rate_limit and attempt < max_retries - 1: - time.sleep(retry_base_delay * (attempt + 1)) - continue - break - if last_err is not None: - logger.warning("fetch_close_panel failed for %s: %s", tickers, last_err) - is_rate_limit = "rate" in str(last_err).lower() or "too many" in str(last_err).lower() - _log_dq( - "yf_client.fetch_close_panel", - "rate_limited" if is_rate_limit else "download_error", - str(last_err), - ticker=",".join(tickers), - ) - return pd.DataFrame() - - -# --------------------------------------------------------------------------- -# Daily OHLCV (for core/market/data_context.py) -# --------------------------------------------------------------------------- -def fetch_daily_ohlcv( - ticker: str, - start, - end, - *, - auto_adjust: bool = False, - max_retries: int = 2, - retry_base_delay: float = 3.0, -) -> pd.DataFrame: - """Download daily OHLCV bars for ``ticker`` in ``[start, end)``. +from data_pipeline.providers.yf_snapshot import fetch_option_chain, fetch_spot, fetch_spots_bulk +from data_pipeline.providers.yfinance_provider import fetch_close_panel, fetch_daily_ohlcv - Returns a DataFrame indexed by Date with columns - ``[Open, High, Low, Close, Adj Close, Volume]``. Empty DataFrame on - failure. Includes simple retry loop for transient empty responses. - """ - last_err: Exception | None = None - for attempt in range(max_retries): - try: - yf_throttle() - df = yf.download( - ticker, - start=start, - end=end, - interval="1d", - progress=False, - auto_adjust=auto_adjust, - ) - if df is None or df.empty: - if attempt < max_retries - 1: - time.sleep(retry_base_delay * (attempt + 1)) - continue - return pd.DataFrame() - if isinstance(df.columns, pd.MultiIndex): - df.columns = df.columns.droplevel(1) - df.index = pd.DatetimeIndex(df.index) - return df - except Exception as exc: # noqa: BLE001 - last_err = exc - is_rate_limit = "rate" in str(exc).lower() or "too many" in str(exc).lower() - if is_rate_limit and attempt < max_retries - 1: - time.sleep(retry_base_delay * (attempt + 1)) - continue - break - if last_err is not None: - logger.warning("fetch_daily_ohlcv failed for %s: %s", ticker, last_err) - is_rate_limit = "rate" in str(last_err).lower() or "too many" in str(last_err).lower() - _log_dq( - "yf_client.fetch_daily_ohlcv", - "rate_limited" if is_rate_limit else "download_error", - str(last_err), - ticker=ticker, - ) - return pd.DataFrame() +__all__ = [ + "fetch_close_panel", + "fetch_daily_ohlcv", + "fetch_option_chain", + "fetch_spot", + "fetch_spots_bulk", +] diff --git a/docs/architecture_review.md b/docs/architecture_review.md index ce6f1ae..cc95d93 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -57,19 +57,23 @@ for the live list. State at registration: | `services/market/health.py` — **resolved 2026-09-03**, `services/portfolio/facade.py` — **resolved 2026-09-03** (`get_conn`) | ad-hoc health/inventory SQL predates `repos.py` coverage | queries moved into `data_pipeline/repos.py` | | `services/regime/facade.py`, `services/regime/ops/_bootstrap.py`, `services/regime/ops/_persistence.py` (`fetch_df`, `init_db`, `upsert_many`) — **resolved 2026-09-03** | regime log writes were split across service and ops modules | consolidated behind `data_pipeline/repos.py` (regime-log + clean-row ops) | -### single-yf-exit (only `yf_client.py` may import yfinance) — 1 marker +### single-yf-exit (only `data_pipeline/providers/` may import yfinance) — 0 markers + +Rescoped in batch B1 of [ADR 0011](decisions/0011-pluggable-data-provider-seam.md) +(2026-09-10): the chokepoint moved from `yf_client.py` into the provider package, and +`yf_client.py` is now a compatibility shim that no longer imports yfinance. | Location | Why it exists | Exit condition | |---|---|---| -| `data_pipeline/downloader.py` | DB-aware gap-detection bulk downloads; documented chokepoint alongside `yf_client` (see `yf_client` module docstring) | fold the gap logic into `yf_client` | -| `data_pipeline/data_ops/_query.py::get_latest_spot` — **resolved 2026-09-03** | former spot fast-path fetched yfinance internally | now routes through `yf_client.fetch_spot` | +| `data_pipeline/downloader.py` — **resolved 2026-09-10 (B1)** | DB-aware gap-detection bulk downloads; it used to call `yf.download` directly as a registered second exit point | the download call moved to `providers/yfinance_provider.py::download_daily_frame`; `downloader.py` keeps only gap detection + `raw_prices` upsert, so it no longer imports yfinance | +| `data_pipeline/data_ops/_query.py::get_latest_spot` — **resolved 2026-09-03** | former spot fast-path fetched yfinance internally | now routes through `fetch_spot` (provider, re-exported by `yf_client`) | ### Watch list (pre-debt, no marker yet) | Location | Concern | Trigger to act | |---|---|---| | `services/market/analysis/summary.py` (fan-in 0, tracked as `dead_code_candidates=1` in baseline) | `generate_summary_analysis` lost its caller when the streaming refactor removed the server-rendered `summary_data` template variable; the Summary tab button is gated off in `templates/index.html` and `summary_pending` in `routes/core.py` is vestigial | any request to ship the multi-ticker Summary tab ⇒ add a `summary` slice to `_RENDER_KIND_SLICES` (aggregates across the job's tickers, not per-ticker) ; otherwise delete the module + `partials/tab_summary.html` + the `summary_pending` flag in the same commit and reset the baseline | -| `data_pipeline/yf_client.py` (391 lines, fan-in 11) | 9 lines below the 400-line god-file threshold; the throttle wrapper itself already lives in `utils/network.py::yf_throttle`, but each new yfinance endpoint (option greeks feeds, dividends/splits, etc.) grows the file | any edit that pushes it past 400 lines ⇒ extract the option-chain section (~150 lines, `fetch_option_chain` + `_fetch_option_chain_serial` + `_OPT_NUMERIC_COLS`) into `data_pipeline/yf_option_chain.py` in the same commit | +| ~~`data_pipeline/yf_client.py` (391 lines, fan-in 11)~~ — **resolved 2026-09-10 (B1)** | it sat 9 lines below the 400-line god-file threshold | the option-chain section was extracted pre-emptively, as prescribed, into `providers/yf_snapshot.py`; `yf_client.py` is now a ~35-line shim. The pressure moved to `providers/yf_snapshot.py` (≈340 lines) and `providers/yfinance_provider.py` (≈290 lines) — watch them before adding endpoints | ## 3. Guardrails (how the score is kept) diff --git a/docs/automation.md b/docs/automation.md index 5341d48..85eca96 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -28,7 +28,7 @@ What it checks (each rule produces a non-zero exit code on violation): | Rule | What it catches | Why | | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | | `tag-syntax` | `WHY:`/`CONSTRAINT:`/etc. used outside the canonical vocabulary, or with malformed forms (lowercase, missing colon). | Keeps the tag set stable so AI grep is reliable. | -| `yfinance-throttle` | Any new `yf.download` / `yf.Ticker(...)` call site not preceded by `yf_throttle()` or routed through `data_pipeline/yf_client.py`. | Hard architectural invariant from ADR 0005. | +| `yfinance-throttle` | Any new `yf.download` / `yf.Ticker(...)` call site not preceded by `yf_throttle()` or routed through `data_pipeline/providers/` (the chokepoint since ADR 0011 / batch B1). | Hard architectural invariant from ADR 0005. | | `yfinance-session-kwarg` | Any call passing `session=` to a yfinance API. | Silent failure mode (curl_cffi). See `docs/constraints.md` §2. | | `sqlite-bypass` | New `sqlite3.connect(` outside `data_pipeline/db.py`. | Bypasses WAL pragmas (ADR 0003). | | `import-direction` | Imports from `services/` inside `core/` or `data_pipeline/`; from `core/` inside `data_pipeline/`. | ADR 0001 — already enforced by an existing hook; doc-guard is the safety net. | diff --git a/docs/constraints.md b/docs/constraints.md index 2c1d664..6d5ccda 100644 --- a/docs/constraints.md +++ b/docs/constraints.md @@ -12,6 +12,11 @@ is usually a workaround for one of the items below. ## 1. yfinance is the only data source - **Constraint**: This is a personal-use research tool. We do not pay for Bloomberg / Polygon / Tradier. +- **Amended by [ADR 0011](decisions/0011-pluggable-data-provider-seam.md) (2026-09-10)**: yfinance is + the only data-source **implementation**, not the only possible one. It lives behind the provider + seam in `data_pipeline/providers/` and maps its fields onto one canonical internal schema, so a + second vendor is one provider module + one registry line. The option-history caveats below are + unchanged — a second vendor does not conjure option history that no free API publishes. - **Implications**: - No SLA, no support, no stable schema. yfinance can break on any release. - Aggressive rate limiting (HTTP 429) — see §2. @@ -24,9 +29,9 @@ is usually a workaround for one of the items below. - **Do NOT pass `session=requests.Session()`** to any yfinance call — it silently fails to make the request. - Proxy must be set via `HTTP_PROXY` / `HTTPS_PROXY` env vars; we read `YF_PROXY` and propagate to both. See [utils/network.py](../utils/network.py) `init_yf_proxy`. - **Dead-proxy poisoning**: an unreachable proxy makes curl_cffi hang. We TCP-probe before activating; falls back to direct connect. -- **Global throttle**: token-bucket limiter (default 5 req/s, burst 5) in [utils/network.py](../utils/network.py)::`yf_throttle`. Every yfinance call MUST be routed through `data_pipeline/yf_client.py` (the single chokepoint) rather than calling `yf_throttle()` directly at each call site — see ADR 0005. +- **Global throttle**: token-bucket limiter (default 5 req/s, burst 5) in [utils/network.py](../utils/network.py)::`yf_throttle`. Every yfinance call MUST be routed through `data_pipeline/providers/` (the single chokepoint since batch B1 of ADR 0011) rather than calling `yf_throttle()` directly at each call site — see ADR 0005. - **DB-first pattern**: never re-download data already in `clean_prices`. The 60-second cooldown in `DataService` exists to prevent thundering herd from concurrent UI requests. -- **Single yfinance exit point**: only `data_pipeline/yf_client.py` may `import yfinance`. Any other module needs a `# doc-guard: allow=single-yf-exit` marker, which is tracked as architecture debt (see [architecture_review.md](architecture_review.md) §2). Enforced by `scripts/doc_guard.py` rules `single-yf-exit`, `import-direction`, `core-purity` and `db-access`; trend-gated in CI by `scripts/arch_metrics.py --check`. +- **Single yfinance exit point**: only `data_pipeline/providers/` may `import yfinance`. The chokepoint moved there from `yf_client.py` in batch B1; `yf_client.py` is now a one-release compatibility shim over the provider package. Any other module needs a `# doc-guard: allow=single-yf-exit` marker (tracked as architecture debt — see [architecture_review.md](architecture_review.md) §2). `tests/` and `scripts/` are exempt by design (test doubles patch `yfinance.download` on the module object). Enforced by `scripts/doc_guard.py` rules `single-yf-exit`, `import-direction`, `core-purity` and `db-access`; trend-gated in CI by `scripts/arch_metrics.py --check`. ## 3. SQLite, single-machine deployment diff --git a/docs/decisions/0011-pluggable-data-provider-seam.md b/docs/decisions/0011-pluggable-data-provider-seam.md index 2458d45..faef1d7 100644 --- a/docs/decisions/0011-pluggable-data-provider-seam.md +++ b/docs/decisions/0011-pluggable-data-provider-seam.md @@ -100,6 +100,29 @@ objects. `transform/`, `read/`, `services/` or `core/`." The option-history caveats (no IV rank / percentile / backtests — ADR 0004) are unchanged. +### Protocol shape — decision gate §8 Q5 (resolved 2026-09-10) + +`providers/base.py` was reviewed against **two** field maps before being written, so the +protocol is not accidentally yfinance-shaped +(`archive/futu_integration/field_mapping.md` is the second map): + +| Concern | yfinance | futu | Protocol decision | +|---|---|---|---| +| IV unit | decimal (`0.2436`) | percent (`24.359`) | canonical `iv` is a **decimal**; the provider normalises at its own boundary | +| bid / ask | present per expiry | absent without an `ORDER_BOOK` subscription | canonical `bid`/`ask` are **nullable**; a provider must not invent a quote | +| `inTheMoney` | present | absent (must be derived from strike vs spot) | **dropped** from the canonical leg — derivable, so not a contract | +| Expiries | `tk.options` list | `get_option_expiration_date()` frame | canonical = `expiries: tuple[str, ...]` (ISO dates) | +| Greeks | absent | native (`delta`/`gamma`/…) | out of scope for B1; the protocol has no Greek fields yet | +| History | `yf.download` frame | `get_stock_quote` / history API | canonical `history()` returns `CANONICAL_BAR_COLUMNS` (`open…adj_close, volume`) | + +A provider id is also a first-class value (`MarketDataProvider.name`), because a second +vendor means two providers coexist rather than one replacing the other. + +**Implementation status**: B1 (seam extraction, no behaviour change) landed 2026-09-10 — +`data_pipeline/providers/{base,_log,_registry,yfinance_provider,yf_snapshot}.py`; +`yf_client.py` is a compatibility shim and `downloader.py` no longer imports yfinance. +Batch ledger: [`docs/plans/business_line_reorg.md`](../plans/business_line_reorg.md) §0. + ## Consequences - Positive: a second provider is one file + one registry line + a field-map test. diff --git a/docs/l0_architecture.md b/docs/l0_architecture.md index f9e9cbd..639f577 100644 --- a/docs/l0_architecture.md +++ b/docs/l0_architecture.md @@ -38,7 +38,7 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ | `routes/` | 909 lines · 8 files | 7 blueprints + `__init__.py` aggregate export; no business logic | good | | `services/` | 3 540 lines · 5 domain packages | `market` (incl. `analysis/` slice factory), `market_review`, `options`, `portfolio`, `regime` | good | | `core/` | 6 372 lines · 8 sub-packages + `_shared` | Pure computation — no Flask, no DB, no network | good | -| `data_pipeline/` | 2 645 lines · 12 files | The only I/O boundary: `yf_client`, `db`/`repos`, `data_ops`, `scheduler`, `job_cache` | good | +| `data_pipeline/` | 3 154 lines · 22 files | The only I/O boundary: `providers/` (the yfinance seam, ADR 0011), `yf_client` (one-release shim), `db`/`repos`, `data_ops`, `scheduler`, `job_cache` | good | | `utils/` | 756 lines · 7 files | Leaf layer; highest fan-in (`ticker_utils.py` = 11) | good | | `templates/` | 1 546 lines · 17 files | `index.html` skeleton + `partials/fragments/*` (HTMX swap targets) | good | | `static/` | 5 473 lines · 31 JS/CSS | `state/` · `sim/` · `components/` · `features/` + tab entry files | fair (see §4 P3-1) | @@ -78,8 +78,10 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ ## 2. Measured shape (`scripts/arch_metrics.py`) +_Refreshed 2026-09-10 after batch B1 (provider seam extraction); the L1 inventory in §1 above is otherwise the 2026-09-08 snapshot._ + ``` -modules=146 import_edges=300 +modules=152 import_edges=312 Layer-edge violations : (none) Import cycles : 0 God files (>400 lines): (none) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index e056727..3b7263f 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -38,7 +38,7 @@ | Batch | State | PR | Landed (commit · date) | Notes | |---|---|---|---|---| | — (planning + ADRs) | 🔨 in review (PR #7) | #7 | branch `worktree-business-line-reorg` · 2026-09-10 | plan, ADR 0011/0012 (Accepted), scaffolding (ledger, gates, AI-guide pointers, memory) | -| B1 — provider seam extraction | ⬜ not started | — | — | delivers the "pluggable API" seam on its own | +| B1 — provider seam extraction | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | delivers the "pluggable API" seam on its own. Actual shape / deviations recorded in §8; `_ALLOWED_DEPS` promotion of `providers` deferred to B3 | | B2 — canonical raw store | ⬜ not started | — | — | gate: §8 Q4 | | B3 — package re-home | ⬜ not started | — | — | resets `arch_baseline.json` | | B4 — close L1 (`core-purity`) | ⬜ not started | — | — | — | @@ -423,7 +423,32 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. | Q2 | **Config tab fate** — is there *any* genuine global setting to keep? Risk-free rate is the only candidate (hard-coded in `static/sim/` and again in `core/options/greeks`). Yes → tab shrinks to it; no → tab deleted. | before **B8** | keep risk-free rate, delete the rest | | Q3 | **`positions` block** — Portfolio Analysis is its only consumer. Move into a dedicated "Portfolio" panel/tab, or keep as a section the bar's Run ignores? | before **B6** | dedicated Portfolio panel | | Q4 | **Table rename vs. reshape** — `raw_prices`→`raw_bars` with identical columns (minimal), or also move the yfinance-ism `adj_close` handling into the provider during the rename? | before **B2** | minimal rename | -| Q5 | **Second-provider protocol shape** — not in scope to *implement*, but `providers/base.py` (written in B1) must be sketched against *both* yfinance and `archive/futu_integration/field_mapping.md` so the protocol is not accidentally yfinance-shaped (IV unit, bid/ask availability, `inTheMoney` derivation all differ). | before **B1** | design review of `base.py` against both field maps | +| Q5 | **Second-provider protocol shape** — not in scope to *implement*, but `providers/base.py` (written in B1) must be sketched against *both* yfinance and `archive/futu_integration/field_mapping.md` so the protocol is not accidentally yfinance-shaped (IV unit, bid/ask availability, `inTheMoney` derivation all differ). | ✅ resolved 2026-09-10 (B1) — outcome table in ADR 0011 §"Protocol shape" | design review of `base.py` against both field maps: IV → decimal, bid/ask nullable, `inTheMoney` dropped (derivable), expiries ISO strings | + +### Batch notes (actuals + deviations, recorded as batches land) + +**B1 (2026-09-10) — provider seam extraction, no behaviour change.** + +- **Files**: `data_pipeline/providers/{__init__,base,_log,_registry,yfinance_provider,yf_snapshot}.py`. + Five modules instead of the three §6 named, for two reasons: (a) `yfinance_provider.py` would have + blown the 400-line god-file cap, so the option-chain section was extracted exactly as + `architecture_review.md` §2 had pre-registered — into `providers/yf_snapshot.py` (which also owns + the spot lookup, because `fetch_option_chain` calls it and a separate module would have created an + import cycle); (b) `_log.py` holds the best-effort failure-log wrapper that both provider modules + need and neither may import from the other. +- **Compatibility**: `yf_client.py` is now a re-export shim; `downloader.py` keeps only gap detection + + `raw_prices` upsert and no longer imports yfinance. No `routes/` or `services/` file changed. +- **`_ALLOWED_DEPS` deferred**: §6 B1 wanted `providers` added to `_ALLOWED_DEPS`, but promoting a + `data_pipeline/` subpackage to a layer needs `_layer_of` / `layer_of` sub-layer resolution in + **both** `doc_guard.py` and `arch_metrics.py`. That is B3's job (its §6 row already owns "new + layer-edge rules" + the layer-table rewrite). In B1 `providers/` stays inside the `data_pipeline` + layer; the new invariant that *does* hold now — "only `providers/` imports yfinance" — is enforced + by the rescoped `single-yf-exit` rule and pinned by `tests/test_provider_seam.py`. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 459 passed / 5 skipped; + `doc_guard.py` clean; `arch_metrics.py --check` ok (no baseline reset needed); + `audit_tags.py` unchanged (16 uncovered vs baseline 16). Production-code + `import yfinance` hits: exactly the two `providers/` modules. (`tests/test_yf_download.py` and + `tests/e2e/conftest.py` also import it as test doubles — `doc_guard` exempts `tests/` by design.) --- diff --git a/scripts/doc_guard.py b/scripts/doc_guard.py index 2f50c25..0366420 100755 --- a/scripts/doc_guard.py +++ b/scripts/doc_guard.py @@ -76,6 +76,24 @@ def _is_suppressed(line: str, rule: str) -> bool: return rule in {x.strip() for x in m.group(1).split(",")} +# ADR 0011: acquisition moved behind this package; yfinance stays the sole +# implementation and the only import site. +_PROVIDER_DIR = REPO_ROOT / "data_pipeline" / "providers" + + +def _in_provider_seam(path: Path) -> bool: + """True when ``path`` lives under ``data_pipeline/providers/`` (ADR 0011). + + INVARIANT: the provider package is the single place an external market-data + SDK is touched, and every call there is throttled by construction. Both the + ``yfinance-throttle`` and ``single-yf-exit`` rules are scoped to it. + """ + try: + return path.resolve().is_relative_to(_PROVIDER_DIR) + except OSError: + return False + + # ── Rule: tag-syntax ───────────────────────────────────────────── def rule_tag_syntax(ctx: Context) -> None: for path in ctx.files: @@ -108,18 +126,14 @@ def rule_tag_syntax(ctx: Context) -> None: def rule_yfinance_throttle(ctx: Context) -> None: - """Each yf.download / yf.Ticker call outside yf_client.py and downloader.py - must have a yf_throttle() call within the previous 5 lines, OR be marked - with `# doc-guard: allow=yfinance-throttle`. + """Each yf.download / yf.Ticker call outside data_pipeline/providers/ must + have a yf_throttle() call within the previous 5 lines, OR be marked with + `# doc-guard: allow=yfinance-throttle`. """ - allowed_files = { - REPO_ROOT / "data_pipeline" / "yf_client.py", - REPO_ROOT / "data_pipeline" / "downloader.py", - } for path in ctx.files: if path.suffix != ".py": continue - if path.resolve() in allowed_files: + if _in_provider_seam(path): continue if "tests/" in str(path): continue @@ -332,15 +346,15 @@ def rule_db_access(ctx: Context) -> None: # ── Rule: single-yf-exit ───────────────────────────────────────── -# INVARIANT: yf_client.py is the single module allowed to talk to yfinance, so -# proxy setup and the token-bucket throttle can never be bypassed (ADR 0005). +# INVARIANT: data_pipeline/providers/ is the single place yfinance is imported +# (batch B1 of ADR 0011 moved the chokepoint here from yf_client.py), so proxy +# setup and the token-bucket throttle can never be bypassed (ADR 0005). _YF_IMPORT_RE = re.compile(r"^\s*(import\s+yfinance\b|from\s+yfinance\b)") -_YF_SINGLE_EXIT = REPO_ROOT / "data_pipeline" / "yf_client.py" def rule_single_yf_exit(ctx: Context) -> None: for path in ctx.files: - if path.suffix != ".py" or path.resolve() == _YF_SINGLE_EXIT: + if path.suffix != ".py" or _in_provider_seam(path): continue if "tests/" in str(path) or "scripts/" in str(path): continue @@ -350,7 +364,8 @@ def rule_single_yf_exit(ctx: Context) -> None: "single-yf-exit", path, i, - "only data_pipeline/yf_client.py may import yfinance — see docs/constraints.md §2 / ADR 0005", + "only data_pipeline/providers/ may import yfinance — " + "see docs/constraints.md §2 / ADR 0005 / ADR 0011", ) diff --git a/tests/test_downloader_gap.py b/tests/test_downloader_gap.py index e945573..3ff7518 100644 --- a/tests/test_downloader_gap.py +++ b/tests/test_downloader_gap.py @@ -108,8 +108,8 @@ def test_weekends_are_not_missing(self): class TestUpsertRawPricesGapAware: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_interior_gap_triggers_download(self, mock_throttle, mock_dl): """The NVDA regression: existing rows on edges + interior hole → must download.""" start = dt.date(2024, 1, 1) @@ -132,8 +132,8 @@ def test_interior_gap_triggers_download(self, mock_throttle, mock_dl): # No remaining business-day gap after upsert. assert find_missing_business_days("NVDA_REGRESSION", start, end) == [] - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_full_coverage_skips_download(self, mock_throttle, mock_dl): start = dt.date(2024, 1, 1) end = dt.date(2024, 1, 5) @@ -146,8 +146,8 @@ def test_full_coverage_skips_download(self, mock_throttle, mock_dl): mock_dl.assert_not_called() mock_throttle.assert_not_called() - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_download_start_expanded_to_earliest_gap_within_request(self, mock_throttle, mock_dl): """When the requested window contains an earlier gap, the actual download `start` is widened to that gap (so we don't waste a round-trip on the tail).""" @@ -172,8 +172,8 @@ def test_download_start_expanded_to_earliest_gap_within_request(self, mock_throt class TestManualUpdateGapScan: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_old_gap_within_scan_window_triggers_download(self, mock_throttle, mock_dl): """`manual_update(days=7)` must still back-fill a gap older than 7 days when it falls within `GAP_SCAN_DAYS`.""" @@ -200,8 +200,8 @@ def test_old_gap_within_scan_window_triggers_download(self, mock_throttle, mock_ called_start = mock_dl.call_args.kwargs.get("start") assert called_start <= old_gap - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_no_gaps_no_download(self, mock_throttle, mock_dl): """When the gap-scan window is fully covered, `manual_update` skips the network.""" end = dt.date.today() diff --git a/tests/test_provider_seam.py b/tests/test_provider_seam.py new file mode 100644 index 0000000..afbf41f --- /dev/null +++ b/tests/test_provider_seam.py @@ -0,0 +1,248 @@ +"""Contract tests for the data-provider seam (ADR 0011, batch B1). + +Domain: Tests — Provider Seam +Context: + - Batch B1 moved every yfinance call behind ``data_pipeline/providers/`` and + introduced the canonical schema + registry. These tests pin the parts of + that contract that are otherwise invisible: the single-import invariant, the + canonical mapping units, and the registry's resolution rules. +Contracts: + - Only production code under ``data_pipeline/providers/`` imports yfinance. + - ``to_canonical_bars`` / ``to_option_chain_snapshot`` apply the unit rules in + ``providers/base.py`` (decimal IV, nullable bid/ask, no ``inTheMoney``). + - ``get_provider`` defaults to yfinance and rejects unknown names loudly. + - ``data_pipeline.yf_client`` still re-exports the legacy callables unchanged. +Dependencies UPWARD: + - (none — stdlib + pytest + the package under test) +""" + +from __future__ import annotations + +import ast +import subprocess +import sys +from pathlib import Path + +import pandas as pd +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +PROVIDER_DIR = REPO_ROOT / "data_pipeline" / "providers" + +# Production roots only: doc_guard's `single-yf-exit` rule deliberately exempts +# tests/ and scripts/ (test doubles patch `yfinance.download` on the module). +PRODUCTION_ROOTS = ("app.py", "routes", "core", "data_pipeline", "services", "utils") + + +def _production_python_files() -> list[Path]: + out: list[Path] = [] + for sub in PRODUCTION_ROOTS: + p = REPO_ROOT / sub + if p.is_file(): + out.append(p) + elif p.is_dir(): + out.extend(x for x in p.rglob("*.py") if "__pycache__" not in x.parts) + return sorted(out) + + +def _imports_yfinance(path: Path) -> bool: + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except SyntaxError: + return False + for node in ast.walk(tree): + if isinstance(node, ast.Import): + if any(a.name.split(".", 1)[0] == "yfinance" for a in node.names): + return True + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + if node.module.split(".", 1)[0] == "yfinance": + return True + return False + + +def test_only_provider_seam_imports_yfinance(): + """B1 exit criterion: production code imports yfinance only under providers/.""" + offenders = [str(p.relative_to(REPO_ROOT)) for p in _production_python_files() if _imports_yfinance(p)] + assert offenders, "expected at least one importer inside data_pipeline/providers/" + for rel in offenders: + assert Path(rel).is_relative_to(Path("data_pipeline") / "providers"), ( + f"{rel} imports yfinance outside data_pipeline/providers/ — see ADR 0011" + ) + + +def test_doc_guard_single_yf_exit_allows_provider_modules(): + """The rescoped guard must not flag the provider package itself.""" + result = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "doc_guard.py"), + "--json", + "--rule", + "single-yf-exit", + "--files", + str(PROVIDER_DIR / "yfinance_provider.py"), + str(PROVIDER_DIR / "yf_snapshot.py"), + ], + capture_output=True, + text=True, + ) + assert "single-yf-exit" not in (result.stdout or ""), result.stdout + assert result.returncode == 0, result.stdout + result.stderr + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- +def test_registry_defaults_to_yfinance(): + from data_pipeline.providers import available_providers, get_provider + from data_pipeline.providers.base import MarketDataProvider + + assert available_providers() == ("yfinance",) + provider = get_provider() + assert provider.name == "yfinance" + assert isinstance(provider, MarketDataProvider) + # Instances are cached per resolved name. + assert get_provider("yfinance") is provider + + +def test_registry_rejects_unknown_provider(): + from data_pipeline.providers import get_provider + + with pytest.raises(ValueError, match="unknown data provider"): + get_provider("does-not-exist") + + +def test_registry_honours_env_override(monkeypatch): + from data_pipeline.providers import get_provider + + monkeypatch.setenv("MARKET_DATA_PROVIDER", "nope") + with pytest.raises(ValueError): + get_provider() + + +# --------------------------------------------------------------------------- +# Canonical mapping +# --------------------------------------------------------------------------- +def _yf_bars_frame() -> pd.DataFrame: + idx = pd.DatetimeIndex(["2026-01-02", "2026-01-05"]) + return pd.DataFrame( + { + "Open": [100.0, 101.0], + "High": [102.0, 103.0], + "Low": [99.0, 100.0], + "Close": [101.0, 102.0], + "Adj Close": [100.5, 101.5], + "Volume": [1_000_000, 1_100_000], + }, + index=idx, + ) + + +def test_to_canonical_bars_renames_and_orders_columns(): + from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS + from data_pipeline.providers.yfinance_provider import to_canonical_bars + + out = to_canonical_bars(_yf_bars_frame()) + + assert tuple(out.columns) == CANONICAL_BAR_COLUMNS + assert out["adj_close"].tolist() == [100.5, 101.5] + assert out["close"].tolist() == [101.0, 102.0] + assert out.index.is_monotonic_increasing + + +def test_to_canonical_bars_handles_empty_and_missing_columns(): + from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS + from data_pipeline.providers.yfinance_provider import to_canonical_bars + + assert tuple(to_canonical_bars(None).columns) == CANONICAL_BAR_COLUMNS + assert tuple(to_canonical_bars(pd.DataFrame()).columns) == CANONICAL_BAR_COLUMNS + + partial = pd.DataFrame({"Close": [1.0]}, index=pd.DatetimeIndex(["2026-01-02"])) + out = to_canonical_bars(partial) + assert tuple(out.columns) == CANONICAL_BAR_COLUMNS + assert out["adj_close"].isna().all() + + +def _legacy_chain_payload() -> dict: + calls = pd.DataFrame( + { + "strike": [100.0, 105.0], + "bid": [2.0, float("nan")], + "ask": [2.2, float("nan")], + "lastPrice": [2.1, 0.4], + "impliedVolatility": [0.25, 0.30], + "openInterest": [500.0, 120.0], + "volume": [10.0, 0.0], + "inTheMoney": [True, False], + } + ) + puts = pd.DataFrame( + { + "strike": [100.0], + "bid": [1.5], + "ask": [1.7], + "lastPrice": [1.6], + "impliedVolatility": [0.28], + "openInterest": [300.0], + "volume": [5.0], + "inTheMoney": [False], + } + ) + return { + "ticker": "AAPL", + "spot": 101.0, + "expiries": ["2026-01-16"], + "chain": {"2026-01-16": {"calls": calls, "puts": puts}}, + } + + +def test_to_option_chain_snapshot_applies_canonical_units(): + from data_pipeline.providers.yf_snapshot import to_option_chain_snapshot + + snap = to_option_chain_snapshot(_legacy_chain_payload()) + + assert snap.provider == "yfinance" + assert snap.symbol == "AAPL" + assert snap.spot == 101.0 + assert snap.expiries == ("2026-01-16",) + + calls = snap.legs("2026-01-16", "calls") + assert [leg.strike for leg in calls] == [100.0, 105.0] + # iv stays a decimal (0.25 == 25 %); futu's percent form is normalised at the + # provider boundary — see providers/base.py. + assert calls[0].iv == 0.25 + assert calls[1].iv == pytest.approx(0.30) + # NaN quotes become None rather than 0 — "absent" must stay expressible. + assert calls[1].bid is None + assert calls[1].ask is None + assert calls[0].open_interest == 500.0 + + # inTheMoney is deliberately NOT canonical (derivable, and futu has none). + assert not hasattr(calls[0], "in_the_money") + + assert len(snap.legs("2026-01-16", "puts")) == 1 + assert snap.legs("1999-01-01", "calls") == () + + +def test_to_option_chain_snapshot_tolerates_empty_payload(): + from data_pipeline.providers.base import OptionChainSnapshot + from data_pipeline.providers.yf_snapshot import to_option_chain_snapshot + + snap = to_option_chain_snapshot({"ticker": "MSFT", "spot": None, "expiries": [], "chain": {}}) + assert isinstance(snap, OptionChainSnapshot) + assert snap.expiries == () + assert snap.spot is None + + +# --------------------------------------------------------------------------- +# Compatibility shim +# --------------------------------------------------------------------------- +def test_yf_client_reexports_legacy_callables_unchanged(): + from data_pipeline import yf_client + from data_pipeline.providers import yf_snapshot, yfinance_provider + + assert yf_client.fetch_spot is yf_snapshot.fetch_spot + assert yf_client.fetch_spots_bulk is yf_snapshot.fetch_spots_bulk + assert yf_client.fetch_option_chain is yf_snapshot.fetch_option_chain + assert yf_client.fetch_close_panel is yfinance_provider.fetch_close_panel + assert yf_client.fetch_daily_ohlcv is yfinance_provider.fetch_daily_ohlcv diff --git a/tests/test_yf_failure_injection.py b/tests/test_yf_failure_injection.py index ca1836f..78a42a3 100644 --- a/tests/test_yf_failure_injection.py +++ b/tests/test_yf_failure_injection.py @@ -99,8 +99,8 @@ class _FakeRateLimitError(Exception): class TestDownloadExceptions: """Network / yfinance exceptions must be caught and reported, not propagated.""" - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_rate_limit_429_returns_failed_result(self, mock_throttle, mock_dl): mock_dl.side_effect = _FakeRateLimitError("429 Too Many Requests") # Use a far-past start so the staleness check can't short-circuit. @@ -116,8 +116,8 @@ def test_rate_limit_429_returns_failed_result(self, mock_throttle, mock_dl): # Throttle must have been called once before the doomed download. assert mock_throttle.call_count == 1 - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_connection_timeout_returns_failed_result(self, mock_throttle, mock_dl): mock_dl.side_effect = TimeoutError("Connection timed out") end = dt.date(2024, 1, 10) @@ -130,8 +130,8 @@ def test_connection_timeout_returns_failed_result(self, mock_throttle, mock_dl): assert "timed out" in (result.error or "").lower() assert result.rows == 0 - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_generic_exception_does_not_crash(self, mock_throttle, mock_dl): mock_dl.side_effect = RuntimeError("yfinance internal boom") end = dt.date(2024, 1, 10) @@ -150,8 +150,8 @@ def test_generic_exception_does_not_crash(self, mock_throttle, mock_dl): class TestDownloadEmptyData: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_empty_dataframe_records_warning_no_crash(self, mock_throttle, mock_dl): mock_dl.return_value = pd.DataFrame() end = dt.date(2024, 1, 10) @@ -165,8 +165,8 @@ def test_empty_dataframe_records_warning_no_crash(self, mock_throttle, mock_dl): assert result.rows == 0 assert any("No new data" in w for w in result.warnings) - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_none_response_treated_as_empty(self, mock_throttle, mock_dl): mock_dl.return_value = None end = dt.date(2024, 1, 10) @@ -183,8 +183,8 @@ def test_none_response_treated_as_empty(self, mock_throttle, mock_dl): class TestStalenessSkip: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_fresh_db_skips_download(self, mock_throttle, mock_dl): """If every business day in [start, end] is already in raw_prices, no yfinance call is made.""" end = dt.date.today() @@ -203,8 +203,8 @@ def test_fresh_db_skips_download(self, mock_throttle, mock_dl): mock_dl.assert_not_called() mock_throttle.assert_not_called() - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_stale_db_triggers_download(self, mock_throttle, mock_dl): """If DB only has very old data, the download proceeds (and gets rate-limited in this test, but that's fine — we only assert that yf.download was attempted).""" @@ -225,8 +225,8 @@ def test_stale_db_triggers_download(self, mock_throttle, mock_dl): class TestDbSurvivesFailure: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_existing_rows_survive_429(self, mock_throttle, mock_dl): """A 429 during update must NOT delete or corrupt existing DB rows.""" end = dt.date.today() @@ -262,8 +262,8 @@ def test_throttle_called_before_download(self): parent.dl.return_value = _make_yf_frame(dt.date(2024, 1, 1), days=3) with ( - patch("data_pipeline.downloader.yf_throttle", parent.throttle), - patch("data_pipeline.downloader.yf.download", parent.dl), + patch("data_pipeline.providers.yfinance_provider.yf_throttle", parent.throttle), + patch("data_pipeline.providers.yfinance_provider.yf.download", parent.dl), ): _download_yf("ORDER_TKR", dt.date(2024, 1, 1), dt.date(2024, 1, 5)) @@ -279,8 +279,8 @@ def test_throttle_called_before_download(self): class TestManualUpdateGracefulFailure: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_manual_update_returns_false_on_429(self, mock_throttle, mock_dl): """`DataService.manual_update` must report False, not raise, on a 429.""" init_db() From 8a53f31b88220dd3d55a90be11fcc78605f53be4 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Thu, 10 Sep 2026 16:51:30 +0800 Subject: [PATCH 02/15] =?UTF-8?q?refactor(data-pipeline):=20B2=20canonical?= =?UTF-8?q?=20store=20=E8=A1=A8=E5=90=8D=E8=90=BD=E5=9C=B0=EF=BC=88raw=5Fb?= =?UTF-8?q?ars/clean=5Fbars/feature=5Fbars=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 业务线重构计划书 §6 B2；先过 §8 决策闸 Q4（name-only rename）。 - db.py：新增 canonical 三表，列集与旧名逐一相同（由同一列元组生成， 结构上杜绝漂移）；旧名保留为 shadow 表一个 release，upsert_many 双向镜像 - 读写全部切到 canonical 名（downloader/cleaning/processing/repos/data_ops） - ingest 经 provider registry 取数：download_bars() 取代 _download_yf，返回 CANONICAL_BAR_COLUMNS，yfinance 列名映射只存在于 provider - scripts/migrate_canonical_tables.py：一次性、幂等、永不覆盖 canonical 的回填 - 测试：新增 tests/test_canonical_tables.py（列一致性 / 双向镜像 / 一次 pipeline 运行两族均有数据 / health 读 raw_bars / 迁移脚本回填且幂等）； test_processing 改为 seed clean_bars、读 feature_bars - 文档：l0 schema、glossary（新增 feature_bars + 兼容期说明）、constraints §2、 .github 诊断与提示文档表名、ADR 0011 加 B2 修订（symbol 列延后） 验收：pytest -m "not network" --ignore=tests/e2e → 468 passed / 5 skipped； doc_guard clean；arch_metrics --check ok（无 god-file 回归）；audit_tags 不变。 --- .github/agents/pipeline-doctor.agent.md | 10 +- .github/prompts/diagnose.prompt.md | 2 +- .github/prompts/pipeline-status.prompt.md | 12 +- .github/skills/debug-pipeline/SKILL.md | 24 +- .../references/pipeline-stages.md | 20 +- .github/skills/test-escalation/SKILL.md | 8 +- .../references/escalation-levels.md | 2 +- CLAUDE.md | 7 +- CODEBUDDY.md | 7 +- data_pipeline/cleaning.py | 16 +- data_pipeline/data_ops/_query.py | 8 +- data_pipeline/data_ops/_range.py | 6 +- data_pipeline/data_ops/facade.py | 4 +- data_pipeline/db.py | 219 +++++++++++------- data_pipeline/downloader.py | 77 +++--- data_pipeline/processing.py | 8 +- data_pipeline/repos.py | 6 +- docs/architecture_review.md | 2 +- docs/constraints.md | 2 +- .../0011-pluggable-data-provider-seam.md | 10 + docs/glossary.md | 13 +- docs/guides/README.md | 4 +- docs/l0_architecture.md | 6 +- docs/plans/business_line_reorg.md | 31 ++- scripts/migrate_canonical_tables.py | 92 ++++++++ services/market/facade.py | 2 +- tests/e2e/conftest.py | 2 +- tests/test_canonical_tables.py | 173 ++++++++++++++ tests/test_db.py | 5 + tests/test_db_errors.py | 5 + tests/test_health_service.py | 2 +- tests/test_nvda_analysis.py | 30 +-- tests/test_processing.py | 16 +- tests/test_yf_failure_injection.py | 4 +- utils/ticker_utils.py | 2 +- 35 files changed, 610 insertions(+), 227 deletions(-) create mode 100644 scripts/migrate_canonical_tables.py create mode 100644 tests/test_canonical_tables.py diff --git a/.github/agents/pipeline-doctor.agent.md b/.github/agents/pipeline-doctor.agent.md index 1730d48..b3cc561 100644 --- a/.github/agents/pipeline-doctor.agent.md +++ b/.github/agents/pipeline-doctor.agent.md @@ -9,9 +9,9 @@ You are a data pipeline diagnostician for the OptionView project. Your job is to ## Architecture ``` -data_pipeline/downloader.py → raw_prices table -data_pipeline/cleaning.py → clean_prices table -data_pipeline/processing.py → processed_prices table +data_pipeline/downloader.py → raw_bars table +data_pipeline/cleaning.py → clean_bars table +data_pipeline/processing.py → feature_bars table core/price_dynamic.py → features DataFrame core/market_analyzer.py → chart generation services/market/analysis/facade.py → base64 images to frontend @@ -26,8 +26,8 @@ services/market/analysis/facade.py → base64 images to frontend ## Approach 1. **Clarify symptom**: What's the user seeing? Empty chart, wrong data, error message? -2. **Check DB tables** (raw_prices → clean_prices → processed_prices) for the target ticker -3. **Look for NaN-only rows**: `SELECT count(*) FROM raw_prices WHERE ticker=? AND open IS NULL AND close IS NULL` +2. **Check DB tables** (raw_bars → clean_bars → feature_bars) for the target ticker +3. **Look for NaN-only rows**: `SELECT count(*) FROM raw_bars WHERE ticker=? AND open IS NULL AND close IS NULL` 4. **Check logs**: Look for yfinance errors (429, timeout), "No new data", pipeline warnings 5. **Trace the failure**: Which stage first produced invalid data? Follow downstream 6. **Check connectivity**: If download is suspected, verify proxy and throttle state diff --git a/.github/prompts/diagnose.prompt.md b/.github/prompts/diagnose.prompt.md index 9474bec..bfb2885 100644 --- a/.github/prompts/diagnose.prompt.md +++ b/.github/prompts/diagnose.prompt.md @@ -8,7 +8,7 @@ argument-hint: "Ticker symbol or symptom (e.g., 'NVDA empty charts', 'stale TLT Diagnose why a specific ticker's data is missing, stale, or showing errors in the OptionView dashboard. Steps: -1. Check the DB for the ticker: query `raw_prices`, `clean_prices`, `processed_prices` for recent rows +1. Check the DB for the ticker: query `raw_bars`, `clean_bars`, `feature_bars` for recent rows 2. Look for NaN-only filler rows (root cause of empty charts) 3. Check yfinance download logs for errors (429, timeout) 4. Trace data flow through the 5-stage pipeline to find the failure point diff --git a/.github/prompts/pipeline-status.prompt.md b/.github/prompts/pipeline-status.prompt.md index 92fa30a..69ccbf7 100644 --- a/.github/prompts/pipeline-status.prompt.md +++ b/.github/prompts/pipeline-status.prompt.md @@ -10,22 +10,22 @@ Check the current health of the OptionView data pipeline: 1. Query the SQLite database (`market_data.sqlite`) for: ```sql -- Row counts per table - SELECT 'raw_prices' as tbl, count(*) as rows FROM raw_prices - UNION SELECT 'clean_prices', count(*) FROM clean_prices - UNION SELECT 'processed_prices', count(*) FROM processed_prices; + SELECT 'raw_bars' as tbl, count(*) as rows FROM raw_bars + UNION SELECT 'clean_bars', count(*) FROM clean_bars + UNION SELECT 'feature_bars', count(*) FROM feature_bars; -- Latest data per ticker - SELECT ticker, MAX(date) as latest, COUNT(*) as rows FROM raw_prices GROUP BY ticker; + SELECT ticker, MAX(date) as latest, COUNT(*) as rows FROM raw_bars GROUP BY ticker; -- NaN-only filler rows (problematic) - SELECT ticker, count(*) as nan_rows FROM raw_prices + SELECT ticker, count(*) as nan_rows FROM raw_bars WHERE open IS NULL AND high IS NULL AND low IS NULL AND close IS NULL GROUP BY ticker HAVING nan_rows > 0; -- Data freshness (days since last update) SELECT ticker, MAX(date) as latest, julianday('now') - julianday(MAX(date)) as days_stale - FROM raw_prices GROUP BY ticker ORDER BY days_stale DESC; + FROM raw_bars GROUP BY ticker ORDER BY days_stale DESC; ``` 2. Report: diff --git a/.github/skills/debug-pipeline/SKILL.md b/.github/skills/debug-pipeline/SKILL.md index d7251f6..7453315 100644 --- a/.github/skills/debug-pipeline/SKILL.md +++ b/.github/skills/debug-pipeline/SKILL.md @@ -32,17 +32,17 @@ Classify the user's report: Query the database for the target ticker using the terminal: ```sql --- Check raw_prices for recent data -SELECT ticker, date, close FROM raw_prices WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; +-- Check raw_bars for recent data +SELECT ticker, date, close FROM raw_bars WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; -- Check for NaN-only filler rows (the root cause of empty charts) -SELECT count(*) FROM raw_prices WHERE ticker='{TICKER}' AND open IS NULL AND high IS NULL AND low IS NULL AND close IS NULL; +SELECT count(*) FROM raw_bars WHERE ticker='{TICKER}' AND open IS NULL AND high IS NULL AND low IS NULL AND close IS NULL; --- Check clean_prices status -SELECT ticker, date, missing_any, price_jump_flag FROM clean_prices WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; +-- Check clean_bars status +SELECT ticker, date, missing_any, price_jump_flag FROM clean_bars WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; --- Check processed_prices -SELECT ticker, date, frequency FROM processed_prices WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; +-- Check feature_bars +SELECT ticker, date, frequency FROM feature_bars WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; ``` ### Step 3: Check yfinance Connectivity @@ -66,10 +66,10 @@ print(df.tail() if not df.empty else "EMPTY - download failed") Follow the data through each stage, checking for where it breaks. See [pipeline stages reference](./references/pipeline-stages.md) for expected inputs/outputs at each stage. -1. **downloader.py** → `upsert_raw_prices()` → writes to `raw_prices` -2. **cleaning.py** → `clean_range()` → reads `raw_prices`, writes to `clean_prices` -3. **processing.py** → `build_features()` → reads `clean_prices`, writes to `processed_prices` -4. **core/price_dynamic.py** → `_fetch_daily_from_db()` → reads `processed_prices` +1. **downloader.py** → `upsert_raw_prices()` → writes to `raw_bars` +2. **cleaning.py** → `clean_range()` → reads `raw_bars`, writes to `clean_bars` +3. **processing.py** → `build_features()` → reads `clean_bars`, writes to `feature_bars` +4. **core/price_dynamic.py** → `_fetch_daily_from_db()` → reads `feature_bars` 5. **core/market_analyzer.py** → uses PriceDynamic features for charts 6. **services/market/analysis/facade.py** → calls chart methods, returns base64 images @@ -78,7 +78,7 @@ Follow the data through each stage, checking for where it breaks. See [pipeline Common root causes: | Root Cause | Evidence | Fix | |-----------|----------|-----| -| NaN-only filler rows from failed download | `raw_prices` has NULL in all price columns | Re-download with `yf_throttle()`, delete filler rows | +| NaN-only filler rows from failed download | `raw_bars` has NULL in all price columns | Re-download with `yf_throttle()`, delete filler rows | | 60s cooldown blocking retry | Download skipped, log says "No new data" | Wait 60s or reset cooldown in `DataService._ticker_locks` | | Proxy unreachable | `curl: (28) Operation timed out` | Check `YF_PROXY` in `.env`, verify proxy is running | | yfinance 429 rate limit | `YFRateLimitError` in logs | Wait 30s, ensure `yf_throttle()` is called everywhere | diff --git a/.github/skills/debug-pipeline/references/pipeline-stages.md b/.github/skills/debug-pipeline/references/pipeline-stages.md index ca3aa67..422ea1b 100644 --- a/.github/skills/debug-pipeline/references/pipeline-stages.md +++ b/.github/skills/debug-pipeline/references/pipeline-stages.md @@ -7,34 +7,34 @@ Data flows through 5 stages. A failure at any stage can propagate downstream as **Function**: `upsert_raw_prices(ticker, start, end)` **Input**: Ticker symbol, date range **Output**: `PipelineResult(ok=True, rows=N)` or `PipelineResult(ok=False, error="...")` -**Side effect**: Writes to `raw_prices` table +**Side effect**: Writes to `raw_bars` table **What can fail**: - yfinance returns empty DataFrame (rate-limit, invalid ticker, network error) - Proxy unreachable (curl_cffi timeout) - Staleness check incorrectly skips download -**Check**: `SELECT count(*) FROM raw_prices WHERE ticker=? AND date BETWEEN ? AND ?` +**Check**: `SELECT count(*) FROM raw_bars WHERE ticker=? AND date BETWEEN ? AND ?` ## Stage 2: Clean (`data_pipeline/cleaning.py`) **Function**: `clean_range(ticker, start, end)` -**Input**: Reads from `raw_prices` table -**Output**: `PipelineResult` — writes to `clean_prices` table +**Input**: Reads from `raw_bars` table +**Output**: `PipelineResult` — writes to `clean_bars` table **Side effect**: Adds anomaly flags (price_jump_flag, vol_anom_flag, ohlc_inconsistent) **What can fail**: -- Source `raw_prices` has NaN-only filler rows → cleans "pass through" NaN +- Source `raw_bars` has NaN-only filler rows → cleans "pass through" NaN - `pd.to_numeric()` coerces strings to NaN silently - Anomaly flag thresholds are heuristic — may miss or over-flag -**Check**: `SELECT date, missing_any, price_jump_flag FROM clean_prices WHERE ticker=? ORDER BY date DESC LIMIT 10` +**Check**: `SELECT date, missing_any, price_jump_flag FROM clean_bars WHERE ticker=? ORDER BY date DESC LIMIT 10` ## Stage 3: Process (`data_pipeline/processing.py`) **Function**: `build_features(ticker, frequency)` -**Input**: Reads from `clean_prices` table -**Output**: `PipelineResult` — writes to `processed_prices` table +**Input**: Reads from `clean_bars` table +**Output**: `PipelineResult` — writes to `feature_bars` table **Side effect**: Computes MA, returns, volatility features **What can fail**: @@ -42,12 +42,12 @@ Data flows through 5 stages. A failure at any stage can propagate downstream as - Wrong frequency conversion (D→W→M) drops rows - `object` dtype from DB causes numpy math errors -**Check**: `SELECT date, frequency, ma_20, ma_50 FROM processed_prices WHERE ticker=? AND frequency=? ORDER BY date DESC LIMIT 5` +**Check**: `SELECT date, frequency, ma_20, ma_50 FROM feature_bars WHERE ticker=? AND frequency=? ORDER BY date DESC LIMIT 5` ## Stage 4: Core Analysis (`core/price_dynamic.py`, `core/market_analyzer.py`) **Function**: `PriceDynamic._fetch_daily_from_db()` → `MarketAnalyzer` methods -**Input**: Reads from `processed_prices` (or `clean_prices` for some features) +**Input**: Reads from `feature_bars` (or `clean_bars` for some features) **Output**: DataFrames for chart generation **What can fail**: diff --git a/.github/skills/test-escalation/SKILL.md b/.github/skills/test-escalation/SKILL.md index 06c6dbd..50583a2 100644 --- a/.github/skills/test-escalation/SKILL.md +++ b/.github/skills/test-escalation/SKILL.md @@ -63,7 +63,7 @@ def test_download_empty(mock_dl): assert result.ok # Rows=0 is valid assert result.rows == 0 # Verify no NaN filler rows were created - df = fetch_df("SELECT * FROM raw_prices WHERE ticker='NVDA'") + df = fetch_df("SELECT * FROM raw_bars WHERE ticker='NVDA'") assert df.empty ``` @@ -78,7 +78,7 @@ def test_full_pipeline_with_nan_data(tmp_path, monkeypatch): # Seed NaN-only filler rows (simulates failed download) upsert_many( - "raw_prices", + "raw_bars", ["ticker", "date", "open", "high", "low", "close"], [("NVDA", "2026-03-28", None, None, None, None)], ) @@ -86,8 +86,8 @@ def test_full_pipeline_with_nan_data(tmp_path, monkeypatch): # Run cleaning — should NOT propagate NaN rows result = clean_range("NVDA", dt.date(2026, 3, 28), dt.date(2026, 3, 28)) - # Verify: clean_prices should be empty (NaN rows filtered) - df = fetch_df("SELECT * FROM clean_prices WHERE ticker='NVDA'") + # Verify: clean_bars should be empty (NaN rows filtered) + df = fetch_df("SELECT * FROM clean_bars WHERE ticker='NVDA'") assert df.empty or df["close"].notna().all() ``` diff --git a/.github/skills/test-escalation/references/escalation-levels.md b/.github/skills/test-escalation/references/escalation-levels.md index acaaf29..c1edde5 100644 --- a/.github/skills/test-escalation/references/escalation-levels.md +++ b/.github/skills/test-escalation/references/escalation-levels.md @@ -32,7 +32,7 @@ Is the bug reproducible with a simple unit test? - **Symptom**: Empty charts, 0 historical data points - **Root cause**: NaN-only filler rows from failed download survive cleaning - **Effective level**: Level 2 (integration — needs real DB to reproduce the chain) -- **Key assertion**: After pipeline, `processed_prices` has no NaN-only rows +- **Key assertion**: After pipeline, `feature_bars` has no NaN-only rows ### Pattern B: yfinance Silent Failure - **Symptom**: Data appears stale, "No new data" in logs diff --git a/CLAUDE.md b/CLAUDE.md index 96c3f09..0bcb48b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -158,8 +158,11 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is one-release compatibility shim over the package and `downloader.py` keeps only gap detection + upsert. **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails (ADR 0005). -- **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). - `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, +- **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). Tables are + named canonically (`raw_bars` / `clean_bars` / `feature_bars`); the pre-rename names + (`raw_prices` / `clean_prices` / `processed_prices`) are kept as shadows for one release — every + `upsert_many` writes both families, and `scripts/migrate_canonical_tables.py` backfills an existing + DB. `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, `busy_timeout=5000`) and does **not** close on exit. `repos.py` is the only place that builds SQL. - **`cleaning.py` / `processing.py`** — align to business days, mark gaps NA with **no interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. diff --git a/CODEBUDDY.md b/CODEBUDDY.md index 96c3f09..0bcb48b 100644 --- a/CODEBUDDY.md +++ b/CODEBUDDY.md @@ -158,8 +158,11 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is one-release compatibility shim over the package and `downloader.py` keeps only gap detection + upsert. **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails (ADR 0005). -- **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). - `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, +- **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). Tables are + named canonically (`raw_bars` / `clean_bars` / `feature_bars`); the pre-rename names + (`raw_prices` / `clean_prices` / `processed_prices`) are kept as shadows for one release — every + `upsert_many` writes both families, and `scripts/migrate_canonical_tables.py` backfills an existing + DB. `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, `busy_timeout=5000`) and does **not** close on exit. `repos.py` is the only place that builds SQL. - **`cleaning.py` / `processing.py`** — align to business days, mark gaps NA with **no interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. diff --git a/data_pipeline/cleaning.py b/data_pipeline/cleaning.py index d492f60..05e980f 100644 --- a/data_pipeline/cleaning.py +++ b/data_pipeline/cleaning.py @@ -52,7 +52,7 @@ def _flag_anomalies(df: pd.DataFrame) -> pd.DataFrame: def clean_range(ticker: str, start: dt.date | None = None, end: dt.date | None = None) -> PipelineResult: """ Clean data for [start, end). Align to business days, mark missing days as NA - (no interpolation for full missing days), flag anomalies, and upsert to clean_prices. + (no interpolation for full missing days), flag anomalies, and upsert to clean_bars. Returns a PipelineResult with row count and any warnings. INVARIANT: missing trading days remain NA. We do NOT interpolate prices @@ -65,27 +65,27 @@ def clean_range(ticker: str, start: dt.date | None = None, end: dt.date | None = # Inclusive end date query df = fetch_df( - "SELECT * FROM raw_prices WHERE ticker=? AND date>=? AND date<=?", + "SELECT * FROM raw_bars WHERE ticker=? AND date>=? AND date<=?", (ticker, start.isoformat(), end.isoformat()), ) - # WHY: If raw_prices has zero rows for this ticker (e.g. the ticker was + # WHY: If raw_bars has zero rows for this ticker (e.g. the ticker was # invalid and yfinance returned nothing), do NOT generate a business-day - # aligned all-NaN frame and upsert it. Doing so pollutes clean_prices + # aligned all-NaN frame and upsert it. Doing so pollutes clean_bars # with phantom rows for arbitrary user input — including XSS payloads — # and turns a read-only "validate_ticker" call into a DB writer. if df.empty: - # Sanity check: only short-circuit when the ticker has no clean_prices + # Sanity check: only short-circuit when the ticker has no clean_bars # history at all. Established tickers might legitimately have a quiet # period (e.g. exchange holiday week) where the requested raw range # is empty; in that case fall through and align as before so existing # downstream guarantees about business-day alignment are preserved. existing = fetch_df( - "SELECT 1 FROM clean_prices WHERE ticker=? LIMIT 1", + "SELECT 1 FROM clean_bars WHERE ticker=? LIMIT 1", (ticker,), ) if existing.empty: logger.info( - "clean_range: skipping upsert for %s — no raw rows and no existing clean_prices", + "clean_range: skipping upsert for %s — no raw rows and no existing clean_bars", ticker, ) return PipelineResult(ok=True, rows=0, warnings=["no_data_for_ticker"]) @@ -180,7 +180,7 @@ def clean_range(ticker: str, start: dt.date | None = None, end: dt.date | None = ) if rows: upsert_many( - "clean_prices", + "clean_bars", [ "ticker", "date", diff --git a/data_pipeline/data_ops/_query.py b/data_pipeline/data_ops/_query.py index 51e5d9f..464c71f 100644 --- a/data_pipeline/data_ops/_query.py +++ b/data_pipeline/data_ops/_query.py @@ -97,7 +97,7 @@ def get_cleaned_daily(ticker: str, start: dt.date | None = None, end: dt.date | _r.ensure_range(ticker, start, end) init_db() df = fetch_df( - "SELECT date, open, high, low, close, adj_close, volume FROM clean_prices WHERE ticker=? AND date>=? AND date<=?", + "SELECT date, open, high, low, close, adj_close, volume FROM clean_bars WHERE ticker=? AND date>=? AND date<=?", (ticker, start.isoformat(), end.isoformat()), ) # Never memoise a partial read: while the background backfill is running @@ -121,7 +121,7 @@ def get_processed( _u.manual_update(ticker, days=7) init_db() df = fetch_df( - "SELECT * FROM processed_prices WHERE ticker=? AND frequency=? AND date>=? AND date<=?", + "SELECT * FROM feature_bars WHERE ticker=? AND frequency=? AND date>=? AND date<=?", (ticker, frequency, start.isoformat(), end.isoformat()), ) # Never memoise an empty read: a not-yet-generated frequency/range would @@ -143,10 +143,10 @@ def get_processed_data(ticker: str, start: dt.date, end: dt.date, frequency: str def get_latest_spot(ticker: str) -> float | None: - """Return latest close price for *ticker* from clean_prices (Yahoo-sourced).""" + """Return latest close price for *ticker* from clean_bars (provider-sourced).""" init_db() df = fetch_df( - "SELECT close FROM clean_prices WHERE ticker=? AND close IS NOT NULL ORDER BY date DESC LIMIT 1", + "SELECT close FROM clean_bars WHERE ticker=? AND close IS NOT NULL ORDER BY date DESC LIMIT 1", (ticker,), ) if not df.empty: diff --git a/data_pipeline/data_ops/_range.py b/data_pipeline/data_ops/_range.py index 2cfe256..e994f80 100644 --- a/data_pipeline/data_ops/_range.py +++ b/data_pipeline/data_ops/_range.py @@ -36,7 +36,7 @@ def needs_backfill(ticker: str, start: dt.date, end: dt.date) -> bool: if (now - last_ts) < _ENSURE_RANGE_TTL and last_start <= start and last_end >= end: return False cov = _db.fetch_df( - "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_prices WHERE ticker=?", + "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_bars WHERE ticker=?", (ticker,), ) if cov.empty or not cov.iloc[0]["n"]: @@ -50,7 +50,7 @@ def needs_backfill(ticker: str, start: dt.date, end: dt.date) -> bool: def ensure_range(ticker: str, start: dt.date, end: dt.date) -> bool: - """Ensure clean_prices covers [start, end]. + """Ensure clean_bars covers [start, end]. NOTE: only *successful* coverage is memoised. Memoising a failure would make every caller within the TTL believe the range is covered and silently @@ -108,7 +108,7 @@ def _ensure_range_impl(ticker: str, start: dt.date, end: dt.date, now: float, wa from data_pipeline.downloader import MAX_AUTO_BACKFILL_DAYS cov = _db.fetch_df( - "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_prices WHERE ticker=?", + "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_bars WHERE ticker=?", (ticker,), ) existing_min = None diff --git a/data_pipeline/data_ops/facade.py b/data_pipeline/data_ops/facade.py index 69feaba..ec652c1 100644 --- a/data_pipeline/data_ops/facade.py +++ b/data_pipeline/data_ops/facade.py @@ -38,13 +38,13 @@ def has_data_for_date(ticker: str, date) -> bool: from data_pipeline.db import fetch_df df = fetch_df( - "SELECT * FROM clean_prices WHERE ticker=? AND date=?", + "SELECT * FROM clean_bars WHERE ticker=? AND date=?", (ticker, date.isoformat()), ) if not df.empty: return True df2 = fetch_df( - "SELECT * FROM raw_prices WHERE ticker=? AND date=?", + "SELECT * FROM raw_bars WHERE ticker=? AND date=?", (ticker, date.isoformat()), ) return not df2.empty diff --git a/data_pipeline/db.py b/data_pipeline/db.py index ec78c6e..0ec52ae 100644 --- a/data_pipeline/db.py +++ b/data_pipeline/db.py @@ -12,6 +12,14 @@ Do NOT add: a migration framework, an ORM, connection pooling beyond the thread-local cache. Each was considered and rejected as overkill for this project's scale. + +Canonical table names (ADR 0011, batch B2): +- The pipeline reads and writes ``raw_bars`` / ``clean_bars`` / ``feature_bars``. + The pre-rename names (``raw_prices`` / ``clean_prices`` / ``processed_prices``) + are kept for one release as shadow tables: ``upsert_many`` writes both, so a + ``git revert`` of batch B2 loses no rows. Column sets are deliberately + identical (decision gate Q4 = minimal rename); ``tests/test_canonical_tables.py`` + asserts that and ``scripts/migrate_canonical_tables.py`` backfills old DBs. """ import logging @@ -104,86 +112,104 @@ def close_all_conns() -> None: _thread_local.conns.clear() +# ── Schema: one column tuple per table shape ──────────────────────── +# INVARIANT: a canonical table and its pre-rename shadow are created from the +# SAME tuple, so their column sets cannot drift while both exist (the rename is +# a pure name change — decision gate §8 Q4). +# INVARIANT: ``frequency`` is one of D / W / ME / QE (see +# ``core/_shared/types.Frequency``). +_BARS_COLUMNS: tuple[str, ...] = ( + "ticker TEXT NOT NULL", + "date TEXT NOT NULL", + "open REAL", + "high REAL", + "low REAL", + "close REAL", + "adj_close REAL", + "volume REAL", + "provider TEXT DEFAULT 'yfinance'", + "PRIMARY KEY (ticker, date)", +) + +_CLEAN_BARS_COLUMNS: tuple[str, ...] = ( + "ticker TEXT NOT NULL", + "date TEXT NOT NULL", + "open REAL", + "high REAL", + "low REAL", + "close REAL", + "adj_close REAL", + "volume REAL", + "is_trading_day INTEGER DEFAULT 1", + "missing_any INTEGER DEFAULT 0", + "price_jump_flag INTEGER DEFAULT 0", + "vol_anom_flag INTEGER DEFAULT 0", + "ohlc_inconsistent INTEGER DEFAULT 0", + "PRIMARY KEY (ticker, date)", +) + +_FEATURE_BARS_COLUMNS: tuple[str, ...] = ( + "ticker TEXT NOT NULL", + "date TEXT NOT NULL", + "frequency TEXT NOT NULL", + "open REAL", + "high REAL", + "low REAL", + "close REAL", + "adj_close REAL", + "volume REAL", + "last_close REAL", + "log_return REAL", + "amplitude REAL", + "log_hl_spread REAL", + "parkinson_var REAL", + "gk_var REAL", + "log_vol_delta REAL", + "vol_zscore REAL", + "ma_5 REAL", + "ma_10 REAL", + "ma_20 REAL", + "ma_60 REAL", + "ma_120 REAL", + "ma_250 REAL", + "mom_10 REAL", + "mom_20 REAL", + "mom_60 REAL", + "osc_high REAL", + "osc_low REAL", + "osc REAL", + "PRIMARY KEY (ticker, date, frequency)", +) + + +def _create_table(cur, name: str, columns: tuple[str, ...]) -> None: + """Create ``name`` if absent, from ``columns``. + + CONSTRAINT: ``name`` is interpolated into SQL; it is only ever a literal from + this module (never caller input) — mirrors ``upsert_many``'s table validation. + """ + body = ",\n ".join(columns) + cur.execute(f"CREATE TABLE IF NOT EXISTS {name} (\n {body}\n)") + + def init_db(db_path: str | None = None): path = db_path or DB_PATH Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True) conn = _get_or_create_conn(path) cur = conn.cursor() - # Raw OHLCV data - cur.execute( - """ - CREATE TABLE IF NOT EXISTS raw_prices ( - ticker TEXT NOT NULL, - date TEXT NOT NULL, - open REAL, - high REAL, - low REAL, - close REAL, - adj_close REAL, - volume REAL, - provider TEXT DEFAULT 'yfinance', - PRIMARY KEY (ticker, date) - ) - """ - ) - # Cleaned daily OHLCV with flags - cur.execute( - """ - CREATE TABLE IF NOT EXISTS clean_prices ( - ticker TEXT NOT NULL, - date TEXT NOT NULL, - open REAL, - high REAL, - low REAL, - close REAL, - adj_close REAL, - volume REAL, - is_trading_day INTEGER DEFAULT 1, - missing_any INTEGER DEFAULT 0, - price_jump_flag INTEGER DEFAULT 0, - vol_anom_flag INTEGER DEFAULT 0, - ohlc_inconsistent INTEGER DEFAULT 0, - PRIMARY KEY (ticker, date) - ) - """ - ) - # Processed features per frequency - cur.execute( - """ - CREATE TABLE IF NOT EXISTS processed_prices ( - ticker TEXT NOT NULL, - date TEXT NOT NULL, - frequency TEXT NOT NULL, -- D/W/M - open REAL, - high REAL, - low REAL, - close REAL, - adj_close REAL, - volume REAL, - last_close REAL, - log_return REAL, - amplitude REAL, - log_hl_spread REAL, - parkinson_var REAL, - gk_var REAL, - log_vol_delta REAL, - vol_zscore REAL, - ma_5 REAL, - ma_10 REAL, - ma_20 REAL, - ma_60 REAL, - ma_120 REAL, - ma_250 REAL, - mom_10 REAL, - mom_20 REAL, - mom_60 REAL, - osc_high REAL, - osc_low REAL, - osc REAL, - PRIMARY KEY (ticker, date, frequency) - ) - """ - ) + # Canonical store (ADR 0011) — the pipeline reads and writes these names. + _create_table(cur, "raw_bars", _BARS_COLUMNS) + _create_table(cur, "clean_bars", _CLEAN_BARS_COLUMNS) + _create_table(cur, "feature_bars", _FEATURE_BARS_COLUMNS) + # ── Compatibility shadows (transitional — one release) ────────────── + # TRADEOFF: the pre-rename names are kept, created from the same column + # tuples, so reverting batch B2 loses no rows and a DB written before the + # rename keeps working until scripts/migrate_canonical_tables.py has run + # (and afterwards: `upsert_many` writes both families). Drop these three + # lines + `_TABLE_SHADOWS` one release after the rename. + _create_table(cur, "raw_prices", _BARS_COLUMNS) + _create_table(cur, "clean_prices", _CLEAN_BARS_COLUMNS) + _create_table(cur, "processed_prices", _FEATURE_BARS_COLUMNS) # Market review benchmark close prices cur.execute( """ @@ -274,6 +300,11 @@ def get_conn(db_path: str | None = None): _UPSERTABLE_TABLES = frozenset( { + # canonical (ADR 0011) + "raw_bars", + "clean_bars", + "feature_bars", + # compatibility shadows — removable one release after the rename "raw_prices", "clean_prices", "processed_prices", @@ -284,14 +315,41 @@ def get_conn(db_path: str | None = None): } ) +# ── Canonical ↔ legacy table naming (transitional) ────────────────── +# INVARIANT: each pair has an identical column set — enforced by +# tests/test_canonical_tables.py, because the two families must stay +# interchangeable for the compatibility window to be safe. +CANONICAL_TABLES: dict[str, str] = { + "raw_prices": "raw_bars", + "clean_prices": "clean_bars", + "processed_prices": "feature_bars", +} + +# WHY bidirectional: writes may arrive under either name during the window (old +# call sites, tests seeding fixtures directly). Mirroring both ways keeps the two +# families identical no matter which name a caller uses. +_TABLE_SHADOWS: dict[str, str] = { + **CANONICAL_TABLES, + **{canonical: legacy for legacy, canonical in CANONICAL_TABLES.items()}, +} + + +def canonical_table(name: str) -> str: + """Return the canonical table name for a legacy name (identity otherwise).""" + return CANONICAL_TABLES.get(name, name) + def upsert_many(table: str, columns: Iterable[str], rows: Iterable[Iterable], db_path: str | None = None): - """Bulk-upsert *rows* into *table*. + """Bulk-upsert *rows* into *table*, plus its shadow table if it has one. CONSTRAINT: the table name is interpolated into SQL (values are still parameterised), so it is validated against the known schema tables — a typo or caller-supplied name fails fast instead of building a malformed (or, with hostile input, malicious) statement. + TRADEOFF (transitional — one release): the canonical and pre-rename table + families are written together (see ``_TABLE_SHADOWS``) so that either name + can be read during the rename. The extra write is one more executemany over + the same small batches; drop it with the shadow tables. """ if table not in _UPSERTABLE_TABLES: raise ValueError(f"upsert_many: unknown table {table!r}; expected one of {sorted(_UPSERTABLE_TABLES)}") @@ -301,11 +359,16 @@ def upsert_many(table: str, columns: Iterable[str], rows: Iterable[Iterable], db cols = list(columns) placeholders = ",".join(["?"] * len(cols)) updates = ",".join([f"{c}=excluded.{c}" for c in cols if c not in ("ticker", "date", "frequency")]) - sql = f"INSERT INTO {table} ({','.join(cols)}) VALUES ({placeholders}) ON CONFLICT DO UPDATE SET {updates}" + targets = [table, *((_TABLE_SHADOWS[table],) if table in _TABLE_SHADOWS else ())] + statements = [ + f"INSERT INTO {target} ({','.join(cols)}) VALUES ({placeholders}) ON CONFLICT DO UPDATE SET {updates}" + for target in targets + ] conn = _get_or_create_conn(db_path or DB_PATH) try: conn.execute("BEGIN") - conn.executemany(sql, rows) + for sql in statements: + conn.executemany(sql, rows) conn.execute("COMMIT") except Exception: try: diff --git a/data_pipeline/downloader.py b/data_pipeline/downloader.py index be3ec96..6cfdfb2 100644 --- a/data_pipeline/downloader.py +++ b/data_pipeline/downloader.py @@ -4,7 +4,7 @@ Context: - Acquisition itself lives in ``data_pipeline/providers/``. This module keeps only the DB-aware parts: business-day gap detection, the auto-backfill cap, - and the ``raw_prices`` upsert. Batch B1 (see + and the ``raw_bars`` upsert. Batch B1 (see docs/plans/business_line_reorg.md §6) moved the ``yf.download`` call behind ``providers.yfinance_provider.download_daily_frame``, so this module no longer imports yfinance. @@ -25,7 +25,9 @@ import pandas as pd -from data_pipeline.providers.yfinance_provider import download_daily_frame +from data_pipeline.providers import get_provider +from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS +from data_pipeline.providers.yfinance_provider import to_canonical_bars from . import PipelineResult from .db import fetch_df, upsert_many @@ -54,10 +56,10 @@ def _last_business_day_on_or_before(d: dt.date) -> dt.date: def _load_test_fixture(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: """Synthesise OHLCV for ``TEST_*`` tickers without touching the network. - Matches the column shape produced by ``_download_yf`` so the rest of the - pipeline is fixture-agnostic. If a CSV exists at - ``tests/fixtures/yf/.csv`` it is used verbatim; otherwise a - deterministic synthetic series is generated. + Matches the yfinance column shape (Title Case + ``Adj_Close``) so the + canonical mapping is exercised identically for fixtures and real downloads. + If a CSV exists at ``tests/fixtures/yf/.csv`` it is used verbatim; + otherwise a deterministic synthetic series is generated. """ csv_path = _FIXTURE_DIR / f"{ticker}.csv" if csv_path.exists(): @@ -85,7 +87,7 @@ def _load_test_fixture(ticker: str, start: dt.date, end: dt.date) -> pd.DataFram def find_missing_business_days(ticker: str, start: dt.date, end: dt.date) -> list[dt.date]: - """Return business days in [start, end] (inclusive) that have no row in raw_prices. + """Return business days in [start, end] (inclusive) that have no row in raw_bars. Uses the same Mon-Fri business-day calendar as `cleaning._get_business_days` so gaps map 1:1 with cleaning's expected index. Holidays are intentionally @@ -95,7 +97,7 @@ def find_missing_business_days(ticker: str, start: dt.date, end: dt.date) -> lis if len(expected) == 0: return [] df = fetch_df( - "SELECT date FROM raw_prices WHERE ticker=? AND date>=? AND date<=?", + "SELECT date FROM raw_bars WHERE ticker=? AND date>=? AND date<=?", (ticker, start.isoformat(), end.isoformat()), ) have: set[dt.date] = set() @@ -111,24 +113,28 @@ def find_missing_business_days(ticker: str, start: dt.date, end: dt.date) -> lis return [ts.date() for ts in expected if ts.date() not in have] -def _download_yf(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: - """Download daily OHLCV for ``[start, end]`` (inclusive), fixture-aware. +def download_bars(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: + """Acquire daily bars for ``[start, end]`` (inclusive) in the canonical schema. ``TEST_*`` tickers never hit the network (useful for unit tests + ad-hoc smoke tests under rate-limit conditions; see ``_load_test_fixture``). - Everything else is delegated to the yfinance provider. + Everything else goes through the provider registry, which returns canonical + bars already (ADR 0011). """ if ticker.startswith("TEST_"): logger.info("Loading fixture data for test ticker %s (%s..%s)", ticker, start, end) - return _load_test_fixture(ticker, start, end) - return download_daily_frame(ticker, start, end) + # WHY the yfinance mapper for fixtures: the fixture frames deliberately + # mimic `yf.download`'s Title-Case shape, so they need the same mapping + # the provider applies to a real download. + return to_canonical_bars(_load_test_fixture(ticker, start, end)) + return get_provider().history(ticker, start, end) def upsert_raw_prices( ticker: str, start: dt.date | None = None, end: dt.date | None = None, days: int = 7 ) -> PipelineResult: """ - Download OHLCV for [start, end) and upsert into raw_prices. + Download OHLCV for [start, end) and upsert into raw_bars (canonical store). If df for a day is entirely NA, skip and keep existing row. Returns a PipelineResult with row count and any warnings. """ @@ -145,7 +151,7 @@ def upsert_raw_prices( # ── Gap-aware coverage check ── # Skip the network only when every business day in [start, end] is already - # present in raw_prices. Otherwise expand the download range to cover the + # present in raw_bars. Otherwise expand the download range to cover the # earliest gap so back-fills happen automatically after an outage. missing = find_missing_business_days(ticker, start, end) if not missing: @@ -171,7 +177,7 @@ def upsert_raw_prices( start = effective_start try: - df_new = _download_yf(ticker, start, end) + df_new = download_bars(ticker, start, end) except Exception as e: logger.error(f"Download failed for {ticker}: {e}", exc_info=True) return PipelineResult(ok=False, error=f"download_failed: {e}") @@ -188,42 +194,33 @@ def upsert_raw_prices( df_new.index = idx.tz_localize(None) if idx.tz is None else idx.tz_convert(None) df_new["date"] = df_new.index.date + # INVARIANT: bars arrive canonical (ADR 0011), so the ingest stage never + # touches vendor column names — the mapping lives in the provider. + bar_cols = list(CANONICAL_BAR_COLUMNS) + provider_name = get_provider().name + rows = [] for _d, row in df_new.iterrows(): date_str = row["date"].isoformat() # If all new values are NA, retain old data (skip insert) and log - if row[["Open", "High", "Low", "Close", "Adj_Close", "Volume"]].isna().all(): + if row[bar_cols].isna().all(): msg = f"Blank data for {ticker} on {date_str}; retaining old data if exists" logger.warning(msg) result.warnings.append(msg) continue - tup = ( - ticker, - date_str, - float(row.get("Open", pd.NA)) if pd.notna(row.get("Open")) else None, - float(row.get("High", pd.NA)) if pd.notna(row.get("High")) else None, - float(row.get("Low", pd.NA)) if pd.notna(row.get("Low")) else None, - float(row.get("Close", pd.NA)) if pd.notna(row.get("Close")) else None, - float(row.get("Adj_Close", pd.NA)) if pd.notna(row.get("Adj_Close")) else None, - float(row.get("Volume", pd.NA)) if pd.notna(row.get("Volume")) else None, - "yfinance", + rows.append( + ( + ticker, + date_str, + *[float(row[col]) if pd.notna(row.get(col)) else None for col in bar_cols], + provider_name, + ) ) - rows.append(tup) if rows: upsert_many( - "raw_prices", - [ - "ticker", - "date", - "open", - "high", - "low", - "close", - "adj_close", - "volume", - "provider", - ], + "raw_bars", + ["ticker", "date", *bar_cols, "provider"], rows, ) result.rows = len(rows) diff --git a/data_pipeline/processing.py b/data_pipeline/processing.py index b808e8f..d91a866 100644 --- a/data_pipeline/processing.py +++ b/data_pipeline/processing.py @@ -1,8 +1,8 @@ """Feature engineering for cleaned daily price series. Context: -- Reads from ``clean_prices`` and emits resampled bars + indicator columns to - ``processed_prices``. Pure pandas; no I/O outside the DB helpers in +- Reads from ``clean_bars`` and emits resampled bars + indicator columns to + ``feature_bars``. Pure pandas; no I/O outside the DB helpers in ``data_pipeline.db``. """ @@ -73,7 +73,7 @@ def process_frequencies(ticker: str, start: dt.date | None = None, end: dt.date start = start or (end - dt.timedelta(days=90)) daily = fetch_df( - "SELECT date, open, high, low, close, adj_close, volume FROM clean_prices WHERE ticker=? AND date>=? AND date<=?", + "SELECT date, open, high, low, close, adj_close, volume FROM clean_bars WHERE ticker=? AND date>=? AND date<=?", (ticker, start.isoformat(), end.isoformat()), ) if daily.empty: @@ -131,7 +131,7 @@ def process_frequencies(ticker: str, start: dt.date | None = None, end: dt.date ) if rows: upsert_many( - "processed_prices", + "feature_bars", [ "ticker", "date", diff --git a/data_pipeline/repos.py b/data_pipeline/repos.py index 941b58e..5f2d710 100644 --- a/data_pipeline/repos.py +++ b/data_pipeline/repos.py @@ -22,7 +22,7 @@ # ── Health / data-quality inventory ───────────────────────────────── def fetch_ticker_inventory() -> list[tuple[Any, ...]]: - """Return one row per ticker from ``raw_prices`` with row counts + NaN tallies. + """Return one row per ticker from ``raw_bars`` with row counts + NaN tallies. Columns: ``(ticker, rows, latest_date, earliest_date, null_close, null_volume)`` ordered by ticker. @@ -35,7 +35,7 @@ def fetch_ticker_inventory() -> list[tuple[Any, ...]]: MIN(date) AS earliest_date, SUM(CASE WHEN close IS NULL THEN 1 ELSE 0 END) AS null_close, SUM(CASE WHEN volume IS NULL THEN 1 ELSE 0 END) AS null_volume - FROM raw_prices + FROM raw_bars GROUP BY ticker ORDER BY ticker """ @@ -172,7 +172,7 @@ def count_clean_rows(ticker: str) -> int: """Return how many priced rows the DB holds for ``ticker``.""" ensure_schema() df = fetch_df( - "SELECT COUNT(*) AS n FROM clean_prices WHERE ticker=? AND close IS NOT NULL", + "SELECT COUNT(*) AS n FROM clean_bars WHERE ticker=? AND close IS NOT NULL", (ticker,), ) if df.empty: diff --git a/docs/architecture_review.md b/docs/architecture_review.md index cc95d93..aa6e562 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -65,7 +65,7 @@ Rescoped in batch B1 of [ADR 0011](decisions/0011-pluggable-data-provider-seam.m | Location | Why it exists | Exit condition | |---|---|---| -| `data_pipeline/downloader.py` — **resolved 2026-09-10 (B1)** | DB-aware gap-detection bulk downloads; it used to call `yf.download` directly as a registered second exit point | the download call moved to `providers/yfinance_provider.py::download_daily_frame`; `downloader.py` keeps only gap detection + `raw_prices` upsert, so it no longer imports yfinance | +| `data_pipeline/downloader.py` — **resolved 2026-09-10 (B1)** | DB-aware gap-detection bulk downloads; it used to call `yf.download` directly as a registered second exit point | the download call moved to `providers/yfinance_provider.py::download_daily_frame`; `downloader.py` keeps only gap detection + `raw_bars` upsert, so it no longer imports yfinance | | `data_pipeline/data_ops/_query.py::get_latest_spot` — **resolved 2026-09-03** | former spot fast-path fetched yfinance internally | now routes through `fetch_spot` (provider, re-exported by `yf_client`) | ### Watch list (pre-debt, no marker yet) diff --git a/docs/constraints.md b/docs/constraints.md index 6d5ccda..0f0b7ee 100644 --- a/docs/constraints.md +++ b/docs/constraints.md @@ -30,7 +30,7 @@ is usually a workaround for one of the items below. - Proxy must be set via `HTTP_PROXY` / `HTTPS_PROXY` env vars; we read `YF_PROXY` and propagate to both. See [utils/network.py](../utils/network.py) `init_yf_proxy`. - **Dead-proxy poisoning**: an unreachable proxy makes curl_cffi hang. We TCP-probe before activating; falls back to direct connect. - **Global throttle**: token-bucket limiter (default 5 req/s, burst 5) in [utils/network.py](../utils/network.py)::`yf_throttle`. Every yfinance call MUST be routed through `data_pipeline/providers/` (the single chokepoint since batch B1 of ADR 0011) rather than calling `yf_throttle()` directly at each call site — see ADR 0005. -- **DB-first pattern**: never re-download data already in `clean_prices`. The 60-second cooldown in `DataService` exists to prevent thundering herd from concurrent UI requests. +- **DB-first pattern**: never re-download data already in `clean_bars`. The 60-second cooldown in `DataService` exists to prevent thundering herd from concurrent UI requests. - **Single yfinance exit point**: only `data_pipeline/providers/` may `import yfinance`. The chokepoint moved there from `yf_client.py` in batch B1; `yf_client.py` is now a one-release compatibility shim over the provider package. Any other module needs a `# doc-guard: allow=single-yf-exit` marker (tracked as architecture debt — see [architecture_review.md](architecture_review.md) §2). `tests/` and `scripts/` are exempt by design (test doubles patch `yfinance.download` on the module object). Enforced by `scripts/doc_guard.py` rules `single-yf-exit`, `import-direction`, `core-purity` and `db-access`; trend-gated in CI by `scripts/arch_metrics.py --check`. ## 3. SQLite, single-machine deployment diff --git a/docs/decisions/0011-pluggable-data-provider-seam.md b/docs/decisions/0011-pluggable-data-provider-seam.md index faef1d7..ad13c74 100644 --- a/docs/decisions/0011-pluggable-data-provider-seam.md +++ b/docs/decisions/0011-pluggable-data-provider-seam.md @@ -123,6 +123,16 @@ vendor means two providers coexist rather than one replacing the other. `yf_client.py` is a compatibility shim and `downloader.py` no longer imports yfinance. Batch ledger: [`docs/plans/business_line_reorg.md`](../plans/business_line_reorg.md) §0. +**Amendment (batch B2, 2026-09-10) — canonical tables are a name-only rename.** B2 landed +`raw_bars` / `clean_bars` / `feature_bars` carrying *exactly* the column sets of the tables they +replace, including the `ticker` column. Renaming `ticker` → `symbol` (and introducing a +`symbol_map`) is deferred until a second provider actually needs a provider-native identifier: +today it is a cross-cutting rename through `repos.py`, `data_ops/`, `services/` and the test +fixtures with no consumer, which batch scope forbids riding along with. The target-state column +lists in the Decision section above stay the reference for when that provider lands. +Compatibility: the pre-rename names remain as shadow tables for one release (`upsert_many` +writes both families) and `scripts/migrate_canonical_tables.py` backfills an existing DB. + ## Consequences - Positive: a second provider is one file + one registry line + a field-map test. diff --git a/docs/glossary.md b/docs/glossary.md index 2764a44..f19e95d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -64,11 +64,16 @@ User-supplied directional preference (Bull / Bear / Neutral) used to filter stra ## Data Pipeline -### `raw_prices` -Untouched OHLCV pulled from yfinance. Indexed by `(ticker, date)`. +### `raw_bars` +Untouched OHLCV pulled from a provider (yfinance today) and mapped onto the canonical schema. Indexed by `(ticker, date)`. -### `clean_prices` -`raw_prices` aligned to business days, anomalies flagged, missing days = NA. **NO interpolation** — see [constraints.md §4](constraints.md#4-the-machine-is-not-247). +### `clean_bars` +`raw_bars` aligned to business days, anomalies flagged, missing days = NA. **NO interpolation** — see [constraints.md §4](constraints.md#4-the-machine-is-not-247). + +### `feature_bars` +`clean_bars` resampled per frequency (D/W/ME/QE) with engineered features (returns, MA, HV, oscillation). Indexed by `(ticker, date, frequency)`. + +> **Compatibility (one release)**: the pre-rename names `raw_prices` / `clean_prices` / `processed_prices` still exist as shadow tables — every write goes to both families (see `data_pipeline/db.py`) — so an un-migrated DB and a `git revert` of the rename keep working. See [ADR 0011](decisions/0011-pluggable-data-provider-seam.md). ### Anomaly Flags - `price_jump_flag`: |log return| > 5σ. diff --git a/docs/guides/README.md b/docs/guides/README.md index 4167468..e3724f1 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -50,7 +50,7 @@ services/ # 请求编排层（按业务域分包） facade.py # 状态标注与持久化 ops/ # 历史回填 + regime_log 写入 data_pipeline/ # 数据管道（下载 → 清洗 → 加工 → 服务） - downloader.py # 通过 yfinance 下载 OHLCV 并写入 raw_prices + downloader.py # 通过 yfinance 下载 OHLCV 并写入 raw_bars cleaning.py # 对齐交易日、标记异常（5σ 波动、成交量异常）、前向填充 processing.py # 日/周/月级聚合及衍生指标（收益率、振幅、Parkinson/GK 方差、动量等） data_service.py # 数据门面：初始化 DB，按需 7 日增量刷新，60s 并发节流 @@ -64,7 +64,7 @@ tests/ # 回归测试 ## 数据管道 ``` -Yahoo Finance ──▶ downloader (upsert raw_prices) +Yahoo Finance ──▶ downloader (upsert raw_bars) │ ▼ cleaning (对齐交易日, 异常标记, 前向填充) diff --git a/docs/l0_architecture.md b/docs/l0_architecture.md index 639f577..714b732 100644 --- a/docs/l0_architecture.md +++ b/docs/l0_architecture.md @@ -68,7 +68,7 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ | Item | State | Note | |---|---|---| -| `market_data.sqlite` (11.7 MB) | git-ignored, still in repo root | code default is now `data/market_data.sqlite` (`data_pipeline/db.py:27`); the local `.env` overrides it back to the root file — move the file into `data/` whenever convenient | +| `market_data.sqlite` (11.7 MB) | git-ignored, still in repo root | code default is now `data/market_data.sqlite` (`data_pipeline/db.py:27`); the local `.env` overrides it back to the root file — move the file into `data/` whenever convenient. Schema: canonical `raw_bars` / `clean_bars` / `feature_bars`, plus the one-release shadows `raw_prices` / `clean_prices` / `processed_prices` — see [ADR 0011](decisions/0011-pluggable-data-provider-seam.md) | | `site/` | **inputs committed (9 files) · build output ignored** | tracked: `fixtures/` (7) · `snapshot/snapshot.json` · `pages-shim.js`; ignored: `index.html`, 5 feature + 6 showcase redirects, `static/**` (42 generated files) — see §5 P1-1 | | `archive/` (8 files · 1 070 lines) | committed | retired code still in tree — P3-3 | | `test.ipynb` (58 lines) | git-ignored | leftover scratch file — P2-2 | @@ -78,10 +78,10 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ ## 2. Measured shape (`scripts/arch_metrics.py`) -_Refreshed 2026-09-10 after batch B1 (provider seam extraction); the L1 inventory in §1 above is otherwise the 2026-09-08 snapshot._ +_Refreshed 2026-09-10 after batches B1 (provider seam) and B2 (canonical table names); the L1 inventory in §1 above is otherwise the 2026-09-08 snapshot._ ``` -modules=152 import_edges=312 +modules=152 import_edges=314 Layer-edge violations : (none) Import cycles : 0 God files (>400 lines): (none) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index 3b7263f..fcc9a65 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -39,7 +39,7 @@ |---|---|---|---|---| | — (planning + ADRs) | 🔨 in review (PR #7) | #7 | branch `worktree-business-line-reorg` · 2026-09-10 | plan, ADR 0011/0012 (Accepted), scaffolding (ledger, gates, AI-guide pointers, memory) | | B1 — provider seam extraction | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | delivers the "pluggable API" seam on its own. Actual shape / deviations recorded in §8; `_ALLOWED_DEPS` promotion of `providers` deferred to B3 | -| B2 — canonical raw store | ⬜ not started | — | — | gate: §8 Q4 | +| B2 — canonical raw store | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | gate §8 Q4 resolved (name-only rename). Actuals in §8; `symbol` column deferred (ADR 0011 amendment) | | B3 — package re-home | ⬜ not started | — | — | resets `arch_baseline.json` | | B4 — close L1 (`core-purity`) | ⬜ not started | — | — | — | | B5 — readiness plan + prefetch | ⬜ not started | — | — | gate: §8 Q1 | @@ -422,7 +422,7 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. | Q1 | **Submit contract** — does `POST /` carry a `modules` manifest with per-module params attached to each `/render` call, or do the streaming market tabs move fully to client-fired `/api/*` like Option Chain? Manifest keeps the streaming model; full client-fired is more uniform but a bigger diff. | before **B5** (locks how B7 wires params) | manifest | | Q2 | **Config tab fate** — is there *any* genuine global setting to keep? Risk-free rate is the only candidate (hard-coded in `static/sim/` and again in `core/options/greeks`). Yes → tab shrinks to it; no → tab deleted. | before **B8** | keep risk-free rate, delete the rest | | Q3 | **`positions` block** — Portfolio Analysis is its only consumer. Move into a dedicated "Portfolio" panel/tab, or keep as a section the bar's Run ignores? | before **B6** | dedicated Portfolio panel | -| Q4 | **Table rename vs. reshape** — `raw_prices`→`raw_bars` with identical columns (minimal), or also move the yfinance-ism `adj_close` handling into the provider during the rename? | before **B2** | minimal rename | +| Q4 | **Table rename vs. reshape** — `raw_prices`→`raw_bars` with identical columns (minimal), or also move the yfinance-ism `adj_close` handling into the provider during the rename? | ✅ resolved 2026-09-10 (B2) | **minimal rename** — identical columns on both sides of each pair (structurally enforced: one column tuple per shape, used to create both names). The `adj_close` normalisation is already inside the provider (B1's `to_canonical_bars`), and ingest now consumes canonical bars, so no reshape is needed. ADR 0011's `symbol` column stays the target state but is deferred — see the B2 note below | | Q5 | **Second-provider protocol shape** — not in scope to *implement*, but `providers/base.py` (written in B1) must be sketched against *both* yfinance and `archive/futu_integration/field_mapping.md` so the protocol is not accidentally yfinance-shaped (IV unit, bid/ask availability, `inTheMoney` derivation all differ). | ✅ resolved 2026-09-10 (B1) — outcome table in ADR 0011 §"Protocol shape" | design review of `base.py` against both field maps: IV → decimal, bid/ask nullable, `inTheMoney` dropped (derivable), expiries ISO strings | ### Batch notes (actuals + deviations, recorded as batches land) @@ -450,6 +450,33 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. `import yfinance` hits: exactly the two `providers/` modules. (`tests/test_yf_download.py` and `tests/e2e/conftest.py` also import it as test doubles — `doc_guard` exempts `tests/` by design.) +**B2 (2026-09-10) — canonical raw store.** + +- **Shape**: `raw_bars` / `clean_bars` / `feature_bars` added; all reads *and* writes in + `data_pipeline/` switched to them. The pre-rename names are kept as shadow tables and + `upsert_many` mirrors **both** directions, so an old seeding path, an un-migrated DB and a + `git revert` all keep working. `scripts/migrate_canonical_tables.py` backfills an existing DB + (idempotent, `INSERT OR IGNORE`, never clobbers the canonical table). +- **Ingest is now canonical**: `downloader.download_bars()` (was `_download_yf`) acquires through + `providers.get_provider().history()` — i.e. the registry, not a concrete vendor module — and + returns `CANONICAL_BAR_COLUMNS`. The yfinance-ism (`Adj Close`→`Adj_Close`) is now confined to + the provider's mapping, and the `provider` column is written from `get_provider().name`. +- **No column reshape** (Q4 above). To keep that true by construction rather than by review, + `init_db` builds each canonical/legacy pair from one shared column tuple — which also kept + `db.py` under the 400-line god-file cap after the 3 extra tables (+0 tracked metrics). +- **Deliberate non-change**: the function name `upsert_raw_prices` is kept (it is called from + `data_ops/{_update,_range}.py`, `services/regime/ops/_bootstrap.py` and ~12 test patch targets); + renaming it is a cross-cutting edit that belongs with the B3 re-home, not with the rename. +- **Tests**: new `tests/test_canonical_tables.py` pins column parity per pair, bidirectional + mirroring, "one pipeline run populates both families", that `fetch_ticker_inventory` (the /health + read) hits `raw_bars`, and that the migration script backfills + is idempotent. + `test_processing.py` now seeds `clean_bars` and reads `feature_bars` (the §6 exit criterion); + `test_health_service.py` / `test_nvda_analysis.py` seeded via raw SQL and therefore had to move. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 468 passed / 5 skipped; + `doc_guard.py` clean; `arch_metrics.py --check` ok (no baseline reset needed); + `audit_tags.py` unchanged (16 vs baseline 16); `routes/` untouched (only the one-line comment + fix in `services/market/facade.py` outside `data_pipeline/`). + --- ## 9. References diff --git a/scripts/migrate_canonical_tables.py b/scripts/migrate_canonical_tables.py new file mode 100644 index 0000000..4403fae --- /dev/null +++ b/scripts/migrate_canonical_tables.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""One-shot backfill of the canonical tables from the pre-rename ones. + +Domain: Data Pipeline — canonical table names (ADR 0011, batch B2) +Context: + - Batch B2 renamed the store tables (: ``raw_prices``/``clean_prices``/ + ``processed_prices`` → ``raw_bars``/``clean_bars``/``feature_bars``). The + pipeline now reads and writes the canonical names, and writes the legacy + names too for one release (see ``data_pipeline/db.py::_TABLE_SHADOWS``), but + a DB created *before* B2 only has rows under the legacy names. + - This script copies legacy → canonical so an existing ``market_data.sqlite`` + becomes readable by the new code without waiting for a re-download. + - It is deliberately NOT a migration framework (constraints §3): no version + table, no ordering, no schema changes — just an idempotent row copy. +Contracts: + - Idempotent: ``INSERT OR IGNORE``, so re-running is a no-op. + - Never overwrites the canonical table (canonical wins on a conflict), so it + is safe to run after the pipeline has already written new data. + - Column sets must match; a mismatch aborts with a non-zero exit code instead + of silently dropping columns. +Usage: + python scripts/migrate_canonical_tables.py [--db PATH] [--dry-run] +Dependencies: + - data_pipeline.db (schema + connection pragmas); stdlib only otherwise. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from data_pipeline.db import CANONICAL_TABLES, get_conn, init_db # noqa: E402 + + +def _columns(conn, table: str) -> list[str]: + """Return the column names of ``table`` (empty when it does not exist).""" + return [row[1] for row in conn.execute(f"PRAGMA table_info({table})")] + + +def _count(conn, table: str) -> int: + return int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + + +def _copy_table(conn, legacy: str, canonical: str) -> int: + """Copy ``legacy`` → ``canonical``; return how many rows were added.""" + legacy_cols = _columns(conn, legacy) + canonical_cols = _columns(conn, canonical) + if not legacy_cols: + print(f" {legacy}: absent — nothing to do") + return 0 + dropped = [c for c in legacy_cols if c not in canonical_cols] + if dropped: + sys.exit( + f"[migrate] {canonical} is missing column(s) {dropped} present in {legacy}; " + "refusing to drop data — fix the schema first" + ) + before = _count(conn, canonical) + cols = ",".join(legacy_cols) + conn.execute(f"INSERT OR IGNORE INTO {canonical} ({cols}) SELECT {cols} FROM {legacy}") + conn.commit() + added = _count(conn, canonical) - before + print(f" {legacy} → {canonical}: +{added} row(s) (canonical now {before + added})") + return added + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--db", default=None, help="override the DB path (default: MARKET_DB_PATH / data/)") + ap.add_argument("--dry-run", action="store_true", help="report what would be copied, change nothing") + args = ap.parse_args() + + init_db(args.db) + with get_conn(args.db) as conn: + if args.dry_run: + for legacy, canonical in CANONICAL_TABLES.items(): + legacy_n = _count(conn, legacy) if _columns(conn, legacy) else 0 + print(f" {legacy} ({legacy_n} rows) → {canonical}") + print("[migrate] dry run — nothing written") + return 0 + print("[migrate] copying pre-rename tables into the canonical store") + total = sum(_copy_table(conn, legacy, canonical) for legacy, canonical in CANONICAL_TABLES.items()) + print(f"[migrate] done — {total} row(s) backfilled") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/services/market/facade.py b/services/market/facade.py index 6133491..1491109 100644 --- a/services/market/facade.py +++ b/services/market/facade.py @@ -34,7 +34,7 @@ def validate_ticker(ticker): # WHY: Reject obvious junk (XSS payloads, SQL fragments, lowercase, etc.) # before hitting the data layer. Otherwise a single call would # trigger DataService.manual_update() which writes one NaN row per - # business day to clean_prices for the bogus ticker. + # business day to clean_bars for the bogus ticker. if not is_valid_ticker_format(ticker): return False, "invalid_ticker_or_no_data_available" try: diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 30a38b9..99f9779 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -98,7 +98,7 @@ def _e2e_db(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: # * `yfinance.Ticker(...).option_chain(exp)` — calls/puts DataFrames # # Combined with the existing `TEST_*` ticker bypass in -# `data_pipeline.downloader._download_yf`, real `TEST_AAPL` form submissions +# `data_pipeline.downloader.download_bars`, real `TEST_AAPL` form submissions # never hit the network. # --------------------------------------------------------------------------- def _synthetic_ohlcv(ticker: str, start: dt.date, end: dt.date): diff --git a/tests/test_canonical_tables.py b/tests/test_canonical_tables.py new file mode 100644 index 0000000..60112ba --- /dev/null +++ b/tests/test_canonical_tables.py @@ -0,0 +1,173 @@ +"""Contract tests for the canonical store tables (ADR 0011, batch B2). + +Domain: Tests — Canonical Store +Context: + - Batch B2 renamed the store tables to ``raw_bars`` / ``clean_bars`` / + ``feature_bars`` and kept the pre-rename names as shadow tables for one + release. These tests pin the two properties that make the compatibility + window safe: the pairs have identical column sets, and a write under either + name reaches both. + - They also pin the *direction* of the migration — reads must hit the + canonical names, and ``scripts/migrate_canonical_tables.py`` must backfill a + DB that only has legacy rows. +Contracts: + - Column-set parity per pair (decision gate §8 Q4 = minimal rename). + - ``upsert_many`` mirrors canonical ↔ legacy in both directions. + - A pipeline run (download → clean → process) populates both families. + - ``fetch_ticker_inventory`` (the /health data source) reads the canonical table. + - The one-shot migration backfills legacy-only rows into the canonical table. +Dependencies UPWARD: + - (none — stdlib + pytest + the package under test) +""" + +from __future__ import annotations + +import datetime as dt +import subprocess +import sys +from pathlib import Path + +import pytest + +from data_pipeline.db import CANONICAL_TABLES, canonical_table, get_conn, init_db, upsert_many + +REPO_ROOT = Path(__file__).resolve().parent.parent + +_RAW_COLS = ["ticker", "date", "open", "high", "low", "close", "adj_close", "volume", "provider"] + + +def _table_info(table: str) -> list[tuple]: + with get_conn() as conn: + return [tuple(row) for row in conn.execute(f"PRAGMA table_info({table})")] + + +def _count(table: str, ticker: str | None = None) -> int: + with get_conn() as conn: + if ticker is None: + return int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + return int(conn.execute(f"SELECT COUNT(*) FROM {table} WHERE ticker=?", (ticker,)).fetchone()[0]) + + +# --------------------------------------------------------------------------- +# Column parity +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("legacy,canonical", sorted(CANONICAL_TABLES.items())) +def test_canonical_table_mirrors_legacy_columns(legacy, canonical): + """Decision gate Q4 = minimal rename, so the column sets must be identical.""" + init_db() + canon_info = _table_info(canonical) + assert canon_info, f"{canonical} was not created by init_db()" + assert canon_info == _table_info(legacy) + + +def test_canonical_table_is_identity_for_names_without_a_pair(): + assert canonical_table("raw_prices") == "raw_bars" + assert canonical_table("raw_bars") == "raw_bars" + assert canonical_table("regime_log") == "regime_log" + + +# --------------------------------------------------------------------------- +# Transitional dual-write +# --------------------------------------------------------------------------- +def test_upsert_many_writes_both_table_families(): + init_db() + upsert_many("raw_bars", _RAW_COLS, [("DUAL_CANON", "2026-01-02", 1.0, 1.0, 1.0, 1.0, 1.0, 10.0, "yfinance")]) + assert _count("raw_bars", "DUAL_CANON") == 1 + assert _count("raw_prices", "DUAL_CANON") == 1 + + # A legacy name must also reach the canonical table: old call sites and test + # fixtures seed under the pre-rename names during the window. + upsert_many("raw_prices", _RAW_COLS, [("DUAL_LEGACY", "2026-01-02", 2.0, 2.0, 2.0, 2.0, 2.0, 20.0, "yfinance")]) + assert _count("raw_prices", "DUAL_LEGACY") == 1 + assert _count("raw_bars", "DUAL_LEGACY") == 1 + + +def test_pipeline_run_populates_both_table_families(): + """B2 exit criterion: one pipeline run leaves both families populated.""" + from data_pipeline.cleaning import clean_range + from data_pipeline.downloader import upsert_raw_prices + from data_pipeline.processing import process_frequencies + + ticker = "TEST_CANON" + end = dt.date.today() + start = end - dt.timedelta(days=45) + + init_db() + assert upsert_raw_prices(ticker, start, end).ok + assert clean_range(ticker, start, end).ok + assert process_frequencies(ticker, start, end).ok + + for legacy, canonical in CANONICAL_TABLES.items(): + canonical_rows = _count(canonical, ticker) + assert canonical_rows > 0, f"{canonical} was not written for {ticker}" + assert canonical_rows == _count(legacy, ticker), f"{legacy} / {canonical} diverged" + + +# --------------------------------------------------------------------------- +# Reads target the canonical tables +# --------------------------------------------------------------------------- +def test_health_inventory_reads_canonical_table(): + """A row that exists only in ``raw_bars`` must be visible to the health read.""" + from data_pipeline.repos import fetch_ticker_inventory + + init_db() + with get_conn() as conn: + conn.execute( + "INSERT OR REPLACE INTO raw_bars (ticker,date,open,high,low,close,adj_close,volume) " + "VALUES (?,?,?,?,?,?,?,?)", + ("CANON_ONLY", "2026-01-06", 7.0, 7.0, 7.0, 7.0, 7.0, 70.0), + ) + conn.execute("DELETE FROM raw_prices WHERE ticker=?", ("CANON_ONLY",)) + conn.commit() + + rows = [r for r in fetch_ticker_inventory() if r[0] == "CANON_ONLY"] + assert rows, "fetch_ticker_inventory did not read raw_bars" + assert rows[0][1] == 1 + + +# --------------------------------------------------------------------------- +# One-shot backfill script +# --------------------------------------------------------------------------- +def test_migration_script_backfills_legacy_only_rows(): + init_db() + with get_conn() as conn: + conn.execute( + "INSERT OR REPLACE INTO raw_prices (ticker,date,close) VALUES (?,?,?)", + ("LEGACY_ONLY", "2026-01-05", 42.0), + ) + conn.execute("DELETE FROM raw_bars WHERE ticker=?", ("LEGACY_ONLY",)) + conn.commit() + + result = subprocess.run( + [sys.executable, str(REPO_ROOT / "scripts" / "migrate_canonical_tables.py")], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + with get_conn() as conn: + row = conn.execute("SELECT close FROM raw_bars WHERE ticker=?", ("LEGACY_ONLY",)).fetchone() + assert row is not None and row[0] == pytest.approx(42.0) + + +def test_migration_script_is_idempotent(): + """Re-running the backfill must not duplicate or clobber canonical rows.""" + init_db() + with get_conn() as conn: + conn.execute( + "INSERT OR REPLACE INTO raw_prices (ticker,date,close) VALUES (?,?,?)", + ("IDEMPOTENT", "2026-01-05", 1.0), + ) + conn.commit() + + for _ in range(2): + result = subprocess.run( + [sys.executable, str(REPO_ROOT / "scripts" / "migrate_canonical_tables.py")], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + assert _count("raw_bars", "IDEMPOTENT") == 1 diff --git a/tests/test_db.py b/tests/test_db.py index bc6babf..1ea8781 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -14,6 +14,11 @@ def test_creates_tables(self, tmp_path): init_db(db) with sqlite3.connect(db) as conn: tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()} + # canonical store (ADR 0011) + assert "raw_bars" in tables + assert "clean_bars" in tables + assert "feature_bars" in tables + # compatibility shadows — removable one release after the rename assert "raw_prices" in tables assert "clean_prices" in tables assert "processed_prices" in tables diff --git a/tests/test_db_errors.py b/tests/test_db_errors.py index c80c2fd..6383988 100644 --- a/tests/test_db_errors.py +++ b/tests/test_db_errors.py @@ -15,6 +15,11 @@ def test_creates_tables(self, tmp_path): init_db(db_path) with sqlite3.connect(db_path) as conn: tables = [r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()] + # canonical store (ADR 0011) + assert "raw_bars" in tables + assert "clean_bars" in tables + assert "feature_bars" in tables + # compatibility shadows — removable one release after the rename assert "raw_prices" in tables assert "clean_prices" in tables assert "processed_prices" in tables diff --git a/tests/test_health_service.py b/tests/test_health_service.py index 36793bb..414122b 100644 --- a/tests/test_health_service.py +++ b/tests/test_health_service.py @@ -14,7 +14,7 @@ def _seed(ticker: str, dates: list[str], close_vals: list[float | None]) -> None init_db() with get_conn() as conn: conn.executemany( - "INSERT OR REPLACE INTO raw_prices (ticker,date,open,high,low,close,adj_close,volume) " + "INSERT OR REPLACE INTO raw_bars (ticker,date,open,high,low,close,adj_close,volume) " "VALUES (?,?,?,?,?,?,?,?)", [(ticker, d, 1.0, 1.0, 1.0, c, c, 100.0) for d, c in zip(dates, close_vals, strict=True)], ) diff --git a/tests/test_nvda_analysis.py b/tests/test_nvda_analysis.py index 63aafb5..f1bf55b 100644 --- a/tests/test_nvda_analysis.py +++ b/tests/test_nvda_analysis.py @@ -30,8 +30,8 @@ def _extract_job_id(html: str) -> str: # --------------------------------------------------------------------------- -def _seed_clean_prices(ticker: str, n_rows: int = 30, *, nan_only: bool = False): - """Insert synthetic price rows into clean_prices. +def _seed_clean_bars(ticker: str, n_rows: int = 30, *, nan_only: bool = False): + """Insert synthetic price rows into clean_bars. Wipes any previous rows for `ticker` first so the seeded distribution is deterministic regardless of test ordering, and invalidates the in-memory @@ -46,18 +46,18 @@ def _seed_clean_prices(ticker: str, n_rows: int = 30, *, nan_only: bool = False) np.random.seed(42) close = 120.0 + np.cumsum(np.random.randn(n_rows) * 0.5) with get_conn() as conn: - conn.execute("DELETE FROM clean_prices WHERE ticker = ?", (ticker,)) + conn.execute("DELETE FROM clean_bars WHERE ticker = ?", (ticker,)) for i, d in enumerate(dates): date_str = d.strftime("%Y-%m-%d") if nan_only: conn.execute( - "INSERT OR REPLACE INTO clean_prices (ticker, date, is_trading_day, missing_any) VALUES (?,?,?,?)", + "INSERT OR REPLACE INTO clean_bars (ticker, date, is_trading_day, missing_any) VALUES (?,?,?,?)", (ticker, date_str, 0, 1), ) else: c = float(close[i]) conn.execute( - "INSERT OR REPLACE INTO clean_prices " + "INSERT OR REPLACE INTO clean_bars " "(ticker, date, open, high, low, close, adj_close, volume) " "VALUES (?,?,?,?,?,?,?,?)", (ticker, date_str, c - 0.5, c + 1.0, c - 1.0, c, c, 1_000_000), @@ -93,7 +93,7 @@ class TestFeaturesDF: def test_good_data_produces_nonempty_features(self, _patch_downloads): """With 30 rows of price data, features_df should have ~29 rows.""" - _seed_clean_prices("NVDA", 30) + _seed_clean_bars("NVDA", 30) from core.market.analyzer import MarketAnalyzer analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") @@ -103,7 +103,7 @@ def test_good_data_produces_nonempty_features(self, _patch_downloads): def test_nan_only_filler_rows_produce_empty_features(self, _patch_downloads): """NaN-only filler rows from clean_range should not fool is_valid.""" - _seed_clean_prices("NVDA", 5, nan_only=True) + _seed_clean_bars("NVDA", 5, nan_only=True) from core.market.analyzer import MarketAnalyzer analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") @@ -131,14 +131,14 @@ def test_mixed_real_and_nan_rows(self, _patch_downloads): if i < 7: # 7 real rows c = float(close[i]) conn.execute( - "INSERT OR REPLACE INTO clean_prices " + "INSERT OR REPLACE INTO clean_bars " "(ticker, date, open, high, low, close, adj_close, volume) " "VALUES (?,?,?,?,?,?,?,?)", ("NVDA", date_str, c - 0.5, c + 1.0, c - 1.0, c, c, 1_000_000), ) else: # 3 NaN filler rows conn.execute( - "INSERT OR REPLACE INTO clean_prices " + "INSERT OR REPLACE INTO clean_bars " "(ticker, date, is_trading_day, missing_any) VALUES (?,?,?,?)", ("NVDA", date_str, 0, 1), ) @@ -153,7 +153,7 @@ def test_mixed_real_and_nan_rows(self, _patch_downloads): def test_single_row_produces_empty_features(self, _patch_downloads): """Only 1 row of data → shift(1) creates NaN → no valid features.""" - _seed_clean_prices("NVDA", 1) + _seed_clean_bars("NVDA", 1) from core.market.analyzer import MarketAnalyzer analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") @@ -162,7 +162,7 @@ def test_single_row_produces_empty_features(self, _patch_downloads): def test_futu_format_ticker_normalized(self, _patch_downloads): """build_data_context normalizes US.NVDA → NVDA for DB lookup.""" - _seed_clean_prices("NVDA", 10) + _seed_clean_bars("NVDA", 10) from core.market.data_context import build_data_context ctx = build_data_context("US.NVDA", dt.date(2026, 1, 1), "D") @@ -201,7 +201,7 @@ class TestFlaskAnalysisPost: def test_nvda_post_returns_charts(self, client): """POST returns a skeleton; GET /render/statistical produces charts.""" - _seed_clean_prices("NVDA", 60) + _seed_clean_bars("NVDA", 60) resp = client.post( "/", @@ -230,7 +230,7 @@ def test_nvda_post_returns_charts(self, client): def test_nvda_post_futu_format_works(self, client): """POST with US.NVDA (futu format) should also work end-to-end.""" - _seed_clean_prices("NVDA", 60) + _seed_clean_bars("NVDA", 60) resp = client.post( "/", @@ -287,7 +287,7 @@ def test_failed_download_shows_error(self, client): def test_nan_only_db_shows_error(self, client): """DB with NaN-only rows should produce an error fragment from /render/statistical, not blank charts.""" - _seed_clean_prices("NVDA", 5, nan_only=True) + _seed_clean_bars("NVDA", 5, nan_only=True) resp = client.post( "/", data={ @@ -314,7 +314,7 @@ def test_nan_only_db_shows_error(self, client): def test_analysis_service_direct(self, _patch_downloads): """Direct AnalysisService call with good data produces charts.""" - _seed_clean_prices("NVDA", 60) + _seed_clean_bars("NVDA", 60) from services.market.analysis import AnalysisService form_data = { diff --git a/tests/test_processing.py b/tests/test_processing.py index 923f0e8..8606cae 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -30,8 +30,8 @@ def _make_daily(n: int = 30, base_close: float = 100.0) -> pd.DataFrame: return df -def _seed_clean_prices(ticker: str, df: pd.DataFrame) -> None: - """Insert rows into clean_prices table for testing.""" +def _seed_clean_bars(ticker: str, df: pd.DataFrame) -> None: + """Insert rows into the clean_bars table for testing.""" init_db() rows = [] for d, r in df.iterrows(): @@ -53,7 +53,7 @@ def _seed_clean_prices(ticker: str, df: pd.DataFrame) -> None: ) ) upsert_many( - "clean_prices", + "clean_bars", [ "ticker", "date", @@ -176,7 +176,7 @@ class TestProcessFrequencies: def test_basic_pipeline(self): """Process 30 days of synthetic data through all frequencies.""" df = _make_daily(30) - _seed_clean_prices("TEST", df) + _seed_clean_bars("TEST", df) start = df.index[0].date() end = df.index[-1].date() result = process_frequencies("TEST", start, end) @@ -196,14 +196,14 @@ def test_all_frequencies_present(self): from data_pipeline.db import fetch_df df = _make_daily(30) - _seed_clean_prices("FREQ", df) + _seed_clean_bars("FREQ", df) start = df.index[0].date() end = df.index[-1].date() process_frequencies("FREQ", start, end) for freq in ("D", "W", "ME"): out = fetch_df( - "SELECT * FROM processed_prices WHERE ticker=? AND frequency=?", + "SELECT * FROM feature_bars WHERE ticker=? AND frequency=?", ("FREQ", freq), ) assert not out.empty, f"No rows for frequency {freq}" @@ -213,13 +213,13 @@ def test_feature_columns_in_db(self): from data_pipeline.db import fetch_df df = _make_daily(30) - _seed_clean_prices("COLS", df) + _seed_clean_bars("COLS", df) start = df.index[0].date() end = df.index[-1].date() process_frequencies("COLS", start, end) out = fetch_df( - "SELECT * FROM processed_prices WHERE ticker=? AND frequency='D'", + "SELECT * FROM feature_bars WHERE ticker=? AND frequency='D'", ("COLS",), ) for col in ("log_return", "ma_5", "ma_20", "mom_10", "osc"): diff --git a/tests/test_yf_failure_injection.py b/tests/test_yf_failure_injection.py index 78a42a3..d04744f 100644 --- a/tests/test_yf_failure_injection.py +++ b/tests/test_yf_failure_injection.py @@ -38,7 +38,7 @@ _update_locks, ) from data_pipeline.db import fetch_df, init_db, upsert_many -from data_pipeline.downloader import _download_yf, upsert_raw_prices +from data_pipeline.downloader import download_bars, upsert_raw_prices # --------------------------------------------------------------------------- # Helpers @@ -265,7 +265,7 @@ def test_throttle_called_before_download(self): patch("data_pipeline.providers.yfinance_provider.yf_throttle", parent.throttle), patch("data_pipeline.providers.yfinance_provider.yf.download", parent.dl), ): - _download_yf("ORDER_TKR", dt.date(2024, 1, 1), dt.date(2024, 1, 5)) + download_bars("ORDER_TKR", dt.date(2024, 1, 1), dt.date(2024, 1, 5)) # First parent call must be throttle, then download. names = [c[0] for c in parent.mock_calls if c[0] in {"throttle", "dl"}] diff --git a/utils/ticker_utils.py b/utils/ticker_utils.py index fa051ce..26466da 100644 --- a/utils/ticker_utils.py +++ b/utils/ticker_utils.py @@ -32,7 +32,7 @@ # Syntactic whitelist for any ticker we are willing to forward to yfinance / DB. # WHY: Without this, validate_ticker() accepts arbitrary strings (including XSS -# payloads and SQL fragments) and persists rows for them in clean_prices — +# payloads and SQL fragments) and persists rows for them in clean_bars — # turning the DB into an attacker-writable surface and a yfinance request # amplifier. The pattern intentionally permits the formats that yfinance/our # code actually use: From 206f03fe334ca03701ea7f037d7410d869bd7ae3 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Thu, 10 Sep 2026 17:23:40 +0800 Subject: [PATCH 03/15] =?UTF-8?q?fix(migrate):=20dry-run=20=E9=9B=B6?= =?UTF-8?q?=E5=89=AF=E4=BD=9C=E7=94=A8=20+=20=E8=A1=A5=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit review 发现 --dry-run 仍会走 init_db() 建表，与「dry」语义矛盾：改为 先校验 DB 文件存在，只读地报告两族行数；建表与回填仅在非 dry-run 分支执行。 补 tests/test_canonical_tables.py::test_migration_dry_run_has_no_side_effects （断言 raw_bars 未被创建），并在计划书 §8 B2 记录。 --- docs/plans/business_line_reorg.md | 3 ++- scripts/migrate_canonical_tables.py | 23 +++++++++++++++------- tests/test_canonical_tables.py | 30 +++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index fcc9a65..8ff4336 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -456,7 +456,8 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. `data_pipeline/` switched to them. The pre-rename names are kept as shadow tables and `upsert_many` mirrors **both** directions, so an old seeding path, an un-migrated DB and a `git revert` all keep working. `scripts/migrate_canonical_tables.py` backfills an existing DB - (idempotent, `INSERT OR IGNORE`, never clobbers the canonical table). + (idempotent, `INSERT OR IGNORE`, never clobbers the canonical table; `--dry-run` reports without + creating anything — it does not even run `init_db`). - **Ingest is now canonical**: `downloader.download_bars()` (was `_download_yf`) acquires through `providers.get_provider().history()` — i.e. the registry, not a concrete vendor module — and returns `CANONICAL_BAR_COLUMNS`. The yfinance-ism (`Adj Close`→`Adj_Close`) is now confined to diff --git a/scripts/migrate_canonical_tables.py b/scripts/migrate_canonical_tables.py index 4403fae..208e724 100644 --- a/scripts/migrate_canonical_tables.py +++ b/scripts/migrate_canonical_tables.py @@ -34,7 +34,7 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from data_pipeline.db import CANONICAL_TABLES, get_conn, init_db # noqa: E402 +from data_pipeline.db import CANONICAL_TABLES, DB_PATH, get_conn, init_db # noqa: E402 def _columns(conn, table: str) -> list[str]: @@ -74,14 +74,23 @@ def main() -> int: ap.add_argument("--dry-run", action="store_true", help="report what would be copied, change nothing") args = ap.parse_args() - init_db(args.db) - with get_conn(args.db) as conn: - if args.dry_run: + if args.dry_run: + # WHY: a dry run must have NO side effects — in particular it must not + # create the canonical tables, otherwise "dry" would already have + # changed the database it is reporting on. + db_file = Path(args.db) if args.db else Path(DB_PATH) + if not db_file.exists(): + sys.exit(f"[migrate] no database at {db_file}") + with get_conn(str(db_file)) as conn: for legacy, canonical in CANONICAL_TABLES.items(): legacy_n = _count(conn, legacy) if _columns(conn, legacy) else 0 - print(f" {legacy} ({legacy_n} rows) → {canonical}") - print("[migrate] dry run — nothing written") - return 0 + canonical_n = _count(conn, canonical) if _columns(conn, canonical) else 0 + print(f" {legacy} ({legacy_n} rows) → {canonical} ({canonical_n} rows)") + print(f"[migrate] dry run on {db_file} — nothing written") + return 0 + + init_db(args.db) + with get_conn(args.db) as conn: print("[migrate] copying pre-rename tables into the canonical store") total = sum(_copy_table(conn, legacy, canonical) for legacy, canonical in CANONICAL_TABLES.items()) print(f"[migrate] done — {total} row(s) backfilled") diff --git a/tests/test_canonical_tables.py b/tests/test_canonical_tables.py index 60112ba..22fc8bd 100644 --- a/tests/test_canonical_tables.py +++ b/tests/test_canonical_tables.py @@ -151,6 +151,36 @@ def test_migration_script_backfills_legacy_only_rows(): assert row is not None and row[0] == pytest.approx(42.0) +def test_migration_dry_run_has_no_side_effects(): + """`--dry-run` must report without creating or copying anything.""" + import os + + db_file = os.environ["MARKET_DB_PATH"] + init_db(db_file) + with get_conn() as conn: + conn.execute("DROP TABLE IF EXISTS raw_bars") + conn.execute( + "INSERT OR REPLACE INTO raw_prices (ticker,date,close) VALUES (?,?,?)", + ("DRY_RUN", "2026-01-05", 3.0), + ) + conn.commit() + + result = subprocess.run( + [sys.executable, str(REPO_ROOT / "scripts" / "migrate_canonical_tables.py"), "--db", db_file, "--dry-run"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "nothing written" in result.stdout + + with get_conn() as conn: + created = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='raw_bars'").fetchone()[ + 0 + ] + assert created == 0, "dry run created the canonical table" + + def test_migration_script_is_idempotent(): """Re-running the backfill must not duplicate or clobber canonical rows.""" init_db() From a1104aaef033b3e4009d2d880f55e9d6329478f5 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Thu, 10 Sep 2026 17:45:52 +0800 Subject: [PATCH 04/15] =?UTF-8?q?refactor(data-pipeline):=20B3=20=E5=8C=85?= =?UTF-8?q?=E9=87=8D=E6=8E=92=E4=B8=BA=E5=85=AD=E9=98=B6=E6=AE=B5=EF=BC=88?= =?UTF-8?q?providers/store/ingest/transform/read/orchestrate=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 业务线重构计划书 §6 B3；ADR 0011 的目标布局落地。 - data_pipeline/ 拆为六个单向阶段 + _state.py；data_ops/ 删除； yf_client.py 移到 providers/（仍是一 release 兼容 shim，但 services→providers 这条边现在可见） - 守卫升级为子层感知：doc_guard._layer_of / _imported_heads 与 arch_metrics.layer_of 把 data_pipeline// 解析为独立层； _ALLOWED_DEPS 补上 B1 欠下的 providers 层；sqlite-bypass 与 db-access 重新指向 store/db.py、store/repos.py - 断环：orchestrate/scheduler 改调 orchestrate.update.manual_update （原先经 DataService，会与 read→orchestrate 形成环） - tests/test_architecture_purity.py 新增 3 项契约：分层图与声明一致、 transform 不得 import providers、两份层表必须相等 - 文档：l0（新增 data_pipeline 分节 + 度量）、architecture_review §2/§3、 CODEBUDDY/CLAUDE、README、constraints/glossary/automation/ frontend_architecture、.github 指令/技能/agents/failure-registry 路径； tag_baseline 重生成（路径变了、计数仍为 16） 验收：pytest -m "not network" --ignore=tests/e2e → 472 passed / 5 skipped； pytest tests/e2e → 38 passed；ruff check/format clean；doc_guard clean； arch_metrics --check ok（layer 0 / cycles 0 / god 0 / dead 1，无需重置基线）。 --- .github/agents/pipeline-doctor.agent.md | 6 +- .github/copilot-instructions.md | 2 +- .github/data/failure-registry.yaml | 14 +-- .github/data/tag_baseline.json | 24 ++--- .../data-pipeline.instructions.md | 2 +- .github/prompts/new-test.prompt.md | 2 +- .github/skills/debug-pipeline/SKILL.md | 4 +- .../references/pipeline-stages.md | 6 +- .github/skills/fix-review/SKILL.md | 2 +- CLAUDE.md | 55 ++++++----- CODEBUDDY.md | 55 ++++++----- README.md | 51 +++++----- app.py | 4 +- core/market/data_context.py | 4 +- core/options/chain/analyzer.py | 4 +- .../{data_ops/_globals.py => _state.py} | 23 ++++- data_pipeline/data_ops/__init__.py | 32 ------- data_pipeline/ingest/__init__.py | 20 ++++ .../{downloader.py => ingest/ohlcv.py} | 7 +- data_pipeline/orchestrate/__init__.py | 22 +++++ .../_range.py => orchestrate/backfill.py} | 12 +-- data_pipeline/{ => orchestrate}/job_cache.py | 0 data_pipeline/{ => orchestrate}/scheduler.py | 9 +- .../_update.py => orchestrate/update.py} | 17 ++-- data_pipeline/providers/_log.py | 6 +- data_pipeline/{ => providers}/yf_client.py | 4 +- data_pipeline/providers/yf_snapshot.py | 4 +- data_pipeline/providers/yfinance_provider.py | 4 +- data_pipeline/read/__init__.py | 22 +++++ data_pipeline/{data_ops => read}/_query.py | 25 +++-- data_pipeline/{data_ops => read}/facade.py | 50 ++++++---- data_pipeline/store/__init__.py | 18 ++++ data_pipeline/{ => store}/db.py | 0 data_pipeline/{ => store}/quality_log.py | 2 +- data_pipeline/{ => store}/repos.py | 6 +- data_pipeline/transform/__init__.py | 19 ++++ data_pipeline/{ => transform}/cleaning.py | 20 +++- data_pipeline/{ => transform}/processing.py | 8 +- docs/architecture_review.md | 33 ++++--- docs/automation.md | 2 +- docs/constraints.md | 6 +- docs/frontend_architecture.md | 2 +- docs/glossary.md | 2 +- docs/guides/USER_GUIDE.md | 2 +- docs/l0_architecture.md | 37 ++++++-- docs/plans/business_line_reorg.md | 45 ++++++++- routes/core.py | 2 +- routes/data.py | 2 +- scripts/arch_metrics.py | 36 ++++++-- scripts/doc_guard.py | 92 +++++++++++++++---- scripts/migrate_canonical_tables.py | 6 +- scripts/seed_history.py | 2 +- services/market/analysis/summary.py | 2 +- services/market/dispatch.py | 6 +- services/market/facade.py | 2 +- services/market/health.py | 4 +- services/market/signals.py | 2 +- services/market_review/fetch.py | 6 +- services/options/builder.py | 4 +- services/options/chain.py | 2 +- services/options/preload.py | 6 +- services/options/simulation.py | 4 +- services/portfolio/analysis.py | 2 +- services/portfolio/facade.py | 4 +- services/regime/facade.py | 4 +- services/regime/ops/_bootstrap.py | 10 +- services/regime/ops/_persistence.py | 6 +- tests/conftest.py | 4 +- tests/e2e/conftest.py | 12 +-- tests/e2e/test_form_submit_flow.py | 2 +- tests/test_architecture_purity.py | 83 ++++++++++++++++- tests/test_background_backfill.py | 45 +++++---- tests/test_canonical_tables.py | 10 +- tests/test_cleaning.py | 4 +- tests/test_concurrency.py | 68 +++++++------- tests/test_db.py | 4 +- tests/test_db_errors.py | 4 +- tests/test_downloader_gap.py | 15 +-- tests/test_health_service.py | 4 +- tests/test_job_cache.py | 4 +- tests/test_market_review.py | 2 +- tests/test_nvda_analysis.py | 14 +-- tests/test_portfolio.py | 4 +- tests/test_processing.py | 10 +- tests/test_provider_seam.py | 5 +- tests/test_quality_log.py | 8 +- tests/test_regime.py | 2 +- tests/test_render_streaming.py | 2 +- tests/test_route_param_validation.py | 6 +- tests/test_scheduler_lock.py | 2 +- tests/test_scheduler_optional_dep.py | 4 +- tests/test_strategy_builder.py | 4 +- tests/test_yf_failure_injection.py | 15 +-- 93 files changed, 800 insertions(+), 435 deletions(-) rename data_pipeline/{data_ops/_globals.py => _state.py} (60%) delete mode 100644 data_pipeline/data_ops/__init__.py create mode 100644 data_pipeline/ingest/__init__.py rename data_pipeline/{downloader.py => ingest/ohlcv.py} (98%) create mode 100644 data_pipeline/orchestrate/__init__.py rename data_pipeline/{data_ops/_range.py => orchestrate/backfill.py} (95%) rename data_pipeline/{ => orchestrate}/job_cache.py (100%) rename data_pipeline/{ => orchestrate}/scheduler.py (93%) rename data_pipeline/{data_ops/_update.py => orchestrate/update.py} (87%) rename data_pipeline/{ => providers}/yf_client.py (90%) create mode 100644 data_pipeline/read/__init__.py rename data_pipeline/{data_ops => read}/_query.py (90%) rename data_pipeline/{data_ops => read}/facade.py (54%) create mode 100644 data_pipeline/store/__init__.py rename data_pipeline/{ => store}/db.py (100%) rename data_pipeline/{ => store}/quality_log.py (98%) rename data_pipeline/{ => store}/repos.py (96%) create mode 100644 data_pipeline/transform/__init__.py rename data_pipeline/{ => transform}/cleaning.py (92%) rename data_pipeline/{ => transform}/processing.py (96%) diff --git a/.github/agents/pipeline-doctor.agent.md b/.github/agents/pipeline-doctor.agent.md index b3cc561..4dd140f 100644 --- a/.github/agents/pipeline-doctor.agent.md +++ b/.github/agents/pipeline-doctor.agent.md @@ -9,9 +9,9 @@ You are a data pipeline diagnostician for the OptionView project. Your job is to ## Architecture ``` -data_pipeline/downloader.py → raw_bars table -data_pipeline/cleaning.py → clean_bars table -data_pipeline/processing.py → feature_bars table +data_pipeline/ingest/ohlcv.py → raw_bars table +data_pipeline/transform/cleaning.py → clean_bars table +data_pipeline/transform/processing.py → feature_bars table core/price_dynamic.py → features DataFrame core/market_analyzer.py → chart generation services/market/analysis/facade.py → base64 images to frontend diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 5d8e7b1..57cba8e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -25,7 +25,7 @@ Flask-based market analysis dashboard with options strategy tools. ## Database -- SQLite via `data_pipeline/db.py` — always use `get_conn()` context manager +- SQLite via `data_pipeline/store/db.py` — always use `get_conn()` context manager - WAL mode enabled; `PRAGMA synchronous=NORMAL` - DB path from `MARKET_DB_PATH` env var, default `./market_data.sqlite` diff --git a/.github/data/failure-registry.yaml b/.github/data/failure-registry.yaml index 15ce8c5..f101e95 100644 --- a/.github/data/failure-registry.yaml +++ b/.github/data/failure-registry.yaml @@ -25,8 +25,8 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/cleaning.py - - data_pipeline/downloader.py + - data_pipeline/transform/cleaning.py + - data_pipeline/ingest/ohlcv.py - core/price_dynamic.py empty-dataframe: @@ -39,8 +39,8 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/downloader.py - - data_pipeline/processing.py + - data_pipeline/ingest/ohlcv.py + - data_pipeline/transform/processing.py - services/market/analysis/facade.py dtype-mismatch: @@ -53,7 +53,7 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/db.py + - data_pipeline/store/db.py - core/options_greeks.py - core/market_analyzer.py @@ -67,7 +67,7 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/downloader.py + - data_pipeline/ingest/ohlcv.py - utils/utils.py db-error: @@ -80,7 +80,7 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/db.py + - data_pipeline/store/db.py greeks-edge-case: hook_regex: "greeks|black.?scholes|delta|gamma|theta|vega.*nan" diff --git a/.github/data/tag_baseline.json b/.github/data/tag_baseline.json index cd9e9e8..9ca2ba0 100644 --- a/.github/data/tag_baseline.json +++ b/.github/data/tag_baseline.json @@ -1,23 +1,23 @@ { - "files_scanned": 136, + "files_scanned": 147, "tags_by_type": { - "WHY": 12, - "CONSTRAINT": 16, - "TRADEOFF": 2, - "INVARIANT": 3, + "WHY": 13, + "CONSTRAINT": 19, + "TRADEOFF": 3, + "INVARIANT": 9, "DOMAIN": 10, "HACK": 0, "WORKAROUND": 0 }, "uncovered_constants_count": 16, "uncovered_constants": [ - "data_pipeline/data_ops/_globals.py:14: _UPDATE_COOLDOWN=60", - "data_pipeline/data_ops/_globals.py:16: _QUERY_CACHE_TTL=60", - "data_pipeline/data_ops/_range.py:12: _ENSURE_RANGE_TTL=300", - "data_pipeline/data_ops/_range.py:19: _SENTINEL_GAP_THRESHOLD_DAYS=365", - "data_pipeline/data_ops/_range.py:20: _SENTINEL_MIN_DB_SPAN_DAYS=365", - "services/market/charts.py:30: _CACHE_MAX_ENTRIES=64", - "services/options/preload.py:32: CACHE_TTL_MINUTES=15", + "data_pipeline/_state.py:29: _UPDATE_COOLDOWN=60", + "data_pipeline/_state.py:31: _QUERY_CACHE_TTL=60", + "data_pipeline/orchestrate/backfill.py:12: _ENSURE_RANGE_TTL=300", + "data_pipeline/orchestrate/backfill.py:19: _SENTINEL_GAP_THRESHOLD_DAYS=365", + "data_pipeline/orchestrate/backfill.py:20: _SENTINEL_MIN_DB_SPAN_DAYS=365", + "services/market/charts.py:29: _CACHE_MAX_ENTRIES=64", + "services/options/preload.py:34: CACHE_TTL_MINUTES=15", "services/options/simulation.py:36: MAX_STRIKES=15", "services/options/simulation.py:37: MAX_EXPIRIES=6", "services/options/simulation.py:38: MAX_IVS=5", diff --git a/.github/instructions/data-pipeline.instructions.md b/.github/instructions/data-pipeline.instructions.md index b16800b..3894f74 100644 --- a/.github/instructions/data-pipeline.instructions.md +++ b/.github/instructions/data-pipeline.instructions.md @@ -6,7 +6,7 @@ applyTo: "data_pipeline/**" # Data Pipeline Rules ## DB Access -- Always use `get_conn()` context manager from `data_pipeline/db.py` — never raw `sqlite3.connect()` +- Always use `get_conn()` context manager from `data_pipeline/store/db.py` — never raw `sqlite3.connect()` - Use `fetch_df()` for reads, `upsert_many()` for writes - Convert DB-sourced columns with `pd.to_numeric(col, errors='coerce')` before any math — SQLite returns `object` dtype diff --git a/.github/prompts/new-test.prompt.md b/.github/prompts/new-test.prompt.md index 8bc2f71..f89c056 100644 --- a/.github/prompts/new-test.prompt.md +++ b/.github/prompts/new-test.prompt.md @@ -2,7 +2,7 @@ description: "Generate a test following OptionView project patterns for a specific module or function." agent: "agent" tools: [read, search, edit] -argument-hint: "Module or function to test (e.g., 'data_pipeline/cleaning.py clean_range')" +argument-hint: "Module or function to test (e.g., 'data_pipeline/transform/cleaning.py clean_range')" --- Generate a pytest test for the specified module/function following OptionView test conventions: diff --git a/.github/skills/debug-pipeline/SKILL.md b/.github/skills/debug-pipeline/SKILL.md index 7453315..946d307 100644 --- a/.github/skills/debug-pipeline/SKILL.md +++ b/.github/skills/debug-pipeline/SKILL.md @@ -23,10 +23,10 @@ Classify the user's report: | Symptom | Likely Layer | |---------|-------------| | Empty chart panels | core/ (PriceDynamic) or data_pipeline/ (NaN filler rows) | -| "No data for TICKER" message | data_pipeline/downloader.py (download failed) | +| "No data for TICKER" message | data_pipeline/ingest/ohlcv.py (download failed) | | Stale prices (dates from days ago) | data_pipeline/data_service.py (cooldown blocking refresh) | | 429 / timeout errors | yfinance rate-limiting or proxy issue | -| Wrong values in analysis | data_pipeline/cleaning.py or processing.py | +| Wrong values in analysis | data_pipeline/transform/cleaning.py or processing.py | ### Step 2: Check DB State diff --git a/.github/skills/debug-pipeline/references/pipeline-stages.md b/.github/skills/debug-pipeline/references/pipeline-stages.md index 422ea1b..8147f7a 100644 --- a/.github/skills/debug-pipeline/references/pipeline-stages.md +++ b/.github/skills/debug-pipeline/references/pipeline-stages.md @@ -2,7 +2,7 @@ Data flows through 5 stages. A failure at any stage can propagate downstream as empty/NaN data. -## Stage 1: Download (`data_pipeline/downloader.py`) +## Stage 1: Download (`data_pipeline/ingest/ohlcv.py`) **Function**: `upsert_raw_prices(ticker, start, end)` **Input**: Ticker symbol, date range @@ -16,7 +16,7 @@ Data flows through 5 stages. A failure at any stage can propagate downstream as **Check**: `SELECT count(*) FROM raw_bars WHERE ticker=? AND date BETWEEN ? AND ?` -## Stage 2: Clean (`data_pipeline/cleaning.py`) +## Stage 2: Clean (`data_pipeline/transform/cleaning.py`) **Function**: `clean_range(ticker, start, end)` **Input**: Reads from `raw_bars` table @@ -30,7 +30,7 @@ Data flows through 5 stages. A failure at any stage can propagate downstream as **Check**: `SELECT date, missing_any, price_jump_flag FROM clean_bars WHERE ticker=? ORDER BY date DESC LIMIT 10` -## Stage 3: Process (`data_pipeline/processing.py`) +## Stage 3: Process (`data_pipeline/transform/processing.py`) **Function**: `build_features(ticker, frequency)` **Input**: Reads from `clean_bars` table diff --git a/.github/skills/fix-review/SKILL.md b/.github/skills/fix-review/SKILL.md index c6ec761..300dcb8 100644 --- a/.github/skills/fix-review/SKILL.md +++ b/.github/skills/fix-review/SKILL.md @@ -61,7 +61,7 @@ Verify the fix uses the correct error pattern for its layer: For each changed production file, check that a corresponding test exists: ```bash # Map production file to test file -# data_pipeline/downloader.py → tests/test_yf_download.py or tests/test_processing.py +# data_pipeline/ingest/ohlcv.py → tests/test_yf_download.py or tests/test_processing.py # core/market_analyzer.py → tests/test_market_review.py # services/market/validation.py → tests/test_validation.py ``` diff --git a/CLAUDE.md b/CLAUDE.md index 0bcb48b..9e99b74 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -123,7 +123,7 @@ Always reference and import them package-qualified (`from core.options.greeks im `POST /` computes **nothing**. `routes/core.py::index` normalises the form (`FormService.extract_form_data` → `ValidationService.validate_input_data`), registers a job via -`data_pipeline/job_cache.py::create_job` (TTL default 90 s), and renders `templates/index.html` +`data_pipeline/orchestrate/job_cache.py::create_job` (TTL default 90 s), and renders `templates/index.html` with `streaming_mode=True`. Each tab shell emits an HTMX placeholder (`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the browser fans out parallel requests. @@ -146,26 +146,37 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is ### `data_pipeline/` specifics -- **`data_ops/` — `DataService` (facade)** is the single read entry point. `ensure_range(ticker, - start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which stops concurrent UI - requests from stampeding Yahoo. `_query.py` calls `_update`/`_range` module functions directly - (never the facade) to avoid an import cycle. +Re-homed in batch B3 of ADR 0011 into six one-way stages; the authoritative layer table is +`docs/architecture_review.md` §3 and is enforced by `doc_guard` + `tests/test_architecture_purity.py`: + +- **`read/` — `DataService` (facade)** is the single read entry point above the package. + `ensure_range(ticker, start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which + stops concurrent UI requests from stampeding Yahoo. The facade and `read/_query.py` call the + `orchestrate` drivers directly (never each other's facade) — `read → orchestrate` is why + `orchestrate` must not import `read` (no cycle). - **`providers/`** is the **only** package allowed to call yfinance (the chokepoint moved here from - `yf_client.py` in batch B1 of ADR 0011; enforced by `doc_guard` `single-yf-exit`, exceptions - registered in `docs/architecture_review.md` §2). It owns the mapping from the vendor's fields onto - one canonical schema (`providers/base.py`) — IV as a decimal, nullable bid/ask, no `inTheMoney`. - Every call goes through `yf_throttle()` (token bucket, 5 req/s, burst 5). `yf_client.py` is a - one-release compatibility shim over the package and `downloader.py` keeps only gap detection + - upsert. **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently - fails (ADR 0005). -- **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). Tables are - named canonically (`raw_bars` / `clean_bars` / `feature_bars`); the pre-rename names - (`raw_prices` / `clean_prices` / `processed_prices`) are kept as shadows for one release — every - `upsert_many` writes both families, and `scripts/migrate_canonical_tables.py` backfills an existing - DB. `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, - `busy_timeout=5000`) and does **not** close on exit. `repos.py` is the only place that builds SQL. -- **`cleaning.py` / `processing.py`** — align to business days, mark gaps NA with **no - interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. + `yf_client.py` in batch B1; enforced by `doc_guard` `single-yf-exit`, exceptions registered in + `docs/architecture_review.md` §2). It owns the mapping from the vendor's fields onto one canonical + schema (`providers/base.py`) — IV as a decimal, nullable bid/ask, no `inTheMoney`. Every call goes + through `yf_throttle()` (token bucket, 5 req/s, burst 5). `providers/yf_client.py` is a one-release + compatibility shim over the package; `_registry.py` is the `MARKET_DATA_PROVIDER` seam. + **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails + (ADR 0005). +- **`store/`** — `db.py` (`init_db()` uses `CREATE TABLE IF NOT EXISTS`; no migration framework), + `repos.py` (the only place that builds SQL) and `quality_log.py`. Tables are named canonically + (`raw_bars` / `clean_bars` / `feature_bars`); the pre-rename names (`raw_prices` / `clean_prices` / + `processed_prices`) are kept as shadows for one release — every `upsert_many` writes both families, + and `scripts/migrate_canonical_tables.py` backfills an existing DB. `get_conn()` yields a + **thread-local** WAL connection (`synchronous=NORMAL`, `busy_timeout=5000`) and does **not** close + on exit. +- **`ingest/ohlcv.py`** — business-day gap detection + `raw_bars` upsert; acquisition goes through + `providers.get_provider().history()`, so this module never names a vendor. +- **`transform/cleaning.py` / `transform/processing.py`** — align to business days, mark gaps NA with + **no interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. + Never imports `providers/` (asserted by a test). +- **`orchestrate/`** — `update.py` (incremental/full drivers), `backfill.py` (chunked coverage + repair), `job_cache.py` (streaming slice memo), `scheduler.py` (optional APScheduler). +- **`_state.py`** — process-local query cache + update locks, shared by `read` and `orchestrate`. - No option-chain history exists from yfinance — no IV rank/percentile/backtests; HV percentile is the deliberate substitute (ADR 0004). @@ -211,8 +222,8 @@ Pages-only. Rendered `site/index.html` / `site/static/` are build artefacts — | Question | File | |---|---| | How does a request get served? | `routes/core.py` → `services/market/dispatch.py` | -| Where does data come from? | `data_pipeline/data_ops/facade.py`, `_range.py`, `yf_client.py` | -| Schema / SQL | `data_pipeline/db.py` (`init_db`), `repos.py` | +| Where does data come from? | `data_pipeline/read/facade.py`, `orchestrate/backfill.py`, `providers/` | +| Schema / SQL | `data_pipeline/store/db.py` (`init_db`), `store/repos.py` | | Chart / analysis maths | `core/market/analyzer.py`, `core/options/`, `core/strategies/` | | Frontend contract | `docs/frontend_architecture.md` | | Why is this weird? | `docs/constraints.md`, then `docs/decisions/` | diff --git a/CODEBUDDY.md b/CODEBUDDY.md index 0bcb48b..9e99b74 100644 --- a/CODEBUDDY.md +++ b/CODEBUDDY.md @@ -123,7 +123,7 @@ Always reference and import them package-qualified (`from core.options.greeks im `POST /` computes **nothing**. `routes/core.py::index` normalises the form (`FormService.extract_form_data` → `ValidationService.validate_input_data`), registers a job via -`data_pipeline/job_cache.py::create_job` (TTL default 90 s), and renders `templates/index.html` +`data_pipeline/orchestrate/job_cache.py::create_job` (TTL default 90 s), and renders `templates/index.html` with `streaming_mode=True`. Each tab shell emits an HTMX placeholder (`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the browser fans out parallel requests. @@ -146,26 +146,37 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is ### `data_pipeline/` specifics -- **`data_ops/` — `DataService` (facade)** is the single read entry point. `ensure_range(ticker, - start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which stops concurrent UI - requests from stampeding Yahoo. `_query.py` calls `_update`/`_range` module functions directly - (never the facade) to avoid an import cycle. +Re-homed in batch B3 of ADR 0011 into six one-way stages; the authoritative layer table is +`docs/architecture_review.md` §3 and is enforced by `doc_guard` + `tests/test_architecture_purity.py`: + +- **`read/` — `DataService` (facade)** is the single read entry point above the package. + `ensure_range(ticker, start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which + stops concurrent UI requests from stampeding Yahoo. The facade and `read/_query.py` call the + `orchestrate` drivers directly (never each other's facade) — `read → orchestrate` is why + `orchestrate` must not import `read` (no cycle). - **`providers/`** is the **only** package allowed to call yfinance (the chokepoint moved here from - `yf_client.py` in batch B1 of ADR 0011; enforced by `doc_guard` `single-yf-exit`, exceptions - registered in `docs/architecture_review.md` §2). It owns the mapping from the vendor's fields onto - one canonical schema (`providers/base.py`) — IV as a decimal, nullable bid/ask, no `inTheMoney`. - Every call goes through `yf_throttle()` (token bucket, 5 req/s, burst 5). `yf_client.py` is a - one-release compatibility shim over the package and `downloader.py` keeps only gap detection + - upsert. **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently - fails (ADR 0005). -- **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). Tables are - named canonically (`raw_bars` / `clean_bars` / `feature_bars`); the pre-rename names - (`raw_prices` / `clean_prices` / `processed_prices`) are kept as shadows for one release — every - `upsert_many` writes both families, and `scripts/migrate_canonical_tables.py` backfills an existing - DB. `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, - `busy_timeout=5000`) and does **not** close on exit. `repos.py` is the only place that builds SQL. -- **`cleaning.py` / `processing.py`** — align to business days, mark gaps NA with **no - interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. + `yf_client.py` in batch B1; enforced by `doc_guard` `single-yf-exit`, exceptions registered in + `docs/architecture_review.md` §2). It owns the mapping from the vendor's fields onto one canonical + schema (`providers/base.py`) — IV as a decimal, nullable bid/ask, no `inTheMoney`. Every call goes + through `yf_throttle()` (token bucket, 5 req/s, burst 5). `providers/yf_client.py` is a one-release + compatibility shim over the package; `_registry.py` is the `MARKET_DATA_PROVIDER` seam. + **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails + (ADR 0005). +- **`store/`** — `db.py` (`init_db()` uses `CREATE TABLE IF NOT EXISTS`; no migration framework), + `repos.py` (the only place that builds SQL) and `quality_log.py`. Tables are named canonically + (`raw_bars` / `clean_bars` / `feature_bars`); the pre-rename names (`raw_prices` / `clean_prices` / + `processed_prices`) are kept as shadows for one release — every `upsert_many` writes both families, + and `scripts/migrate_canonical_tables.py` backfills an existing DB. `get_conn()` yields a + **thread-local** WAL connection (`synchronous=NORMAL`, `busy_timeout=5000`) and does **not** close + on exit. +- **`ingest/ohlcv.py`** — business-day gap detection + `raw_bars` upsert; acquisition goes through + `providers.get_provider().history()`, so this module never names a vendor. +- **`transform/cleaning.py` / `transform/processing.py`** — align to business days, mark gaps NA with + **no interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. + Never imports `providers/` (asserted by a test). +- **`orchestrate/`** — `update.py` (incremental/full drivers), `backfill.py` (chunked coverage + repair), `job_cache.py` (streaming slice memo), `scheduler.py` (optional APScheduler). +- **`_state.py`** — process-local query cache + update locks, shared by `read` and `orchestrate`. - No option-chain history exists from yfinance — no IV rank/percentile/backtests; HV percentile is the deliberate substitute (ADR 0004). @@ -211,8 +222,8 @@ Pages-only. Rendered `site/index.html` / `site/static/` are build artefacts — | Question | File | |---|---| | How does a request get served? | `routes/core.py` → `services/market/dispatch.py` | -| Where does data come from? | `data_pipeline/data_ops/facade.py`, `_range.py`, `yf_client.py` | -| Schema / SQL | `data_pipeline/db.py` (`init_db`), `repos.py` | +| Where does data come from? | `data_pipeline/read/facade.py`, `orchestrate/backfill.py`, `providers/` | +| Schema / SQL | `data_pipeline/store/db.py` (`init_db`), `store/repos.py` | | Chart / analysis maths | `core/market/analyzer.py`, `core/options/`, `core/strategies/` | | Frontend contract | `docs/frontend_architecture.md` | | Why is this weird? | `docs/constraints.md`, then `docs/decisions/` | diff --git a/README.md b/README.md index bff8620..3a7b5b7 100644 --- a/README.md +++ b/README.md @@ -82,17 +82,14 @@ app.py Flask entry point — registers blueprints, middlew ├── decision/ Put-selling candidate scoring pipeline ├── correlation_validator.py Rolling pairwise correlations └── _shared/ Plotting helpers, types, validators -└── data_pipeline/ Download · clean · process · persist - ├── data_ops/ DataService facade (DB-first cache, 60 s freshness) - ├── db.py SQLite context manager (WAL, synchronous=NORMAL) - ├── repos.py SQL builders (prices, regime, positions, …) - ├── yf_client.py Single chokepoint for yfinance (token-bucket throttle, proxy) - ├── downloader.py Raw bar / chain fetch with gap detection - ├── cleaning.py Time-series alignment; gaps marked NA (no interpolation) - ├── processing.py Feature engineering (returns, MAs, HV) - ├── scheduler.py APScheduler daily + monthly correlation refresh - ├── job_cache.py In-process TTL cache for /render/ payloads - └── quality_log.py Pipeline anomaly persistence +└── data_pipeline/ Acquire · process · serve (six one-way stages, ADR 0011) + ├── providers/ The ONLY `import yfinance`: vendor adapter + canonical schema + registry + ├── store/ Canonical schema, the only SQL, the failure log + ├── ingest/ Business-day gap detection + raw_bars upsert + ├── transform/ Alignment + anomaly flags + feature engineering (provider-agnostic) + ├── read/ DataService facade (DB-first cache, 60 s freshness) + ├── orchestrate/ Update/backfill drivers, job cache, optional scheduler + └── _state.py Process-local query cache + update locks └── utils/ Shared helpers (ticker normalisation, error envelopes, …) ├── constants.py Domain defaults (DEFAULT_TICKER, FREQUENCY_DISPLAY, …) ├── date_helpers.py parse_month_str, exclusive_month_end @@ -146,12 +143,12 @@ Packaged by business domain; each package exposes a `facade.py` entry point. | File | Role | Pulls from | |---|---|---| | [`services/market/facade.py`](services/market/facade.py) | Ticker validation + market-review summary for `/api/validate_*` and `/render/market_review`. | `core/market_review`, `core/market.data_context` | -| [`services/market/analysis/facade.py`](services/market/analysis/facade.py) | Top-level "run a full market analysis" facade for `/render/statistical` and `/render/assessment`. | `core/market.analyzer`, `core/market.correlation_validator`, `data_pipeline/data_ops` | -| [`services/market/charts.py`](services/market/charts.py) | Builds matplotlib figures and returns base64 PNGs; caches by `(ticker, kind, params)`. | `core/*`, `data_pipeline/data_ops` | -| [`services/market/signals.py`](services/market/signals.py) | Wraps `core/signals` over DB-cached daily bars for `/api/signals`. | `core/signals`, `data_pipeline/data_ops` | +| [`services/market/analysis/facade.py`](services/market/analysis/facade.py) | Top-level "run a full market analysis" facade for `/render/statistical` and `/render/assessment`. | `core/market.analyzer`, `core/market.correlation_validator`, `data_pipeline/read` | +| [`services/market/charts.py`](services/market/charts.py) | Builds matplotlib figures and returns base64 PNGs; caches by `(ticker, kind, params)`. | `core/*`, `data_pipeline/read` | +| [`services/market/signals.py`](services/market/signals.py) | Wraps `core/signals` over DB-cached daily bars for `/api/signals`. | `core/signals`, `data_pipeline/read` | | [`services/market/form.py`](services/market/form.py) | Extracts and normalises POST form fields, applying defaults from `utils/constants.py`. | `utils/constants`, `utils/date_helpers` | | [`services/market/validation.py`](services/market/validation.py) | Pure form-value validation rules (date ranges, frequency, …). | (none) | -| [`services/market/health.py`](services/market/health.py) | Aggregates DB freshness / row-count / NaN metrics for `/health/*`. | `data_pipeline/db`, `data_pipeline/repos` | +| [`services/market/health.py`](services/market/health.py) | Aggregates DB freshness / row-count / NaN metrics for `/health/*`. | `data_pipeline/store` | | [`services/market/dispatch.py`](services/market/dispatch.py) | Shared `/render/` handler: job lookup, memoisation, fragment render. | `services/market/analysis`, `services/options/chain` | **`services/options/`** @@ -168,15 +165,15 @@ Packaged by business domain; each package exposes a `facade.py` entry point. | File | Role | Pulls from | |---|---|---| -| [`services/portfolio/facade.py`](services/portfolio/facade.py) | CRUD for tracked positions in SQLite; computes live P&L via `data_pipeline/repos`. | `core/portfolio`, `data_pipeline/repos` | +| [`services/portfolio/facade.py`](services/portfolio/facade.py) | CRUD for tracked positions in SQLite; computes live P&L via `data_pipeline/store/repos`. | `core/portfolio`, `data_pipeline/store/repos` | | [`services/portfolio/analysis.py`](services/portfolio/analysis.py) | Stateless "analyse this basket of legs" endpoint backing `/api/portfolio_analysis`. | `core/options/greeks/portfolio`, `core/strategies` | **`services/regime/`** | File | Role | Pulls from | |---|---|---| -| [`services/regime/facade.py`](services/regime/facade.py) | Labels & persists market regimes; serves `/api/regime/*`. | `core/regime`, `data_pipeline/repos` | -| [`services/regime/ops/`](services/regime/ops/) | History bootstrap + `regime_log` read/write helpers. | `data_pipeline/db`, `data_pipeline/downloader` | +| [`services/regime/facade.py`](services/regime/facade.py) | Labels & persists market regimes; serves `/api/regime/*`. | `core/regime`, `data_pipeline/store/repos` | +| [`services/regime/ops/`](services/regime/ops/) | History bootstrap + `regime_log` read/write helpers. | `data_pipeline/store/db`, `data_pipeline/downloader` | ### `core/` — pure computation (no Flask, no I/O) @@ -201,16 +198,12 @@ Packaged by business domain; each package exposes a `facade.py` entry point. | File | Role | |---|---| -| [`data_pipeline/yf_client.py`](data_pipeline/yf_client.py) | Single chokepoint for `yfinance` calls: token-bucket throttle, proxy probe, error mapping. | -| [`data_pipeline/downloader.py`](data_pipeline/downloader.py) | Uses `yf_client` to fetch raw bars / option chains, with gap detection against the DB. | -| [`data_pipeline/cleaning.py`](data_pipeline/cleaning.py) | Aligns to business days and drops broken rows; missing gaps are marked NA — never interpolated. | -| [`data_pipeline/processing.py`](data_pipeline/processing.py) | Feature engineering (returns, MAs, HV) on cleaned bars. | -| [`data_pipeline/data_ops/`](data_pipeline/data_ops/) | `DataService` facade — DB-first cache with a 60 s freshness window, the single read entry-point. | -| [`data_pipeline/db.py`](data_pipeline/db.py) | `get_conn()` context manager, schema bootstrap, WAL pragmas, thread-local connection pooling. | -| [`data_pipeline/repos.py`](data_pipeline/repos.py) | Repository wrappers — the only modules that build SQL. | -| [`data_pipeline/scheduler.py`](data_pipeline/scheduler.py) | APScheduler wrapper: daily backfill + monthly correlation refresh, gated by a leader-lock file. APScheduler is imported lazily — it is optional and only needed when `AUTO_UPDATE_TICKERS` is set. | -| [`data_pipeline/job_cache.py`](data_pipeline/job_cache.py) | TTL'd in-process map keyed by `job_id`; lets `/render/` partials share the same form payload. | -| [`data_pipeline/quality_log.py`](data_pipeline/quality_log.py) | Persists pipeline anomalies for `/health/data`. | +| [`data_pipeline/providers/`](data_pipeline/providers/) | The single chokepoint for `yfinance` (adapter + canonical mapping + registry) and the token-bucket throttle / proxy probe. | +| [`data_pipeline/ingest/`](data_pipeline/ingest/) | Business-day gap detection and the `raw_bars` upsert, acquisition via `providers.get_provider()`. | +| [`data_pipeline/transform/`](data_pipeline/transform/) | Business-day alignment, anomaly flags (gaps NA — never interpolated) and feature engineering (returns, MAs, HV). | +| [`data_pipeline/read/`](data_pipeline/read/) | `DataService` facade — DB-first cache with a 60 s freshness window, the single read entry-point. | +| [`data_pipeline/store/`](data_pipeline/store/) | `db.py` (`get_conn()`, schema bootstrap, WAL pragmas, thread-local pooling), `repos.py` (the only SQL), `quality_log.py`. | +| [`data_pipeline/orchestrate/`](data_pipeline/orchestrate/) | Update/seed drivers, chunked backfill, the `/render/` TTL cache, and the optional APScheduler wrapper (lazy import; only needed when `AUTO_UPDATE_TICKERS` is set). | ### `utils/` @@ -369,7 +362,7 @@ See [`.env.example`](.env.example) for the full list. The most relevant ones: with a token bucket (default 5 req/s, burst 5) and uses a DB-first cache to avoid redundant downloads. Do not pass `session=requests.Session()` — yfinance uses `curl_cffi` and silently breaks otherwise. -- **DB layer**: always go through `data_pipeline/db.py::get_conn()`; it +- **DB layer**: always go through `data_pipeline/store/db.py::get_conn()`; it enables WAL mode, sets `synchronous=NORMAL`, and is safe to share across threads. - **Logging**: use `logging.getLogger(__name__)`; no `print()` in production diff --git a/app.py b/app.py index 2a9be9b..92c70c9 100644 --- a/app.py +++ b/app.py @@ -13,8 +13,8 @@ from dotenv import load_dotenv from flask import Flask -from data_pipeline.data_ops import DataService -from data_pipeline.scheduler import UpdateScheduler, acquire_scheduler_lock +from data_pipeline.orchestrate.scheduler import UpdateScheduler, acquire_scheduler_lock +from data_pipeline.read import DataService from utils.network import init_yf_proxy load_dotenv() diff --git a/core/market/data_context.py b/core/market/data_context.py index 44450e3..d7e105d 100644 --- a/core/market/data_context.py +++ b/core/market/data_context.py @@ -63,7 +63,7 @@ def _validate_inputs(ticker, start_date, frequency, end_date=None): def _fetch_daily_from_db(ticker: str, download_start: dt.date): - from data_pipeline.data_ops import DataService # doc-guard: allow=core-purity + from data_pipeline.read import DataService # doc-guard: allow=core-purity try: DataService.initialize() @@ -96,7 +96,7 @@ def _fetch_daily_from_db(ticker: str, download_start: dt.date): def _download_data(ticker: str, download_start: dt.date): - from data_pipeline.yf_client import fetch_daily_ohlcv # doc-guard: allow=core-purity + from data_pipeline.providers.yf_client import fetch_daily_ohlcv # doc-guard: allow=core-purity yf_end = dt.date.today() + dt.timedelta(days=1) df = fetch_daily_ohlcv( diff --git a/core/options/chain/analyzer.py b/core/options/chain/analyzer.py index efe4d80..03b1147 100644 --- a/core/options/chain/analyzer.py +++ b/core/options/chain/analyzer.py @@ -113,7 +113,7 @@ class OptionsChainAnalyzer: """Analyses an option chain snapshot. INVARIANT: this class performs no I/O. Callers fetch the snapshot upstream - (``data_pipeline.yf_client.fetch_option_chain``) and inject it via + (``data_pipeline.providers.yf_client.fetch_option_chain``) and inject it via ``snapshot=``. WHY: keeping ``core/`` pure means the analyzer can be driven entirely by fixture data in tests, and every network call stays behind the single yfinance exit point where proxy setup and throttling are enforced. @@ -123,7 +123,7 @@ def __init__(self, ticker: str = "^SPX", *, snapshot: dict): if snapshot is None: raise ValueError( "OptionsChainAnalyzer requires snapshot=... — fetch it upstream via " - "data_pipeline.yf_client.fetch_option_chain (core/ must stay pure)" + "data_pipeline.providers.yf_client.fetch_option_chain (core/ must stay pure)" ) self.ticker = ticker self._init_from_snapshot(snapshot) diff --git a/data_pipeline/data_ops/_globals.py b/data_pipeline/_state.py similarity index 60% rename from data_pipeline/data_ops/_globals.py rename to data_pipeline/_state.py index 933a9ff..e65313f 100644 --- a/data_pipeline/data_ops/_globals.py +++ b/data_pipeline/_state.py @@ -1,8 +1,23 @@ -"""Shared globals for data operations (locks, caches, TTLs). +"""Process-local shared state for the read + orchestrate layers. -All heavy-lifting state (cooldown locks, query cache, TTL constants) is -co-located here so that ``data_ops.facade.DataService`` and tests operate -on the **same** underlying objects. +Domain: Data Pipeline — Shared State +Context: + - All heavy-lifting state (update cooldown locks, the query cache, TTL + constants) is co-located here so ``read`` and ``orchestrate`` — and the + tests that inspect them — operate on the **same** underlying objects. + - It lives at the ``data_pipeline/`` root rather than in ``read/`` or + ``orchestrate/`` because both layers need it and neither may import the + other's internals (``read`` imports ``orchestrate``, so ``orchestrate`` + must not import ``read``). See the layer table in + docs/architecture_review.md §3. +Why not in ``store/``: this is in-memory state, not persistence. +Contracts: + - ``_query_cache`` / ``_cache_get`` / ``_cache_set`` / ``_cache_invalidate`` + - ``_update_locks`` / ``_update_lock_mutex`` / ``_UPDATE_COOLDOWN`` / ``GAP_SCAN_DAYS`` +Dependencies UPWARD: + - (none — stdlib + pandas only) +Dependencies DOWNWARD: + - read/ (query cache), orchestrate/ (update locks), tests """ import os diff --git a/data_pipeline/data_ops/__init__.py b/data_pipeline/data_ops/__init__.py deleted file mode 100644 index b038b93..0000000 --- a/data_pipeline/data_ops/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Data operations — DataService facade and shared cache/lock globals. - -All heavy-lifting state (cooldown locks, query cache, TTL constants) is -co-located in ``_globals`` so that DataService and tests operate on the -same underlying objects. -""" - -from ._globals import ( - _QUERY_CACHE_TTL, - GAP_SCAN_DAYS, - _cache_get, - _cache_invalidate, - _cache_set, - _query_cache, - _query_cache_lock, - _update_lock_mutex, - _update_locks, -) -from .facade import DataService - -__all__ = [ - "DataService", - "_query_cache", - "_query_cache_lock", - "_update_lock_mutex", - "_update_locks", - "_cache_get", - "_cache_set", - "_cache_invalidate", - "_QUERY_CACHE_TTL", - "GAP_SCAN_DAYS", -] diff --git a/data_pipeline/ingest/__init__.py b/data_pipeline/ingest/__init__.py new file mode 100644 index 0000000..d12cb27 --- /dev/null +++ b/data_pipeline/ingest/__init__.py @@ -0,0 +1,20 @@ +"""INGEST — acquisition → store glue. + +Domain: Data Pipeline — Ingest +Context: + - ADR 0011: ``providers/`` touches the external API and returns canonical + data; this package is the glue that decides *what* to fetch (business-day + gap detection, the auto-backfill cap) and writes it to ``raw_bars``. + - Live snapshots (spot / option chain) have no ingest module on purpose: they + are never persisted (ADR 0004), so there is nothing to ingest — callers go + to ``providers`` directly. +Contracts: + - ``ohlcv.download_bars`` — canonical bars for a window, provider-agnostic. + - ``ohlcv.upsert_raw_prices`` — never raises; degrades through ``PipelineResult``. +Dependencies UPWARD: + - providers (acquisition), store (raw_bars), data_pipeline (PipelineResult) +Dependencies DOWNWARD: + - orchestrate (the "make it ready" driver) +""" + +from __future__ import annotations diff --git a/data_pipeline/downloader.py b/data_pipeline/ingest/ohlcv.py similarity index 98% rename from data_pipeline/downloader.py rename to data_pipeline/ingest/ohlcv.py index 6cfdfb2..0106393 100644 --- a/data_pipeline/downloader.py +++ b/data_pipeline/ingest/ohlcv.py @@ -15,7 +15,7 @@ Dependencies UPWARD: - providers.yfinance_provider (download), .db (fetch_df / upsert_many) Dependencies DOWNWARD: - - data_pipeline/data_ops (_update / _range), services/regime/ops/_bootstrap.py + - data_pipeline/orchestrate (update / backfill), services/regime/ops/_bootstrap.py """ import datetime as dt @@ -25,12 +25,11 @@ import pandas as pd +from data_pipeline import PipelineResult from data_pipeline.providers import get_provider from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS from data_pipeline.providers.yfinance_provider import to_canonical_bars - -from . import PipelineResult -from .db import fetch_df, upsert_many +from data_pipeline.store.db import fetch_df, upsert_many logger = logging.getLogger(__name__) diff --git a/data_pipeline/orchestrate/__init__.py b/data_pipeline/orchestrate/__init__.py new file mode 100644 index 0000000..eba4189 --- /dev/null +++ b/data_pipeline/orchestrate/__init__.py @@ -0,0 +1,22 @@ +"""ORCHESTRATE — the "make the data ready" layer. + +Domain: Data Pipeline — Orchestrate +Context: + - ADR 0011: this package owns everything that *sequences* the pipeline: + the incremental/full update drivers, the chunked backfill loop, the + streaming slice memo, and the optional cron scheduler. It is the only + layer allowed to call ingest + transform + store together. + - CONSTRAINT (docs/constraints.md §6): no job queue. Work either runs inside + a request or on a bounded daemon thread with an 8s grace window. +Contracts: + - ``update.manual_update`` / ``update.seed_history`` — the pipeline drivers. + - ``backfill.ensure_range`` / ``backfill.needs_backfill`` — chunked coverage repair. + - ``job_cache`` — TTL'd streaming slice memo (``create_job`` / ``compute_or_get``). + - ``scheduler`` — optional APScheduler entry point (leader-locked). +Dependencies UPWARD: + - ingest, transform, store, data_pipeline (PipelineResult + _state) +Dependencies DOWNWARD: + - services/, routes/, app.py +""" + +from __future__ import annotations diff --git a/data_pipeline/data_ops/_range.py b/data_pipeline/orchestrate/backfill.py similarity index 95% rename from data_pipeline/data_ops/_range.py rename to data_pipeline/orchestrate/backfill.py index e994f80..236be6f 100644 --- a/data_pipeline/data_ops/_range.py +++ b/data_pipeline/orchestrate/backfill.py @@ -5,7 +5,7 @@ import threading import time -import data_pipeline.db as _db +import data_pipeline.store.db as _db logger = logging.getLogger(__name__) @@ -102,10 +102,10 @@ def ensure_range(ticker: str, start: dt.date, end: dt.date) -> bool: def _ensure_range_impl(ticker: str, start: dt.date, end: dt.date, now: float, was_sentinel: bool = False) -> bool: """Internal: actual backfill. Caller must hold the in-flight slot.""" - import data_pipeline.cleaning as _cl - import data_pipeline.downloader as _dl - import data_pipeline.processing as _pr - from data_pipeline.downloader import MAX_AUTO_BACKFILL_DAYS + import data_pipeline.ingest.ohlcv as _dl + import data_pipeline.transform.cleaning as _cl + import data_pipeline.transform.processing as _pr + from data_pipeline.ingest.ohlcv import MAX_AUTO_BACKFILL_DAYS cov = _db.fetch_df( "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_bars WHERE ticker=?", @@ -177,7 +177,7 @@ def _ensure_range_impl(ticker: str, start: dt.date, end: dt.date, now: float, wa if not pr.ok: logger.warning("ensure_range processing failed for %s: %s", ticker, pr.error) return False - from . import _globals as _g + from data_pipeline import _state as _g _g._cache_invalidate(ticker) with _ensure_range_lock: diff --git a/data_pipeline/job_cache.py b/data_pipeline/orchestrate/job_cache.py similarity index 100% rename from data_pipeline/job_cache.py rename to data_pipeline/orchestrate/job_cache.py diff --git a/data_pipeline/scheduler.py b/data_pipeline/orchestrate/scheduler.py similarity index 93% rename from data_pipeline/scheduler.py rename to data_pipeline/orchestrate/scheduler.py index 212b77c..23aaabe 100644 --- a/data_pipeline/scheduler.py +++ b/data_pipeline/orchestrate/scheduler.py @@ -11,7 +11,10 @@ import os from pathlib import Path -from .data_ops import DataService +# WHY (no DataService): the scheduler drives the pipeline, it does not read. +# Going through the read facade here would create read <-> orchestrate cycle +# (read/_query.py already imports this package to trigger refreshes). +from data_pipeline.orchestrate.update import manual_update logger = logging.getLogger(__name__) @@ -93,7 +96,7 @@ def start_daily_update(self, tickers: list[str]): def job(): for t in tickers: try: - DataService.manual_update(t, days=7) + manual_update(t, days=7) logger.info(f"Auto-updated {t}") except Exception as e: logger.exception(f"Auto-update failed for {t}: {e}") @@ -115,7 +118,7 @@ def correlation_job(): for t in tickers: try: # Trigger a full data update which includes correlation recalculation - DataService.manual_update(t, days=30) + manual_update(t, days=30) logger.info(f"Monthly correlation update completed for {t}") except Exception as e: logger.exception(f"Monthly correlation update failed for {t}: {e}") diff --git a/data_pipeline/data_ops/_update.py b/data_pipeline/orchestrate/update.py similarity index 87% rename from data_pipeline/data_ops/_update.py rename to data_pipeline/orchestrate/update.py index 15806a6..8f7ee59 100644 --- a/data_pipeline/data_ops/_update.py +++ b/data_pipeline/orchestrate/update.py @@ -4,10 +4,9 @@ import logging import time +from data_pipeline import _state as _g from utils.ticker_utils import is_valid_ticker_format -from . import _globals as _g - logger = logging.getLogger(__name__) @@ -38,7 +37,7 @@ def manual_update(ticker: str, days: int = 7) -> bool: start = end - dt.timedelta(days=days - 1) scan_start = end - dt.timedelta(days=_g.GAP_SCAN_DAYS) - from data_pipeline.downloader import find_missing_business_days + from data_pipeline.ingest.ohlcv import find_missing_business_days gaps = find_missing_business_days(ticker, scan_start, end) if gaps and min(gaps) < start: @@ -51,9 +50,9 @@ def manual_update(ticker: str, days: int = 7) -> bool: ) start = min(gaps) - import data_pipeline.cleaning as _cl - import data_pipeline.downloader as _dl - import data_pipeline.processing as _pr + import data_pipeline.ingest.ohlcv as _dl + import data_pipeline.transform.cleaning as _cl + import data_pipeline.transform.processing as _pr dl_result = _dl.upsert_raw_prices(ticker, start, end) if not dl_result.ok: @@ -81,9 +80,9 @@ def seed_history(ticker: str, years: int = 5) -> None: """One-time helper to seed multi-year history for a ticker into the DB.""" end = dt.date.today() start = end - dt.timedelta(days=years * 365) - import data_pipeline.cleaning as _cl - import data_pipeline.downloader as _dl - import data_pipeline.processing as _pr + import data_pipeline.ingest.ohlcv as _dl + import data_pipeline.transform.cleaning as _cl + import data_pipeline.transform.processing as _pr _dl.upsert_raw_prices(ticker, start, end) _cl.clean_range(ticker, start, end) diff --git a/data_pipeline/providers/_log.py b/data_pipeline/providers/_log.py index a89ae90..682b824 100644 --- a/data_pipeline/providers/_log.py +++ b/data_pipeline/providers/_log.py @@ -3,7 +3,7 @@ Domain: Data Pipeline — Providers (shared) Context: - Every acquisition failure worth surfacing in ``/health/data`` lands in - ``data_quality_log`` (see ``data_pipeline/quality_log.py``). Both option-chain + ``data_quality_log`` (see ``data_pipeline/store/quality_log.py``). Both option-chain and general yfinance provider modules need to record failures, and neither may import the other, so the best-effort wrapper lives here. Why the ``source`` strings still read ``yf_client.*``: @@ -11,7 +11,7 @@ rows; renaming the source labels is a separate, observable change and is deliberately deferred (see docs/plans/business_line_reorg.md §6 B1). Dependencies UPWARD: - - data_pipeline.quality_log (imported lazily — keeps package import cheap and + - data_pipeline.store.quality_log (imported lazily — keeps package import cheap and avoids a cycle at import time) """ @@ -21,7 +21,7 @@ def _log_dq(source: str, error_class: str, message: str, *, ticker: str | None = None) -> None: """Best-effort write to ``data_quality_log``. Never raises.""" try: - from data_pipeline.quality_log import log_failure + from data_pipeline.store.quality_log import log_failure log_failure(source, error_class, message, ticker=ticker) except Exception: # noqa: BLE001 diff --git a/data_pipeline/yf_client.py b/data_pipeline/providers/yf_client.py similarity index 90% rename from data_pipeline/yf_client.py rename to data_pipeline/providers/yf_client.py index caf65e2..b132571 100644 --- a/data_pipeline/yf_client.py +++ b/data_pipeline/providers/yf_client.py @@ -5,7 +5,7 @@ - Batch B1 (docs/plans/business_line_reorg.md §6) moved every yfinance call into ``data_pipeline/providers/``. This module is kept for one release so the existing importers (``services/``, ``core/market/data_context.py``, - ``data_pipeline/data_ops/``) do not have to change in the same PR as the + ``data_pipeline/read/``) do not have to change in the same PR as the extraction. See ADR 0011. - New code should import from ``data_pipeline.providers`` (canonical shapes) instead of here. @@ -19,7 +19,7 @@ Dependencies UPWARD: - providers/yf_snapshot (live snapshots), providers/yfinance_provider (bars) Dependencies DOWNWARD: - - services/*, core/market/data_context.py, data_pipeline/data_ops/* + - services/*, core/market/data_context.py, data_pipeline/read/* """ from __future__ import annotations diff --git a/data_pipeline/providers/yf_snapshot.py b/data_pipeline/providers/yf_snapshot.py index 15796ef..bb5a698 100644 --- a/data_pipeline/providers/yf_snapshot.py +++ b/data_pipeline/providers/yf_snapshot.py @@ -15,7 +15,7 @@ - ``fetch_spot(ticker)`` / ``fetch_spots_bulk(tickers)`` -> ``float`` / ``dict``. - ``fetch_option_chain(ticker)`` keeps the legacy payload contract (``ticker`` / ``spot`` / ``expiries`` / ``chain{expiry:{calls,puts}}``) for - callers that still import it through the ``data_pipeline.yf_client`` shim. + callers that still import it through the ``data_pipeline.providers.yf_client`` shim. - ``to_option_chain_snapshot(payload)`` -> canonical ``OptionChainSnapshot``. Design rules: - CONSTRAINT: every public function calls ``yf_throttle()`` before each @@ -24,7 +24,7 @@ chain and decide how to surface the error. Raising here would cascade into unhandled 500s from several routes. Dependencies UPWARD: - - utils.network (throttle), data_pipeline.quality_log (via providers._log) + - utils.network (throttle), data_pipeline.store.quality_log (via providers._log) Dependencies DOWNWARD: - providers/base, providers/_log """ diff --git a/data_pipeline/providers/yfinance_provider.py b/data_pipeline/providers/yfinance_provider.py index df53dec..973a791 100644 --- a/data_pipeline/providers/yfinance_provider.py +++ b/data_pipeline/providers/yfinance_provider.py @@ -26,7 +26,7 @@ - CONSTRAINT: never pass ``session=requests.Session()`` — yfinance ≥0.2.50 uses curl_cffi and silently fails (ADR 0005 / docs/constraints.md §2). Dependencies UPWARD: - - utils.network (throttle), data_pipeline.quality_log (via providers._log) + - utils.network (throttle), data_pipeline.store.quality_log (via providers._log) Dependencies DOWNWARD: - providers/base, providers/_log, providers/yf_snapshot """ @@ -197,7 +197,7 @@ def download_daily_frame(ticker: str, start: dt.date, end: dt.date) -> pd.DataFr """Download daily OHLCV for ``[start, end]`` (inclusive) from yfinance. Returns a frame with Title-Case columns plus ``Adj_Close`` — the shape - ``data_pipeline.downloader.upsert_raw_prices`` consumes. ``history()`` on + ``data_pipeline.ingest.ohlcv.upsert_raw_prices`` consumes. ``history()`` on ``YFinanceProvider`` is the canonical equivalent. """ # yfinance 'end' is exclusive, so pass end + 1 day to include the requested end date diff --git a/data_pipeline/read/__init__.py b/data_pipeline/read/__init__.py new file mode 100644 index 0000000..2228e99 --- /dev/null +++ b/data_pipeline/read/__init__.py @@ -0,0 +1,22 @@ +"""READ — the DB-first read API services call. + +Domain: Data Pipeline — Read +Context: + - ADR 0011: ``DataService`` is the single entry point above ``data_pipeline/``. + It is DB-first and, when coverage is missing, triggers the orchestration + layer rather than downloading inline (that is why this package may import + ``orchestrate`` — see the layer table in docs/architecture_review.md §3). +Contracts: + - ``DataService`` — the facade used by services/, routes/ and app.py. + - ``_query`` — memoised reads with in-flight de-duplication. +Dependencies UPWARD: + - store, orchestrate (refresh triggers), providers (spot fallback) +Dependencies DOWNWARD: + - services/, routes/ +""" + +from __future__ import annotations + +from data_pipeline.read.facade import DataService + +__all__ = ["DataService"] diff --git a/data_pipeline/data_ops/_query.py b/data_pipeline/read/_query.py similarity index 90% rename from data_pipeline/data_ops/_query.py rename to data_pipeline/read/_query.py index 464c71f..55c428c 100644 --- a/data_pipeline/data_ops/_query.py +++ b/data_pipeline/read/_query.py @@ -8,11 +8,10 @@ import pandas as pd -from data_pipeline.db import fetch_df, init_db - -from . import _globals as _g -from . import _range as _r -from . import _update as _u +from data_pipeline import _state as _g +from data_pipeline.orchestrate import backfill as _bf +from data_pipeline.orchestrate import update as _u +from data_pipeline.store.db import fetch_df, init_db logger = logging.getLogger(__name__) @@ -42,13 +41,13 @@ def _kick_backfill(ticker: str, start, end) -> None: def _run_backfill(ticker, start, end, key) -> None: try: - _r.ensure_range(ticker, start, end) + _bf.ensure_range(ticker, start, end) except Exception as e: logger.warning("background backfill failed for %s: %s", ticker, e) finally: # Daemon threads never re-run dispatch's finally-cleanups; drop this # thread's SQLite connection so _all_conns doesn't grow per backfill. - from data_pipeline.db import close_thread_conn + from data_pipeline.store.db import close_thread_conn close_thread_conn() with _backfill_lock: @@ -66,10 +65,10 @@ def _join_backfills(timeout: float | None = None) -> None: def _wait_for_coverage(ticker, start, end, timeout: float) -> bool: deadline = time.monotonic() + timeout while time.monotonic() < deadline: - if not _r.needs_backfill(ticker, start, end): + if not _bf.needs_backfill(ticker, start, end): return True time.sleep(0.25) - return not _r.needs_backfill(ticker, start, end) + return not _bf.needs_backfill(ticker, start, end) def get_cleaned_daily(ticker: str, start: dt.date | None = None, end: dt.date | None = None) -> pd.DataFrame: @@ -87,14 +86,14 @@ def get_cleaned_daily(ticker: str, start: dt.date | None = None, end: dt.date | # ensure_range path. return cached _u.manual_update(ticker, days=7) - if _r.needs_backfill(ticker, start, end): + if _bf.needs_backfill(ticker, start, end): # Missing span ⇒ keep the heavy download off the request thread; give # it a short grace period so "one chunk missing" still returns full # data, then fall through to whatever coverage the DB has now. _kick_backfill(ticker, start, end) _wait_for_coverage(ticker, start, end, _BACKFILL_WAIT_SECONDS) else: - _r.ensure_range(ticker, start, end) + _bf.ensure_range(ticker, start, end) init_db() df = fetch_df( "SELECT date, open, high, low, close, adj_close, volume FROM clean_bars WHERE ticker=? AND date>=? AND date<=?", @@ -104,7 +103,7 @@ def get_cleaned_daily(ticker: str, start: dt.date | None = None, end: dt.date | # this df may lack the requested span. ensure_range invalidates the # ticker's cache entries on success, so the completed data becomes visible # on the next request. - if not _r.needs_backfill(ticker, start, end): + if not _bf.needs_backfill(ticker, start, end): _g._cache_set(cache_key, df) return df @@ -156,7 +155,7 @@ def get_latest_spot(ticker: str) -> float | None: except (TypeError, ValueError): pass - from data_pipeline.yf_client import fetch_spot + from data_pipeline.providers.yf_client import fetch_spot try: price = fetch_spot(ticker) diff --git a/data_pipeline/data_ops/facade.py b/data_pipeline/read/facade.py similarity index 54% rename from data_pipeline/data_ops/facade.py rename to data_pipeline/read/facade.py index ec652c1..657e9ea 100644 --- a/data_pipeline/data_ops/facade.py +++ b/data_pipeline/read/facade.py @@ -1,24 +1,40 @@ -"""DataService facade — thin orchestrator over data_ops submodules.""" - -from data_pipeline.db import init_db +"""DataService facade — the DB-first read entry point (ADR 0011). + +Domain: Data Pipeline — Read +Context: + - This is the only ``data_pipeline`` surface above the package: services/, + routes/ and app.py talk to ``DataService`` and never to store/ingest/ + transform directly. Reads come from ``read/_query.py``; anything that has to + *make data ready* is delegated to ``orchestrate``. +Contracts: + - ``DataService`` staticmethods — initialize, manual_update, seed_history, + has_data_for_date, ensure_range, get_cleaned_daily, get_processed, + get_processed_data, get_latest_spot. +Dependencies UPWARD: + - store (db), orchestrate (backfill / update) +Dependencies DOWNWARD: + - services/, routes/, app.py +""" + +from data_pipeline.orchestrate import backfill as _bf +from data_pipeline.orchestrate import update as _u +from data_pipeline.store.db import init_db from . import _query as _q -from . import _range as _r -from . import _update as _u class DataService: """Facade for data operations.""" # Re-export class-level attributes for backward compat - _ENSURE_RANGE_TTL = _r._ENSURE_RANGE_TTL - _ensure_range_memo = _r._ensure_range_memo - _ensure_range_lock = _r._ensure_range_lock - _ensure_range_inflight = _r._ensure_range_inflight - _ensure_range_inflight_lock = _r._ensure_range_inflight_lock - _BACKFILL_MIN_DATE = _r._BACKFILL_MIN_DATE - _SENTINEL_GAP_THRESHOLD_DAYS = _r._SENTINEL_GAP_THRESHOLD_DAYS - _SENTINEL_MIN_DB_SPAN_DAYS = _r._SENTINEL_MIN_DB_SPAN_DAYS + _ENSURE_RANGE_TTL = _bf._ENSURE_RANGE_TTL + _ensure_range_memo = _bf._ensure_range_memo + _ensure_range_lock = _bf._ensure_range_lock + _ensure_range_inflight = _bf._ensure_range_inflight + _ensure_range_inflight_lock = _bf._ensure_range_inflight_lock + _BACKFILL_MIN_DATE = _bf._BACKFILL_MIN_DATE + _SENTINEL_GAP_THRESHOLD_DAYS = _bf._SENTINEL_GAP_THRESHOLD_DAYS + _SENTINEL_MIN_DB_SPAN_DAYS = _bf._SENTINEL_MIN_DB_SPAN_DAYS @staticmethod def initialize(): @@ -35,7 +51,7 @@ def seed_history(ticker: str, years: int = 5): @staticmethod def has_data_for_date(ticker: str, date) -> bool: init_db() - from data_pipeline.db import fetch_df + from data_pipeline.store.db import fetch_df df = fetch_df( "SELECT * FROM clean_bars WHERE ticker=? AND date=?", @@ -51,7 +67,7 @@ def has_data_for_date(ticker: str, date) -> bool: @staticmethod def ensure_range(ticker: str, start, end) -> bool: - return _r.ensure_range(ticker, start, end) + return _bf.ensure_range(ticker, start, end) @staticmethod def get_cleaned_daily(ticker: str, start=None, end=None): @@ -76,5 +92,5 @@ def clear_ensure_range_memo(ticker: str) -> None: Call this before ``seed_history`` when you want to bypass a previously cached failure and force a fresh backfill. """ - with _r._ensure_range_lock: - _r._ensure_range_memo.pop(ticker, None) + with _bf._ensure_range_lock: + _bf._ensure_range_memo.pop(ticker, None) diff --git a/data_pipeline/store/__init__.py b/data_pipeline/store/__init__.py new file mode 100644 index 0000000..b2934c7 --- /dev/null +++ b/data_pipeline/store/__init__.py @@ -0,0 +1,18 @@ +"""STORE — schema, SQL, and the failure log. + +Domain: Data Pipeline — Store +Context: + - ADR 0011 split ``data_pipeline/`` into stages. This package owns the SQLite + boundary: the schema, the only SQL builder, and the ``data_quality_log`` + writer. Nothing here knows about vendors, pandas transforms, or Flask. +Contracts: + - ``db`` — connections (thread-local WAL), ``init_db``, ``upsert_many``, ``fetch_df``. + - ``repos`` — the only place that builds SQL. + - ``quality_log`` — the append-only fetch-failure log. +Dependencies UPWARD: + - (none — stdlib + pandas only) +Dependencies DOWNWARD: + - everything above: providers, ingest, transform, read, orchestrate +""" + +from __future__ import annotations diff --git a/data_pipeline/db.py b/data_pipeline/store/db.py similarity index 100% rename from data_pipeline/db.py rename to data_pipeline/store/db.py diff --git a/data_pipeline/quality_log.py b/data_pipeline/store/quality_log.py similarity index 98% rename from data_pipeline/quality_log.py rename to data_pipeline/store/quality_log.py index 15feee1..750ba9e 100644 --- a/data_pipeline/quality_log.py +++ b/data_pipeline/store/quality_log.py @@ -15,7 +15,7 @@ from datetime import UTC, datetime from typing import Any -from data_pipeline.db import get_conn +from data_pipeline.store.db import get_conn _logger = logging.getLogger(__name__) diff --git a/data_pipeline/repos.py b/data_pipeline/store/repos.py similarity index 96% rename from data_pipeline/repos.py rename to data_pipeline/store/repos.py index 5f2d710..83e03f3 100644 --- a/data_pipeline/repos.py +++ b/data_pipeline/store/repos.py @@ -1,11 +1,11 @@ """Repository layer — the only place that builds SQL. INVARIANT (doc_guard `db-access`): upper layers (routes/services) must import -these functions instead of touching ``data_pipeline.db`` connection primitives +these functions instead of touching ``data_pipeline.store.db`` connection primitives directly, so WAL pragmas and the query cache apply uniformly (ADR 0003). This module is part of ``data_pipeline`` (an I/O layer); importing -``data_pipeline.db`` here is the intended single exception and is not flagged +``data_pipeline.store.db`` here is the intended single exception and is not flagged by the guardrail. """ @@ -17,7 +17,7 @@ import pandas as pd -from data_pipeline.db import fetch_df, get_conn, init_db, upsert_many +from data_pipeline.store.db import fetch_df, get_conn, init_db, upsert_many # ── Health / data-quality inventory ───────────────────────────────── diff --git a/data_pipeline/transform/__init__.py b/data_pipeline/transform/__init__.py new file mode 100644 index 0000000..e7fd076 --- /dev/null +++ b/data_pipeline/transform/__init__.py @@ -0,0 +1,19 @@ +"""TRANSFORM — raw → clean → features. + +Domain: Data Pipeline — Transform +Context: + - ADR 0011: the processing stage is provider-agnostic — it reads canonical + ``raw_bars`` / ``clean_bars`` frames and never imports ``providers/``. + - Domain rules live here: business-day alignment with **no interpolation** + (invented prices are worse than missing ones — docs/constraints.md §4), + anomaly flagging, then per-frequency feature engineering. +Contracts: + - ``cleaning.clean_range`` — raw_bars → clean_bars. + - ``processing.process_frequencies`` — clean_bars → feature_bars (D/W/ME/QE). +Dependencies UPWARD: + - store (canonical tables), data_pipeline (PipelineResult) +Dependencies DOWNWARD: + - orchestrate +""" + +from __future__ import annotations diff --git a/data_pipeline/cleaning.py b/data_pipeline/transform/cleaning.py similarity index 92% rename from data_pipeline/cleaning.py rename to data_pipeline/transform/cleaning.py index 05e980f..ba544c7 100644 --- a/data_pipeline/cleaning.py +++ b/data_pipeline/transform/cleaning.py @@ -1,4 +1,18 @@ -"""Data cleaning utilities for raw market price data.""" +"""Data cleaning: raw_prices → clean_bars. + +Domain: Data Pipeline — Transform (cleaning) +Context: + - Aligns raw bars to business days, flags anomalies, and marks missing days as + NA. INVARIANT: **no interpolation** — inventing prices that never traded + would corrupt every downstream indicator (HV, MA, regime). See + docs/constraints.md §4. +Contracts: + - ``clean_range(ticker, start, end) -> PipelineResult`` — raw_bars → clean_bars. +Dependencies UPWARD: + - store (db), data_pipeline (PipelineResult) +Dependencies DOWNWARD: + - orchestrate/update.py, orchestrate/backfill.py +""" import datetime as dt import logging @@ -6,8 +20,8 @@ import numpy as np import pandas as pd -from . import PipelineResult -from .db import fetch_df, upsert_many +from data_pipeline import PipelineResult +from data_pipeline.store.db import fetch_df, upsert_many logger = logging.getLogger(__name__) diff --git a/data_pipeline/processing.py b/data_pipeline/transform/processing.py similarity index 96% rename from data_pipeline/processing.py rename to data_pipeline/transform/processing.py index d91a866..b3b9273 100644 --- a/data_pipeline/processing.py +++ b/data_pipeline/transform/processing.py @@ -3,7 +3,9 @@ Context: - Reads from ``clean_bars`` and emits resampled bars + indicator columns to ``feature_bars``. Pure pandas; no I/O outside the DB helpers in - ``data_pipeline.db``. + ``data_pipeline.store.db``. INVARIANT (ADR 0011 §5.2): never imports + ``providers/`` — the processing stage only ever sees canonical tables. + ``data_pipeline.store.db``. """ import datetime as dt @@ -12,8 +14,8 @@ import numpy as np import pandas as pd -from . import PipelineResult -from .db import fetch_df, upsert_many +from data_pipeline import PipelineResult +from data_pipeline.store.db import fetch_df, upsert_many logger = logging.getLogger(__name__) diff --git a/docs/architecture_review.md b/docs/architecture_review.md index aa6e562..26dd932 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -54,8 +54,8 @@ for the live list. State at registration: | Location | Why it exists | Exit condition | |---|---|---| -| `services/market/health.py` — **resolved 2026-09-03**, `services/portfolio/facade.py` — **resolved 2026-09-03** (`get_conn`) | ad-hoc health/inventory SQL predates `repos.py` coverage | queries moved into `data_pipeline/repos.py` | -| `services/regime/facade.py`, `services/regime/ops/_bootstrap.py`, `services/regime/ops/_persistence.py` (`fetch_df`, `init_db`, `upsert_many`) — **resolved 2026-09-03** | regime log writes were split across service and ops modules | consolidated behind `data_pipeline/repos.py` (regime-log + clean-row ops) | +| `services/market/health.py` — **resolved 2026-09-03**, `services/portfolio/facade.py` — **resolved 2026-09-03** (`get_conn`) | ad-hoc health/inventory SQL predates `repos.py` coverage | queries moved into `data_pipeline/store/repos.py` | +| `services/regime/facade.py`, `services/regime/ops/_bootstrap.py`, `services/regime/ops/_persistence.py` (`fetch_df`, `init_db`, `upsert_many`) — **resolved 2026-09-03** | regime log writes were split across service and ops modules | consolidated behind `data_pipeline/store/repos.py` (regime-log + clean-row ops) | ### single-yf-exit (only `data_pipeline/providers/` may import yfinance) — 0 markers @@ -65,33 +65,40 @@ Rescoped in batch B1 of [ADR 0011](decisions/0011-pluggable-data-provider-seam.m | Location | Why it exists | Exit condition | |---|---|---| -| `data_pipeline/downloader.py` — **resolved 2026-09-10 (B1)** | DB-aware gap-detection bulk downloads; it used to call `yf.download` directly as a registered second exit point | the download call moved to `providers/yfinance_provider.py::download_daily_frame`; `downloader.py` keeps only gap detection + `raw_bars` upsert, so it no longer imports yfinance | -| `data_pipeline/data_ops/_query.py::get_latest_spot` — **resolved 2026-09-03** | former spot fast-path fetched yfinance internally | now routes through `fetch_spot` (provider, re-exported by `yf_client`) | +| `data_pipeline/ingest/ohlcv.py` — **resolved 2026-09-10 (B1)** | DB-aware gap-detection bulk downloads; it used to call `yf.download` directly as a registered second exit point | the download call moved to `providers/yfinance_provider.py::download_daily_frame`; `downloader.py` keeps only gap detection + `raw_bars` upsert, so it no longer imports yfinance | +| `data_pipeline/read/_query.py::get_latest_spot` — **resolved 2026-09-03** | former spot fast-path fetched yfinance internally | now routes through `fetch_spot` (provider, re-exported by `providers/yf_client.py`) | ### Watch list (pre-debt, no marker yet) | Location | Concern | Trigger to act | |---|---|---| | `services/market/analysis/summary.py` (fan-in 0, tracked as `dead_code_candidates=1` in baseline) | `generate_summary_analysis` lost its caller when the streaming refactor removed the server-rendered `summary_data` template variable; the Summary tab button is gated off in `templates/index.html` and `summary_pending` in `routes/core.py` is vestigial | any request to ship the multi-ticker Summary tab ⇒ add a `summary` slice to `_RENDER_KIND_SLICES` (aggregates across the job's tickers, not per-ticker) ; otherwise delete the module + `partials/tab_summary.html` + the `summary_pending` flag in the same commit and reset the baseline | -| ~~`data_pipeline/yf_client.py` (391 lines, fan-in 11)~~ — **resolved 2026-09-10 (B1)** | it sat 9 lines below the 400-line god-file threshold | the option-chain section was extracted pre-emptively, as prescribed, into `providers/yf_snapshot.py`; `yf_client.py` is now a ~35-line shim. The pressure moved to `providers/yf_snapshot.py` (≈340 lines) and `providers/yfinance_provider.py` (≈290 lines) — watch them before adding endpoints | +| ~~`data_pipeline/providers/yf_client.py` (391 lines, fan-in 11)~~ — **resolved 2026-09-10 (B1)** | it sat 9 lines below the 400-line god-file threshold | the option-chain section was extracted pre-emptively, as prescribed, into `providers/yf_snapshot.py`; `yf_client.py` is now a ~35-line shim. The pressure moved to `providers/yf_snapshot.py` (≈340 lines) and `providers/yfinance_provider.py` (≈290 lines) — watch them before adding endpoints | ## 3. Guardrails (how the score is kept) | Tool | Role | Run where | |---|---|---| -| `scripts/doc_guard.py` | blocks violating edits: `import-direction`, `core-purity`, `db-access`, `single-yf-exit`, `sqlite-bypass`, `yfinance-throttle`, `yfinance-session-kwarg`, `tag-syntax`, `module-docstring`, ADR rules | pre-commit + CI, per changed file | +| `scripts/doc_guard.py` | blocks violating edits: `import-direction` (sub-package aware since B3), `core-purity`, `db-access`, `single-yf-exit`, `sqlite-bypass`, `yfinance-throttle`, `yfinance-session-kwarg`, `tag-syntax`, `module-docstring`, ADR rules | pre-commit + CI, per changed file | | `scripts/arch_metrics.py` | trend metrics: layer-edge violations, import cycles (Tarjan), god files, dead-code candidates, fan-in/out Top-5; `--check` fails CI on regression vs `.github/data/arch_baseline.json` | CI, whole repo | -| `tests/test_architecture_purity.py` | contract test re-asserting core purity at the test layer so suppressed markers stay visible in the test report | pytest | +| `tests/test_architecture_purity.py` | contract tests at the test layer: core purity, the `data_pipeline/` layer graph, transform↛providers, and that the two copies of the layer table agree | pytest | **Layer allow-list** (single source of truth: `doc_guard.py::_ALLOWED_DEPS`, -mirrored in `arch_metrics.py`): +mirrored in `arch_metrics.py`; asserted equal by +`tests/test_architecture_purity.py::test_layer_tables_are_in_sync`): ``` -app → routes, services, core, data_pipeline, utils -routes → services, data_pipeline, utils (never core directly) -services → core, data_pipeline, utils -core → utils (data_pipeline only via §2 markers) -data_pipeline → utils +app → routes, services, core, data_pipeline, utils, read, orchestrate +routes → services, data_pipeline, utils, store, read, orchestrate (never core directly) +services → core, data_pipeline, utils, providers, store, ingest, transform, read, orchestrate +core → utils, read, providers (data_pipeline* only via §2 markers; B4 removes both) +data_pipeline → utils # root: PipelineResult (types) + _state.py + store → (nothing upward) + providers → store, utils # store = quality_log; see plan §8 B3 + ingest → data_pipeline, providers, store, utils + transform → data_pipeline, store, utils # never providers (asserted by test) + read → data_pipeline, orchestrate, providers, store, utils + orchestrate → data_pipeline, ingest, store, transform, utils utils → (leaf: nothing upward) ``` diff --git a/docs/automation.md b/docs/automation.md index 85eca96..5ebe2fa 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -30,7 +30,7 @@ What it checks (each rule produces a non-zero exit code on violation): | `tag-syntax` | `WHY:`/`CONSTRAINT:`/etc. used outside the canonical vocabulary, or with malformed forms (lowercase, missing colon). | Keeps the tag set stable so AI grep is reliable. | | `yfinance-throttle` | Any new `yf.download` / `yf.Ticker(...)` call site not preceded by `yf_throttle()` or routed through `data_pipeline/providers/` (the chokepoint since ADR 0011 / batch B1). | Hard architectural invariant from ADR 0005. | | `yfinance-session-kwarg` | Any call passing `session=` to a yfinance API. | Silent failure mode (curl_cffi). See `docs/constraints.md` §2. | -| `sqlite-bypass` | New `sqlite3.connect(` outside `data_pipeline/db.py`. | Bypasses WAL pragmas (ADR 0003). | +| `sqlite-bypass` | New `sqlite3.connect(` outside `data_pipeline/store/db.py`. | Bypasses WAL pragmas (ADR 0003). | | `import-direction` | Imports from `services/` inside `core/` or `data_pipeline/`; from `core/` inside `data_pipeline/`. | ADR 0001 — already enforced by an existing hook; doc-guard is the safety net. | | `adr-link-integrity` | Markdown links from `docs/decisions/` / `docs/constraints.md` / `docs/glossary.md` that point at non-existent files or anchors. | ADRs must stay reachable. | | `adr-index-fresh` | `docs/decisions/README.md` index does not match the actual ADR files in the folder. | Auto-fixable; CI fails if not regenerated. | diff --git a/docs/constraints.md b/docs/constraints.md index 0f0b7ee..100ac42 100644 --- a/docs/constraints.md +++ b/docs/constraints.md @@ -37,14 +37,14 @@ is usually a workaround for one of the items below. - **Constraint**: this app runs on one developer machine, occasionally a small VPS. Postgres is overkill. - **WAL mode + `synchronous=NORMAL`**: chosen for read concurrency. Do not switch to `FULL` (latency) or remove WAL (locks block reads during scheduler writes). -- **Thread-local connections** (`data_pipeline/db.py`): SQLite connections are not thread-safe to share, but per-query reconnects are wasteful. We cache one connection per (thread, path) and apply PRAGMAs once. +- **Thread-local connections** (`data_pipeline/store/db.py`): SQLite connections are not thread-safe to share, but per-query reconnects are wasteful. We cache one connection per (thread, path) and apply PRAGMAs once. - **No migration framework**: schemas are created via `CREATE TABLE IF NOT EXISTS`. Breaking changes require manual `.sqlite` migration scripts in `scripts/`. ## 4. The machine is not 24/7 - Snapshot cadence (scheduler) **will have gaps**: laptop sleeps, weekends off, network outages. - Any feature that consumes time-series data must tolerate **sparse, non-contiguous days**. Do NOT assume daily continuity. -- `data_pipeline/cleaning.py` aligns to business days and marks missing days as NA — **no interpolation**, by design. Filling gaps would invent prices that didn't trade. +- `data_pipeline/transform/cleaning.py` aligns to business days and marks missing days as NA — **no interpolation**, by design. Filling gaps would invent prices that didn't trade. ## 5. Financial domain "magic numbers" are intentional @@ -60,7 +60,7 @@ These are NOT magic numbers — they encode domain knowledge. Do not "DRY" them ## 6. Computation must finish in one HTTP request - **No background job queue** (no Celery, no RQ). The Flask process serves the UI and runs the scheduler in-thread. -- **APScheduler is optional and lazily imported.** The scheduler only starts when `AUTO_UPDATE_TICKERS` is set, and `data_pipeline/scheduler.py` imports APScheduler inside `UpdateScheduler.__init__` (plus a lazy `CronTrigger` import) so the rest of the app — and `acquire_scheduler_lock`'s unit tests — run without the package installed. **Do not move that import back to module scope**: an optional feature must not become a hard startup dependency. +- **APScheduler is optional and lazily imported.** The scheduler only starts when `AUTO_UPDATE_TICKERS` is set, and `data_pipeline/orchestrate/scheduler.py` imports APScheduler inside `UpdateScheduler.__init__` (plus a lazy `CronTrigger` import) so the rest of the app — and `acquire_scheduler_lock`'s unit tests — run without the package installed. **Do not move that import back to module scope**: an optional feature must not become a hard startup dependency. - Long-running computations either: - Run inside a request and respond synchronously (fine for <2s), or - Are pre-computed by the scheduler and read from DB. diff --git a/docs/frontend_architecture.md b/docs/frontend_architecture.md index b0048e6..f8df57d 100644 --- a/docs/frontend_architecture.md +++ b/docs/frontend_architecture.md @@ -86,7 +86,7 @@ Flask ── HTML fragment ───────────────── ``` Key files: -- `data_pipeline/job_cache.py` — in-process JobCache (TTL 90 s). +- `data_pipeline/orchestrate/job_cache.py` — in-process JobCache (TTL 90 s). - `app.py::_render_streaming_slice` — shared `/render/` handler. - `services/market/analysis/facade.py::generate_*_slice` — per-tab compute. - `templates/partials/fragments/*.html` — rendered fragments. diff --git a/docs/glossary.md b/docs/glossary.md index f19e95d..bfb0939 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -73,7 +73,7 @@ Untouched OHLCV pulled from a provider (yfinance today) and mapped onto the cano ### `feature_bars` `clean_bars` resampled per frequency (D/W/ME/QE) with engineered features (returns, MA, HV, oscillation). Indexed by `(ticker, date, frequency)`. -> **Compatibility (one release)**: the pre-rename names `raw_prices` / `clean_prices` / `processed_prices` still exist as shadow tables — every write goes to both families (see `data_pipeline/db.py`) — so an un-migrated DB and a `git revert` of the rename keep working. See [ADR 0011](decisions/0011-pluggable-data-provider-seam.md). +> **Compatibility (one release)**: the pre-rename names `raw_prices` / `clean_prices` / `processed_prices` still exist as shadow tables — every write goes to both families (see `data_pipeline/store/db.py`) — so an un-migrated DB and a `git revert` of the rename keep working. See [ADR 0011](decisions/0011-pluggable-data-provider-seam.md). ### Anomaly Flags - `price_jump_flag`: |log return| > 5σ. diff --git a/docs/guides/USER_GUIDE.md b/docs/guides/USER_GUIDE.md index 3eda2a4..2ae045e 100644 --- a/docs/guides/USER_GUIDE.md +++ b/docs/guides/USER_GUIDE.md @@ -615,7 +615,7 @@ The combined regime is the cartesian product (e.g. *"High vol / Down"*). The her | `GET /api/regime/history` | Full labelled time-series | | `POST /api/regime/backfill` | Recompute and persist the series | -Computation lives in the `core/regime/` package (`classify.py`, `series.py`, `models.py`); persistence goes through `data_pipeline/repos.py`. +Computation lives in the `core/regime/` package (`classify.py`, `series.py`, `models.py`); persistence goes through `data_pipeline/store/repos.py`. --- diff --git a/docs/l0_architecture.md b/docs/l0_architecture.md index 714b732..db7c326 100644 --- a/docs/l0_architecture.md +++ b/docs/l0_architecture.md @@ -38,7 +38,7 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ | `routes/` | 909 lines · 8 files | 7 blueprints + `__init__.py` aggregate export; no business logic | good | | `services/` | 3 540 lines · 5 domain packages | `market` (incl. `analysis/` slice factory), `market_review`, `options`, `portfolio`, `regime` | good | | `core/` | 6 372 lines · 8 sub-packages + `_shared` | Pure computation — no Flask, no DB, no network | good | -| `data_pipeline/` | 3 154 lines · 22 files | The only I/O boundary: `providers/` (the yfinance seam, ADR 0011), `yf_client` (one-release shim), `db`/`repos`, `data_ops`, `scheduler`, `job_cache` | good | +| `data_pipeline/` | 3 331 lines · 26 files | The only I/O boundary, re-homed into six one-way stages (ADR 0011, batch B3): `providers/` · `store/` · `ingest/` · `transform/` · `read/` · `orchestrate/` (+ `_state.py`) | good | | `utils/` | 756 lines · 7 files | Leaf layer; highest fan-in (`ticker_utils.py` = 11) | good | | `templates/` | 1 546 lines · 17 files | `index.html` skeleton + `partials/fragments/*` (HTMX swap targets) | good | | `static/` | 5 473 lines · 31 JS/CSS | `state/` · `sim/` · `components/` · `features/` + tab entry files | fair (see §4 P3-1) | @@ -68,7 +68,7 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ | Item | State | Note | |---|---|---| -| `market_data.sqlite` (11.7 MB) | git-ignored, still in repo root | code default is now `data/market_data.sqlite` (`data_pipeline/db.py:27`); the local `.env` overrides it back to the root file — move the file into `data/` whenever convenient. Schema: canonical `raw_bars` / `clean_bars` / `feature_bars`, plus the one-release shadows `raw_prices` / `clean_prices` / `processed_prices` — see [ADR 0011](decisions/0011-pluggable-data-provider-seam.md) | +| `market_data.sqlite` (11.7 MB) | git-ignored, still in repo root | code default is now `data/market_data.sqlite` (`data_pipeline/store/db.py:27`); the local `.env` overrides it back to the root file — move the file into `data/` whenever convenient. Schema: canonical `raw_bars` / `clean_bars` / `feature_bars`, plus the one-release shadows `raw_prices` / `clean_prices` / `processed_prices` — see [ADR 0011](decisions/0011-pluggable-data-provider-seam.md) | | `site/` | **inputs committed (9 files) · build output ignored** | tracked: `fixtures/` (7) · `snapshot/snapshot.json` · `pages-shim.js`; ignored: `index.html`, 5 feature + 6 showcase redirects, `static/**` (42 generated files) — see §5 P1-1 | | `archive/` (8 files · 1 070 lines) | committed | retired code still in tree — P3-3 | | `test.ipynb` (58 lines) | git-ignored | leftover scratch file — P2-2 | @@ -78,17 +78,17 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ ## 2. Measured shape (`scripts/arch_metrics.py`) -_Refreshed 2026-09-10 after batches B1 (provider seam) and B2 (canonical table names); the L1 inventory in §1 above is otherwise the 2026-09-08 snapshot._ +_Refreshed 2026-09-10 after batches B1 (provider seam), B2 (canonical table names) and B3 (data_pipeline re-home); the L1 inventory in §1 above is otherwise the 2026-09-08 snapshot._ ``` -modules=152 import_edges=314 +modules=156 import_edges=313 Layer-edge violations : (none) Import cycles : 0 God files (>400 lines): (none) Top fan-out : core/market/charts/facade.py(14) · core/market/data_context.py(7) routes/__init__.py(7) · routes/core.py(7) · app.py(6) -Top fan-in : core/_shared/plotting.py(13) · data_pipeline/yf_client.py(11) - utils/ticker_utils.py(11) · data_pipeline/db.py(9) +Top fan-in : core/_shared/plotting.py(13) · data_pipeline/providers/yf_client.py(11) + utils/ticker_utils.py(11) · data_pipeline/store/db.py(9) Dead code : services/market/analysis/summary.py (only one; already on the watch list in docs/architecture_review.md §2) ``` @@ -115,6 +115,31 @@ OptionLab/ └─ archive/ # mark read-only or move out ``` +### Inside `data_pipeline/` — the six stages (ADR 0011, achieved in B3) + +``` +data_pipeline/ + providers/ ACQUIRE yfinance adapter + canonical schema + registry — the ONLY `import yfinance` + store/ SERVE schema, the only SQL, the failure log + ingest/ GLUE business-day gap detection + raw_bars upsert + transform/ PROCESS raw_bars → clean_bars → feature_bars (never imports providers) + read/ SERVE DataService facade + memoised queries + orchestrate/ DRIVERS manual/seed update, chunked backfill, job cache, scheduler + _state.py process-local shared state (query cache, update locks) +``` + +Call direction (same allow-list in `scripts/doc_guard.py` and +`scripts/arch_metrics.py`): + +``` +services → read → {store, orchestrate, providers} +services → orchestrate → {ingest, transform, store} +ingest → {providers, store} +transform→ store +providers→ {store, utils} # store = the failure log, see plan §8 B3 +read → orchestrate # the read path triggers refreshes +``` + --- ## 4. Open findings (summary; details in the 2026-09-08 review) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index 8ff4336..9ef6fca 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -40,7 +40,7 @@ | — (planning + ADRs) | 🔨 in review (PR #7) | #7 | branch `worktree-business-line-reorg` · 2026-09-10 | plan, ADR 0011/0012 (Accepted), scaffolding (ledger, gates, AI-guide pointers, memory) | | B1 — provider seam extraction | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | delivers the "pluggable API" seam on its own. Actual shape / deviations recorded in §8; `_ALLOWED_DEPS` promotion of `providers` deferred to B3 | | B2 — canonical raw store | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | gate §8 Q4 resolved (name-only rename). Actuals in §8; `symbol` column deferred (ADR 0011 amendment) | -| B3 — package re-home | ⬜ not started | — | — | resets `arch_baseline.json` | +| B3 — package re-home | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | six stages + `_state.py`; first sub-layer guard table; `arch_baseline.json` **not** reset (no tracked drift). Actuals + deviations in §8 | | B4 — close L1 (`core-purity`) | ⬜ not started | — | — | — | | B5 — readiness plan + prefetch | ⬜ not started | — | — | gate: §8 Q1 | | B6 — `ticker`-only Parameters bar | ⬜ not started | — | — | gate: §8 Q3; depends on B5 | @@ -478,6 +478,49 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. `audit_tags.py` unchanged (16 vs baseline 16); `routes/` untouched (only the one-line comment fix in `services/market/facade.py` outside `data_pipeline/`). +**B3 (2026-09-10) — package re-home.** + +- **Layout achieved** (`data_pipeline/`): `providers/` (ACQUIRE) · `store/` · `ingest/` · + `transform/` · `read/` · `orchestrate/` + `_state.py`. `data_ops/` is gone; `yf_client.py` moved + to `providers/yf_client.py` (kept, not deleted, so its one-release shim promise holds while + *services*→*providers* becomes the visible edge). Path map for anyone following older docs: + + | old | new | + |---|---| + | `db.py` / `repos.py` / `quality_log.py` | `store/…` | + | `downloader.py` | `ingest/ohlcv.py` | + | `cleaning.py` / `processing.py` | `transform/…` | + | `data_ops/{facade,_query}.py` | `read/…` | + | `data_ops/{_update,_range}.py` | `orchestrate/{update,backfill}.py` | + | `job_cache.py` / `scheduler.py` | `orchestrate/…` | + | `data_ops/_globals.py` | `_state.py` (package root) | + | `yf_client.py` | `providers/yf_client.py` | + +- **Guards now sub-package aware**: `doc_guard._layer_of` / `_imported_heads` and + `arch_metrics.layer_of` resolve `data_pipeline//…` to ``; `_ALLOWED_DEPS` carries + the six new keys **plus** the `providers` key B1 deferred; `sqlite-bypass` and `db-access` were + rescoped to `store/db.py` / `store/repos.py`. `tests/test_architecture_purity.py` gained three + tests: the layer graph matches the table, `transform/` never imports `providers/`, and the two + copies of the layer table agree. +- **Deviations from §5.2's sketch** (all deliberate): + 1. `orchestrate/update.py` exists (the sketch listed four files) — `manual_update` / + `seed_history` is a distinct "make it ready" entry point from chunked backfill. + 2. **No `ingest/snapshots.py`**: live snapshots are never persisted (ADR 0004), so there is no + ingest glue to move — callers reach `providers` directly. + 3. `_state.py` sits at the package root (the sketch put nothing there): `read` and `orchestrate` + both need the query cache / update locks, and the two must not import each other. + 4. **`read → orchestrate`** is kept (the read path triggers refreshes), which is the reverse of + the sketch's `orchestrate → read`. Consequence: `orchestrate` may not import `read`, so + `orchestrate/scheduler.py` now calls `orchestrate.update.manual_update` instead of + `DataService.manual_update` (behaviour-identical; it removed the last cycle candidate). + 5. **`providers → store`** (the provider writes its own failures to `store/quality_log.py`) + instead of being a pure leaf; the alternative was inventing a callback for a diagnostic write. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 472 passed / 5 skipped; + full `pytest tests/e2e` → 38 passed; `ruff check` + `format --check` clean; `doc_guard.py` clean; + `arch_metrics.py --check` ok — layer violations 0, cycles 0, god files 0, dead code 1, so + **no baseline reset was needed** (the §6 row anticipated one); `audit_tags.py` regenerated + (`--update-baseline`) because the uncovered-constant *paths* moved while the count stayed 16. + --- ## 9. References diff --git a/routes/core.py b/routes/core.py index 110daaf..1546a74 100644 --- a/routes/core.py +++ b/routes/core.py @@ -18,7 +18,7 @@ from flask import Blueprint, current_app, jsonify, render_template, request -from data_pipeline.job_cache import create_job +from data_pipeline.orchestrate.job_cache import create_job from services.market.dispatch import render_streaming_slice from services.market.facade import MarketService from services.market.form import FormService diff --git a/routes/data.py b/routes/data.py index e9f88fa..a40d5a0 100644 --- a/routes/data.py +++ b/routes/data.py @@ -13,7 +13,7 @@ from flask import Blueprint, jsonify, request -from data_pipeline.data_ops import DataService +from data_pipeline.read import DataService from services.market.health import overall_summary from utils.rate_limit import client_ip, rate_limit from utils.ticker_utils import normalize_ticker diff --git a/scripts/arch_metrics.py b/scripts/arch_metrics.py index e59d5e7..c8a19f7 100644 --- a/scripts/arch_metrics.py +++ b/scripts/arch_metrics.py @@ -50,15 +50,34 @@ GOD_FILE_LINES = 400 -# KEEP IN SYNC with scripts/doc_guard.py::_ALLOWED_DEPS. +# KEEP IN SYNC with scripts/doc_guard.py::_ALLOWED_DEPS (same invariant, two +# consumers: doc_guard blocks edits, this script tracks trend). ALLOWED_DEPS: dict[str, set[str]] = { - "app": {"routes", "services", "core", "data_pipeline", "utils"}, - "routes": {"services", "data_pipeline", "utils"}, - "services": {"core", "data_pipeline", "utils"}, - "core": {"data_pipeline", "utils"}, + "app": {"routes", "services", "core", "data_pipeline", "utils", "read", "orchestrate"}, + "routes": {"services", "data_pipeline", "utils", "store", "read", "orchestrate"}, + "services": { + "core", + "data_pipeline", + "utils", + "providers", + "store", + "ingest", + "transform", + "read", + "orchestrate", + }, + "core": {"data_pipeline", "utils", "read", "providers"}, "data_pipeline": {"utils"}, + "store": set(), + "providers": {"store", "utils"}, + "ingest": {"data_pipeline", "providers", "store", "utils"}, + "transform": {"data_pipeline", "store", "utils"}, + "read": {"data_pipeline", "orchestrate", "providers", "store", "utils"}, + "orchestrate": {"data_pipeline", "ingest", "store", "transform", "utils"}, "utils": set(), } +# KEEP IN SYNC with scripts/doc_guard.py::DATA_PIPELINE_SUBLAYERS. +DATA_PIPELINE_SUBLAYERS = frozenset({"providers", "store", "ingest", "transform", "read", "orchestrate"}) BUSINESS_LAYERS = set(ALLOWED_DEPS) @@ -74,7 +93,12 @@ def collect_files() -> list[Path]: def layer_of(rel: Path) -> str: - return "app" if rel.as_posix() == "app.py" else rel.parts[0] + parts = rel.parts + if rel.as_posix() == "app.py": + return "app" + if parts[0] == "data_pipeline" and len(parts) > 2 and parts[1] in DATA_PIPELINE_SUBLAYERS: + return parts[1] + return parts[0] def resolve_import(module: str | None, name: str | None, level: int, src: Path) -> Path | None: diff --git a/scripts/doc_guard.py b/scripts/doc_guard.py index 0366420..64171b1 100755 --- a/scripts/doc_guard.py +++ b/scripts/doc_guard.py @@ -178,7 +178,7 @@ def rule_yfinance_session_kwarg(ctx: Context) -> None: def rule_sqlite_bypass(ctx: Context) -> None: - allowed = {REPO_ROOT / "data_pipeline" / "db.py"} + allowed = {REPO_ROOT / "data_pipeline" / "store" / "db.py"} for path in ctx.files: if path.suffix != ".py": continue @@ -192,14 +192,24 @@ def rule_sqlite_bypass(ctx: Context) -> None: "sqlite-bypass", path, i, - "direct sqlite3.connect outside data_pipeline/db.py — bypasses WAL pragmas (ADR 0003)", + "direct sqlite3.connect outside data_pipeline/store/db.py — bypasses WAL pragmas (ADR 0003)", ) # ── Rule: import-direction ─────────────────────────────────────── # INVARIANT: the layer order is app → routes → services → core → data_pipeline -# → utils. A layer may only depend on layers *below* it, and ``routes`` may not -# skip across ``services`` into ``core``. +# → utils, and inside data_pipeline/ the acquire → process → serve stages are +# their own layers (ADR 0011, batch B3): +# +# data_pipeline/providers/ ACQUIRE (the only `import yfinance` site) +# data_pipeline/store/ schema + the only SQL +# data_pipeline/ingest/ acquisition → store glue +# data_pipeline/transform/ raw → clean → features (never imports providers) +# data_pipeline/read/ the DB-first read API services call +# data_pipeline/orchestrate/ "make the data ready" drivers +# +# A layer may only depend on layers *below* it, and ``routes`` may not skip +# across ``services`` into ``core``. # # WHY an explicit allow-list instead of the previous numeric comparison: the # numeric form compared layer numbers and therefore @@ -209,32 +219,72 @@ def rule_sqlite_bypass(ctx: Context) -> None: # (c) exempted ``utils`` wholesale via a sentinel value. # Those three blind spots let the declared architecture drift from the real # import graph. The allow-list states the intended edges directly. +# +# Two deviations from the plan's sketch, both recorded in +# docs/plans/business_line_reorg.md §8 (B3 note): +# * ``read → orchestrate``: the read path triggers refreshes, so the edge +# exists (orchestrate must therefore never import read). +# * ``providers → store``: the provider writes its own failures to +# ``store/quality_log``; keeping providers a pure leaf would mean inventing +# a callback for a one-line diagnostic write. _ALLOWED_DEPS: dict[str, set[str]] = { - "app": {"routes", "services", "core", "data_pipeline", "utils"}, - "routes": {"services", "data_pipeline", "utils"}, - "services": {"core", "data_pipeline", "utils"}, - # TRADEOFF: core→data_pipeline is directionally legal but breaks core's + "app": {"routes", "services", "core", "data_pipeline", "utils", "read", "orchestrate"}, + "routes": {"services", "data_pipeline", "utils", "store", "read", "orchestrate"}, + "services": { + "core", + "data_pipeline", + "utils", + "providers", + "store", + "ingest", + "transform", + "read", + "orchestrate", + }, + # TRADEOFF: core→data_pipeline* is directionally legal but breaks core's # purity contract. It is policed by the separate ``core-purity`` rule so the # two concerns (direction vs. purity) can be whitelisted and paid down at - # different paces. - "core": {"data_pipeline", "utils"}, + # different paces. B4 removes these two edges entirely. + "core": {"data_pipeline", "utils", "read", "providers"}, + # data_pipeline/ root: shared types (PipelineResult) + process-local state. "data_pipeline": {"utils"}, + "store": set(), + "providers": {"store", "utils"}, + "ingest": {"data_pipeline", "providers", "store", "utils"}, + "transform": {"data_pipeline", "store", "utils"}, + "read": {"data_pipeline", "orchestrate", "providers", "store", "utils"}, + "orchestrate": {"data_pipeline", "ingest", "store", "transform", "utils"}, # utils is a leaf: it may not reach back into any business layer. "utils": set(), } +# INVARIANT (KEEP IN SYNC with scripts/arch_metrics.py): the data_pipeline +# sub-packages that get their own layer key. +DATA_PIPELINE_SUBLAYERS = frozenset({"providers", "store", "ingest", "transform", "read", "orchestrate"}) + def _layer_of(path: Path) -> str | None: try: rel = path.relative_to(REPO_ROOT) if path.is_absolute() else path except ValueError: return None - head = rel.parts[0] if rel.parts else "" + parts = rel.parts + head = parts[0] if parts else "" if head == "app.py": return "app" + if head == "data_pipeline" and len(parts) > 2 and parts[1] in DATA_PIPELINE_SUBLAYERS: + return parts[1] return head if head in _ALLOWED_DEPS else None +def _import_layer(module: str) -> str: + """Map an imported module path to its layer key (see ``_layer_of``).""" + parts = module.split(".") + if parts[0] == "data_pipeline" and len(parts) > 1 and parts[1] in DATA_PIPELINE_SUBLAYERS: + return parts[1] + return parts[0] + + def _is_suppressed_at(path: Path, lineno: int, rule: str) -> bool: lines = _read(path) if not 1 <= lineno <= len(lines): @@ -243,11 +293,15 @@ def _is_suppressed_at(path: Path, lineno: int, rule: str) -> bool: def _imported_heads(path: Path) -> list[tuple[int, str]]: - """Every absolutely-imported top-level package with its line number. + """Every absolutely-imported module's *layer key*, with its line number. WHY ast.walk and not a scan of top-level statements: ``routes/`` historically hid its service imports inside function bodies, which kept them invisible to static review. Function-local imports are dependencies just the same. + + WHY a layer key rather than the top-level package: since batch B3 the six + ``data_pipeline/`` sub-packages are separate layers, so + ``data_pipeline.store.db`` must resolve to ``store``, not ``data_pipeline``. """ try: tree = ast.parse(path.read_text(encoding="utf-8")) @@ -261,7 +315,7 @@ def _imported_heads(path: Path) -> list[tuple[int, str]]: elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: mods = [node.module] for m in mods: - out.append((getattr(node, "lineno", 1), m.split(".", 1)[0])) + out.append((getattr(node, "lineno", 1), _import_layer(m))) return out @@ -299,7 +353,7 @@ def rule_core_purity(ctx: Context) -> None: if path.suffix != ".py" or _layer_of(path) != "core": continue for lineno, head in _imported_heads(path): - if head != "data_pipeline": + if head != "data_pipeline" and head not in DATA_PIPELINE_SUBLAYERS: continue if _is_suppressed_at(path, lineno, "core-purity"): continue @@ -313,10 +367,10 @@ def rule_core_purity(ctx: Context) -> None: # ── Rule: db-access ────────────────────────────────────────────── -# INVARIANT: data_pipeline/repos.py is the only place that builds SQL and -# data_pipeline/db.py the only place that owns connections. Upper layers must go +# INVARIANT: data_pipeline/store/repos.py is the only place that builds SQL and +# data_pipeline/store/db.py the only place that owns connections. Upper layers must go # through repos.py / DataService so WAL pragmas and the query cache apply. -_DB_IMPORT_RE = re.compile(r"^\s*from\s+data_pipeline\.db\s+import\s+(.+)$") +_DB_IMPORT_RE = re.compile(r"^\s*from\s+data_pipeline\.store\.db\s+import\s+(.+)$") # Connection lifecycle helpers are not SQL access: they have no repos.py # equivalent and every threaded render path must call them to avoid leaking the # thread-local connection. @@ -340,8 +394,8 @@ def rule_db_access(ctx: Context) -> None: "db-access", path, i, - "do not touch data_pipeline.db primitives — go through " - "data_pipeline/repos.py or DataService (ADR 0003)", + "do not touch data_pipeline.store.db primitives — go through " + "data_pipeline/store/repos.py or DataService (ADR 0003)", ) diff --git a/scripts/migrate_canonical_tables.py b/scripts/migrate_canonical_tables.py index 208e724..e05ca4c 100644 --- a/scripts/migrate_canonical_tables.py +++ b/scripts/migrate_canonical_tables.py @@ -6,7 +6,7 @@ - Batch B2 renamed the store tables (: ``raw_prices``/``clean_prices``/ ``processed_prices`` → ``raw_bars``/``clean_bars``/``feature_bars``). The pipeline now reads and writes the canonical names, and writes the legacy - names too for one release (see ``data_pipeline/db.py::_TABLE_SHADOWS``), but + names too for one release (see ``data_pipeline/store/db.py::_TABLE_SHADOWS``), but a DB created *before* B2 only has rows under the legacy names. - This script copies legacy → canonical so an existing ``market_data.sqlite`` becomes readable by the new code without waiting for a re-download. @@ -21,7 +21,7 @@ Usage: python scripts/migrate_canonical_tables.py [--db PATH] [--dry-run] Dependencies: - - data_pipeline.db (schema + connection pragmas); stdlib only otherwise. + - data_pipeline.store.db (schema + connection pragmas); stdlib only otherwise. """ from __future__ import annotations @@ -34,7 +34,7 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from data_pipeline.db import CANONICAL_TABLES, DB_PATH, get_conn, init_db # noqa: E402 +from data_pipeline.store.db import CANONICAL_TABLES, DB_PATH, get_conn, init_db # noqa: E402 def _columns(conn, table: str) -> list[str]: diff --git a/scripts/seed_history.py b/scripts/seed_history.py index e14150a..c1fc9df 100755 --- a/scripts/seed_history.py +++ b/scripts/seed_history.py @@ -7,7 +7,7 @@ import logging import sys -from data_pipeline.data_ops import DataService +from data_pipeline.read import DataService logging.basicConfig(level=logging.INFO) diff --git a/services/market/analysis/summary.py b/services/market/analysis/summary.py index 6e30136..f9f5836 100644 --- a/services/market/analysis/summary.py +++ b/services/market/analysis/summary.py @@ -37,7 +37,7 @@ def generate_summary_analysis(tickers: list, results_by_ticker: dict) -> dict: # Correlation matrix try: - from data_pipeline.yf_client import fetch_close_panel + from data_pipeline.providers.yf_client import fetch_close_panel data = fetch_close_panel(tickers, period="90d") if data is None or data.empty: diff --git a/services/market/dispatch.py b/services/market/dispatch.py index 876289b..f2e8f8d 100644 --- a/services/market/dispatch.py +++ b/services/market/dispatch.py @@ -11,7 +11,7 @@ Contracts: - render_streaming_slice(kind) -> Response | tuple[str, int] Dependencies UPWARD: - - data_pipeline.job_cache, data_pipeline.db + - data_pipeline.orchestrate.job_cache, data_pipeline.store.db - services.market.analysis - utils.constants, utils.render_helpers Dependencies DOWNWARD: @@ -26,8 +26,8 @@ from flask import render_template, request -from data_pipeline.db import close_thread_conn -from data_pipeline.job_cache import compute_or_get, get_job +from data_pipeline.orchestrate.job_cache import compute_or_get, get_job +from data_pipeline.store.db import close_thread_conn from services.market.analysis import AnalysisService from utils.constants import ( DEFAULT_FREQUENCY, diff --git a/services/market/facade.py b/services/market/facade.py index 1491109..33326b3 100644 --- a/services/market/facade.py +++ b/services/market/facade.py @@ -10,7 +10,7 @@ import logging from core.market.data_context import build_data_context -from data_pipeline.yf_client import fetch_spot as _fetch_spot +from data_pipeline.providers.yf_client import fetch_spot as _fetch_spot from services.market_review import market_review, market_review_timeseries from utils.date_helpers import exclusive_month_end from utils.ticker_utils import is_valid_ticker_format diff --git a/services/market/health.py b/services/market/health.py index e92c816..8dc74ee 100644 --- a/services/market/health.py +++ b/services/market/health.py @@ -11,7 +11,7 @@ from datetime import UTC, date, datetime, timedelta from typing import Any -from data_pipeline.repos import fetch_ticker_inventory +from data_pipeline.store.repos import fetch_ticker_inventory _FRESHNESS_DAYS = int(os.environ.get("DATA_FRESHNESS_DAYS", "5")) @@ -56,7 +56,7 @@ def overall_summary() -> dict[str, Any]: # Recent yfinance / pipeline failures from data_quality_log. try: - from data_pipeline.quality_log import failure_counts, recent_failures + from data_pipeline.store.quality_log import failure_counts, recent_failures failures_24h = failure_counts(hours=24) recent = recent_failures(hours=24, limit=20) diff --git a/services/market/signals.py b/services/market/signals.py index f886ba7..d084441 100644 --- a/services/market/signals.py +++ b/services/market/signals.py @@ -12,7 +12,7 @@ import logging from core import signals as sig -from data_pipeline.data_ops import DataService +from data_pipeline.read import DataService logger = logging.getLogger(__name__) diff --git a/services/market_review/fetch.py b/services/market_review/fetch.py index ad12f96..bad52c8 100644 --- a/services/market_review/fetch.py +++ b/services/market_review/fetch.py @@ -14,7 +14,7 @@ Contracts: - fetch_market_data(instrument, start_date, end_date) -> tuple[pd.DataFrame, pd.DataFrame, list] Dependencies: - - data_pipeline.yf_client, data_pipeline.db + - data_pipeline.providers.yf_client, data_pipeline.store.db - core.market_review.constants (BENCHMARKS) """ @@ -28,13 +28,13 @@ import pandas as pd from core.market_review.constants import BENCHMARKS -from data_pipeline.repos import ( +from data_pipeline.providers.yf_client import fetch_close_panel +from data_pipeline.store.repos import ( ensure_schema, fetch_market_review_latest_dates, fetch_market_review_panel, upsert_market_review_prices, ) -from data_pipeline.yf_client import fetch_close_panel logger = logging.getLogger(__name__) diff --git a/services/options/builder.py b/services/options/builder.py index f6bd902..93b6453 100644 --- a/services/options/builder.py +++ b/services/options/builder.py @@ -12,7 +12,7 @@ import pandas as pd from core import strategies as strategies_mod -from data_pipeline.yf_client import fetch_option_chain +from data_pipeline.providers.yf_client import fetch_option_chain from utils.api_errors import ApiError logger = logging.getLogger(__name__) @@ -101,7 +101,7 @@ def _vol_context(ticker: str, current_iv_pct: float | None) -> dict[str, Any]: from datetime import date, timedelta from core import signals as signals_mod - from data_pipeline.data_ops import DataService + from data_pipeline.read import DataService start = date.today() - timedelta(days=400) df = DataService.get_cleaned_daily(ticker, start=start) diff --git a/services/options/chain.py b/services/options/chain.py index 76ab595..2e5c7d2 100644 --- a/services/options/chain.py +++ b/services/options/chain.py @@ -12,7 +12,7 @@ liquidity_score, ) from core.options.chain.filters import filter_option_chain -from data_pipeline.yf_client import fetch_option_chain +from data_pipeline.providers.yf_client import fetch_option_chain logger = logging.getLogger(__name__) diff --git a/services/options/preload.py b/services/options/preload.py index ba27bc8..a961085 100644 --- a/services/options/preload.py +++ b/services/options/preload.py @@ -10,7 +10,7 @@ Dependencies UPWARD: - core._shared.dates (dte) - core.options.chain.analyzer (OptionsChainAnalyzer) - - data_pipeline.yf_client + - data_pipeline.providers.yf_client Dependencies DOWNWARD: - routes/options.py """ @@ -25,7 +25,7 @@ from core._shared.dates import dte from core.options.chain.analyzer import OptionsChainAnalyzer -from data_pipeline.yf_client import fetch_option_chain +from data_pipeline.providers.yf_client import fetch_option_chain logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ # CONSTRAINT: bound on the cache key space. Each payload is a full option # chain (up to MBs) and keys come from a public endpoint, so ticker # enumeration would otherwise grow memory without limit (mirrors -# data_pipeline/data_ops/_globals.py::_cache_set). +# data_pipeline/_state.py::_cache_set). _OPTION_CHAIN_CACHE_MAX = 128 diff --git a/services/options/simulation.py b/services/options/simulation.py index 9ac6087..f1d34bf 100644 --- a/services/options/simulation.py +++ b/services/options/simulation.py @@ -10,7 +10,7 @@ Contracts: - run_simulation(payload) -> dict Dependencies UPWARD: - - utils.api_errors, utils.ticker_utils, data_pipeline.yf_client + - utils.api_errors, utils.ticker_utils, data_pipeline.providers.yf_client - core.options.simulation Dependencies DOWNWARD: - routes.options, tests @@ -118,7 +118,7 @@ def resolve_spot(ticker: str, override: Any) -> float: if not ticker: raise ApiError("ticker (or an explicit spot) is required", code="ticker_required") - from data_pipeline.yf_client import fetch_spot + from data_pipeline.providers.yf_client import fetch_spot spot = fetch_spot(ticker) if spot is None or not math.isfinite(spot) or spot <= 0: diff --git a/services/portfolio/analysis.py b/services/portfolio/analysis.py index f4c0355..ec7433f 100644 --- a/services/portfolio/analysis.py +++ b/services/portfolio/analysis.py @@ -241,7 +241,7 @@ def _calc_var(positions, spots, greeks_totals, confidence=0.95): def _get_spots(positions: list) -> dict: tickers = list({p["ticker"] for p in positions}) try: - from data_pipeline.yf_client import fetch_spots_bulk + from data_pipeline.providers.yf_client import fetch_spots_bulk return fetch_spots_bulk(tickers) except Exception as e: diff --git a/services/portfolio/facade.py b/services/portfolio/facade.py index 2589d3b..e17f25e 100644 --- a/services/portfolio/facade.py +++ b/services/portfolio/facade.py @@ -16,12 +16,12 @@ from core.portfolio import Position, aggregate_greeks, attribute_pnl from core.strategies import Leg -from data_pipeline.repos import ( +from data_pipeline.providers.yf_client import fetch_spots_bulk +from data_pipeline.store.repos import ( insert_tracked_strategy, select_tracked_strategies, update_tracked_strategy_closed, ) -from data_pipeline.yf_client import fetch_spots_bulk from utils.api_errors import ApiError logger = logging.getLogger(__name__) diff --git a/services/regime/facade.py b/services/regime/facade.py index 3c00a10..762bd0a 100644 --- a/services/regime/facade.py +++ b/services/regime/facade.py @@ -26,8 +26,8 @@ label_series, regime_transitions, ) -from data_pipeline.data_ops import DataService -from data_pipeline.repos import fetch_regime_log_window +from data_pipeline.read import DataService +from data_pipeline.store.repos import fetch_regime_log_window from services.regime.ops._bootstrap import ( BOOTSTRAP_DAYS, MIN_TRADING_ROWS, diff --git a/services/regime/ops/_bootstrap.py b/services/regime/ops/_bootstrap.py index cdeb785..5e7c459 100644 --- a/services/regime/ops/_bootstrap.py +++ b/services/regime/ops/_bootstrap.py @@ -4,11 +4,11 @@ import logging from core.regime import SLOPE_LOOKBACK, SMA_WINDOW -from data_pipeline.cleaning import clean_range -from data_pipeline.data_ops import _cache_invalidate -from data_pipeline.downloader import upsert_raw_prices -from data_pipeline.processing import process_frequencies -from data_pipeline.repos import count_clean_rows +from data_pipeline._state import _cache_invalidate +from data_pipeline.ingest.ohlcv import upsert_raw_prices +from data_pipeline.store.repos import count_clean_rows +from data_pipeline.transform.cleaning import clean_range +from data_pipeline.transform.processing import process_frequencies logger = logging.getLogger(__name__) diff --git a/services/regime/ops/_persistence.py b/services/regime/ops/_persistence.py index 96dd530..6ac940e 100644 --- a/services/regime/ops/_persistence.py +++ b/services/regime/ops/_persistence.py @@ -1,6 +1,6 @@ -"""Regime log persistence: thin delegates to ``data_pipeline.repos``. +"""Regime log persistence: thin delegates to ``data_pipeline.store.repos``. -The actual SQL lives in ``data_pipeline.repos`` (the single SQL-building home, +The actual SQL lives in ``data_pipeline.store.repos`` (the single SQL-building home, ADR 0003 / architecture review §2 `db-access`); these wrappers keep the ``services.regime`` call surface stable. """ @@ -9,7 +9,7 @@ import pandas as pd -from data_pipeline.repos import ( +from data_pipeline.store.repos import ( load_regime_log, previous_regime_log_row, upsert_regime_log_rows, diff --git a/tests/conftest.py b/tests/conftest.py index 5132da6..422aa74 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,7 +57,7 @@ def _isolate_db(monkeypatch, tmp_path): db_file = str(tmp_path / "test_market_data.sqlite") monkeypatch.setenv("MARKET_DB_PATH", db_file) try: - import data_pipeline.db as db_mod + import data_pipeline.store.db as db_mod except ImportError: # Project deps not installed in the current interpreter — let # tests that actually need the DB fail with their own clear error @@ -70,7 +70,7 @@ def _isolate_db(monkeypatch, tmp_path): # test (which used a different DB file) would otherwise mask freshly # seeded data within the 60-second TTL. try: - import data_pipeline.data_ops as ds_mod + import data_pipeline._state as ds_mod with ds_mod._query_cache_lock: ds_mod._query_cache.clear() diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 99f9779..07097ab 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -10,7 +10,7 @@ The Flask process runs every route normally, but `yfinance.Ticker`, `yf.download`, and `fast_info` are monkey-patched in the backend process to return synthetic data. Combined with the existing - `TEST_*` ticker fixture mechanism in `data_pipeline.downloader`, this + `TEST_*` ticker fixture mechanism in `data_pipeline.ingest.ohlcv`, this lets e2e tests exercise real form submission, real DataService pipeline, real chart rendering — without network. @@ -69,11 +69,11 @@ def _e2e_db(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: """ db_file = str(tmp_path_factory.mktemp("e2e-db") / "market.sqlite") os.environ["MARKET_DB_PATH"] = db_file - # Patch the module attr in case data_pipeline.db was already imported + # Patch the module attr in case data_pipeline.store.db was already imported # by a previous test module in the same pytest session (DB_PATH is # captured at import time). try: - import data_pipeline.db as db_mod + import data_pipeline.store.db as db_mod db_mod.DB_PATH = db_file # Ensure schema exists at the new path even if app was pre-imported. @@ -91,14 +91,14 @@ def _e2e_db(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: # → /api/option_chain → /api/validate_tickers) opt in by using the `yf_stub` # fixture *instead of* `mock_apis`. The patch covers: # -# * `yfinance.download` — used by data_pipeline.downloader and +# * `yfinance.download` — used by data_pipeline.ingest.ohlcv and # core.market_review # * `yfinance.Ticker(...).fast_info` — used for spot price lookups # * `yfinance.Ticker(...).options` — option expirations list # * `yfinance.Ticker(...).option_chain(exp)` — calls/puts DataFrames # # Combined with the existing `TEST_*` ticker bypass in -# `data_pipeline.downloader.download_bars`, real `TEST_AAPL` form submissions +# `data_pipeline.ingest.ohlcv.download_bars`, real `TEST_AAPL` form submissions # never hit the network. # --------------------------------------------------------------------------- def _synthetic_ohlcv(ticker: str, start: dt.date, end: dt.date): @@ -221,7 +221,7 @@ def seed_test_data(_e2e_db: str, yf_stub: None) -> Iterator[None]: Uses the production downloader's `TEST_*` fixture branch — no network. """ try: - from data_pipeline.data_ops import DataService + from data_pipeline.read import DataService # `manual_update` will route to the synthetic fixture for TEST_* DataService.manual_update("TEST_AAPL", days=120) diff --git a/tests/e2e/test_form_submit_flow.py b/tests/e2e/test_form_submit_flow.py index fcde223..f28e915 100644 --- a/tests/e2e/test_form_submit_flow.py +++ b/tests/e2e/test_form_submit_flow.py @@ -4,7 +4,7 @@ template render → table visible. The yfinance layer is patched at the backend process level via the `yf_stub` fixture; the synthetic ticker ``TEST_AAPL`` routes through the existing fixture branch in -`data_pipeline.downloader`. +`data_pipeline.ingest.ohlcv`. """ from __future__ import annotations diff --git a/tests/test_architecture_purity.py b/tests/test_architecture_purity.py index f99adeb..5e903d2 100644 --- a/tests/test_architecture_purity.py +++ b/tests/test_architecture_purity.py @@ -1,4 +1,4 @@ -"""Architecture contract tests: core/ subpackages stay pure. +"""Architecture contract tests: core/ purity and the data_pipeline layer graph. Domain: Tests — Architecture Purity Contracts Context: @@ -8,10 +8,19 @@ remaining violation stays visible in the test report and can be counted down to zero instead of being forgotten. - The violation registry lives in docs/architecture_review.md §2. + - Batch B3 (ADR 0011) split ``data_pipeline/`` into six layers + (providers / store / ingest / transform / read / orchestrate). The declared + edges live in ``scripts/doc_guard.py::_ALLOWED_DEPS`` and are mirrored in + ``scripts/arch_metrics.py`` — two copies, hence the sync test below. Contracts: - test_core_subpackage_has_no_io_or_framework_imports: for every core/ subpackage, no absolute import of an I/O or framework package, except lines explicitly carrying ``doc-guard: allow=core-purity``. + - test_data_pipeline_import_graph_matches_declared_layers: the real import + graph conforms to the declared layer table. + - test_transform_never_imports_providers: processing stays provider-agnostic + (ADR 0011 §5.2) — it only sees canonical tables. + - test_layer_tables_are_in_sync: doc_guard and arch_metrics agree. Dependencies UPWARD: - (none — stdlib + pytest only) """ @@ -19,12 +28,33 @@ from __future__ import annotations import ast +import importlib.util +import sys from pathlib import Path import pytest REPO_ROOT = Path(__file__).resolve().parent.parent CORE = REPO_ROOT / "core" +DATA_PIPELINE = REPO_ROOT / "data_pipeline" + + +def _load_script(name: str): + """Import a ``scripts/*.py`` module (scripts/ is not a package). + + WHY exec_module: the two guard scripts are standalone (no third-party + imports, runnable from pre-commit) and must stay that way, so the tests + reach into them rather than the other way round. + """ + spec = importlib.util.spec_from_file_location(f"_guard_{name}", REPO_ROOT / "scripts" / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + # WHY register first: doc_guard defines dataclasses, and @dataclass resolves + # type hints through sys.modules[cls.__module__] at class-creation time. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + # INVARIANT: core/ is pure computation — no DB, no network, no Flask, no app. FORBIDDEN_ROOTS = { @@ -70,3 +100,54 @@ def test_core_subpackage_has_no_io_or_framework_imports(pkg): continue offenders.append(f"{py.relative_to(REPO_ROOT)}:{lineno} imports '{head}'") assert not offenders, "core/ purity violated (fetch upstream and pass data in, ADR 0001):\n" + "\n".join(offenders) + + +def test_data_pipeline_import_graph_matches_declared_layers(): + """Every data_pipeline/ import must point at an allowed layer. + + This is the test-layer twin of doc_guard's ``import-direction`` rule, with + sub-package granularity: ``data_pipeline.store.db`` counts as ``store``. + """ + guard = _load_script("doc_guard") + offenders: list[str] = [] + for py in sorted(DATA_PIPELINE.rglob("*.py")): + if "__pycache__" in py.parts: + continue + layer = guard._layer_of(py) + if layer is None: + continue + allowed = guard._ALLOWED_DEPS[layer] + for lineno, head in guard._imported_heads(py): + if head not in guard._ALLOWED_DEPS or head == layer: + continue + if head in allowed or guard._is_suppressed_at(py, lineno, "import-direction"): + continue + offenders.append(f"{py.relative_to(REPO_ROOT)}:{lineno}: {layer} -> {head}") + assert not offenders, "data_pipeline layer graph violated (see scripts/doc_guard.py::_ALLOWED_DEPS):\n" + "\n".join( + offenders + ) + + +def test_transform_never_imports_providers(): + """INVARIANT (ADR 0011 §5.2): processing is provider-agnostic. + + ``transform/`` reads canonical tables only. ``read/`` is allowed to reach + the provider for the spot fallback (recorded as a deviation in the plan §8). + """ + guard = _load_script("doc_guard") + offenders: list[str] = [] + for py in sorted((DATA_PIPELINE / "transform").rglob("*.py")): + if "__pycache__" in py.parts: + continue + for lineno, head in guard._imported_heads(py): + if head == "providers": + offenders.append(f"{py.relative_to(REPO_ROOT)}:{lineno}") + assert not offenders, "transform/ must not import providers/:\n" + "\n".join(offenders) + + +def test_layer_tables_are_in_sync(): + """``doc_guard`` and ``arch_metrics`` each carry a copy of the layer table.""" + doc_guard = _load_script("doc_guard") + arch_metrics = _load_script("arch_metrics") + assert doc_guard._ALLOWED_DEPS == arch_metrics.ALLOWED_DEPS + assert doc_guard.DATA_PIPELINE_SUBLAYERS == arch_metrics.DATA_PIPELINE_SUBLAYERS diff --git a/tests/test_background_backfill.py b/tests/test_background_backfill.py index 6ad9d10..27df42b 100644 --- a/tests/test_background_backfill.py +++ b/tests/test_background_backfill.py @@ -14,14 +14,15 @@ import pandas as pd import pytest -import data_pipeline.data_ops._query as _q +import data_pipeline.read._query as _q from data_pipeline import PipelineResult -from data_pipeline.data_ops import _cache_get, _cache_invalidate -from data_pipeline.data_ops._query import ( +from data_pipeline._state import _cache_get, _cache_invalidate +from data_pipeline.orchestrate import backfill as _bf +from data_pipeline.read._query import ( _join_backfills, _kick_backfill, ) -from data_pipeline.db import init_db +from data_pipeline.store.db import init_db TICKER = "BGTEST1" @@ -48,9 +49,11 @@ def test_wide_range_request_returns_without_full_backfill(self, monkeypatch): whatever exists (nothing), while the backfill continues in background.""" init_db() _dl, calls = _slow_downloader(delay=1.0) - monkeypatch.setattr("data_pipeline.downloader.upsert_raw_prices", _dl) - monkeypatch.setattr("data_pipeline.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) - monkeypatch.setattr("data_pipeline.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr("data_pipeline.ingest.ohlcv.upsert_raw_prices", _dl) + monkeypatch.setattr("data_pipeline.transform.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr( + "data_pipeline.transform.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1) + ) start = dt.date(2021, 1, 1) end = dt.date.today() @@ -68,9 +71,11 @@ def test_wide_range_request_returns_without_full_backfill(self, monkeypatch): def test_partial_read_is_not_cached(self, monkeypatch): init_db() _dl, _calls = _slow_downloader(delay=1.0) - monkeypatch.setattr("data_pipeline.downloader.upsert_raw_prices", _dl) - monkeypatch.setattr("data_pipeline.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) - monkeypatch.setattr("data_pipeline.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr("data_pipeline.ingest.ohlcv.upsert_raw_prices", _dl) + monkeypatch.setattr("data_pipeline.transform.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr( + "data_pipeline.transform.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1) + ) start = dt.date(2021, 1, 1) end = dt.date.today() @@ -88,11 +93,13 @@ def test_completed_backfill_becomes_visible_and_cached(self, monkeypatch): def _fast_dl(ticker, start, end): # noqa: ARG001 return PipelineResult(rows=10) - monkeypatch.setattr("data_pipeline.downloader.upsert_raw_prices", _fast_dl) - monkeypatch.setattr("data_pipeline.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) - monkeypatch.setattr("data_pipeline.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr("data_pipeline.ingest.ohlcv.upsert_raw_prices", _fast_dl) + monkeypatch.setattr("data_pipeline.transform.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) monkeypatch.setattr( - "data_pipeline.data_ops._query.fetch_df", + "data_pipeline.transform.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1) + ) + monkeypatch.setattr( + "data_pipeline.read._query.fetch_df", lambda sql, params: pd.DataFrame( { "date": ["2026-01-05"], @@ -123,14 +130,16 @@ def test_needs_backfill_probe(self): start = dt.date(2021, 1, 1) end = dt.date.today() # Empty DB → backfill needed. - assert _q._r.needs_backfill(TICKER + "-PROBE", start, end) is True + assert _bf.needs_backfill(TICKER + "-PROBE", start, end) is True def test_kick_dedupes_concurrent_kicks(self, monkeypatch): init_db() _dl, calls = _slow_downloader(delay=0.3) - monkeypatch.setattr("data_pipeline.downloader.upsert_raw_prices", _dl) - monkeypatch.setattr("data_pipeline.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) - monkeypatch.setattr("data_pipeline.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr("data_pipeline.ingest.ohlcv.upsert_raw_prices", _dl) + monkeypatch.setattr("data_pipeline.transform.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr( + "data_pipeline.transform.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1) + ) start, end = dt.date(2021, 1, 1), dt.date.today() for _ in range(5): diff --git a/tests/test_canonical_tables.py b/tests/test_canonical_tables.py index 22fc8bd..253e21a 100644 --- a/tests/test_canonical_tables.py +++ b/tests/test_canonical_tables.py @@ -29,7 +29,7 @@ import pytest -from data_pipeline.db import CANONICAL_TABLES, canonical_table, get_conn, init_db, upsert_many +from data_pipeline.store.db import CANONICAL_TABLES, canonical_table, get_conn, init_db, upsert_many REPO_ROOT = Path(__file__).resolve().parent.parent @@ -84,9 +84,9 @@ def test_upsert_many_writes_both_table_families(): def test_pipeline_run_populates_both_table_families(): """B2 exit criterion: one pipeline run leaves both families populated.""" - from data_pipeline.cleaning import clean_range - from data_pipeline.downloader import upsert_raw_prices - from data_pipeline.processing import process_frequencies + from data_pipeline.ingest.ohlcv import upsert_raw_prices + from data_pipeline.transform.cleaning import clean_range + from data_pipeline.transform.processing import process_frequencies ticker = "TEST_CANON" end = dt.date.today() @@ -108,7 +108,7 @@ def test_pipeline_run_populates_both_table_families(): # --------------------------------------------------------------------------- def test_health_inventory_reads_canonical_table(): """A row that exists only in ``raw_bars`` must be visible to the health read.""" - from data_pipeline.repos import fetch_ticker_inventory + from data_pipeline.store.repos import fetch_ticker_inventory init_db() with get_conn() as conn: diff --git a/tests/test_cleaning.py b/tests/test_cleaning.py index de18559..1191ced 100644 --- a/tests/test_cleaning.py +++ b/tests/test_cleaning.py @@ -1,11 +1,11 @@ -"""Tests for data_pipeline/cleaning.py — anomaly flags and business-day alignment.""" +"""Tests for data_pipeline/transform/cleaning.py — anomaly flags and business-day alignment.""" import datetime as dt import numpy as np import pandas as pd -from data_pipeline.cleaning import _flag_anomalies, _get_business_days +from data_pipeline.transform.cleaning import _flag_anomalies, _get_business_days class TestGetBusinessDays: diff --git a/tests/test_concurrency.py b/tests/test_concurrency.py index 6bb7cc7..a1bddea 100644 --- a/tests/test_concurrency.py +++ b/tests/test_concurrency.py @@ -8,9 +8,8 @@ import pytest from data_pipeline import PipelineResult -from data_pipeline.data_ops import ( +from data_pipeline._state import ( _QUERY_CACHE_TTL, - DataService, _cache_get, _cache_invalidate, _cache_set, @@ -19,7 +18,8 @@ _update_lock_mutex, _update_locks, ) -from data_pipeline.db import init_db +from data_pipeline.read import DataService +from data_pipeline.store.db import init_db @pytest.fixture(autouse=True) @@ -40,18 +40,18 @@ def _reset_state(): class TestCooldown: - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_first_call_runs_pipeline(self, mock_dl, mock_cl, mock_pr): init_db() result = DataService.manual_update("COOL1") assert result is True mock_dl.assert_called_once() - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_second_call_within_cooldown_skips(self, mock_dl, mock_cl, mock_pr): init_db() DataService.manual_update("COOL2") @@ -59,9 +59,9 @@ def test_second_call_within_cooldown_skips(self, mock_dl, mock_cl, mock_pr): assert result is False assert mock_dl.call_count == 1 # only first call - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_different_tickers_not_blocked(self, mock_dl, mock_cl, mock_pr): init_db() DataService.manual_update("TCKR_A") @@ -69,7 +69,9 @@ def test_different_tickers_not_blocked(self, mock_dl, mock_cl, mock_pr): assert result is True assert mock_dl.call_count == 2 - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(ok=False, error="download_failed")) + @patch( + "data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(ok=False, error="download_failed") + ) def test_failed_pipeline_clears_cooldown(self, mock_dl): """If download fails, cooldown should NOT prevent retry (since we return False, not raise).""" init_db() @@ -84,9 +86,9 @@ def test_failed_pipeline_clears_cooldown(self, mock_dl): class TestConcurrentUpdates: - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_concurrent_same_ticker_only_one_runs(self, mock_dl, mock_cl, mock_pr): """Two threads updating same ticker: only first should actually run.""" init_db() @@ -105,9 +107,9 @@ def update(): # One True (ran), one False (cooldown) assert sorted(results) == [False, True] - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_concurrent_different_tickers_both_run(self, mock_dl, mock_cl, mock_pr): """Two threads updating different tickers: both should run.""" init_db() @@ -188,10 +190,10 @@ def setup_method(self): DataService._ensure_range_memo.clear() DataService._ensure_range_inflight.clear() - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.db.fetch_df") + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.store.db.fetch_df") def test_concurrent_calls_run_backfill_only_once(self, mock_fetch_df, mock_dl, mock_cl, mock_pr): import datetime as dt @@ -239,8 +241,8 @@ def test_sentinel_start_skips_backfill_when_db_has_coverage(self): DataService._ensure_range_memo.clear() with ( - patch("data_pipeline.db.fetch_df") as mock_fetch_df, - patch("data_pipeline.downloader.upsert_raw_prices") as mock_dl, + patch("data_pipeline.store.db.fetch_df") as mock_fetch_df, + patch("data_pipeline.ingest.ohlcv.upsert_raw_prices") as mock_dl, ): # DB has 2021-01-01 .. today coverage already. mock_fetch_df.return_value = pd.DataFrame( @@ -254,10 +256,10 @@ def test_sentinel_start_skips_backfill_when_db_has_coverage(self): assert ok is True assert mock_dl.call_count == 0, "must NOT walk yfinance back to 1990" - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.db.fetch_df") + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.store.db.fetch_df") def test_explicit_multiyear_request_does_backfill(self, mock_fetch_df, mock_dl, mock_cl, mock_pr): """Regression for the 'NVDA only has 30 days, user asked for 5 years, sentinel short-circuit silently lied' bug. A user-explicit @@ -281,10 +283,10 @@ def test_explicit_multiyear_request_does_backfill(self, mock_fetch_df, mock_dl, "user-explicit 5-year range must trigger backfill — sentinel short-circuit must NOT apply here" ) - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.db.fetch_df") + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.store.db.fetch_df") def test_sentinel_with_thin_db_still_backfills(self, mock_fetch_df, mock_dl, mock_cl, mock_pr): """Regression: sentinel start (PriceDynamic uses 1900-01-01 always) but DB has only ~1 month of recent data MUST backfill. The sentinel diff --git a/tests/test_db.py b/tests/test_db.py index 1ea8781..045f16b 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,11 +1,11 @@ -"""Tests for data_pipeline/db.py — init, get_conn, upsert, fetch.""" +"""Tests for data_pipeline/store/db.py — init, get_conn, upsert, fetch.""" import sqlite3 import threading import pytest -from data_pipeline.db import close_thread_conn, fetch_df, get_conn, init_db, upsert_many +from data_pipeline.store.db import close_thread_conn, fetch_df, get_conn, init_db, upsert_many class TestInitDb: diff --git a/tests/test_db_errors.py b/tests/test_db_errors.py index 6383988..27eced1 100644 --- a/tests/test_db_errors.py +++ b/tests/test_db_errors.py @@ -1,4 +1,4 @@ -"""Tests for data_pipeline.db — error scenarios and edge cases.""" +"""Tests for data_pipeline.store.db — error scenarios and edge cases.""" import os import sqlite3 @@ -6,7 +6,7 @@ import pandas as pd import pytest -from data_pipeline.db import fetch_df, get_conn, init_db, upsert_many +from data_pipeline.store.db import fetch_df, get_conn, init_db, upsert_many class TestInitDb: diff --git a/tests/test_downloader_gap.py b/tests/test_downloader_gap.py index 3ff7518..f9f173f 100644 --- a/tests/test_downloader_gap.py +++ b/tests/test_downloader_gap.py @@ -1,4 +1,4 @@ -"""Tests for the gap-aware downloader logic in `data_pipeline/downloader.py`. +"""Tests for the gap-aware downloader logic in `data_pipeline/ingest/ohlcv.py`. These lock in the behavior fix for the NVDA-style outage: when historical business days are missing from `raw_prices` (e.g. after a yfinance rate-limit @@ -14,15 +14,10 @@ import pandas as pd import pytest -from data_pipeline.data_ops import ( - DataService, - _query_cache, - _query_cache_lock, - _update_lock_mutex, - _update_locks, -) -from data_pipeline.db import init_db, upsert_many -from data_pipeline.downloader import find_missing_business_days, upsert_raw_prices +from data_pipeline._state import _query_cache, _query_cache_lock, _update_lock_mutex, _update_locks +from data_pipeline.ingest.ohlcv import find_missing_business_days, upsert_raw_prices +from data_pipeline.read import DataService +from data_pipeline.store.db import init_db, upsert_many @pytest.fixture(autouse=True) diff --git a/tests/test_health_service.py b/tests/test_health_service.py index 414122b..ed831be 100644 --- a/tests/test_health_service.py +++ b/tests/test_health_service.py @@ -4,12 +4,12 @@ import pandas as pd -from data_pipeline.db import get_conn +from data_pipeline.store.db import get_conn from services.market.health import overall_summary, per_ticker_summary def _seed(ticker: str, dates: list[str], close_vals: list[float | None]) -> None: - from data_pipeline.db import init_db + from data_pipeline.store.db import init_db init_db() with get_conn() as conn: diff --git a/tests/test_job_cache.py b/tests/test_job_cache.py index f14ffcf..d036ae4 100644 --- a/tests/test_job_cache.py +++ b/tests/test_job_cache.py @@ -1,11 +1,11 @@ -"""Tests for data_pipeline/job_cache.py.""" +"""Tests for data_pipeline/orchestrate/job_cache.py.""" import threading import time import pytest -from data_pipeline import job_cache as jc +from data_pipeline.orchestrate import job_cache as jc @pytest.fixture(autouse=True) diff --git a/tests/test_market_review.py b/tests/test_market_review.py index fdda1e3..7c9fa53 100644 --- a/tests/test_market_review.py +++ b/tests/test_market_review.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from data_pipeline.db import get_conn, init_db +from data_pipeline.store.db import get_conn, init_db # ── Helpers ─────────────────────────────────────────────────────── diff --git a/tests/test_nvda_analysis.py b/tests/test_nvda_analysis.py index f1bf55b..0f35cac 100644 --- a/tests/test_nvda_analysis.py +++ b/tests/test_nvda_analysis.py @@ -14,7 +14,7 @@ import pandas as pd import pytest -from data_pipeline.db import get_conn, init_db +from data_pipeline.store.db import get_conn, init_db _JOB_ID_RE = re.compile(r'STREAMING_JOB_ID\s*=\s*"([^"]+)"') @@ -39,7 +39,7 @@ def _seed_clean_bars(ticker: str, n_rows: int = 30, *, nan_only: bool = False): """ init_db() # Drop the cross-test query cache that DataService maintains (TTL 60s). - from data_pipeline.data_ops import _cache_invalidate + from data_pipeline._state import _cache_invalidate _cache_invalidate(ticker) dates = pd.bdate_range(end=dt.date.today(), periods=n_rows) @@ -68,17 +68,17 @@ def _seed_clean_bars(ticker: str, n_rows: int = 30, *, nan_only: bool = False): @pytest.fixture() def _patch_downloads(monkeypatch): """Disable all real yfinance download paths for unit tests.""" - from data_pipeline.data_ops import DataService + from data_pipeline.read import DataService # Block the manual_update → pipeline path. Patch BOTH the DataService # facade and the module-level function: _query.py calls the _update module # directly because facade imports _query (the reverse edge would be an # import cycle), so patching only the class would be bypassed. monkeypatch.setattr(DataService, "manual_update", staticmethod(lambda *a, **kw: None)) - monkeypatch.setattr("data_pipeline.data_ops._update.manual_update", lambda *a, **kw: None) + monkeypatch.setattr("data_pipeline.orchestrate.update.manual_update", lambda *a, **kw: None) # Block the ensure_range → chunked backfill path (same dual patching). monkeypatch.setattr(DataService, "ensure_range", staticmethod(lambda *a, **kw: True)) - monkeypatch.setattr("data_pipeline.data_ops._range.ensure_range", lambda *a, **kw: True) + monkeypatch.setattr("data_pipeline.orchestrate.backfill.ensure_range", lambda *a, **kw: True) # Block the data_context fallback to yfinance monkeypatch.setattr("core.market.data_context._download_data", lambda *a, **kw: None) @@ -180,8 +180,8 @@ def client(_patch_downloads): """Create Flask test client with isolated DB.""" import app as flask_app - from data_pipeline import data_ops as _ds - from data_pipeline import job_cache as _jc + from data_pipeline import _state as _ds + from data_pipeline.orchestrate import job_cache as _jc # Reset module-level caches so prior tests don't leak data into this one. _jc._reset() diff --git a/tests/test_portfolio.py b/tests/test_portfolio.py index ac1e4d3..1796178 100644 --- a/tests/test_portfolio.py +++ b/tests/test_portfolio.py @@ -59,7 +59,7 @@ def test_attribute_pnl_with_iv_drop_hurts_long_vega(): def test_create_and_list_position(): - from data_pipeline.db import init_db + from data_pipeline.store.db import init_db from services.portfolio.facade import create_position, list_positions init_db() @@ -88,7 +88,7 @@ def test_create_position_rejects_missing_ticker(): def test_portfolio_snapshot_uses_mocked_spots(monkeypatch): - from data_pipeline.db import init_db + from data_pipeline.store.db import init_db from services.portfolio import facade as ps init_db() diff --git a/tests/test_processing.py b/tests/test_processing.py index 8606cae..7eedb34 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -1,12 +1,12 @@ -"""Tests for data_pipeline.processing — feature computation correctness.""" +"""Tests for data_pipeline.transform.processing — feature computation correctness.""" import datetime as dt import numpy as np import pandas as pd -from data_pipeline.db import init_db, upsert_many -from data_pipeline.processing import _agg_ohlcv, _features, process_frequencies +from data_pipeline.store.db import init_db, upsert_many +from data_pipeline.transform.processing import _agg_ohlcv, _features, process_frequencies # ── Helpers ─────────────────────────────────────────────────────── @@ -193,7 +193,7 @@ def test_empty_data_returns_zero_rows(self): def test_all_frequencies_present(self): """Check D, W, ME rows are produced.""" - from data_pipeline.db import fetch_df + from data_pipeline.store.db import fetch_df df = _make_daily(30) _seed_clean_bars("FREQ", df) @@ -210,7 +210,7 @@ def test_all_frequencies_present(self): def test_feature_columns_in_db(self): """Verify key feature columns are stored.""" - from data_pipeline.db import fetch_df + from data_pipeline.store.db import fetch_df df = _make_daily(30) _seed_clean_bars("COLS", df) diff --git a/tests/test_provider_seam.py b/tests/test_provider_seam.py index afbf41f..8d71801 100644 --- a/tests/test_provider_seam.py +++ b/tests/test_provider_seam.py @@ -11,7 +11,7 @@ - ``to_canonical_bars`` / ``to_option_chain_snapshot`` apply the unit rules in ``providers/base.py`` (decimal IV, nullable bid/ask, no ``inTheMoney``). - ``get_provider`` defaults to yfinance and rejects unknown names loudly. - - ``data_pipeline.yf_client`` still re-exports the legacy callables unchanged. + - ``data_pipeline.providers.yf_client`` still re-exports the legacy callables unchanged. Dependencies UPWARD: - (none — stdlib + pytest + the package under test) """ @@ -238,8 +238,7 @@ def test_to_option_chain_snapshot_tolerates_empty_payload(): # Compatibility shim # --------------------------------------------------------------------------- def test_yf_client_reexports_legacy_callables_unchanged(): - from data_pipeline import yf_client - from data_pipeline.providers import yf_snapshot, yfinance_provider + from data_pipeline.providers import yf_client, yf_snapshot, yfinance_provider assert yf_client.fetch_spot is yf_snapshot.fetch_spot assert yf_client.fetch_spots_bulk is yf_snapshot.fetch_spots_bulk diff --git a/tests/test_quality_log.py b/tests/test_quality_log.py index 1225811..4a27735 100644 --- a/tests/test_quality_log.py +++ b/tests/test_quality_log.py @@ -1,9 +1,9 @@ -"""Tests for data_pipeline/quality_log.py.""" +"""Tests for data_pipeline/store/quality_log.py.""" from __future__ import annotations -from data_pipeline.db import init_db -from data_pipeline.quality_log import failure_counts, log_failure, recent_failures +from data_pipeline.store.db import init_db +from data_pipeline.store.quality_log import failure_counts, log_failure, recent_failures def test_log_and_query_recent(): @@ -27,7 +27,7 @@ def test_failure_counts_aggregates_by_class(): def test_log_failure_swallows_db_errors(monkeypatch): """Logging path must never raise — caller is in an except block.""" - import data_pipeline.quality_log as ql + import data_pipeline.store.quality_log as ql def boom(*a, **kw): raise RuntimeError("db down") diff --git a/tests/test_regime.py b/tests/test_regime.py index 7d861bc..55a1805 100644 --- a/tests/test_regime.py +++ b/tests/test_regime.py @@ -197,7 +197,7 @@ def test_label_series_empty_inputs_returns_structured_df(): def test_fetch_df_aggregate_query_without_date_column(): """Regression: ``fetch_df`` must not try to index by 'date' on queries that don't select a date column (e.g. ``SELECT MAX(date)``).""" - from data_pipeline.db import fetch_df, init_db + from data_pipeline.store.db import fetch_df, init_db init_db() df = fetch_df("SELECT MAX(date) as max_date FROM raw_prices WHERE ticker=?", ("DOES_NOT_EXIST",)) diff --git a/tests/test_render_streaming.py b/tests/test_render_streaming.py index 9ffec1b..984faae 100644 --- a/tests/test_render_streaming.py +++ b/tests/test_render_streaming.py @@ -11,7 +11,7 @@ import pytest -from data_pipeline import job_cache as jc +from data_pipeline.orchestrate import job_cache as jc @pytest.fixture(autouse=True) diff --git a/tests/test_route_param_validation.py b/tests/test_route_param_validation.py index 2808d28..9aafbb1 100644 --- a/tests/test_route_param_validation.py +++ b/tests/test_route_param_validation.py @@ -96,14 +96,14 @@ def _boom(*args, **kwargs): # noqa: ARG001 class TestReposColumnWhitelist: def test_unknown_column_raises_before_sql(self): - from data_pipeline.repos import select_tracked_strategies + from data_pipeline.store.repos import select_tracked_strategies with pytest.raises(ValueError, match="unknown tracked_strategies columns"): select_tracked_strategies(["id", "notes; DROP TABLE tracked_strategies--"], None) def test_known_columns_accepted(self): - from data_pipeline.db import init_db - from data_pipeline.repos import select_tracked_strategies + from data_pipeline.store.db import init_db + from data_pipeline.store.repos import select_tracked_strategies init_db() rows = select_tracked_strategies(["id", "ticker", "status"], None) diff --git a/tests/test_scheduler_lock.py b/tests/test_scheduler_lock.py index 9c0e30c..3192c5a 100644 --- a/tests/test_scheduler_lock.py +++ b/tests/test_scheduler_lock.py @@ -3,7 +3,7 @@ import os import tempfile -from data_pipeline.scheduler import acquire_scheduler_lock +from data_pipeline.orchestrate.scheduler import acquire_scheduler_lock def test_first_acquire_succeeds_second_returns_none(monkeypatch): diff --git a/tests/test_scheduler_optional_dep.py b/tests/test_scheduler_optional_dep.py index 54b3eb7..71e764c 100644 --- a/tests/test_scheduler_optional_dep.py +++ b/tests/test_scheduler_optional_dep.py @@ -40,7 +40,7 @@ def test_importing_scheduler_module_does_not_import_apscheduler(): """Module import alone must not require the package (see constraints §6).""" result = _run( "import sys\n" - "import data_pipeline.scheduler\n" + "import data_pipeline.orchestrate.scheduler\n" "assert 'apscheduler' not in sys.modules, 'apscheduler imported at module scope'\n" ) @@ -52,7 +52,7 @@ def test_missing_apscheduler_raises_actionable_error(): result = _run( _BLOCK_APSCHEDULER + ( - "from data_pipeline.scheduler import UpdateScheduler\n" + "from data_pipeline.orchestrate.scheduler import UpdateScheduler\n" "try:\n" " UpdateScheduler()\n" "except ModuleNotFoundError as exc:\n" diff --git a/tests/test_strategy_builder.py b/tests/test_strategy_builder.py index b2dcbd8..c1f5f35 100644 --- a/tests/test_strategy_builder.py +++ b/tests/test_strategy_builder.py @@ -49,9 +49,9 @@ def _fake_chain(spot: float = 100.0, expiry: str = "2099-12-31"): def patched(monkeypatch): monkeypatch.setattr(sb, "fetch_option_chain", lambda t: _fake_chain()) # Skip DB lookup for vol context — return None - from data_pipeline import data_ops as _dops + from data_pipeline import read as _read_pkg - monkeypatch.setattr(_dops.DataService, "get_cleaned_daily", staticmethod(lambda *a, **kw: pd.DataFrame())) + monkeypatch.setattr(_read_pkg.DataService, "get_cleaned_daily", staticmethod(lambda *a, **kw: pd.DataFrame())) return monkeypatch diff --git a/tests/test_yf_failure_injection.py b/tests/test_yf_failure_injection.py index d04744f..72cdb96 100644 --- a/tests/test_yf_failure_injection.py +++ b/tests/test_yf_failure_injection.py @@ -30,15 +30,10 @@ import pytest from data_pipeline import PipelineResult -from data_pipeline.data_ops import ( - DataService, - _query_cache, - _query_cache_lock, - _update_lock_mutex, - _update_locks, -) -from data_pipeline.db import fetch_df, init_db, upsert_many -from data_pipeline.downloader import download_bars, upsert_raw_prices +from data_pipeline._state import _query_cache, _query_cache_lock, _update_lock_mutex, _update_locks +from data_pipeline.ingest.ohlcv import download_bars, upsert_raw_prices +from data_pipeline.read import DataService +from data_pipeline.store.db import fetch_df, init_db, upsert_many # --------------------------------------------------------------------------- # Helpers @@ -290,7 +285,7 @@ def test_manual_update_returns_false_on_429(self, mock_throttle, mock_dl): result = DataService.manual_update("E2E_TKR") assert result is False - @patch("data_pipeline.downloader.upsert_raw_prices") + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices") def test_manual_update_returns_false_on_pipeline_error_field(self, mock_upsert): """Even if downloader returns ok=False (rather than raising), we degrade gracefully.""" init_db() From 94447c0360ff7d8145eb55cdb35459455d6b053a Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Thu, 10 Sep 2026 20:43:28 +0800 Subject: [PATCH 05/15] =?UTF-8?q?refactor(core):=20B4=20=E5=85=B3=E9=97=AD?= =?UTF-8?q?=20L1=20core-purity=EF=BC=8C=E5=8F=96=E6=95=B0=E4=B8=8B?= =?UTF-8?q?=E6=B2=89=E5=88=B0=20services?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 业务线重构计划书 §6 B4；core/ 从此零 data_pipeline 依赖。 - 拆分：core/market/data_context.py 的取数部分移到 services/market/data_context_fetch.py::fetch_data_context； core 只保留纯 DataContext + refrequency() + 纯装配 build_data_context(*, ticker, frequency, horizon, raw_data) + empty_data_context() 两个 allow=core-purity 标记删除 - 反转构造方：MarketAnalyzer(data_context) / CorrelationValidator(price_data=…) 接收注入的 context（沿用 2026-09 对 OptionsChainAnalyzer(snapshot=) 的做法）； CorrelationValidator 缺 price_data 时明确报错而不是偷偷取数； 新增 MarketAnalyzer.data_context 公开属性，statistical.py 不再触碰 _ctx - 守卫收紧：core 层允许依赖从 {read, providers} 收紧为 {utils} （doc_guard 与 arch_metrics 两份表同步）；新增 test_core_has_zero_data_pipeline_imports，刻意不认 allow 标记 - 测试迁移：test_frontend_api 改用纯构造器（删掉三处闭包）、 test_nvda_analysis 新增 _analyzer() 助手（5 处）、test_chart_time_range 与 test_ticker_format_integration 同步（后者顺带去掉 __init__ monkeypatch 技巧） - 文档：architecture_review §2（该行关闭、标记数 2→0）与 §3 层表、 §5 开放事项、CODEBUDDY/CLAUDE 的 core 规则、计划书台账/§8 验收：pytest -m "not network" --ignore=tests/e2e → 473 passed / 5 skipped； pytest tests/e2e → 38 passed；ruff clean；doc_guard clean； arch_metrics --check ok（无需重置基线）；grep allow=core-purity 为空。 --- CLAUDE.md | 4 +- CODEBUDDY.md | 4 +- core/market/__init__.py | 2 +- core/market/analyzer.py | 36 ++- core/market/correlation_validator.py | 34 ++- core/market/data_context.py | 288 +++++++-------------- core/options/simulation/expiry_calendar.py | 2 +- docs/architecture_review.md | 13 +- docs/plans/business_line_reorg.md | 29 ++- scripts/arch_metrics.py | 2 +- scripts/doc_guard.py | 9 +- services/market/analysis/facade.py | 31 ++- services/market/analysis/statistical.py | 4 +- services/market/data_context_fetch.py | 170 ++++++++++++ services/market/facade.py | 4 +- services/options/chain.py | 4 +- tests/test_architecture_purity.py | 21 ++ tests/test_chart_time_range.py | 7 +- tests/test_frontend_api.py | 141 ++++------ tests/test_nvda_analysis.py | 42 +-- tests/test_ticker_format_integration.py | 38 +-- 21 files changed, 488 insertions(+), 397 deletions(-) create mode 100644 services/market/data_context_fetch.py diff --git a/CLAUDE.md b/CLAUDE.md index 9e99b74..9bff738 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,7 +109,9 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ are legal; only the *layer* direction is policed. - **`core/`** — pure computation: no Flask, no DB, no network. Data in → numbers/DataFrames out. `core/` and `data_pipeline/` must never import `services/`, `routes/`, or `app.py`; - `data_pipeline/` must never import `core/`. + `data_pipeline/` must never import `core/`. **`core/` must not import `data_pipeline/` either** + (closed in batch B4 — enforced by `doc_guard` `import-direction` and + `tests/test_architecture_purity.py`; acquisition belongs in `services/`). - **`data_pipeline/`** — owns **every** I/O boundary: yfinance, SQLite, the scheduler. - **`utils/`** — leaf helpers only. diff --git a/CODEBUDDY.md b/CODEBUDDY.md index 9e99b74..9bff738 100644 --- a/CODEBUDDY.md +++ b/CODEBUDDY.md @@ -109,7 +109,9 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ are legal; only the *layer* direction is policed. - **`core/`** — pure computation: no Flask, no DB, no network. Data in → numbers/DataFrames out. `core/` and `data_pipeline/` must never import `services/`, `routes/`, or `app.py`; - `data_pipeline/` must never import `core/`. + `data_pipeline/` must never import `core/`. **`core/` must not import `data_pipeline/` either** + (closed in batch B4 — enforced by `doc_guard` `import-direction` and + `tests/test_architecture_purity.py`; acquisition belongs in `services/`). - **`data_pipeline/`** — owns **every** I/O boundary: yfinance, SQLite, the scheduler. - **`utils/`** — leaf helpers only. diff --git a/core/market/__init__.py b/core/market/__init__.py index 4fb02f6..22a60b6 100644 --- a/core/market/__init__.py +++ b/core/market/__init__.py @@ -1,7 +1,7 @@ """Market Analysis Domain. Dependency graph (flows downward): - data_context # PriceDynamic — data fetching & resampling + data_context # pure data container + resampling (no I/O — ADR 0001) features/ # Pure numeric feature computation ├── osc.py ├── returns.py diff --git a/core/market/analyzer.py b/core/market/analyzer.py index 12752d5..62b9cfc 100644 --- a/core/market/analyzer.py +++ b/core/market/analyzer.py @@ -2,24 +2,26 @@ Domain: Market Analysis — Orchestration Context: - - Builds the data context and delegates all chart rendering to - ``core.market.charts.facade.MarketChartAssembly``. + - Wraps an already-fetched ``DataContext`` and delegates all chart rendering + to ``core.market.charts.facade.MarketChartAssembly``. + - INVARIANT (ADR 0001; batch B4): it does **not** fetch. Callers build the + context with ``services.market.data_context_fetch.fetch_data_context`` and + pass it in — see ``OptionsChainAnalyzer(snapshot=…)`` for the same pattern. - Kept intentionally thin: the chart-assembly fan-out (renderers + the feature/projection primitives that feed them) lives in the facade, so this module is no longer the repo's top change-magnet. Dependencies UPWARD: - core.market.data_context, core.market.charts.facade Dependencies DOWNWARD: - - services.market.analysis.facade, tests + - services.market.analysis.facade, services.market.analysis.statistical, tests """ from __future__ import annotations -import datetime as dt import logging from core.market.charts.facade import MarketChartAssembly -from core.market.data_context import build_data_context +from core.market.data_context import DataContext logger = logging.getLogger(__name__) @@ -27,14 +29,28 @@ class MarketAnalyzer: """High-level market analysis — thin orchestrator over core.market submodules.""" - def __init__(self, ticker: str, start_date: dt.date, frequency: str, end_date: dt.date | None = None): - self._ctx = build_data_context(ticker, start_date, frequency, end_date) - self.ticker = ticker - self.frequency = frequency - self.end_date = end_date + def __init__(self, data_context: DataContext): + """Wrap ``data_context``. + + WHY the context is injected rather than built here: constructing it means + reading the DB and possibly the provider, which core/ must not do (ADR + 0001). Services own that decision — see + ``services/market/data_context_fetch.py``. + """ + self._ctx = data_context + self.ticker = data_context.ticker + self.frequency = data_context.frequency + # Backward-compat: the old signature exposed the caller's ``end_date`` + # (None when the horizon end was implicit). + self.end_date = data_context.horizon.end if data_context.horizon.user_provided_end else None self.features_df = self._ctx.features_df self._charts = MarketChartAssembly(self._ctx, self.features_df, self.ticker, self.frequency) + @property + def data_context(self) -> DataContext: + """The wrapped context (public accessor for service-layer callers).""" + return self._ctx + def is_data_valid(self): return self._ctx.is_valid() diff --git a/core/market/correlation_validator.py b/core/market/correlation_validator.py index 11dfae1..04a9f42 100644 --- a/core/market/correlation_validator.py +++ b/core/market/correlation_validator.py @@ -2,6 +2,17 @@ Rolling correlation computation lives here (domain-specific logic); rendering is delegated to core.market.charts.correlation. + +Domain: Market Analysis — Correlation Validation +Context: + - INVARIANT (ADR 0001; batch B4): this module does **not** fetch. The bars + arrive via the injected ``price_data`` (a ``DataContext``, or anything + exposing ``bars``); callers use + ``services.market.data_context_fetch.fetch_data_context``. +Dependencies UPWARD: + - core.market.charts.correlation, core.market.features._horizon +Dependencies DOWNWARD: + - services.market.analysis.statistical """ from __future__ import annotations @@ -13,7 +24,6 @@ import pandas as pd from core.market.charts.correlation import render_correlation -from core.market.data_context import build_data_context from core.market.features._horizon import apply_horizon, compute_effective_end logger = logging.getLogger(__name__) @@ -36,16 +46,18 @@ def __init__( self.user_end_date = end_date or dt.date.today() self._user_provided_end = end_date is not None - # Duck-type support: injected object may be DataContext (has bars) - # or any object exposing a `_data` attribute that returns a DataFrame. - if price_data is not None: - self._raw_data = getattr(price_data, "_data", None) or getattr(price_data, "bars", None) - is_valid_fn = getattr(price_data, "is_valid", None) - self._is_valid = bool(is_valid_fn()) if is_valid_fn else self._raw_data is not None - else: - ctx = build_data_context(ticker, start_date, frequency, end_date) - self._raw_data = ctx.bars - self._is_valid = ctx.is_valid() + if price_data is None: + # WHY raise instead of fetching: the fetch used to live here, which + # made core/ depend on data_pipeline (architecture_review.md §2). + raise ValueError( + "CorrelationValidator requires price_data=; core/ must not fetch — " + "build it with services.market.data_context_fetch.fetch_data_context()" + ) + # Duck-type support: the injected object may be a DataContext (has + # `bars`) or any object exposing a `_data` attribute. + self._raw_data = getattr(price_data, "_data", None) or getattr(price_data, "bars", None) + is_valid_fn = getattr(price_data, "is_valid", None) + self._is_valid = bool(is_valid_fn()) if is_valid_fn else self._raw_data is not None self.data = self._build_data() diff --git a/core/market/data_context.py b/core/market/data_context.py index d7e105d..de9bec9 100644 --- a/core/market/data_context.py +++ b/core/market/data_context.py @@ -1,18 +1,26 @@ -"""Market data context — explicit data container with data-fetching logic. +"""Market data context — pure container + resampling. Domain: Market Analysis — Data Context Context: - - Encapsulates data-fetching/resampling logic previously inside PriceDynamic. - - Returns plain DataFrames so downstream features/charts are fully decoupled. + - The container market analysis works on: ``bars`` (at the requested + frequency) plus the ``daily_bars`` they were derived from. + - INVARIANT (ADR 0001; batch B4 of the reorg): this module performs **no + I/O**. It receives already-fetched bars and resamples them; acquisition + (DB-first read + provider fallback) lives in + ``services/market/data_context_fetch.py``. Closing that leak is what lets + ``tests/test_architecture_purity.py`` require ``core/`` to have zero + ``data_pipeline`` imports. - No feature calculation, no matplotlib, no business logic. Contracts: - - build_data_context(ticker, start_date, frequency, end_date) -> DataContext - - DataContext exposes bars, daily_bars, horizon, ticker, frequency, is_valid + - ``DataContext`` — the container (``bars``, ``daily_bars``, ``horizon``, + ``ticker``, ``frequency``, ``is_valid()``, ``features_df``, ``current_price``). + - ``refrequency(df, frequency)`` — daily bars → bars at ``D``/``W``/``ME``/``QE``. + - ``build_data_context(*, ticker, frequency, horizon, raw_data)`` — pure assembly. Dependencies UPWARD: - - core.market.features, core.market.charts, data_pipeline (I/O boundary, - see the doc-guard: allow=core-purity markers below) + - core.market.features, core.market.models, core._shared.types Dependencies DOWNWARD: - - core.market.analyzer, services.market.analysis.facade + - core.market.analyzer, core.market.correlation_validator, + core.market.charts.facade, services.market.data_context_fetch """ from __future__ import annotations @@ -27,161 +35,6 @@ logger = logging.getLogger(__name__) -# CONSTRAINT: bounded retries prevent transient yfinance failures from crashing the pipeline. -_YF_MAX_RETRIES = 2 - -# CONSTRAINT: sub-second retries hit Yahoo rate-limiting; 3 s is the minimum stable back-off. -_YF_RETRY_BASE_DELAY = 3 # seconds - - -# --------------------------------------------------------------------------- -# Internal helpers (extracted from former PriceDynamic) -# --------------------------------------------------------------------------- - - -def _normalize_ticker(ticker: str) -> str: - from utils.ticker_utils import normalize_ticker - - try: - yahoo_ticker, _ = normalize_ticker(ticker) - return yahoo_ticker or ticker - except (ValueError, ImportError): - return ticker - - -def _validate_inputs(ticker, start_date, frequency, end_date=None): - if not isinstance(ticker, str) or not ticker.strip(): - raise ValueError("Ticker must be a non-empty string") - if not isinstance(start_date, dt.date): - raise ValueError("start_date must be a datetime.date object") - if frequency not in ("D", "W", "ME", "QE"): - raise ValueError("frequency must be one of ['D', 'W', 'ME', 'QE']") - if end_date is not None and not isinstance(end_date, dt.date): - raise ValueError("end_date must be a datetime.date object or None") - if end_date is not None and end_date < start_date: - raise ValueError("end_date must be on or after start_date") - - -def _fetch_daily_from_db(ticker: str, download_start: dt.date): - from data_pipeline.read import DataService # doc-guard: allow=core-purity - - try: - DataService.initialize() - except Exception: - pass - try: - df = DataService.get_cleaned_daily(ticker, download_start, dt.date.today()) - if df is None or df.empty: - return None - df = df.rename( - columns={ - "open": "Open", - "high": "High", - "low": "Low", - "close": "Close", - "adj_close": "Adj Close", - "volume": "Volume", - } - ) - for col in ("Open", "High", "Low", "Close", "Adj Close", "Volume"): - if col in df.columns: - df[col] = pd.to_numeric(df[col], errors="coerce") - price_cols = [c for c in ("Open", "High", "Low", "Close", "Adj Close") if c in df.columns] - if price_cols: - df = df.dropna(subset=price_cols, how="all") - return df if not df.empty else None - except Exception as e: - logger.warning("DB fetch failed for %s: %s", ticker, e) - return None - - -def _download_data(ticker: str, download_start: dt.date): - from data_pipeline.providers.yf_client import fetch_daily_ohlcv # doc-guard: allow=core-purity - - yf_end = dt.date.today() + dt.timedelta(days=1) - df = fetch_daily_ohlcv( - ticker, - download_start, - yf_end, - auto_adjust=False, - max_retries=_YF_MAX_RETRIES, - retry_base_delay=_YF_RETRY_BASE_DELAY, - ) - if df.empty: - logger.warning("No data downloaded for %s", ticker) - return None - required_columns = ["Open", "High", "Low", "Close", "Adj Close", "Volume"] - missing_columns = [col for col in required_columns if col not in df.columns] - if missing_columns: - logger.error("Missing columns for %s: %s", ticker, missing_columns) - return None - return df[required_columns] - - -def _refrequency(df: pd.DataFrame | None, frequency: str) -> pd.DataFrame | None: - if df is None or df.empty: - return None - try: - if frequency == "D": - df = df.copy() - df["LastClose"] = df["Close"].shift(1) - df["LastAdjClose"] = df["Adj Close"].shift(1) - return df - resampled = ( - df.resample(frequency) - .agg( - { - "Open": "first", - "High": "max", - "Low": "min", - "Close": "last", - "Adj Close": "last", - "Volume": "sum", - } - ) - .dropna() - ) - resampled["LastClose"] = resampled["Close"].shift(1) - resampled["LastAdjClose"] = resampled["Adj Close"].shift(1) - date_agg = df.resample(frequency).agg( - { - "Open": lambda x: x.index[0] if len(x) > 0 else pd.NaT, - "High": lambda x: x.index[x.argmax()] if len(x) > 0 else pd.NaT, - "Low": lambda x: x.index[x.argmin()] if len(x) > 0 else pd.NaT, - "Close": lambda x: x.index[-1] if len(x) > 0 else pd.NaT, - } - ) - resampled["OpenDate"] = date_agg["Open"] - resampled["HighDate"] = date_agg["High"] - resampled["LowDate"] = date_agg["Low"] - resampled["CloseDate"] = date_agg["Close"] - return resampled - except Exception as e: - logger.error("Error resampling data: %s", e) - return None - - -def _fetch_raw_data(ticker: str, user_start_date: dt.date, frequency: str): - """L1: DB L2: yfinance fallback. Returns (daily_df, ticker).""" - download_start = dt.date(1900, 1, 1) - raw_data = _fetch_daily_from_db(ticker, download_start) - db_data = raw_data - db_min = raw_data.index.min().date() if raw_data is not None and not raw_data.empty else None - needs_yfinance = raw_data is None or raw_data.empty or (db_min is not None and db_min > user_start_date) - if needs_yfinance: - yf_data = _download_data(ticker, download_start) - if yf_data is not None and not yf_data.empty: - raw_data = yf_data - elif db_data is not None and not db_data.empty: - logger.warning("yfinance download failed for %s, using available DB data.", ticker) - raw_data = db_data - return raw_data, ticker - - -# --------------------------------------------------------------------------- -# DataContext -# --------------------------------------------------------------------------- - class DataContext: """Immutable-ish container for market data fetched for a given ticker/horizon.""" @@ -255,50 +108,83 @@ def bars_date_range(self) -> tuple[str, str] | None: return None +def refrequency(df: pd.DataFrame | None, frequency: Frequency) -> pd.DataFrame | None: + """Resample daily bars to ``frequency`` and add ``LastClose``/``LastAdjClose``.""" + if df is None or df.empty: + return None + try: + if frequency == "D": + df = df.copy() + df["LastClose"] = df["Close"].shift(1) + df["LastAdjClose"] = df["Adj Close"].shift(1) + return df + resampled = ( + df.resample(frequency) + .agg( + { + "Open": "first", + "High": "max", + "Low": "min", + "Close": "last", + "Adj Close": "last", + "Volume": "sum", + } + ) + .dropna() + ) + resampled["LastClose"] = resampled["Close"].shift(1) + resampled["LastAdjClose"] = resampled["Adj Close"].shift(1) + date_agg = df.resample(frequency).agg( + { + "Open": lambda x: x.index[0] if len(x) > 0 else pd.NaT, + "High": lambda x: x.index[x.argmax()] if len(x) > 0 else pd.NaT, + "Low": lambda x: x.index[x.argmin()] if len(x) > 0 else pd.NaT, + "Close": lambda x: x.index[-1] if len(x) > 0 else pd.NaT, + } + ) + resampled["OpenDate"] = date_agg["Open"] + resampled["HighDate"] = date_agg["High"] + resampled["LowDate"] = date_agg["Low"] + resampled["CloseDate"] = date_agg["Close"] + return resampled + except Exception as e: + logger.error("Error resampling data: %s", e) + return None + + def build_data_context( + *, ticker: str, - start_date: dt.date, - frequency: Frequency = "W", - end_date: dt.date | None = None, + frequency: Frequency, + horizon: Horizon, + raw_data: pd.DataFrame | None, ) -> DataContext: - """Build a DataContext by fetching and resampling market data. + """Assemble a ``DataContext`` from bars that were already fetched. - Data pipeline: - 1. Normalise ticker (futu-format -> yahoo-format). - 2. Validate inputs. - 3. Fetch from DB first; fall back to yfinance if DB coverage is insufficient. - 4. Resample to requested frequency. + WHY keyword-only with an explicit ``raw_data``: the caller (a service) owns + acquisition, so this function stays pure and cannot accidentally re-introduce + the core→data_pipeline edge that batch B4 removed. """ - try: - _validate_inputs(ticker, start_date, frequency, end_date) - norm_ticker = _normalize_ticker(ticker) - raw_data, final_ticker = _fetch_raw_data(norm_ticker, start_date, frequency) - bars = _refrequency(raw_data, frequency) - horizon = Horizon( - start=start_date, - end=end_date or dt.date.today(), - user_provided_end=end_date is not None, - frequency=frequency, - ) - return DataContext( - ticker=final_ticker, - frequency=frequency, - horizon=horizon, - bars=bars, - daily_bars=raw_data, - ) - except Exception as e: - logger.error("Failed to build DataContext for %s: %s", ticker, e) - horizon = Horizon( + return DataContext( + ticker=ticker, + frequency=frequency, + horizon=horizon, + bars=refrequency(raw_data, frequency), + daily_bars=raw_data, + ) + + +def empty_data_context(ticker: str, start_date: dt.date, frequency: Frequency, end_date: dt.date | None) -> DataContext: + """Return an invalid context for a failed acquisition (no bars).""" + return DataContext( + ticker=ticker, + frequency=frequency, + horizon=Horizon( start=start_date, end=end_date or dt.date.today(), user_provided_end=end_date is not None, frequency=frequency, - ) - return DataContext( - ticker=ticker, - frequency=frequency, - horizon=horizon, - bars=None, - daily_bars=None, - ) + ), + bars=None, + daily_bars=None, + ) diff --git a/core/options/simulation/expiry_calendar.py b/core/options/simulation/expiry_calendar.py index 0597dc0..1820580 100644 --- a/core/options/simulation/expiry_calendar.py +++ b/core/options/simulation/expiry_calendar.py @@ -9,7 +9,7 @@ day) series for the short end, plus a *weekly* listed series on every Friday for the longer maturities. When a Friday is an exchange holiday the expiration rolls back to the previous business day (usually Thursday). - - The project otherwise ignores exchange holidays (see data_pipeline/cleaning + - The project otherwise ignores exchange holidays (see data_pipeline/transform/cleaning for the "B" frequency), but the listed-expiration rules *require* them, so we ship a self-contained NYSE approximation here rather than depending on an external calendar package. diff --git a/docs/architecture_review.md b/docs/architecture_review.md index 26dd932..fc653a0 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -42,11 +42,15 @@ count can only go down without an explicit baseline update. Run `grep -rn "doc-guard: allow" --include='*.py' core services data_pipeline` for the live list. State at registration: -### core-purity (core must not import data_pipeline) — 2 markers +### core-purity (core must not import data_pipeline) — 0 markers + +**All registered core-purity debt is closed (batch B4, 2026-09-10).** `core/` has +zero `data_pipeline` imports; `tests/test_architecture_purity.py` now refuses the +suppression marker outright, and the layer table (§3) only allows `core → utils`. | Location | Why it exists | Exit condition | |---|---|---| -| `core/market/data_context.py` (DataService, fetch_daily_ohlcv) | `build_data_context` *is* the DB-first read path; extracting it means inverting who constructs `DataContext` | `DataContext` becomes data-in/data-out; the fetch moves into a service factory | +| `core/market/data_context.py` (DataService, fetch_daily_ohlcv) — **resolved 2026-09-10 (B4)** | `build_data_context` *was* the DB-first read path, so core did its own I/O | split: `core/market/data_context.py` keeps a pure `DataContext` + `refrequency` + the data-in/data-out `build_data_context(*, ticker, frequency, horizon, raw_data)`; acquisition moved to `services/market/data_context_fetch.py::fetch_data_context`. `MarketAnalyzer` / `CorrelationValidator` now take the context instead of building it (same pattern as `OptionsChainAnalyzer(snapshot=…)`) | | `core/market_review/fetch.py`, `core/market_review/__init__.py` (fetch_close_panel, get_conn) — **resolved 2026-09-03** | L1/L2/L3 cache ladder lived beside the computation it feeds | ladder moved to `services/market_review` (`fetch.py` + `facade.py`); `core/market_review` now receives panels via pure `build_review` / `build_timeseries` | | `core/options/chain/analyzer.py` — **resolved 2026-09-03** | former ticker-only constructor fetched yfinance internally | constructor now requires `snapshot=`; fetch lives in `services/options/chain._build_analyzer` | @@ -91,7 +95,7 @@ mirrored in `arch_metrics.py`; asserted equal by app → routes, services, core, data_pipeline, utils, read, orchestrate routes → services, data_pipeline, utils, store, read, orchestrate (never core directly) services → core, data_pipeline, utils, providers, store, ingest, transform, read, orchestrate -core → utils, read, providers (data_pipeline* only via §2 markers; B4 removes both) +core → utils (zero data_pipeline imports — closed in B4) data_pipeline → utils # root: PipelineResult (types) + _state.py store → (nothing upward) providers → store, utils # store = quality_log; see plan §8 B3 @@ -144,7 +148,8 @@ utils → (leaf: nothing upward) 1. **§2 debt paydown** (easiest first): `_query.get_latest_spot` → `yf_client` — **done 2026-09-03**; `health`/`portfolio` SQL → `repos.py` — **done 2026-09-03**; `market_review` cache ladder → `services/market_review` — **done 2026-09-03**; `regime` SQL consolidation → - `repos.py` — **done 2026-09-03**. All registered §2 debt is now resolved. + `repos.py` — **done 2026-09-03**. All registered §2 debt is now resolved, including the last + `core-purity` markers (`core/market/data_context.py` — **done 2026-09-10, batch B4**). 3. **Frontend consolidation** (P3): move the eight loose root-level scripts in `static/` (`option-chain.js`, `position.js`, `regime.js`, `simulation.js`, `market_review.js`, …) into `static/features/`. diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index 9ef6fca..adb116d 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -41,7 +41,7 @@ | B1 — provider seam extraction | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | delivers the "pluggable API" seam on its own. Actual shape / deviations recorded in §8; `_ALLOWED_DEPS` promotion of `providers` deferred to B3 | | B2 — canonical raw store | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | gate §8 Q4 resolved (name-only rename). Actuals in §8; `symbol` column deferred (ADR 0011 amendment) | | B3 — package re-home | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | six stages + `_state.py`; first sub-layer guard table; `arch_baseline.json` **not** reset (no tracked drift). Actuals + deviations in §8 | -| B4 — close L1 (`core-purity`) | ⬜ not started | — | — | — | +| B4 — close L1 (`core-purity`) | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | zero core→data_pipeline edges; markers deleted and refused by test; `core` layer tightened to `{utils}` | | B5 — readiness plan + prefetch | ⬜ not started | — | — | gate: §8 Q1 | | B6 — `ticker`-only Parameters bar | ⬜ not started | — | — | gate: §8 Q3; depends on B5 | | B7 — module-scoped params | ⬜ not started | — | — | gate: §8 Q1; depends on B6 | @@ -521,6 +521,33 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. **no baseline reset was needed** (the §6 row anticipated one); `audit_tags.py` regenerated (`--update-baseline`) because the uncovered-constant *paths* moved while the count stayed 16. +**B4 (2026-09-10) — close L1 (`core-purity`).** + +- **Split**: the fetch half of `core/market/data_context.py` moved to + `services/market/data_context_fetch.py::fetch_data_context`. `core` keeps the pure + `DataContext`, `refrequency()` (was `_refrequency`), a data-in/data-out + `build_data_context(*, ticker, frequency, horizon, raw_data)`, and `empty_data_context()` + for failed acquisitions. The two `# doc-guard: allow=core-purity` markers are gone. +- **Who fetches is now inverted** (the §2 row's exit condition): `MarketAnalyzer(data_context)` + and `CorrelationValidator(price_data=…)` receive the context instead of building it — the same + pattern the 2026-09 remediation applied to `OptionsChainAnalyzer(snapshot=…)`. + `CorrelationValidator` now raises a `ValueError` explaining where to build one instead of + quietly fetching. A public `MarketAnalyzer.data_context` property replaced the + `analyzer._ctx` reach-through in `services/market/analysis/statistical.py`. +- **Guard tightened**: §3's table had allowed `core → {read, providers}` in B3 (a transitional + concession); it is now `core → {utils}` in `doc_guard._ALLOWED_DEPS` **and** + `arch_metrics.ALLOWED_DEPS`. `tests/test_architecture_purity.py` gained + `test_core_has_zero_data_pipeline_imports`, which deliberately ignores the suppression marker — + re-introducing one now fails the test even though `doc_guard` would accept it. +- **Tests migrated**: `test_frontend_api.py` builds contexts with the pure builder (three closures + deleted), `test_nvda_analysis.py` gained an `_analyzer()` helper (5 sites) and stubs the + provider at its new path, `test_chart_time_range.py` / `test_ticker_format_integration.py` follow + the same pattern — the latter also lost its `MarketAnalyzer.__init__` monkeypatch hack. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 473 passed / 5 skipped; + full `pytest tests/e2e` → 38 passed; `ruff check` + `format --check` clean; `doc_guard.py` clean; + `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / dead 1 — no baseline reset); + `grep -rn "allow=core-purity"` returns nothing. + --- ## 9. References diff --git a/scripts/arch_metrics.py b/scripts/arch_metrics.py index c8a19f7..6150a81 100644 --- a/scripts/arch_metrics.py +++ b/scripts/arch_metrics.py @@ -66,7 +66,7 @@ "read", "orchestrate", }, - "core": {"data_pipeline", "utils", "read", "providers"}, + "core": {"utils"}, "data_pipeline": {"utils"}, "store": set(), "providers": {"store", "utils"}, diff --git a/scripts/doc_guard.py b/scripts/doc_guard.py index 64171b1..272ea56 100755 --- a/scripts/doc_guard.py +++ b/scripts/doc_guard.py @@ -241,11 +241,10 @@ def rule_sqlite_bypass(ctx: Context) -> None: "read", "orchestrate", }, - # TRADEOFF: core→data_pipeline* is directionally legal but breaks core's - # purity contract. It is policed by the separate ``core-purity`` rule so the - # two concerns (direction vs. purity) can be whitelisted and paid down at - # different paces. B4 removes these two edges entirely. - "core": {"data_pipeline", "utils", "read", "providers"}, + # INVARIANT (closed in batch B4): core/ has ZERO data_pipeline imports, so + # the entry above is gone. The ``core-purity`` rule remains as the second + # line of defence and the test layer asserts the same thing. + "core": {"utils"}, # data_pipeline/ root: shared types (PipelineResult) + process-local state. "data_pipeline": {"utils"}, "store": set(), diff --git a/services/market/analysis/facade.py b/services/market/analysis/facade.py index 4fa1e6b..4cd5506 100644 --- a/services/market/analysis/facade.py +++ b/services/market/analysis/facade.py @@ -4,6 +4,7 @@ import logging from core.market.analyzer import MarketAnalyzer +from services.market.data_context_fetch import fetch_data_context from services.market.facade import MarketService from services.options.chain import OptionsChainService from utils.date_helpers import exclusive_month_end @@ -14,6 +15,22 @@ logger = logging.getLogger(__name__) +def _build_analyzer(form_data, end_exclusive) -> MarketAnalyzer: + """Build the DataContext (services own I/O) and wrap it in a MarketAnalyzer. + + WHY here and not in core: constructing a context reads the DB and may hit + the provider — see ADR 0001 and the closed `core-purity` row in + docs/architecture_review.md §2 (batch B4). + """ + ctx = fetch_data_context( + form_data["ticker"], + form_data["parsed_start_time"], + form_data["frequency"], + end_exclusive, + ) + return MarketAnalyzer(ctx) + + class AnalysisService: """Service for coordinating all analysis operations.""" @@ -29,12 +46,7 @@ def generate_complete_analysis(form_data): try: end_exclusive = exclusive_month_end(form_data.get("parsed_end_time")) - analyzer = MarketAnalyzer( - ticker=form_data["ticker"], - start_date=form_data["parsed_start_time"], - frequency=form_data["frequency"], - end_date=end_exclusive, - ) + analyzer = _build_analyzer(form_data, end_exclusive) if not analyzer.is_data_valid(): return {"error": f"Failed to download data for {form_data['ticker']}. Please check the ticker symbol."} @@ -60,12 +72,7 @@ def generate_complete_analysis(form_data): def _build_analyzer_or_error(form_data): """Helper: build a MarketAnalyzer or return ({"error": …}, None).""" end_exclusive = exclusive_month_end(form_data.get("parsed_end_time")) - analyzer = MarketAnalyzer( - ticker=form_data["ticker"], - start_date=form_data["parsed_start_time"], - frequency=form_data["frequency"], - end_date=end_exclusive, - ) + analyzer = _build_analyzer(form_data, end_exclusive) if not analyzer.is_data_valid(): return ( {"error": f"Failed to download data for {form_data['ticker']}. Please check the ticker symbol."}, diff --git a/services/market/analysis/statistical.py b/services/market/analysis/statistical.py index 69f8f99..ac83c34 100644 --- a/services/market/analysis/statistical.py +++ b/services/market/analysis/statistical.py @@ -94,7 +94,7 @@ def _generate_statistical_analysis(analyzer, form_data): start_date=form_data["parsed_start_time"], frequency=form_data["frequency"], end_date=form_data.get("parsed_end_time"), - price_data=analyzer._ctx, + price_data=analyzer.data_context, ) if correlation_validator.is_data_valid(): @@ -111,7 +111,7 @@ def _generate_statistical_analysis(analyzer, form_data): all_none = all(v is None for k, v in results.items() if k != "statistical_error") if all_none and analyzer.is_data_valid(): fdf = analyzer.features_df - ctx = getattr(analyzer, "_ctx", None) + ctx = getattr(analyzer, "data_context", None) bars = getattr(ctx, "bars", None) actual_min = actual_max = None if bars is not None and not bars.empty: diff --git a/services/market/data_context_fetch.py b/services/market/data_context_fetch.py new file mode 100644 index 0000000..8be47ce --- /dev/null +++ b/services/market/data_context_fetch.py @@ -0,0 +1,170 @@ +"""Data-context acquisition — the I/O half of the old ``build_data_context``. + +Domain: Market Analysis — Data Context Acquisition +Context: + - Batch B4 of the reorg closed the last ``core-purity`` leak: the DB-first + read + provider fallback that used to live in + ``core/market/data_context.py`` moved here, and ``core`` now only holds the + pure container (``DataContext``) plus ``refrequency``. + - WHY services/ owns it: choosing *what* to read (DB first, provider fallback, + how far back) is orchestration, not computation — exactly the distinction + ADR 0001 draws between ``core/`` and ``services/``. +Contracts: + - ``fetch_data_context(ticker, start_date, frequency="W", end_date=None) -> DataContext`` + — never raises; a failed acquisition yields an invalid ``DataContext``. +Dependencies UPWARD: + - data_pipeline.read (DataService), data_pipeline.providers.yf_client + (provider fallback), core.market.data_context (pure container + builder), + core.market.models (Horizon), utils.ticker_utils +Dependencies DOWNWARD: + - services.market.facade, services.market.analysis.facade, + services.options.chain +""" + +from __future__ import annotations + +import datetime as dt +import logging + +import pandas as pd + +from core.market.data_context import DataContext, build_data_context, empty_data_context +from core.market.models import Horizon + +logger = logging.getLogger(__name__) + +# CONSTRAINT: bounded retries prevent transient yfinance failures from crashing the pipeline. +_YF_MAX_RETRIES = 2 + +# CONSTRAINT: sub-second retries hit Yahoo rate-limiting; 3 s is the minimum stable back-off. +_YF_RETRY_BASE_DELAY = 3 # seconds + + +def _normalize_ticker(ticker: str) -> str: + from utils.ticker_utils import normalize_ticker + + try: + yahoo_ticker, _ = normalize_ticker(ticker) + return yahoo_ticker or ticker + except (ValueError, ImportError): + return ticker + + +def _validate_inputs(ticker, start_date, frequency, end_date=None): + if not isinstance(ticker, str) or not ticker.strip(): + raise ValueError("Ticker must be a non-empty string") + if not isinstance(start_date, dt.date): + raise ValueError("start_date must be a datetime.date object") + if frequency not in ("D", "W", "ME", "QE"): + raise ValueError("frequency must be one of ['D', 'W', 'ME', 'QE']") + if end_date is not None and not isinstance(end_date, dt.date): + raise ValueError("end_date must be a datetime.date object or None") + if end_date is not None and end_date < start_date: + raise ValueError("end_date must be on or after start_date") + + +def _fetch_daily_from_db(ticker: str, download_start: dt.date): + from data_pipeline.read import DataService + + try: + DataService.initialize() + except Exception: + pass + try: + df = DataService.get_cleaned_daily(ticker, download_start, dt.date.today()) + if df is None or df.empty: + return None + df = df.rename( + columns={ + "open": "Open", + "high": "High", + "low": "Low", + "close": "Close", + "adj_close": "Adj Close", + "volume": "Volume", + } + ) + for col in ("Open", "High", "Low", "Close", "Adj Close", "Volume"): + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + price_cols = [c for c in ("Open", "High", "Low", "Close", "Adj Close") if c in df.columns] + if price_cols: + df = df.dropna(subset=price_cols, how="all") + return df if not df.empty else None + except Exception as e: + logger.warning("DB fetch failed for %s: %s", ticker, e) + return None + + +def _download_data(ticker: str, download_start: dt.date): + from data_pipeline.providers.yf_client import fetch_daily_ohlcv + + yf_end = dt.date.today() + dt.timedelta(days=1) + df = fetch_daily_ohlcv( + ticker, + download_start, + yf_end, + auto_adjust=False, + max_retries=_YF_MAX_RETRIES, + retry_base_delay=_YF_RETRY_BASE_DELAY, + ) + if df.empty: + logger.warning("No data downloaded for %s", ticker) + return None + required_columns = ["Open", "High", "Low", "Close", "Adj Close", "Volume"] + missing_columns = [col for col in required_columns if col not in df.columns] + if missing_columns: + logger.error("Missing columns for %s: %s", ticker, missing_columns) + return None + return df[required_columns] + + +def _fetch_raw_data(ticker: str, user_start_date: dt.date, frequency: str): + """L1: DB L2: provider fallback. Returns (daily_df, ticker).""" + download_start = dt.date(1900, 1, 1) + raw_data = _fetch_daily_from_db(ticker, download_start) + db_data = raw_data + db_min = raw_data.index.min().date() if raw_data is not None and not raw_data.empty else None + needs_yfinance = raw_data is None or raw_data.empty or (db_min is not None and db_min > user_start_date) + if needs_yfinance: + yf_data = _download_data(ticker, download_start) + if yf_data is not None and not yf_data.empty: + raw_data = yf_data + elif db_data is not None and not db_data.empty: + logger.warning("yfinance download failed for %s, using available DB data.", ticker) + raw_data = db_data + return raw_data, ticker + + +def fetch_data_context( + ticker: str, + start_date: dt.date, + frequency: str = "W", + end_date: dt.date | None = None, +) -> DataContext: + """Fetch (DB-first) and assemble a ``DataContext``. + + Data pipeline: + 1. Normalise ticker (futu-format -> yahoo-format). + 2. Validate inputs. + 3. Fetch from the DB first; fall back to the provider if coverage is short. + 4. Resample to the requested frequency (pure ``build_data_context``). + """ + try: + _validate_inputs(ticker, start_date, frequency, end_date) + norm_ticker = _normalize_ticker(ticker) + raw_data, final_ticker = _fetch_raw_data(norm_ticker, start_date, frequency) + return build_data_context( + ticker=final_ticker, + frequency=frequency, + horizon=Horizon( + start=start_date, + end=end_date or dt.date.today(), + user_provided_end=end_date is not None, + frequency=frequency, + ), + raw_data=raw_data, + ) + except Exception as e: + logger.error("Failed to build DataContext for %s: %s", ticker, e) + return empty_data_context(ticker, start_date, frequency, end_date) diff --git a/services/market/facade.py b/services/market/facade.py index 33326b3..385a30c 100644 --- a/services/market/facade.py +++ b/services/market/facade.py @@ -9,8 +9,8 @@ import datetime as dt import logging -from core.market.data_context import build_data_context from data_pipeline.providers.yf_client import fetch_spot as _fetch_spot +from services.market.data_context_fetch import fetch_data_context from services.market_review import market_review, market_review_timeseries from utils.date_helpers import exclusive_month_end from utils.ticker_utils import is_valid_ticker_format @@ -38,7 +38,7 @@ def validate_ticker(ticker): if not is_valid_ticker_format(ticker): return False, "invalid_ticker_or_no_data_available" try: - ctx = build_data_context(ticker, dt.date.today() - dt.timedelta(days=30), "D") + ctx = fetch_data_context(ticker, dt.date.today() - dt.timedelta(days=30), "D") is_valid = ctx.is_valid() message = "valid_ticker" if is_valid else "invalid_ticker_or_no_data_available" return is_valid, message diff --git a/services/options/chain.py b/services/options/chain.py index 2e5c7d2..02144f5 100644 --- a/services/options/chain.py +++ b/services/options/chain.py @@ -227,10 +227,10 @@ def generate_options_chain_analysis(ticker: str) -> dict: try: import datetime as dt - from core.market.data_context import build_data_context from core.signals.hv import vol_premium_context + from services.market.data_context_fetch import fetch_data_context - ctx = build_data_context(ticker, dt.date.today() - dt.timedelta(days=365), "D") + ctx = fetch_data_context(ticker, dt.date.today() - dt.timedelta(days=365), "D") if ctx.is_valid() and ctx.daily_bars is not None: # Get nearest-expiry ATM IV atm_iv = None diff --git a/tests/test_architecture_purity.py b/tests/test_architecture_purity.py index 5e903d2..d5ebadc 100644 --- a/tests/test_architecture_purity.py +++ b/tests/test_architecture_purity.py @@ -102,6 +102,27 @@ def test_core_subpackage_has_no_io_or_framework_imports(pkg): assert not offenders, "core/ purity violated (fetch upstream and pass data in, ADR 0001):\n" + "\n".join(offenders) +def test_core_has_zero_data_pipeline_imports(): + """B4 exit criterion: no suppression markers, no core→data_pipeline edge left. + + Stricter than ``test_core_subpackage_has_no_io_or_framework_imports`` above: + that one honours ``# doc-guard: allow=core-purity`` for registered debt. Batch + B4 closed the debt, so this test refuses the marker entirely — re-introducing + one fails here even if doc_guard would accept it. + """ + guard = _load_script("doc_guard") + offenders: list[str] = [] + for py in sorted(CORE.rglob("*.py")): + if "__pycache__" in py.parts: + continue + for lineno, head in guard._imported_heads(py): + if head == "data_pipeline" or head in guard.DATA_PIPELINE_SUBLAYERS: + offenders.append(f"{py.relative_to(REPO_ROOT)}:{lineno} imports {head}") + assert not offenders, ( + "core/ must not import data_pipeline (fetch upstream and pass data in, ADR 0001):\n" + "\n".join(offenders) + ) + + def test_data_pipeline_import_graph_matches_declared_layers(): """Every data_pipeline/ import must point at an allowed layer. diff --git a/tests/test_chart_time_range.py b/tests/test_chart_time_range.py index 7a63afc..0fe011f 100644 --- a/tests/test_chart_time_range.py +++ b/tests/test_chart_time_range.py @@ -48,11 +48,10 @@ def _run_single_case(test_case: dict) -> None: """Assert that a single test case produces sufficient data points and charts.""" + from services.market.data_context_fetch import fetch_data_context + analyzer = MarketAnalyzer( - ticker=test_case["ticker"], - start_date=test_case["start"], - frequency=test_case["frequency"], - end_date=test_case["end"], + fetch_data_context(test_case["ticker"], test_case["start"], test_case["frequency"], test_case["end"]) ) assert analyzer.is_data_valid(), f"{test_case['description']}: No valid data returned" diff --git a/tests/test_frontend_api.py b/tests/test_frontend_api.py index d96e9c2..668a609 100644 --- a/tests/test_frontend_api.py +++ b/tests/test_frontend_api.py @@ -6,7 +6,7 @@ - The /api/option_chain endpoint handles valid/invalid inputs - Config-driven filter parameters (DTE, moneyness) are respected - MarketAnalyzer features_df is non-empty with adequate data - - build_data_context normalizes futu-format tickers to yahoo format + - fetch_data_context normalizes futu-format tickers to yahoo format """ import datetime as dt @@ -185,43 +185,33 @@ def _make_price_df(n_rows=60): index=dates, ) - def test_features_df_nonempty_with_synthetic_data(self, monkeypatch): + def test_features_df_nonempty_with_synthetic_data(self): """features_df should have rows when data context has adequate data.""" from core.market.analyzer import MarketAnalyzer - from core.market.data_context import DataContext, Horizon + from core.market.data_context import Horizon, build_data_context fake_df = self._make_price_df(60) start = fake_df.index[0].date() end = fake_df.index[-1].date() - def _mock_build(ticker, start_date, frequency="D", end_date=None): - df = fake_df.copy() - df["LastClose"] = df["Close"].shift(1) - df["LastAdjClose"] = df["Adj Close"].shift(1) - return DataContext( - ticker=ticker, - frequency=frequency, - horizon=Horizon( - start=start_date, - end=end_date or end, - user_provided_end=end_date is not None, - frequency=frequency, - ), - bars=df, - daily_bars=fake_df.copy(), - ) - - monkeypatch.setattr("core.market.analyzer.build_data_context", _mock_build) - - analyzer = MarketAnalyzer("TEST", start, "D", end_date=end) + # WHY the pure builder: core/ must not fetch (ADR 0001 / batch B4), so + # the context is assembled here from synthetic bars and injected. + ctx = build_data_context( + ticker="TEST", + frequency="D", + horizon=Horizon(start=start, end=end, user_provided_end=True, frequency="D"), + raw_data=fake_df, + ) + + analyzer = MarketAnalyzer(ctx) assert not analyzer.features_df.empty, f"features_df should not be empty, shape={analyzer.features_df.shape}" assert len(analyzer.features_df) >= 50, f"Expected >=50 rows, got {len(analyzer.features_df)}" assert set(analyzer.features_df.columns) == {"Oscillation", "Osc_high", "Osc_low", "Returns", "Difference"} - def test_features_df_tolerates_partial_nan(self, monkeypatch): + def test_features_df_tolerates_partial_nan(self): """features_df should retain rows even when one column has NaN at a few spots.""" from core.market.analyzer import MarketAnalyzer - from core.market.data_context import DataContext, Horizon + from core.market.data_context import Horizon, build_data_context fake_df = self._make_price_df(60) # Introduce NaN in High for a few rows (osc_high will be NaN there) @@ -230,35 +220,23 @@ def test_features_df_tolerates_partial_nan(self, monkeypatch): start = fake_df.index[0].date() end = fake_df.index[-1].date() - def _mock_build(ticker, start_date, frequency="D", end_date=None): - df = fake_df.copy() - df["LastClose"] = df["Close"].shift(1) - df["LastAdjClose"] = df["Adj Close"].shift(1) - return DataContext( - ticker=ticker, - frequency=frequency, - horizon=Horizon( - start=start_date, - end=end_date or end, - user_provided_end=end_date is not None, - frequency=frequency, - ), - bars=df, - daily_bars=fake_df.copy(), - ) - - monkeypatch.setattr("core.market.analyzer.build_data_context", _mock_build) - - analyzer = MarketAnalyzer("TEST", start, "D", end_date=end) + ctx = build_data_context( + ticker="TEST", + frequency="D", + horizon=Horizon(start=start, end=end, user_provided_end=True, frequency="D"), + raw_data=fake_df, + ) + + analyzer = MarketAnalyzer(ctx) # With dropna(how='all'), rows with partial NaN are kept assert not analyzer.features_df.empty # At least most rows should survive — only first row (shift NaN) removed assert len(analyzer.features_df) >= 50 - def test_features_df_empty_when_all_nan(self, monkeypatch): + def test_features_df_empty_when_all_nan(self): """features_df should have 0 rows when all data is NaN.""" from core.market.analyzer import MarketAnalyzer - from core.market.data_context import DataContext, Horizon + from core.market.data_context import Horizon, build_data_context fake_df = self._make_price_df(10) fake_df["Adj Close"] = np.nan @@ -268,61 +246,48 @@ def test_features_df_empty_when_all_nan(self, monkeypatch): start = fake_df.index[0].date() end = fake_df.index[-1].date() - def _mock_build(ticker, start_date, frequency="D", end_date=None): - df = fake_df.copy() - df["LastClose"] = df["Close"].shift(1) - df["LastAdjClose"] = df["Adj Close"].shift(1) - return DataContext( - ticker=ticker, - frequency=frequency, - horizon=Horizon( - start=start_date, - end=end_date or end, - user_provided_end=end_date is not None, - frequency=frequency, - ), - bars=df, - daily_bars=fake_df.copy(), - ) - - monkeypatch.setattr("core.market.analyzer.build_data_context", _mock_build) - - analyzer = MarketAnalyzer("TEST", start, "D", end_date=end) + ctx = build_data_context( + ticker="TEST", + frequency="D", + horizon=Horizon(start=start, end=end, user_provided_end=True, frequency="D"), + raw_data=fake_df, + ) + + analyzer = MarketAnalyzer(ctx) assert analyzer.features_df.empty # ═══════════════════════════════════════════════════════════════════════════ -# 5. build_data_context ticker normalization +# 5. fetch_data_context ticker normalization # ═══════════════════════════════════════════════════════════════════════════ class TestDataContextTickerNorm: - def test_futu_format_normalized(self, monkeypatch): - """build_data_context('US.NVDA', ...) should normalize to 'NVDA'.""" - monkeypatch.setattr("core.market.data_context._fetch_raw_data", lambda ticker, start, freq: (None, ticker)) - - from core.market.data_context import build_data_context + """Normalisation lives in services/market/data_context_fetch.py (batch B4).""" + + def _ctx_for(self, monkeypatch, ticker: str): + # WHY the fetch stub: normalisation happens before the DB/provider hop, + # so stubbing the raw fetch keeps these tests offline and fast. + monkeypatch.setattr( + "services.market.data_context_fetch._fetch_raw_data", + lambda t, start, freq: (None, t), + ) - ctx = build_data_context("US.NVDA", dt.date(2024, 1, 1)) - assert ctx.ticker == "NVDA" + from services.market.data_context_fetch import fetch_data_context - def test_yahoo_format_unchanged(self, monkeypatch): - """build_data_context('NVDA', ...) should keep ticker as 'NVDA'.""" - monkeypatch.setattr("core.market.data_context._fetch_raw_data", lambda ticker, start, freq: (None, ticker)) + return fetch_data_context(ticker, dt.date(2024, 1, 1)) - from core.market.data_context import build_data_context + def test_futu_format_normalized(self, monkeypatch): + """fetch_data_context('US.NVDA', ...) should normalize to 'NVDA'.""" + assert self._ctx_for(monkeypatch, "US.NVDA").ticker == "NVDA" - ctx = build_data_context("NVDA", dt.date(2024, 1, 1)) - assert ctx.ticker == "NVDA" + def test_yahoo_format_unchanged(self, monkeypatch): + """fetch_data_context('NVDA', ...) should keep ticker as 'NVDA'.""" + assert self._ctx_for(monkeypatch, "NVDA").ticker == "NVDA" def test_hk_format_normalized(self, monkeypatch): - """build_data_context('HK.00700', ...) should normalize to '0700.HK'.""" - monkeypatch.setattr("core.market.data_context._fetch_raw_data", lambda ticker, start, freq: (None, ticker)) - - from core.market.data_context import build_data_context - - ctx = build_data_context("HK.00700", dt.date(2024, 1, 1)) - assert ctx.ticker == "0700.HK" + """fetch_data_context('HK.00700', ...) should normalize to '0700.HK'.""" + assert self._ctx_for(monkeypatch, "HK.00700").ticker == "0700.HK" # ═══════════════════════════════════════════════════════════════════════════ diff --git a/tests/test_nvda_analysis.py b/tests/test_nvda_analysis.py index 0f35cac..eb123ea 100644 --- a/tests/test_nvda_analysis.py +++ b/tests/test_nvda_analysis.py @@ -65,6 +65,18 @@ def _seed_clean_bars(ticker: str, n_rows: int = 30, *, nan_only: bool = False): conn.commit() +def _analyzer(ticker: str, start: dt.date, frequency: str = "D"): + """Build a MarketAnalyzer the way production does: services fetch, core wraps. + + WHY: since batch B4 ``core/`` never fetches (ADR 0001), so the DataContext + has to be built in the service layer and injected. + """ + from core.market.analyzer import MarketAnalyzer + from services.market.data_context_fetch import fetch_data_context + + return MarketAnalyzer(fetch_data_context(ticker, start, frequency)) + + @pytest.fixture() def _patch_downloads(monkeypatch): """Disable all real yfinance download paths for unit tests.""" @@ -79,8 +91,8 @@ def _patch_downloads(monkeypatch): # Block the ensure_range → chunked backfill path (same dual patching). monkeypatch.setattr(DataService, "ensure_range", staticmethod(lambda *a, **kw: True)) monkeypatch.setattr("data_pipeline.orchestrate.backfill.ensure_range", lambda *a, **kw: True) - # Block the data_context fallback to yfinance - monkeypatch.setattr("core.market.data_context._download_data", lambda *a, **kw: None) + # Block the data_context fallback to the provider + monkeypatch.setattr("services.market.data_context_fetch._download_data", lambda *a, **kw: None) # --------------------------------------------------------------------------- @@ -94,9 +106,7 @@ class TestFeaturesDF: def test_good_data_produces_nonempty_features(self, _patch_downloads): """With 30 rows of price data, features_df should have ~29 rows.""" _seed_clean_bars("NVDA", 30) - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) assert analyzer.is_data_valid() assert analyzer.features_df.shape[0] >= 20 assert set(analyzer.features_df.columns) == {"Oscillation", "Osc_high", "Osc_low", "Returns", "Difference"} @@ -104,18 +114,14 @@ def test_good_data_produces_nonempty_features(self, _patch_downloads): def test_nan_only_filler_rows_produce_empty_features(self, _patch_downloads): """NaN-only filler rows from clean_range should not fool is_valid.""" _seed_clean_bars("NVDA", 5, nan_only=True) - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) assert not analyzer.is_data_valid() assert analyzer.features_df.empty def test_empty_db_no_download(self, _patch_downloads): """Empty DB + failed download → proper error, no crash.""" init_db() - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) assert not analyzer.is_data_valid() assert analyzer.features_df.empty @@ -144,9 +150,7 @@ def test_mixed_real_and_nan_rows(self, _patch_downloads): ) conn.commit() - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) assert analyzer.is_data_valid() # 7 real rows → shift(1) eats 1 → 6 feature rows assert analyzer.features_df.shape[0] == 6 @@ -154,18 +158,16 @@ def test_mixed_real_and_nan_rows(self, _patch_downloads): def test_single_row_produces_empty_features(self, _patch_downloads): """Only 1 row of data → shift(1) creates NaN → no valid features.""" _seed_clean_bars("NVDA", 1) - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) # 1 row is valid data, but after shift(1) → 0 feature rows assert analyzer.features_df.shape[0] == 0 def test_futu_format_ticker_normalized(self, _patch_downloads): - """build_data_context normalizes US.NVDA → NVDA for DB lookup.""" + """fetch_data_context normalizes US.NVDA → NVDA for DB lookup.""" _seed_clean_bars("NVDA", 10) - from core.market.data_context import build_data_context + from services.market.data_context_fetch import fetch_data_context - ctx = build_data_context("US.NVDA", dt.date(2026, 1, 1), "D") + ctx = fetch_data_context("US.NVDA", dt.date(2026, 1, 1), "D") assert ctx.ticker == "NVDA" assert ctx.is_valid() diff --git a/tests/test_ticker_format_integration.py b/tests/test_ticker_format_integration.py index 4b31dfa..9725a97 100644 --- a/tests/test_ticker_format_integration.py +++ b/tests/test_ticker_format_integration.py @@ -216,45 +216,23 @@ class TestMarketAnalyzerFeaturesNotEmpty: """features_df must have rows when the horizon contains sufficient data.""" def _build_analyzer_with_mock_data(self, start_date, end_date=None, frequency="D", data_days=60): - """Create a MarketAnalyzer with mocked price data routed through the canonical DataContext path.""" + """Create a MarketAnalyzer from synthetic bars — no I/O (batch B4). + + WHY the pure builder: ``core/`` no longer fetches (ADR 0001), so the + context is assembled from the fixture frame and injected. + """ from core.market.analyzer import MarketAnalyzer - from core.market.data_context import DataContext - from core.market.models import Horizon + from core.market.data_context import Horizon, build_data_context fake_df = _make_daily_ohlcv(days=data_days, start="2025-12-01") - - with patch.object( - MarketAnalyzer, - "__init__", - lambda self, *a, **kw: None, - ): - analyzer = MarketAnalyzer.__new__(MarketAnalyzer) - - # Replicate _refrequency for D - resampled = fake_df.copy() - resampled["LastClose"] = resampled["Close"].shift(1) - resampled["LastAdjClose"] = resampled["Adj Close"].shift(1) - horizon = Horizon( start=start_date, end=end_date or dt.date.today(), user_provided_end=end_date is not None, frequency=frequency, ) - ctx = DataContext( - ticker="TEST", - frequency=frequency, - horizon=horizon, - bars=resampled, - daily_bars=fake_df, - ) - - analyzer._ctx = ctx - analyzer.ticker = "TEST" - analyzer.frequency = frequency - analyzer.end_date = end_date - analyzer.features_df = ctx.features_df - return analyzer + ctx = build_data_context(ticker="TEST", frequency=frequency, horizon=horizon, raw_data=fake_df) + return MarketAnalyzer(ctx) def test_features_not_empty_two_month_daily(self): """2-month daily horizon should produce a non-empty features_df.""" From 5c4e0d8aba2d1d255d8183edae4cd517f10f1466 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Thu, 10 Sep 2026 21:04:50 +0800 Subject: [PATCH 06/15] =?UTF-8?q?feat(readiness):=20B5=20=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E5=8D=B3=E8=A7=84=E5=88=92=20+=20=E9=A2=84=E5=8F=96?= =?UTF-8?q?=EF=BC=88ADR=200012=EF=BC=89=EF=BC=8C=C2=A78=20Q1=20=E5=AE=9A?= =?UTF-8?q?=E4=B8=BA=20manifest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 业务线重构计划书 §6 B5；决策闸 Q1 选 A（保留流式 + 模块参数随 /render 带）。 - orchestrate/readiness.py（新）：KIND_DATASETS 模块→数据集映射、plan_datasets 求并集、check_and_kick（每 ticker 一次 DB-only 探测 + 守护线程 kick）、 status_for/hold_seconds_left/should_hold/is_backfill_running；守护线程 backfill kicker 从 read/_query.py 上移到此，POST 侧与分片侧共用一份 - services/market/readiness.py（新）：services 侧一半——调用 plan/kick 后为 实时链模块在守护线程预热 options.preload（orchestrate 不能 import services） - FormService.extract_modules：接受重复或逗号分隔 token，默认全模块（B7 前 前端还不发这个字段），未知 token 丢弃而非报错 - job_cache.create_job(..., plan=) 存下 readiness 结果；POST / 组装后传给 job - dispatch 冷启动保持：plan 说 kicked + 无任何历史 + backfill 线程仍在跑时， 返回自刷新的 partials/fragments/readiness.html，而不是渲染空图； 受 HOLD_SECONDS 与线程存活双重约束 （review 抓到仅靠计时器会让“下载失败”白转 30s 盖住错误，加存活判断修复， 由 test_should_hold_stops_as_soon_as_the_backfill_is_gone 钉住） - 测试：新增 tests/test_readiness.py（18 例）；后台 backfill 测试改从 readiness 导入 kicker - 文档：ADR 0012 补 Q1 决议与 B5 实现状态、frontend_architecture 流式段 （冷启动 + 模块 token 词汇表）、plan §8 Q1/台账/B5 注记、l0 度量、 CODEBUDDY/CLAUDE 流式段 验收：pytest -m "not network" --ignore=tests/e2e → 493 passed / 5 skipped； pytest tests/e2e → 38 passed；ruff clean；doc_guard clean； arch_metrics --check ok（无需重置基线）；audit_tags 16 vs 16。 --- CLAUDE.md | 19 +- CODEBUDDY.md | 19 +- data_pipeline/orchestrate/job_cache.py | 20 +- data_pipeline/orchestrate/readiness.py | 312 ++++++++++++++++++ data_pipeline/read/_query.py | 42 +-- .../0012-parameter-ownership-and-prefetch.md | 15 + docs/frontend_architecture.md | 29 +- docs/l0_architecture.md | 15 +- docs/plans/business_line_reorg.md | 41 ++- routes/core.py | 17 +- services/market/dispatch.py | 35 ++ services/market/form.py | 29 ++ services/market/readiness.py | 96 ++++++ templates/partials/fragments/readiness.html | 14 + tests/test_background_backfill.py | 15 +- tests/test_readiness.py | 265 +++++++++++++++ 16 files changed, 902 insertions(+), 81 deletions(-) create mode 100644 data_pipeline/orchestrate/readiness.py create mode 100644 services/market/readiness.py create mode 100644 templates/partials/fragments/readiness.html create mode 100644 tests/test_readiness.py diff --git a/CLAUDE.md b/CLAUDE.md index 9bff738..4775239 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -124,16 +124,21 @@ Always reference and import them package-qualified (`from core.options.greeks im ### The streaming / lazy-tab model (the key non-obvious flow) `POST /` computes **nothing**. `routes/core.py::index` normalises the form -(`FormService.extract_form_data` → `ValidationService.validate_input_data`), registers a job via -`data_pipeline/orchestrate/job_cache.py::create_job` (TTL default 90 s), and renders `templates/index.html` -with `streaming_mode=True`. Each tab shell emits an HTMX placeholder -(`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the browser fans out parallel requests. +(`FormService.extract_form_data` → `ValidationService.validate_input_data`), resolves the requested +modules (`FormService.extract_modules`), runs the **data-readiness pass** +(`services/market/readiness.py` → `data_pipeline/orchestrate/readiness.py`: plan the datasets the +modules need, probe the DB once, kick missing ranges on daemon threads, warm the live option-chain +preload), registers a job via `data_pipeline/orchestrate/job_cache.py::create_job` (TTL default 90 s, +carrying the readiness plan), and renders `templates/index.html` with `streaming_mode=True`. Each tab +shell emits an HTMX placeholder (`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the +browser fans out parallel requests. All `/render/` routes funnel into **`services/market/dispatch.py::render_streaming_slice`**, which: 1. auto-bootstraps a synthetic job with defaults when `job` is missing (direct URL / refresh / bookmark) instead of erroring; -2. dispatches via `_RENDER_KIND_SLICES` (kind → `(AnalysisService method name as a string, fragment template)`), late-bound with `getattr` so test monkey-patches are honoured; -3. memoises per `(job_id, ticker, kind)` through `compute_or_get`; -4. calls `close_thread_conn()` in `finally` to avoid leaking the thread-local SQLite connection. +2. consults the job's readiness plan and, on a **cold start** (no usable history yet *and* the backfill still running), returns `partials/fragments/readiness.html` — a self-re-firing "正在准备…" fragment — instead of an empty chart (bounded by `readiness.HOLD_SECONDS` and by thread liveness); +3. dispatches via `_RENDER_KIND_SLICES` (kind → `(AnalysisService method name as a string, fragment template)`), late-bound with `getattr` so test monkey-patches are honoured; +4. memoises per `(job_id, ticker, kind)` through `compute_or_get`; +5. calls `close_thread_conn()` in `finally` to avoid leaking the thread-local SQLite connection. Failures return `render_error_fragment` — usually **HTTP 200 on purpose** (expired job) so HTMX swaps a helpful message rather than a browser error toast. diff --git a/CODEBUDDY.md b/CODEBUDDY.md index 9bff738..4775239 100644 --- a/CODEBUDDY.md +++ b/CODEBUDDY.md @@ -124,16 +124,21 @@ Always reference and import them package-qualified (`from core.options.greeks im ### The streaming / lazy-tab model (the key non-obvious flow) `POST /` computes **nothing**. `routes/core.py::index` normalises the form -(`FormService.extract_form_data` → `ValidationService.validate_input_data`), registers a job via -`data_pipeline/orchestrate/job_cache.py::create_job` (TTL default 90 s), and renders `templates/index.html` -with `streaming_mode=True`. Each tab shell emits an HTMX placeholder -(`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the browser fans out parallel requests. +(`FormService.extract_form_data` → `ValidationService.validate_input_data`), resolves the requested +modules (`FormService.extract_modules`), runs the **data-readiness pass** +(`services/market/readiness.py` → `data_pipeline/orchestrate/readiness.py`: plan the datasets the +modules need, probe the DB once, kick missing ranges on daemon threads, warm the live option-chain +preload), registers a job via `data_pipeline/orchestrate/job_cache.py::create_job` (TTL default 90 s, +carrying the readiness plan), and renders `templates/index.html` with `streaming_mode=True`. Each tab +shell emits an HTMX placeholder (`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the +browser fans out parallel requests. All `/render/` routes funnel into **`services/market/dispatch.py::render_streaming_slice`**, which: 1. auto-bootstraps a synthetic job with defaults when `job` is missing (direct URL / refresh / bookmark) instead of erroring; -2. dispatches via `_RENDER_KIND_SLICES` (kind → `(AnalysisService method name as a string, fragment template)`), late-bound with `getattr` so test monkey-patches are honoured; -3. memoises per `(job_id, ticker, kind)` through `compute_or_get`; -4. calls `close_thread_conn()` in `finally` to avoid leaking the thread-local SQLite connection. +2. consults the job's readiness plan and, on a **cold start** (no usable history yet *and* the backfill still running), returns `partials/fragments/readiness.html` — a self-re-firing "正在准备…" fragment — instead of an empty chart (bounded by `readiness.HOLD_SECONDS` and by thread liveness); +3. dispatches via `_RENDER_KIND_SLICES` (kind → `(AnalysisService method name as a string, fragment template)`), late-bound with `getattr` so test monkey-patches are honoured; +4. memoises per `(job_id, ticker, kind)` through `compute_or_get`; +5. calls `close_thread_conn()` in `finally` to avoid leaking the thread-local SQLite connection. Failures return `render_error_fragment` — usually **HTTP 200 on purpose** (expired job) so HTMX swaps a helpful message rather than a browser error toast. diff --git a/data_pipeline/orchestrate/job_cache.py b/data_pipeline/orchestrate/job_cache.py index b07d91a..e33c0cc 100644 --- a/data_pipeline/orchestrate/job_cache.py +++ b/data_pipeline/orchestrate/job_cache.py @@ -4,8 +4,10 @@ ------------ The streaming render flow is: - POST / → JobCache.create_job(form_data, tickers) - returns job_id; render skeleton. + POST / → JobCache.create_job(form_data, tickers, plan) + returns job_id; render skeleton. `plan` is the + readiness status list (batch B5) so /render/* + knows whether its dataset is already covered. GET /render/?job=… → JobCache.compute_or_get(job_id, ticker, kind, fn) runs `fn` (the slice computation) under a per-(job, ticker, kind) lock; subsequent @@ -51,6 +53,7 @@ class _JobEntry: __slots__ = ( "form_data", "tickers", + "plan", "created_at", "last_access", "results", @@ -59,9 +62,12 @@ class _JobEntry: "_master_lock", ) - def __init__(self, form_data: dict, tickers: list[str]): + def __init__(self, form_data: dict, tickers: list[str], plan: list | None = None): self.form_data: dict = form_data self.tickers: list[str] = list(tickers) + # Readiness statuses computed at POST time (batch B5 / ADR 0012). Empty + # for jobs registered without a readiness pass (tests, legacy callers). + self.plan: list = list(plan or []) self.created_at: float = time.monotonic() # TTL counts from the LAST access, not creation: a slice that computes # longer than the TTL must not lose its result to a mid-compute @@ -110,15 +116,17 @@ def _evict_expired(now: float | None = None) -> None: logger.debug("JobCache evicted %d stale job(s)", len(stale)) -def create_job(form_data: dict, tickers: list[str]) -> str: +def create_job(form_data: dict, tickers: list[str], plan: list | None = None) -> str: """Register a new job and return its opaque id. `form_data` is shallow-copied so later mutations by the caller don't - leak into the cache. + leak into the cache. `plan` is the readiness status list from + ``orchestrate/readiness.py`` (optional — ``/render/*`` degrades to its + original behaviour when it is absent). """ _evict_expired() job_id = uuid.uuid4().hex - entry = _JobEntry(form_data=dict(form_data), tickers=list(tickers)) + entry = _JobEntry(form_data=dict(form_data), tickers=list(tickers), plan=plan) with _jobs_lock: _jobs[job_id] = entry logger.info("JobCache created job=%s tickers=%s", job_id[:8], tickers) diff --git a/data_pipeline/orchestrate/readiness.py b/data_pipeline/orchestrate/readiness.py new file mode 100644 index 0000000..b74665c --- /dev/null +++ b/data_pipeline/orchestrate/readiness.py @@ -0,0 +1,312 @@ +"""Data-readiness planning and prefetch (ADR 0012). + +Domain: Data Pipeline — Orchestrate (readiness) +Context: + - Before batch B5, ``POST /`` computed nothing and each ``/render/`` + discovered missing coverage on its own, so whichever tab the user opened + first paid for the whole backfill. This module turns that implicit per-slice + decision into an explicit plan: given the ticker(s) and the modules the user + asked for, work out which datasets they need, probe the DB once, and kick the + missing ranges immediately. + - The module → dataset map is static and lives here so the route never has to + know which slice touches which table. +Constraints: + - CONSTRAINT (docs/constraints.md §6): no job queue. Coverage probes are + DB-only — never network on the request thread — and a missing range is kicked + on a daemon thread. ``check_and_kick`` therefore cannot block for more than a + probe, so ``POST /`` still returns the skeleton in < 1 s. + - INVARIANT: one pipeline run (download → clean → process) fills both + ``clean_bars`` and ``feature_bars``, so coverage is gated on the + ``clean_bars`` probe and a single kick per (ticker, range) covers every + dataset the plan asked for. +Contracts: + - ``plan_datasets(tickers, modules, *, start, end, today=None)`` + - ``check_and_kick(plan, *, kick=None)`` + - ``dataset_for_module(module)`` / ``status_for(plan, ticker, module)`` + - ``kick_backfill(ticker, start, end)`` / ``join_backfills(timeout=None)`` +Dependencies UPWARD: + - data_pipeline.orchestrate.backfill, data_pipeline.store.db +Dependencies DOWNWARD: + - routes/core.py (through services/market/readiness.py), + data_pipeline/read/_query.py +""" + +from __future__ import annotations + +import datetime as dt +import logging +import threading +import time +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + +# INVARIANT: module token (the same token ``/render/`` and the sidebar use) +# → the datasets that module reads. A module with no DB dependency maps to (). +KIND_DATASETS: dict[str, tuple[str, ...]] = { + "market_review": ("clean_bars",), + "statistical": ("feature_bars",), + "assessment": ("feature_bars",), + # The volatility slice renders the HV context next to the live chain, so it + # needs daily bars as well as the live snapshot. + "options_chain": ("clean_bars",), + # Live-only modules: no stored dataset to prefetch (ADR 0004). + "payoff_ratio": (), + "regime": (), + "simulation": (), + "option_pricing_matrix": (), +} + +ALL_MODULES: tuple[str, ...] = tuple(KIND_DATASETS) + +# DOMAIN: default lookback when the caller does not pass a horizon. Mirrors the +# direct-URL bootstrap default in services/market/dispatch.py (2 years). +DEFAULT_LOOKBACK_DAYS = 365 * 2 + +# DOMAIN: how long /render/ will hold a tab in the "data is being +# prepared" state (cold start only) before falling back to computing with +# whatever coverage exists. Bounded so a stuck backfill degrades into the normal +# (graceful) slice error instead of an endless spinner. +HOLD_SECONDS = 30.0 + + +@dataclass(frozen=True) +class DatasetRequest: + """One (ticker, dataset) the requested modules will need.""" + + ticker: str + dataset: str + start: dt.date + end: dt.date + module: str + + +@dataclass(frozen=True) +class ReadinessStatus: + """Outcome of the coverage probe for one request. + + ``kicked_at`` is a ``time.monotonic()`` stamp (0.0 when nothing was kicked); + it lives on the status so ``hold_seconds_left`` needs no extra bookkeeping. + ``has_data`` distinguishes a **cold start** (no rows at all — nothing to show + yet, worth holding the tab for) from a **partial gap** (usable history exists, + so the tab should render now and let the backfill catch up behind it). + """ + + ticker: str + dataset: str + module: str + state: str # "covered" | "kicked" + kicked_at: float = field(default=0.0) + has_data: bool = True + # The (ticker, range) that was probed/kicked — used to ask whether the + # backfill is still running (see should_hold). + start: dt.date | None = None + end: dt.date | None = None + + +def dataset_for_module(module: str) -> str | None: + """Return the first dataset ``module`` reads, or None when it is live-only.""" + datasets = KIND_DATASETS.get(module) or () + return datasets[0] if datasets else None + + +def plan_datasets( + tickers: list[str], + modules: list[str], + *, + start: dt.date | None = None, + end: dt.date | None = None, + today: dt.date | None = None, +) -> list[DatasetRequest]: + """Union over the requested modules, one entry per (ticker, dataset). + + ORDER: preserves the module order the caller passed and, inside a module, the + ticker order — so the caller can prioritise ``tickers[0]`` downstream. + """ + today = today or dt.date.today() + end = end or today + start = start or (end - dt.timedelta(days=DEFAULT_LOOKBACK_DAYS)) + requests: list[DatasetRequest] = [] + seen: set[tuple[str, str]] = set() + for module in modules: + for dataset in KIND_DATASETS.get(module, ()): + for ticker in tickers: + key = (ticker, dataset) + if key in seen: + continue + seen.add(key) + requests.append(DatasetRequest(ticker=ticker, dataset=dataset, start=start, end=end, module=module)) + return requests + + +def check_and_kick(plan: list[DatasetRequest], *, kick=None) -> list[ReadinessStatus]: + """Probe coverage per (ticker, range) and kick the missing ones. + + ``kick`` is injectable so tests can assert the decision without starting + threads. Returns one status per request, in plan order. + """ + from data_pipeline.orchestrate import backfill as _bf + + kick = kick or kick_backfill + statuses: list[ReadinessStatus] = [] + for ticker, group in _by_ticker(plan): + start = min(req.start for req in group) + end = max(req.end for req in group) + try: + missing = _bf.needs_backfill(ticker, start, end) + except Exception as exc: # noqa: BLE001 — a probe must never break the POST + logger.warning("readiness probe failed for %s: %s", ticker, exc) + missing = False + kicked_at = 0.0 + has_data = True + if missing: + has_data = _has_any_prices(ticker) + kicked_at = time.monotonic() + kick(ticker, start, end) + state = "kicked" if missing else "covered" + statuses.extend( + ReadinessStatus( + ticker=req.ticker, + dataset=req.dataset, + module=req.module, + state=state, + kicked_at=kicked_at, + has_data=has_data, + start=start, + end=end, + ) + for req in group + ) + if statuses: + logger.info( + "readiness: %d request(s) — %d covered, %d kicked", + len(statuses), + sum(1 for s in statuses if s.state == "covered"), + sum(1 for s in statuses if s.state == "kicked"), + ) + return statuses + + +def status_for(plan: list[ReadinessStatus] | None, ticker: str, module: str) -> ReadinessStatus | None: + """Return the plan entry for ``(ticker, module)``, or None when not planned.""" + dataset = dataset_for_module(module) + if dataset is None or not plan: + return None + for status in plan: + if status.ticker == ticker and status.dataset == dataset: + return status + return None + + +def hold_seconds_left(status: ReadinessStatus | None, *, now: float | None = None) -> float | None: + """Seconds ``/render/*`` should keep holding this tab, or None to compute now. + + Holds only on a **cold start** (``has_data`` False): with usable history the tab + should paint immediately while the backfill fills the tail, which is what the + per-slice grace period already does. + + WHY bounded: an unbounded hold would spin forever if the backfill failed; after + HOLD_SECONDS the caller computes with whatever coverage exists and the slice's + own graceful-degradation path takes over. + """ + if status is None or status.state != "kicked" or status.has_data or not status.kicked_at: + return None + remaining = HOLD_SECONDS - ((now if now is not None else time.monotonic()) - status.kicked_at) + return remaining if remaining > 0 else None + + +def is_backfill_running(status: ReadinessStatus) -> bool: + """True when the daemon-thread backfill kicked for this entry is still alive.""" + if status.start is None or status.end is None: + return False + key = (status.ticker, str(status.start), str(status.end)) + with _backfill_lock: + thread = _backfill_threads.get(key) + return thread is not None and thread.is_alive() + + +def should_hold(status: ReadinessStatus | None, *, now: float | None = None) -> bool: + """Should ``/render/`` wait for data instead of computing now? + + Holds only while (a) this was a cold start, (b) the HOLD_SECONDS window has not + elapsed, and (c) the backfill thread is **still running**. (c) is what keeps a + failed download honest: once the thread dies the tab stops claiming to be + "preparing data" and the slice reports the real outcome (its own error). + """ + if hold_seconds_left(status, now=now) is None: + return False + return is_backfill_running(status) + + +def _has_any_prices(ticker: str) -> bool: + """True when the DB already holds at least one priced row for ``ticker``.""" + try: + from data_pipeline.store.repos import count_clean_rows + + return count_clean_rows(ticker) > 0 + except Exception as exc: # noqa: BLE001 — a probe must never break the POST + logger.debug("readiness: existing-data probe failed for %s: %s", ticker, exc) + return True + + +def _by_ticker(plan: list[DatasetRequest]) -> list[tuple[str, list[DatasetRequest]]]: + """Group a plan by ticker, preserving first-appearance order.""" + order: list[str] = [] + groups: dict[str, list[DatasetRequest]] = {} + for req in plan: + if req.ticker not in groups: + groups[req.ticker] = [] + order.append(req.ticker) + groups[req.ticker].append(req) + return [(t, groups[t]) for t in order] + + +# ── Daemon-thread backfill kicker ─────────────────────────────────────────── +# WHY it lives here and not in read/: `read` imports `orchestrate` (the read path +# triggers refreshes), so orchestrate may not import read. Both callers now share +# this one kicker instead of read keeping a private copy. +_backfill_lock = threading.Lock() +_backfill_threads: dict[tuple, threading.Thread] = {} + + +def kick_backfill(ticker: str, start: dt.date, end: dt.date) -> None: + """Start a daemon-thread backfill for ``[start, end]`` unless one is running. + + The in-flight map is the same de-duplication ``ensure_range`` applies + internally, hoisted to the thread level so a POST that fans out over N + tickers cannot start N copies of the same work. + """ + key = (ticker, str(start), str(end)) + with _backfill_lock: + existing = _backfill_threads.get(key) + if existing is not None and existing.is_alive(): + return + t = threading.Thread(target=_run_backfill, args=(ticker, start, end, key), daemon=True) + _backfill_threads[key] = t + t.start() + logger.info("background backfill kicked for %s [%s .. %s]", ticker, start, end) + + +def _run_backfill(ticker: str, start: dt.date, end: dt.date, key: tuple) -> None: + from data_pipeline.orchestrate import backfill as _bf + + try: + _bf.ensure_range(ticker, start, end) + except Exception as e: # noqa: BLE001 + logger.warning("background backfill failed for %s: %s", ticker, e) + finally: + # Daemon threads never run dispatch's finally-cleanups; drop this + # thread's SQLite connection so _all_conns doesn't grow per backfill. + from data_pipeline.store.db import close_thread_conn + + close_thread_conn() + with _backfill_lock: + _backfill_threads.pop(key, None) + + +def join_backfills(timeout: float | None = None) -> None: + """Test helper: wait for all in-flight background backfills.""" + with _backfill_lock: + threads = list(_backfill_threads.values()) + for t in threads: + t.join(timeout=timeout) diff --git a/data_pipeline/read/_query.py b/data_pipeline/read/_query.py index 55c428c..05ac146 100644 --- a/data_pipeline/read/_query.py +++ b/data_pipeline/read/_query.py @@ -3,7 +3,6 @@ import datetime as dt import logging import os -import threading import time import pandas as pd @@ -11,6 +10,7 @@ from data_pipeline import _state as _g from data_pipeline.orchestrate import backfill as _bf from data_pipeline.orchestrate import update as _u +from data_pipeline.orchestrate.readiness import kick_backfill as _kick_backfill from data_pipeline.store.db import fetch_df, init_db logger = logging.getLogger(__name__) @@ -23,43 +23,11 @@ # kicks), the request waits a short grace period so the common "one chunk # missing" case still returns full data, then reads whatever coverage exists. _BACKFILL_WAIT_SECONDS = float(os.environ.get("BACKFILL_WAIT_SECONDS", "8")) -_backfill_lock = threading.Lock() -_backfill_threads: dict[tuple, threading.Thread] = {} - -def _kick_backfill(ticker: str, start, end) -> None: - key = (ticker, str(start), str(end)) - with _backfill_lock: - existing = _backfill_threads.get(key) - if existing is not None and existing.is_alive(): - return - t = threading.Thread(target=_run_backfill, args=(ticker, start, end, key), daemon=True) - _backfill_threads[key] = t - t.start() - logger.info("background backfill kicked for %s [%s .. %s]", ticker, start, end) - - -def _run_backfill(ticker, start, end, key) -> None: - try: - _bf.ensure_range(ticker, start, end) - except Exception as e: - logger.warning("background backfill failed for %s: %s", ticker, e) - finally: - # Daemon threads never re-run dispatch's finally-cleanups; drop this - # thread's SQLite connection so _all_conns doesn't grow per backfill. - from data_pipeline.store.db import close_thread_conn - - close_thread_conn() - with _backfill_lock: - _backfill_threads.pop(key, None) - - -def _join_backfills(timeout: float | None = None) -> None: - """Test helper: wait for all in-flight background backfills.""" - with _backfill_lock: - threads = list(_backfill_threads.values()) - for t in threads: - t.join(timeout=timeout) +# NOTE: the kicker itself lives in ``orchestrate/readiness.py`` — batch B5 made it +# shared between this per-slice path and the readiness pass on POST /. It is +# re-exported (as ``_kick_backfill`` / ``_join_backfills``) at the top of this +# module so existing callers and tests keep working. def _wait_for_coverage(ticker, start, end, timeout: float) -> bool: diff --git a/docs/decisions/0012-parameter-ownership-and-prefetch.md b/docs/decisions/0012-parameter-ownership-and-prefetch.md index 966de5b..6ee455c 100644 --- a/docs/decisions/0012-parameter-ownership-and-prefetch.md +++ b/docs/decisions/0012-parameter-ownership-and-prefetch.md @@ -111,6 +111,21 @@ The **submit contract** — whether `POST /` carries a `modules` manifest with per-module params attached to each `/render` call, or the streaming tabs move fully to client-fired `/api/*` — is deferred to the implementation batch. +> **Resolved (batch B5, 2026-09-10): manifest.** `POST /` carries the module tokens; each module's +> parameters travel as query args on its own `/render` call, mirroring +> `/api/option_chain?ticker=…`. Full client-fired was rejected because (a) this prefetch pass needs +> the module list *at submit time*, (b) the four streaming slices return server-rendered HTML + +> base64 PNG, so changing the transport would not change the product, and (c) the diff/revert surface +> would span four templates plus four loaders. It would only win if the charts moved to client-side +> rendering (ADR 0006 / 0008). See the plan §8 gate table and the B5 note. + +**Implementation status (B5)**: `data_pipeline/orchestrate/readiness.py` plans datasets per module, +probes coverage (DB-only) and kicks missing ranges on a daemon thread; +`services/market/readiness.py` adds the live-preload warm; the plan is stored on the job and +`/render/` holds a cold-start tab with a self-re-firing readiness fragment (bounded by +`HOLD_SECONDS` and by backfill-thread liveness). The per-module toolbars and the removal of +`syncConfigToForm` remain B7's work. + ## Consequences - Positive: the always-visible surface is one field; module parameters are diff --git a/docs/frontend_architecture.md b/docs/frontend_architecture.md index f8df57d..b785c8c 100644 --- a/docs/frontend_architecture.md +++ b/docs/frontend_architecture.md @@ -60,10 +60,13 @@ templates/ Heavy analysis no longer runs synchronously inside `POST /`. The flow is: ``` -Browser ── POST / (form data) ─────────────────► Flask +Browser ── POST / (form data + module tokens) ──► Flask │ -Flask creates a JobCache entry (job_id) and immediately renders -`index.html` with `streaming_mode=True`. Each tab partial emits an +Flask resolves the modules, runs the **readiness pass** (ADR 0012): +plan the datasets those modules need, probe the DB once, kick anything +missing on a daemon thread, warm the live option-chain preload — then +creates a JobCache entry (job_id, carrying that plan) and immediately +renders `index.html` with `streaming_mode=True`. Each tab partial emits an HTMX placeholder:
in parallel for each visible ticker: /render/assessment /render/options_chain +Flask ── consults the job's readiness plan ─────► cold start? hold Flask ── compute_or_get(job_id, ticker, kind) ──► AnalysisService.*_slice └─ memoised per (job, ticker, kind) @@ -86,8 +90,10 @@ Flask ── HTML fragment ───────────────── ``` Key files: -- `data_pipeline/orchestrate/job_cache.py` — in-process JobCache (TTL 90 s). -- `app.py::_render_streaming_slice` — shared `/render/` handler. +- `data_pipeline/orchestrate/job_cache.py` — in-process JobCache (TTL 90 s), carries the plan. +- `data_pipeline/orchestrate/readiness.py` — dataset plan, coverage probe, backfill kicker. +- `services/market/readiness.py` — services half (preload warm), called from `routes/core.py::index`. +- `services/market/dispatch.py::render_streaming_slice` — shared `/render/` handler. - `services/market/analysis/facade.py::generate_*_slice` — per-tab compute. - `templates/partials/fragments/*.html` — rendered fragments. @@ -95,6 +101,19 @@ The browser-side HTMX library replaces each placeholder when its fragment arrives, so users see tabs populate as their data is ready instead of waiting for the slowest tab. +**Cold start** (batch B5): if the readiness pass kicked this module's dataset, the ticker has no +usable history yet *and* the backfill is still running, `/render/` returns +`partials/fragments/readiness.html` — a self-re-firing "正在准备…" fragment (`hx-trigger="load +delay:3s"`) — instead of rendering an empty chart. The hold is bounded by +`readiness.HOLD_SECONDS` (30 s) **and** by backfill-thread liveness, so a failed download quickly +falls through to the slice's own error rather than a permanent spinner. + +**Parameter ownership** (batch B7, per §8 Q1): the module tokens above (`market_review`, +`statistical`, `assessment`, `options_chain`, `payoff_ratio`, `regime`, `simulation`, +`option_pricing_matrix`) are the same vocabulary the readiness plan uses. Each module's parameters +travel as **query args on its own `/render` call**, mirroring `/api/option_chain?ticker=…`; the +persistent Parameters bar owns only `ticker`. + --- ## Page Architecture diff --git a/docs/l0_architecture.md b/docs/l0_architecture.md index db7c326..b26dcd0 100644 --- a/docs/l0_architecture.md +++ b/docs/l0_architecture.md @@ -38,7 +38,7 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ | `routes/` | 909 lines · 8 files | 7 blueprints + `__init__.py` aggregate export; no business logic | good | | `services/` | 3 540 lines · 5 domain packages | `market` (incl. `analysis/` slice factory), `market_review`, `options`, `portfolio`, `regime` | good | | `core/` | 6 372 lines · 8 sub-packages + `_shared` | Pure computation — no Flask, no DB, no network | good | -| `data_pipeline/` | 3 331 lines · 26 files | The only I/O boundary, re-homed into six one-way stages (ADR 0011, batch B3): `providers/` · `store/` · `ingest/` · `transform/` · `read/` · `orchestrate/` (+ `_state.py`) | good | +| `data_pipeline/` | 3 618 lines · 27 files | The only I/O boundary, re-homed into six one-way stages (ADR 0011, batch B3): `providers/` · `store/` · `ingest/` · `transform/` · `read/` · `orchestrate/` (+ `_state.py`) | good | | `utils/` | 756 lines · 7 files | Leaf layer; highest fan-in (`ticker_utils.py` = 11) | good | | `templates/` | 1 546 lines · 17 files | `index.html` skeleton + `partials/fragments/*` (HTMX swap targets) | good | | `static/` | 5 473 lines · 31 JS/CSS | `state/` · `sim/` · `components/` · `features/` + tab entry files | fair (see §4 P3-1) | @@ -78,17 +78,18 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ ## 2. Measured shape (`scripts/arch_metrics.py`) -_Refreshed 2026-09-10 after batches B1 (provider seam), B2 (canonical table names) and B3 (data_pipeline re-home); the L1 inventory in §1 above is otherwise the 2026-09-08 snapshot._ +_Refreshed 2026-09-10 after batches B1 (provider seam), B2 (canonical table names), B3 (data_pipeline re-home), B4 (core purity) and B5 (readiness); the L1 inventory in §1 above is otherwise the 2026-09-08 snapshot._ ``` -modules=156 import_edges=313 +modules=159 import_edges=324 Layer-edge violations : (none) Import cycles : 0 God files (>400 lines): (none) -Top fan-out : core/market/charts/facade.py(14) · core/market/data_context.py(7) - routes/__init__.py(7) · routes/core.py(7) · app.py(6) +Top fan-out : core/market/charts/facade.py(14) · routes/core.py(8) + routes/__init__.py(7) · services/market/analysis/facade.py(7) + services/market/dispatch.py(7) Top fan-in : core/_shared/plotting.py(13) · data_pipeline/providers/yf_client.py(11) - utils/ticker_utils.py(11) · data_pipeline/store/db.py(9) + utils/ticker_utils.py(11) · data_pipeline/store/db.py(10) Dead code : services/market/analysis/summary.py (only one; already on the watch list in docs/architecture_review.md §2) ``` @@ -124,7 +125,7 @@ data_pipeline/ ingest/ GLUE business-day gap detection + raw_bars upsert transform/ PROCESS raw_bars → clean_bars → feature_bars (never imports providers) read/ SERVE DataService facade + memoised queries - orchestrate/ DRIVERS manual/seed update, chunked backfill, job cache, scheduler + orchestrate/ DRIVERS manual/seed update, chunked backfill, readiness, job cache, scheduler _state.py process-local shared state (query cache, update locks) ``` diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index adb116d..0bb534b 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -42,7 +42,7 @@ | B2 — canonical raw store | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | gate §8 Q4 resolved (name-only rename). Actuals in §8; `symbol` column deferred (ADR 0011 amendment) | | B3 — package re-home | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | six stages + `_state.py`; first sub-layer guard table; `arch_baseline.json` **not** reset (no tracked drift). Actuals + deviations in §8 | | B4 — close L1 (`core-purity`) | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | zero core→data_pipeline edges; markers deleted and refused by test; `core` layer tightened to `{utils}` | -| B5 — readiness plan + prefetch | ⬜ not started | — | — | gate: §8 Q1 | +| B5 — readiness plan + prefetch | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (**manifest**, params as `/render` query args); readiness plan on the job; cold-start hold fragment. Actuals + deferrals in §8 | | B6 — `ticker`-only Parameters bar | ⬜ not started | — | — | gate: §8 Q3; depends on B5 | | B7 — module-scoped params | ⬜ not started | — | — | gate: §8 Q1; depends on B6 | | B8 — retire / repurpose Config tab | ⬜ not started | — | — | gate: §8 Q2 | @@ -419,7 +419,7 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. | # | Question | Decision gate | Working lean | |---|---|---|---| -| Q1 | **Submit contract** — does `POST /` carry a `modules` manifest with per-module params attached to each `/render` call, or do the streaming market tabs move fully to client-fired `/api/*` like Option Chain? Manifest keeps the streaming model; full client-fired is more uniform but a bigger diff. | before **B5** (locks how B7 wires params) | manifest | +| Q1 | **Submit contract** — does `POST /` carry a `modules` manifest with per-module params attached to each `/render` call, or do the streaming market tabs move fully to client-fired `/api/*` like Option Chain? Manifest keeps the streaming model; full client-fired is more uniform but a bigger diff. | ✅ resolved 2026-09-10 (B5) — **manifest**, per-module params as query args on each `/render` call (sub-option A1) | **manifest.** Decisive reasons: (1) ADR 0012's readiness pass needs the module list *at submit time* — with no POST manifest, B5 would need an extra `/api/ready` protocol; (2) the four streaming slices return server-rendered HTML + base64 PNG, so client-firing changes only the transport, not the product; (3) the diff and the revert surface stay one batch wide. Full client-fired would only win if the charts moved to client-side rendering (ADR 0006/0008 territory). Params travel as query args (not `hx-post` JSON) to match the existing `/api/option_chain?ticker=…` shape and stay bookmark-reproducible — recorded in the B5 note below | | Q2 | **Config tab fate** — is there *any* genuine global setting to keep? Risk-free rate is the only candidate (hard-coded in `static/sim/` and again in `core/options/greeks`). Yes → tab shrinks to it; no → tab deleted. | before **B8** | keep risk-free rate, delete the rest | | Q3 | **`positions` block** — Portfolio Analysis is its only consumer. Move into a dedicated "Portfolio" panel/tab, or keep as a section the bar's Run ignores? | before **B6** | dedicated Portfolio panel | | Q4 | **Table rename vs. reshape** — `raw_prices`→`raw_bars` with identical columns (minimal), or also move the yfinance-ism `adj_close` handling into the provider during the rename? | ✅ resolved 2026-09-10 (B2) | **minimal rename** — identical columns on both sides of each pair (structurally enforced: one column tuple per shape, used to create both names). The `adj_close` normalisation is already inside the provider (B1's `to_canonical_bars`), and ingest now consumes canonical bars, so no reshape is needed. ADR 0011's `symbol` column stays the target state but is deferred — see the B2 note below | @@ -548,6 +548,43 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / dead 1 — no baseline reset); `grep -rn "allow=core-purity"` returns nothing. +**B5 (2026-09-10) — readiness plan + prefetch on submit.** + +- **Q1 = manifest** (see the gate table above). `POST /` now resolves the module list + (`FormService.extract_modules`: repeated or comma-separated tokens; defaults to all known modules + until B7 sends the field; unknown tokens are dropped, not fatal) and stores the readiness plan on + the job. Per-module params still arrive on the existing hidden fields — B7 moves them onto each + `/render` call as query args. +- **`orchestrate/readiness.py`** (new): `KIND_DATASETS` (module → datasets; live-only modules map to + `()`), `plan_datasets` (union over modules, one entry per `(ticker, dataset)`), `check_and_kick` + (one DB-only coverage probe per ticker+range, then a daemon-thread kick), plus `status_for` / + `hold_seconds_left` / `should_hold` / `is_backfill_running`. + The daemon-thread kicker moved here from `read/_query.py` so POST-time readiness and the per-slice + path share one implementation (and `read → orchestrate` keeps the graph acyclic). +- **`services/market/readiness.py`** (new): the services half — calls the plan/kick, then warms + `services.options.preload` for the live-chain modules on daemon threads. The split exists because + `orchestrate` may not import `services`. +- **Cold-start hold**: `/render/` consults the job's plan and, when the plan says "kicked" *and* + there is no usable history yet *and* the backfill thread is still alive, returns a self-re-firing + `partials/fragments/readiness.html` ("正在准备…") instead of an empty chart. Bounded by + `HOLD_SECONDS = 30` **and** by thread liveness — a review pass caught that the timer alone left a + *failed* download showing a spinner for 30 s and hiding the real error + (`tests/test_nvda_analysis.py::test_failed_download_shows_error`); the liveness check fixed it and + is pinned by `test_should_hold_stops_as_soon_as_the_backfill_is_gone`. +- **Tests**: new `tests/test_readiness.py` (18 cases) — plan union / dedupe / live-only emptiness / + horizon defaults, kick decision + probe-failure resilience, hold window + thread-liveness, + `create_job` carrying the plan, and `extract_modules` parsing. `test_background_backfill.py` now + imports the kicker from `readiness`. +- **Deferred to B6/B7 (recorded, not silently dropped)**: (a) the *client* half of "the browser + re-fires" — the held fragment self-refreshes, but the module **toolbars** and the per-module query + args are B7; (b) the market-review benchmark panel (`market_review_prices`, L5) is not in the + dataset map yet — its ladder lives in `services/market_review/fetch.py` and folds into the provider + seam later. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 493 passed / 5 skipped; + full `pytest tests/e2e` → 38 passed; `ruff check` + `format --check` clean; `doc_guard.py` clean; + `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / dead 1 — no baseline reset); + `audit_tags.py` 16 vs baseline 16 after tagging two new domain constants. + --- ## 9. References diff --git a/routes/core.py b/routes/core.py index 1546a74..395dd71 100644 --- a/routes/core.py +++ b/routes/core.py @@ -22,6 +22,7 @@ from services.market.dispatch import render_streaming_slice from services.market.facade import MarketService from services.market.form import FormService +from services.market.readiness import prepare_readiness from services.market.validation import ValidationService from utils.constants import ( DEFAULT_FREQUENCY, @@ -80,7 +81,20 @@ def index(): return render_template("index.html", error="Please enter at least one ticker symbol.") first_ticker = tickers[0] - job_id = create_job({**form_data, "ticker": first_ticker}, tickers) + + # ── Data readiness (ADR 0012 / batch B5) ── + # CONSTRAINT: DB-only probes + daemon-thread prefetch, so this call + # cannot push POST / past the <1 s skeleton budget. See + # services/market/readiness.py for the layer split. + modules = FormService.extract_modules(request) + plan = prepare_readiness( + tickers, + modules, + start=form_data.get("parsed_start_time"), + end=form_data.get("parsed_end_time"), + ) + + job_id = create_job({**form_data, "ticker": first_ticker}, tickers, plan) template_data = { **form_data, @@ -89,6 +103,7 @@ def index(): "tickers_raw": ", ".join(tickers), "streaming_mode": True, "job_id": job_id, + "modules": modules, "summary_pending": len(tickers) > 1, } return render_template("index.html", **template_data) diff --git a/services/market/dispatch.py b/services/market/dispatch.py index f2e8f8d..6fb8986 100644 --- a/services/market/dispatch.py +++ b/services/market/dispatch.py @@ -27,6 +27,7 @@ from flask import render_template, request from data_pipeline.orchestrate.job_cache import compute_or_get, get_job +from data_pipeline.orchestrate.readiness import should_hold, status_for from data_pipeline.store.db import close_thread_conn from services.market.analysis import AnalysisService from utils.constants import ( @@ -52,6 +53,32 @@ } +# DOMAIN: how long the held fragment waits before re-issuing itself. Short +# enough that the user sees the tab fill in promptly, long enough not to hammer +# the server. +_RETRY_DELAY_SECONDS = 3 + + +def render_readiness_fragment(kind: str, job_id: str, ticker: str) -> tuple[str, int]: + """Fragment that says "preparing data" and re-issues its own request. + + WHY HTTP 200: the job is alive and the request is being handled correctly — + the data just is not there yet. HTMX swaps the fragment, and the fragment's own + ``hx-trigger`` re-fires until the data is ready or ``HOLD_SECONDS`` elapses. + """ + return ( + render_template( + "partials/fragments/readiness.html", + kind=kind, + kind_id=kind.replace("_", "-"), + job_id=job_id, + ticker=ticker, + retry_seconds=_RETRY_DELAY_SECONDS, + ), + 200, + ) + + def render_streaming_slice(kind: str) -> Any: """Shared handler for /render/?job=…&ticker=…. @@ -108,6 +135,14 @@ def render_streaming_slice(kind: str) -> Any: # the user knows to re-submit the form. return render_error_fragment(kind, "session expired (job no longer cached); please re-submit the form", 200) + # ── Batch B5: consult the job's readiness plan ── + # On a cold start (no rows at all for this ticker) the slice would render an + # empty chart; hold the tab with a self-re-firing fragment instead. Bounded by + # readiness.HOLD_SECONDS and stops as soon as the backfill thread exits — + # see readiness.should_hold. + if job is not None and should_hold(status_for(job.plan, ticker, kind)): + return render_readiness_fragment(kind, job_id, ticker) + slice_fn_name, template = _RENDER_KIND_SLICES[kind] # The form_data captured at POST time was for the first ticker. When the diff --git a/services/market/form.py b/services/market/form.py index 9d9a56b..c0ac617 100644 --- a/services/market/form.py +++ b/services/market/form.py @@ -24,9 +24,38 @@ class FormService: """ Service for handling form data extraction and processing from Flask request. - extract_form_data: Extracts and parses all dashboard form fields. + - extract_modules: Modules the client asked for (readiness planning). - parse_option_data: Parses option positions from JSON string. """ + @staticmethod + def extract_modules(request) -> list[str]: + """Return the module tokens the client requested (ADR 0012 / batch B5). + + Accepts either repeated fields (``modules=market_review&modules=statistical``) + or one comma-separated value. Unknown tokens are dropped rather than + rejected: an unrecognised module simply has no datasets to plan, and + failing the whole submit for a typo would be hostile. + + WHY default to every known module: the frontend does not send this field + until batch B7, and the streaming tabs are rendered unconditionally — so + "everything" is the honest interpretation of a request that omits it. + """ + from data_pipeline.orchestrate.readiness import ALL_MODULES + + raw = request.form.getlist("modules") + tokens: list[str] = [] + for item in raw: + tokens.extend(part.strip() for part in item.split(",") if part.strip()) + if not tokens: + return list(ALL_MODULES) + known = [t for t in tokens if t in ALL_MODULES] + unknown = sorted(set(tokens) - set(ALL_MODULES)) + if unknown: + logger.warning("extract_modules: ignoring unknown module token(s) %s", unknown) + # De-duplicate while preserving the caller's order. + return list(dict.fromkeys(known)) + @staticmethod def extract_form_data(request): """ diff --git a/services/market/readiness.py b/services/market/readiness.py new file mode 100644 index 0000000..fabe4bd --- /dev/null +++ b/services/market/readiness.py @@ -0,0 +1,96 @@ +"""Readiness on submit: plan the stored datasets and warm the live snapshots. + +Domain: Market Analysis — Readiness +Context: + - ADR 0012 / batch B5. ``routes/core.py::index`` calls this right after + validation: the data_pipeline half plans + kicks the stored datasets, and the + services half warms ``services.options.preload`` so switching to a live option + tab is instant instead of a cold ~2 s fetch. + - WHY the split: ``data_pipeline.orchestrate`` may not import ``services`` + (ADR 0001), and warming an option chain *is* a service concern. +Constraints: + - CONSTRAINT (docs/constraints.md §6): nothing here blocks on the network. The + preload warm runs on a daemon thread, so ``POST /`` still returns the + skeleton in < 1 s and a failed warm is invisible to the user. +Contracts: + - ``prepare_readiness(tickers, modules, *, start, end) -> list[ReadinessStatus]`` + - ``warm_live_snapshots(tickers) -> None`` — fire-and-forget +Dependencies UPWARD: + - data_pipeline.orchestrate.readiness (plan + kick), services.options.preload +Dependencies DOWNWARD: + - routes/core.py +""" + +from __future__ import annotations + +import datetime as dt +import logging +import threading + +from data_pipeline.orchestrate.readiness import ( + KIND_DATASETS, + ReadinessStatus, + check_and_kick, + plan_datasets, +) + +logger = logging.getLogger(__name__) + +# INVARIANT: modules whose first paint needs a live option-chain snapshot +# (ADR 0004 — never persisted), so the only useful prefetch is the in-process +# preload cache. +LIVE_CHAIN_MODULES = frozenset({"options_chain", "payoff_ratio"}) + + +def prepare_readiness( + tickers: list[str], + modules: list[str], + *, + start: dt.date | None = None, + end: dt.date | None = None, +) -> list[ReadinessStatus]: + """Plan + kick the stored datasets, then warm the live snapshots. + + Returns the per-request statuses so the caller can store them on the job and + ``/render/`` can tell "covered" from "still downloading". + """ + plan = plan_datasets(tickers, modules, start=start, end=end) + if not plan: + logger.info("readiness: no stored dataset required for modules=%s", modules) + statuses = check_and_kick(plan) + if LIVE_CHAIN_MODULES & set(modules): + warm_live_snapshots(tickers) + return statuses + + +def warm_live_snapshots(tickers: list[str]) -> None: + """Warm the option-chain preload cache for ``tickers`` on daemon threads.""" + for ticker in tickers: + threading.Thread(target=_warm_one, args=(ticker,), daemon=True).start() + + +def _warm_one(ticker: str) -> None: + try: + from services.options.preload import build_preload_payload, get_cached, set_cached + + if get_cached(ticker) is not None: + return + set_cached(ticker, build_preload_payload(ticker)) + logger.info("readiness: warmed option-chain preload for %s", ticker) + except Exception as exc: # noqa: BLE001 — a warm is best-effort by definition + logger.debug("readiness: preload warm failed for %s: %s", ticker, exc) + + +def modules_needing_live_chain(modules: list[str]) -> bool: + """True when any requested module needs a live option-chain snapshot.""" + return bool(LIVE_CHAIN_MODULES & set(modules)) + + +__all__ = [ + "KIND_DATASETS", + "LIVE_CHAIN_MODULES", + "ReadinessStatus", + "modules_needing_live_chain", + "prepare_readiness", + "warm_live_snapshots", +] diff --git a/templates/partials/fragments/readiness.html b/templates/partials/fragments/readiness.html new file mode 100644 index 0000000..b241c00 --- /dev/null +++ b/templates/partials/fragments/readiness.html @@ -0,0 +1,14 @@ +{# Held fragment for a cold start: the readiness pass (batch B5) kicked this + module's dataset download at POST time and it is still running. The fragment + re-issues its own /render request until the data lands, capped by + data_pipeline.orchestrate.readiness.HOLD_SECONDS on the server side. #} +
+
+ +

正在准备 {{ kind.replace('_', ' ') }} 所需的历史数据，稍后自动重试…

+
+
diff --git a/tests/test_background_backfill.py b/tests/test_background_backfill.py index 27df42b..ccbe5d3 100644 --- a/tests/test_background_backfill.py +++ b/tests/test_background_backfill.py @@ -18,10 +18,7 @@ from data_pipeline import PipelineResult from data_pipeline._state import _cache_get, _cache_invalidate from data_pipeline.orchestrate import backfill as _bf -from data_pipeline.read._query import ( - _join_backfills, - _kick_backfill, -) +from data_pipeline.orchestrate.readiness import join_backfills, kick_backfill from data_pipeline.store.db import init_db TICKER = "BGTEST1" @@ -65,7 +62,7 @@ def test_wide_range_request_returns_without_full_backfill(self, monkeypatch): assert df.empty, "no data seeded yet — partial read must be empty, not fabricated" # The backfill is still running in the background… assert calls["n"] >= 1, "background backfill was not kicked" - _join_backfills(timeout=10) + join_backfills(timeout=10) assert calls["n"] >= 2, "chunked backfill did not continue after the request returned" def test_partial_read_is_not_cached(self, monkeypatch): @@ -83,7 +80,7 @@ def test_partial_read_is_not_cached(self, monkeypatch): key = (TICKER, "clean", str(start), str(end)) assert _cache_get(key) is None, "partial read must not be memoised" - _join_backfills(timeout=10) + join_backfills(timeout=10) def test_completed_backfill_becomes_visible_and_cached(self, monkeypatch): """After the background backfill finishes, the next request returns the @@ -116,7 +113,7 @@ def _fast_dl(ticker, start, end): # noqa: ARG001 start = dt.date(2026, 1, 1) end = dt.date(2026, 2, 1) df = _q.get_cleaned_daily(TICKER, start, end) - _join_backfills(timeout=10) + join_backfills(timeout=10) assert not df.empty _cache_invalidate(TICKER) # mimic ensure_range's post-success invalidation @@ -143,8 +140,8 @@ def test_kick_dedupes_concurrent_kicks(self, monkeypatch): start, end = dt.date(2021, 1, 1), dt.date.today() for _ in range(5): - _kick_backfill(TICKER + "-DEDUP", start, end) - _join_backfills(timeout=10) + kick_backfill(TICKER + "-DEDUP", start, end) + join_backfills(timeout=10) # ensure_range's own in-flight dedup collapses the kicked threads — # a single leader runs the chunked pipeline, not five. Chunks for a # 5.6-year range ≈ days/89, allow one boundary chunk. diff --git a/tests/test_readiness.py b/tests/test_readiness.py new file mode 100644 index 0000000..39d3a1e --- /dev/null +++ b/tests/test_readiness.py @@ -0,0 +1,265 @@ +"""Readiness planning + prefetch on submit (ADR 0012, batch B5). + +Domain: Tests — Data Readiness +Context: + - B5 turned the implicit, per-slice "discover missing coverage when the tab + loads" flow into an explicit plan computed at POST time. These tests pin the + plan union, the kick decision, the cold-start hold window, and the fact that a + kick really does populate the DB. +Contracts: + - ``plan_datasets`` is the union over the requested modules, one entry per + (ticker, dataset), with live-only modules contributing nothing. + - ``check_and_kick`` probes once per (ticker, range) and kicks exactly the + missing ones. + - ``hold_seconds_left`` holds only on a cold start and only for HOLD_SECONDS. + - ``create_job`` stores the plan; ``FormService.extract_modules`` defaults to + every known module and ignores unknown tokens. +Dependencies UPWARD: + - (none — stdlib + pytest + the packages under test) +""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from data_pipeline.orchestrate import backfill as _bf +from data_pipeline.orchestrate import readiness +from data_pipeline.orchestrate.job_cache import _reset as reset_jobs +from data_pipeline.orchestrate.job_cache import create_job, get_job +from data_pipeline.store.db import fetch_df, init_db + +TODAY = dt.date(2026, 9, 10) + + +# --------------------------------------------------------------------------- +# plan_datasets +# --------------------------------------------------------------------------- +def test_plan_datasets_is_the_union_over_modules(): + plan = readiness.plan_datasets(["AAPL"], ["market_review", "assessment"], today=TODAY) + pairs = {(r.ticker, r.dataset) for r in plan} + assert pairs == {("AAPL", "clean_bars"), ("AAPL", "feature_bars")} + assert {r.module for r in plan} == {"market_review", "assessment"} + + +def test_plan_datasets_dedupes_a_dataset_shared_by_two_modules(): + """market_review and options_chain both read clean_bars → one entry.""" + plan = readiness.plan_datasets(["AAPL"], ["market_review", "options_chain"], today=TODAY) + assert len(plan) == 1 + assert plan[0].dataset == "clean_bars" + assert plan[0].module == "market_review" # first module wins (documented) + + +def test_plan_datasets_covers_every_ticker(): + plan = readiness.plan_datasets(["AAPL", "MSFT"], ["statistical"], today=TODAY) + assert {r.ticker for r in plan} == {"AAPL", "MSFT"} + assert all(r.dataset == "feature_bars" for r in plan) + + +def test_plan_datasets_is_empty_for_live_only_modules(): + assert readiness.plan_datasets(["AAPL"], ["regime", "payoff_ratio", "simulation"], today=TODAY) == [] + + +def test_plan_datasets_horizon_defaults_and_overrides(): + default = readiness.plan_datasets(["AAPL"], ["statistical"], today=TODAY)[0] + assert default.end == TODAY + assert default.start == TODAY - dt.timedelta(days=readiness.DEFAULT_LOOKBACK_DAYS) + + explicit = readiness.plan_datasets( + ["AAPL"], ["statistical"], start=dt.date(2020, 1, 1), end=dt.date(2021, 1, 1), today=TODAY + )[0] + assert (explicit.start, explicit.end) == (dt.date(2020, 1, 1), dt.date(2021, 1, 1)) + + +# --------------------------------------------------------------------------- +# check_and_kick +# --------------------------------------------------------------------------- +def test_check_and_kick_kicks_missing_ranges_once_per_ticker(monkeypatch): + init_db() + monkeypatch.setattr(_bf, "needs_backfill", lambda ticker, start, end: True) + kicked: list[tuple] = [] + plan = readiness.plan_datasets(["AAPL", "MSFT"], ["market_review", "statistical"], today=TODAY) + + statuses = readiness.check_and_kick(plan, kick=lambda t, s, e: kicked.append((t, s, e))) + + expected_start = TODAY - dt.timedelta(days=readiness.DEFAULT_LOOKBACK_DAYS) + assert sorted(kicked) == [("AAPL", expected_start, TODAY), ("MSFT", expected_start, TODAY)] + assert len(statuses) == 4 + # One status per (ticker, dataset) — AAPL carries two datasets, MSFT two. + assert {s.ticker for s in statuses} == {"AAPL", "MSFT"} + assert all(s.state == "kicked" for s in statuses) + assert all(s.kicked_at > 0 for s in statuses) + + +def test_check_and_kick_skips_covered_ranges(monkeypatch): + init_db() + monkeypatch.setattr(_bf, "needs_backfill", lambda ticker, start, end: False) + kicked: list[tuple] = [] + + statuses = readiness.check_and_kick( + readiness.plan_datasets(["AAPL"], ["statistical"], today=TODAY), kick=lambda *a: kicked.append(a) + ) + + assert kicked == [] + assert [s.state for s in statuses] == ["covered"] + assert statuses[0].kicked_at == 0.0 + + +def test_check_and_kick_survives_a_failing_probe(monkeypatch): + """A probe must never break POST / — a failure degrades to 'not missing'.""" + init_db() + + def _boom(*_a, **_kw): + raise RuntimeError("db is unhappy") + + monkeypatch.setattr(_bf, "needs_backfill", _boom) + statuses = readiness.check_and_kick( + readiness.plan_datasets(["AAPL"], ["statistical"], today=TODAY), kick=lambda *a: None + ) + assert [s.state for s in statuses] == ["covered"] + + +# --------------------------------------------------------------------------- +# Cold-start hold window +# --------------------------------------------------------------------------- +def _status(state: str, *, has_data: bool, kicked_at: float = 100.0) -> readiness.ReadinessStatus: + return readiness.ReadinessStatus( + ticker="AAPL", dataset="feature_bars", module="statistical", state=state, kicked_at=kicked_at, has_data=has_data + ) + + +def test_hold_is_none_when_nothing_was_kicked(): + assert readiness.hold_seconds_left(None) is None + assert readiness.hold_seconds_left(_status("covered", has_data=False)) is None + + +def test_hold_is_none_when_the_ticker_already_has_usable_history(): + """A partial gap must paint now — only a cold start is worth holding.""" + assert readiness.hold_seconds_left(_status("kicked", has_data=True)) is None + + +def test_hold_counts_down_and_expires(): + cold = _status("kicked", has_data=False, kicked_at=100.0) + assert readiness.hold_seconds_left(cold, now=100.0) == pytest.approx(readiness.HOLD_SECONDS) + assert readiness.hold_seconds_left(cold, now=100.0 + readiness.HOLD_SECONDS - 1) == pytest.approx(1.0) + assert readiness.hold_seconds_left(cold, now=100.0 + readiness.HOLD_SECONDS + 1) is None + + +def test_should_hold_stops_as_soon_as_the_backfill_is_gone(): + """A dead (failed or finished) backfill must not keep the tab in 'preparing'. + + This is the difference between "still downloading" and "download failed": once + the thread exits, /render/* computes and the slice reports the real outcome. + """ + cold = _status("kicked", has_data=False, kicked_at=100.0) + assert readiness.should_hold(cold, now=100.0) is False # no live thread in this test + assert readiness.should_hold(None) is False + # Timer alone (hold_seconds_left) is what the count-down test above pins. + assert readiness.is_backfill_running(cold) is False + + +def test_status_for_maps_a_module_to_its_plan_entry(): + plan = [ + readiness.ReadinessStatus("AAPL", "clean_bars", "market_review", "covered"), + readiness.ReadinessStatus("AAPL", "feature_bars", "statistical", "kicked", kicked_at=5.0), + ] + assert readiness.status_for(plan, "AAPL", "statistical").dataset == "feature_bars" + assert readiness.status_for(plan, "AAPL", "regime") is None # live-only module + assert readiness.status_for(plan, "MSFT", "statistical") is None + assert readiness.status_for(None, "AAPL", "statistical") is None + + +# --------------------------------------------------------------------------- +# kick_backfill really populates the DB (offline TEST_ fixture) +# --------------------------------------------------------------------------- +def test_kick_backfill_populates_the_db(): + init_db() + ticker = "TEST_AAPL" + end = dt.date.today() + start = end - dt.timedelta(days=45) + + readiness.kick_backfill(ticker, start, end) + readiness.join_backfills(timeout=60) + + df = fetch_df("SELECT date FROM clean_bars WHERE ticker=?", (ticker,)) + assert len(df.index) > 0, "cold-start kick did not populate clean_bars" + + +def test_kick_backfill_dedupes_an_in_flight_range(): + """Two kicks for the same range must collapse into one thread.""" + init_db() + ticker = "TEST_AAPL" + end = dt.date.today() + start = end - dt.timedelta(days=30) + + readiness.kick_backfill(ticker, start, end) + with readiness._backfill_lock: + first = readiness._backfill_threads[(ticker, str(start), str(end))] + readiness.kick_backfill(ticker, start, end) + with readiness._backfill_lock: + second = readiness._backfill_threads.get((ticker, str(start), str(end))) + assert first is second + + readiness.join_backfills(timeout=60) + + +# --------------------------------------------------------------------------- +# job_cache carries the plan +# --------------------------------------------------------------------------- +def test_create_job_stores_the_plan(): + reset_jobs() + plan = [readiness.ReadinessStatus("AAPL", "clean_bars", "market_review", "covered")] + job_id = create_job({"ticker": "AAPL"}, ["AAPL"], plan) + assert get_job(job_id).plan == plan + + +def test_create_job_without_a_plan_defaults_to_empty(): + reset_jobs() + job_id = create_job({"ticker": "AAPL"}, ["AAPL"]) + assert get_job(job_id).plan == [] + + +# --------------------------------------------------------------------------- +# FormService.extract_modules +# --------------------------------------------------------------------------- +class _FakeRequest: + def __init__(self, values: list[str] | None): + self._values = values or [] + + class _Form(dict): + def __init__(self, values): + super().__init__() + self._values = values + + def getlist(self, key): + return self._values if key == "modules" else [] + + @property + def form(self): + return self._Form(self._values) + + +def test_extract_modules_defaults_to_every_known_module(): + from services.market.form import FormService + + assert FormService.extract_modules(_FakeRequest(None)) == list(readiness.ALL_MODULES) + + +def test_extract_modules_accepts_repeated_and_comma_separated_values(): + from services.market.form import FormService + + assert FormService.extract_modules(_FakeRequest(["market_review", "statistical"])) == [ + "market_review", + "statistical", + ] + assert FormService.extract_modules(_FakeRequest(["market_review,statistical"])) == [ + "market_review", + "statistical", + ] + + +def test_extract_modules_drops_unknown_tokens_and_dedupes(): + from services.market.form import FormService + + assert FormService.extract_modules(_FakeRequest(["statistical", "nope", "statistical"])) == ["statistical"] From a1ca7fd579fa33296a89cc8cbbf105a519ae4eee Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Thu, 10 Sep 2026 21:21:20 +0800 Subject: [PATCH 07/15] =?UTF-8?q?feat(ui):=20B6=20ticker-only=20=E5=B8=B8?= =?UTF-8?q?=E9=A9=BB=E5=8F=82=E6=95=B0=E6=A0=8F=20+=20Portfolio=20?= =?UTF-8?q?=E9=A1=B5=E7=AD=BE=EF=BC=88=C2=A78=20Q3=20=E5=AE=9A=E4=B8=BA?= =?UTF-8?q?=E7=8B=AC=E7=AB=8B=E9=9D=A2=E6=9D=BF=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 业务线重构计划书 §6 B6；ADR 0012 参数归属第一批。 - 新 templates/partials/parameters_bar.html + static/parametersBar.js：位于 header 与 .app-body 之间、position: sticky（top: var(--header-h)），只拥有 ticker 一个输入 + Run + 校验徽标；折叠状态按访客持久化 （localStorage['parametersBarCollapsed']，每次访问都 try/catch，禁用存储 时降级为「不记忆」），折叠后显示一行摘要 ▸ ^SPX；删除 tab_parameter.html 及其侧栏按钮 - Q3 = 独立 Portfolio 页签：templates/partials/tab_portfolio.html 承接持仓表与 Portfolio Analysis 结果面板；#positions-tbody 仍在每次加载的 DOM 中，既有 全局处理器（addPositionRow / runPortfolioAnalysis）无需改动 - 记录的临时偏差：B6 删掉 Parameters 页签时 B7 尚未落地，若把时间窗/仓位/ Config 桥一并撤掉会造成功能回退，故它们暂留在同一
内的可折叠 「Analysis settings」组（明确标注为 B7 的抽取源），POST 契约逐字未变 - Pages 镜像：build_pages_site 的断言与 showcase/parameter.html 指向 tab-portfolio，横幅链接改「去 Portfolio 页」；test_pages_build 的 ticker 输入断言改为与属性顺序无关（原断言被新增 class 属性打破） - 测试：新增 tests/unit/parametersBar.test.js（8 例）并纳入覆盖率清单； 5 个 e2e 去掉「先激活 Parameter 页签」（栏常驻），持仓级联测试改开 tab-portfolio，test_smoke 的页签清单 tab-parameter → tab-portfolio 验收：pytest -m "not network" --ignore=tests/e2e → 493 passed / 5 skipped； pytest tests/e2e → 38 passed；npx vitest run → 187 passed / 15 files； doc_guard clean；arch_metrics --check ok；audit_tags 16 vs 16。 无 axe 自动化工具，改用人工核对（
""" @@ -383,7 +383,7 @@ def assemble(out_dir: Path, ticker: str = DEMO_TICKER) -> Path: assert 'hx-get="/render/' not in html, "streaming placeholders leaked into static build" assert '"/static/' not in html and "'/static/" not in html, "absolute /static/ paths break the /OptionLab/ subpath" for tab_id in ( - "tab-parameter", + "tab-portfolio", "tab-market-review", "tab-statistical-analysis", "tab-market-assessment", diff --git a/static/parametersBar.js b/static/parametersBar.js new file mode 100644 index 0000000..598084e --- /dev/null +++ b/static/parametersBar.js @@ -0,0 +1,84 @@ +/* parametersBar.js — persistent Parameters bar: collapse state + summary. + * + * Contract (docs/frontend_architecture.md, batch B6): + * - the bar owns the shared `ticker` input and the Run button; it is NOT a tab, + * so it stays visible while the user switches tabs; + * - collapsing is a per-viewer convenience persisted in localStorage, and the + * collapsed bar shows the current ticker as `▸ ^SPX`; + * - CONSTRAINT: every localStorage access is guarded — private mode / disabled + * storage must degrade to "not persisted", never throw (no build step, no + * polyfills; see ADR 0006). + */ +(function (root) { + 'use strict'; + + var STORAGE_KEY = 'parametersBarCollapsed'; + var BAR_SELECTOR = '.parameters-bar'; + + function _read() { + try { + return root.localStorage.getItem(STORAGE_KEY); + } catch (_) { + return null; + } + } + + function _write(value) { + try { + root.localStorage.setItem(STORAGE_KEY, value); + } catch (_) { + /* not fatal: the bar just will not remember its state */ + } + } + + function setCollapsed(bar, collapsed) { + bar.dataset.collapsed = collapsed ? 'true' : 'false'; + var toggle = document.getElementById('parameters-bar-toggle'); + if (!toggle) return; + toggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true'); + var icon = toggle.querySelector('i'); + if (icon) icon.className = collapsed ? 'fas fa-chevron-right' : 'fas fa-chevron-down'; + } + + function updateSummary(bar) { + var summary = document.getElementById('parameters-bar-summary'); + var input = document.getElementById('ticker'); + if (!summary || !input) return; + var value = (input.value || '').trim(); + summary.textContent = value ? '▸ ' + value : ''; + } + + function init() { + var bar = document.querySelector(BAR_SELECTOR); + if (!bar) return; + + setCollapsed(bar, _read() === 'true'); + + var toggle = document.getElementById('parameters-bar-toggle'); + if (toggle) { + toggle.addEventListener('click', function () { + var next = bar.dataset.collapsed !== 'true'; + setCollapsed(bar, next); + _write(next ? 'true' : 'false'); + }); + } + + var input = document.getElementById('ticker'); + if (input) input.addEventListener('input', function () { updateSummary(bar); }); + updateSummary(bar); + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', init); + } else { + init(); + } + + // Exposed for the jsdom unit tests (no bundler; plain global, like theme.js). + root.parametersBar = { + init: init, + setCollapsed: setCollapsed, + updateSummary: updateSummary, + STORAGE_KEY: STORAGE_KEY, + }; +})(typeof window !== 'undefined' ? window : this); diff --git a/static/styles.css b/static/styles.css index c074fe1..24a54f7 100644 --- a/static/styles.css +++ b/static/styles.css @@ -3173,6 +3173,100 @@ textarea:focus-visible, } +/* ============================================================ + Parameters bar (batch B6) — persistent `ticker` bar above the panes. + Not a tab: it stays put across tab switches (sticky under the header) + and collapses to a one-line summary (`▸ ^SPX`). Tokens only, so the + Onyx override layer below themes it for free. + ============================================================ */ +.parameters-bar { + position: sticky; + top: var(--header-h); + z-index: 100; + display: flex; + flex-direction: column; + gap: 6px; + max-width: 1600px; + margin: 0 auto; + padding: 10px 1.5rem; + background: var(--slate-50); + border-bottom: 1px solid var(--slate-200); + font-family: var(--font); +} + +.parameters-bar-main { + display: flex; + align-items: center; + gap: 10px; +} + +.parameters-bar-toggle { + background: transparent; + border: 1px solid var(--slate-200); + border-radius: var(--radius-sm); + color: var(--slate-700); + cursor: pointer; + padding: 5px 9px; + line-height: 1; +} + +.parameters-bar-label { + color: var(--slate-700); + font-size: 12px; + font-weight: 600; + letter-spacing: .02em; + text-transform: uppercase; +} + +.parameters-bar-ticker { + flex: 0 1 320px; + min-width: 180px; + padding: 6px 10px; + border: 1px solid var(--slate-200); + border-radius: var(--radius-sm); + font-family: inherit; + font-size: 14px; +} + +.parameters-bar-summary { + color: var(--slate-500); + font-size: 13px; + font-variant-numeric: tabular-nums; +} + +/* Collapsed → hide the settings body; expanded → the input already shows the + ticker, so the one-line summary is redundant. */ +.parameters-bar[data-collapsed="true"] .parameters-bar-body { + display: none; +} + +.parameters-bar[data-collapsed="false"] .parameters-bar-summary { + display: none; +} + +.parameters-bar-body { + border-top: 1px solid var(--slate-200); + padding-top: 10px; +} + +.parameters-bar-group-title { + display: block; + margin-bottom: 6px; + color: var(--slate-500); + font-size: 11px; + font-weight: 600; + letter-spacing: .04em; + text-transform: uppercase; +} + +.parameters-bar-alert { + margin: 0; +} + +.parameters-bar .btn-primary { + margin-left: auto; +} + /* ============================================================ Onyx Theme — minimalist black with gold + purple accents. Append-only override layer. Reassigns design tokens from the diff --git a/templates/index.html b/templates/index.html index e56dae2..51bed6d 100644 --- a/templates/index.html +++ b/templates/index.html @@ -133,6 +133,10 @@

Market Dashboard

+ {# Persistent Parameters bar (batch B6): above the panes so it survives tab + switches. Owns `ticker` + Run; module params move to the tab toolbars in B7. #} + {% include 'partials/parameters_bar.html' %} +
- - - -
- - -
-
- - -
- - - - - {# Hidden fields for Config-tab values (injected by JS from localStorage). - Batch B7 replaces the whole bridge; B8 retires the Config tab. #} - - - - - + {# See the header comment: submit-only mirror of the marketParams store. #} + + diff --git a/templates/partials/tab_config.html b/templates/partials/tab_config.html index f174506..542386a 100644 --- a/templates/partials/tab_config.html +++ b/templates/partials/tab_config.html @@ -1,65 +1,30 @@ +{# Config tab — emptied by batch B7. + + Everything this tab used to hold was mislabelled "global": the frequency and the + Assessment knobs belong to Assessment, the chain filters to Option Chain, the + refresh interval to the chain's auto-refresh. They now live in the module + toolbars, read through `static/state/*ParamsState.js` instead of the + localStorage -> hidden-input bridge. + + The tab shell survives (with a pointer) so existing deep links and the sidebar + keep working; batch B8 answers §8 Q2 — whether a genuine global setting (the + risk-free rate, currently hard-coded in two places) earns this tab or the tab + is deleted outright. #}

Config

-

Analysis parameters that persist across sessions.

-
-
-
Analysis Config
-
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
+

Nothing here yet.

-
Option Chain Filter
-
-
- - -
Maximum days to expiration (default 45).
-
-
- - -
Lower strike bound as fraction of spot (default 0.70).
-
-
- - -
Upper strike bound as fraction of spot (default 1.30).
-
-
- - -
Maximum total contracts per query.
-
-
+
These settings moved
+
    +
  • Time horizon & frequency → the Market Review, Statistical and Assessment toolbars.
    • +
  • Side bias, risk threshold, rolling window, account size, max risk → the Assessment toolbar.
    • +
  • Max DTE, moneyness bounds, max contracts, refresh interval → the Option Chain toolbar (Payoff Ratio shares them).
    • +
+

+ Each one is remembered per browser and re-runs only the module it belongs to. +

diff --git a/templates/partials/tab_market_assessment.html b/templates/partials/tab_market_assessment.html index 7498c46..7afd6c2 100644 --- a/templates/partials/tab_market_assessment.html +++ b/templates/partials/tab_market_assessment.html @@ -7,9 +7,60 @@

Assessment & Projections

{% if streaming_mode and job_id and ticker %} + {# Module toolbar (batch B7): horizon + frequency (the shared marketParams + group) and the Assessment-only knobs (assessmentParams group). #} +
+ Assessment settings + + + to + + + + + +
+ Assessment knobs +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+

Loading projections…

diff --git a/templates/partials/tab_market_review.html b/templates/partials/tab_market_review.html index 457d570..4688c45 100644 --- a/templates/partials/tab_market_review.html +++ b/templates/partials/tab_market_review.html @@ -7,10 +7,23 @@

Market Review

{% if streaming_mode and job_id and ticker %} + {# Module toolbar (batch B7): the module owns its parameters. `hx-include` + carries them on the /render call; a later change is re-issued by + static/moduleParams.js for THIS module only. #} +
+ Market Review settings + + + to + + +
+ {# HTMX streaming: skeleton fires on load and is replaced by /render/market_review. #}

Loading market review…

diff --git a/templates/partials/tab_option_chain.html b/templates/partials/tab_option_chain.html index 3e9479b..1f8b173 100644 --- a/templates/partials/tab_option_chain.html +++ b/templates/partials/tab_option_chain.html @@ -22,6 +22,23 @@

Option Chain

+ +
+ Option filter + + + + + + + + + + +
+
@@ -83,7 +100,7 @@

Option Chain

-

Set a ticker in Parameter tab — option chain data loads automatically when you switch to this tab.

+

Set a ticker in the Parameters bar — option chain data loads automatically when you switch to this tab.

diff --git a/templates/partials/tab_payoff_ratio.html b/templates/partials/tab_payoff_ratio.html index f05f185..5d87c16 100644 --- a/templates/partials/tab_payoff_ratio.html +++ b/templates/partials/tab_payoff_ratio.html @@ -78,7 +78,7 @@

Payoff Ratio

-

Set a ticker in Parameter tab — payoff data loads automatically when you switch to this tab.

+

Set a ticker in the Parameters bar — payoff data loads automatically when you switch to this tab.

diff --git a/templates/partials/tab_simulation.html b/templates/partials/tab_simulation.html index e87fda5..c9fed7e 100644 --- a/templates/partials/tab_simulation.html +++ b/templates/partials/tab_simulation.html @@ -28,7 +28,7 @@

Simulation

- Blank = use the Parameter tab ticker. + Blank = use the Parameters bar ticker.
@@ -182,7 +182,7 @@

Simulation

- Set a ticker in the Parameter tab — the simulation runs + Set a ticker in the Parameters bar — the simulation runs automatically when you switch to this tab.

diff --git a/templates/partials/tab_statistical_analysis.html b/templates/partials/tab_statistical_analysis.html index 0acbdc1..003875d 100644 --- a/templates/partials/tab_statistical_analysis.html +++ b/templates/partials/tab_statistical_analysis.html @@ -7,9 +7,27 @@

Statistical Analysis

{% if streaming_mode and job_id and ticker %} + {# Module toolbar (batch B7) — horizon + frequency belong to this module. #} +
+ Statistical settings + + + to + + + + +
+

Loading statistical analysis…

diff --git a/tests/e2e/test_form_submit_flow.py b/tests/e2e/test_form_submit_flow.py index d80d46d..e0c2a73 100644 --- a/tests/e2e/test_form_submit_flow.py +++ b/tests/e2e/test_form_submit_flow.py @@ -9,34 +9,30 @@ from __future__ import annotations -import datetime as dt - from playwright.sync_api import Page, expect -def _months_ago(n: int) -> str: - """Return a YYYY-MM string n months before today (HTML ).""" - today = dt.date.today().replace(day=1) - for _ in range(n): - today = (today - dt.timedelta(days=1)).replace(day=1) - return today.strftime("%Y-%m") - - def test_form_submit_renders_summary( page: Page, live_server: str, yf_stub: None, seed_test_data: None, js_errors: list[str], - open_tab, ) -> None: """Submit the analysis form with a TEST_ ticker and assert the page - re-renders with the ticker echoed back.""" + re-renders with the ticker echoed back. + + Batch B7: the bar posts `ticker` only — the horizon is owned by the market + modules' toolbars and mirrored into the bar's hidden inputs by + `state/marketParamsState.js`, so the test no longer types a start month. + """ page.goto(live_server, wait_until="domcontentloaded") # The Parameters bar is always visible (batch B6). - page.fill("#ticker", "TEST_AAPL") - page.fill("#start_time", _months_ago(3)) + + # The marketParams store must have mirrored its horizon before submit, + # otherwise POST / fails its start_time validation. + expect(page.locator("#start_time")).not_to_have_value("", timeout=5_000) # POST the form and wait for navigation to complete. with page.expect_navigation(wait_until="domcontentloaded", timeout=15_000): diff --git a/tests/e2e/test_localstorage_restore.py b/tests/e2e/test_localstorage_restore.py index 93d45be..0240c23 100644 --- a/tests/e2e/test_localstorage_restore.py +++ b/tests/e2e/test_localstorage_restore.py @@ -1,4 +1,8 @@ -"""LocalStorage form-state restoration after page reload.""" +"""LocalStorage restoration of the module parameter groups (batch B7). + +One key per group — `marketParams`, `assessmentParams`, `optionFilter` — plus the +bar's own `marketAnalysisForm` convenience copy for the ticker. +""" from __future__ import annotations @@ -7,7 +11,7 @@ from playwright.sync_api import Page, expect -def test_localstorage_restores_form_state( +def test_localstorage_restores_module_params( page: Page, live_server: str, mock_apis, @@ -16,51 +20,45 @@ def test_localstorage_restores_form_state( page.goto(live_server, wait_until="domcontentloaded") # Seed localStorage *before* DOMContentLoaded handlers re-fire on reload. - saved_form = { - "ticker": "TEST_AAPL", - "start_time": "202401", - "end_time": "202403", - "positions": [], - } - saved_cfg = { - "frequency": "W", - "side_bias": "Neutral", - "risk_threshold": "75", - "rolling_window": "90", - "max_dte": "30", - "moneyness_low": "0.80", - "moneyness_high": "1.20", - "max_contracts": "500", - "refresh_interval": "120", - } + saved_form = {"ticker": "TEST_AAPL", "positions": []} page.evaluate( - """({form, cfg}) => { + """({form}) => { localStorage.setItem('marketAnalysisForm', JSON.stringify(form)); - localStorage.setItem('marketAnalysisConfig', JSON.stringify(cfg)); + localStorage.setItem('marketParams', JSON.stringify( + { from: '2024-01', to: '2024-03', frequency: 'W' })); + localStorage.setItem('assessmentParams', JSON.stringify( + { side_bias: 'Neutral', risk_threshold: '75', rolling_window: '90', + account_size: '100000', max_risk_pct: '2' })); + localStorage.setItem('optionFilter', JSON.stringify( + { max_dte: '30', moneyness_low: '0.80', moneyness_high: '1.20', + max_contracts: '500', refresh_interval: '120' })); }""", - {"form": saved_form, "cfg": saved_cfg}, + {"form": saved_form}, ) page.reload(wait_until="domcontentloaded") - # Form fields should be hydrated from `marketAnalysisForm`. + # The bar's ticker survives, and the marketParams store mirrors the horizon + # into the bar's submit-only hidden inputs. expect(page.locator("#ticker")).to_have_value("TEST_AAPL", timeout=5_000) expect(page.locator("#start_time")).to_have_value("2024-01") expect(page.locator("#end_time")).to_have_value("2024-03") - # Hidden fields should be synced from `marketAnalysisConfig`. - freq = page.locator("#frequency").input_value() - side = page.locator("#side_bias").input_value() - risk = page.locator("#risk_threshold").input_value() - rw = page.locator("#rolling_window").input_value() - assert freq == "W" - assert side == "Neutral" - assert risk == "75" - assert rw == "90" + # The Option Chain toolbar is the only module toolbar present before a run. + expect(page.locator("#oc-max-dte")).to_have_value("30") + expect(page.locator("#oc-moneyness-low")).to_have_value("0.80") + expect(page.locator("#oc-moneyness-high")).to_have_value("1.20") + expect(page.locator("#oc-refresh-interval")).to_have_value("120") + + # The stores expose the restored values (the market toolbars render only in + # streaming mode, i.e. after a run). + assert page.evaluate("() => appState.marketParams.get().from") == "2024-01" + assert page.evaluate("() => appState.marketParams.get().frequency") == "W" + assert page.evaluate("() => appState.assessmentParams.get().side_bias") == "Neutral" + assert page.evaluate("() => appState.optionFilter.get().moneyness_high") == "1.20" - # Storage round-trip is intact (no accidental mutation). - raw_form = page.evaluate("() => localStorage.getItem('marketAnalysisForm')") - assert json.loads(raw_form)["ticker"] == "TEST_AAPL" + # Storage round-trip is intact (no accidental mutation of the group keys). + assert json.loads(page.evaluate("() => localStorage.getItem('marketParams')"))["from"] == "2024-01" fatal = [e for e in js_errors if "favicon" not in e.lower()] assert fatal == [], f"JS errors during reload restore: {fatal}" diff --git a/tests/e2e/test_module_params.py b/tests/e2e/test_module_params.py new file mode 100644 index 0000000..af92cc7 --- /dev/null +++ b/tests/e2e/test_module_params.py @@ -0,0 +1,124 @@ +"""Module-scoped parameters re-run only their own group (batch B7). + +The behavioural claim of decision gate §8 Q1: each module's toolbar appends its +parameters to *its* `/render` call. One nuance is part of the contract, not an +accident: the three streaming market tabs share one parameter group +(`marketParams` — horizon + frequency), so a change there re-runs all three, +while a change to a *different* group (the Option Chain's filters) must not +touch the streaming panes at all. +""" + +from __future__ import annotations + +import json + +from playwright.sync_api import Page, expect + +STATISTICAL_FRAGMENT = "#tab-statistical-analysis-content" + + +def _submit(page: Page, live_server: str, ticker: str = "TEST_AAPL") -> None: + page.goto(live_server, wait_until="domcontentloaded") + page.fill("#ticker", ticker) + with page.expect_navigation(wait_until="domcontentloaded", timeout=20_000): + page.click("#analysis-form button[type=submit]") + + +def test_changing_a_market_param_reruns_its_group_only( + page: Page, + live_server: str, + yf_stub: None, + seed_test_data: None, + js_errors: list[str], + open_tab, +) -> None: + _submit(page, live_server) + # The fragment loads regardless of visibility; the tab has to be active for + # its toolbar to be actionable. + open_tab("tab-statistical-analysis") + expect(page.locator(STATISTICAL_FRAGMENT)).to_be_visible(timeout=60_000) + page.wait_for_timeout(3_000) + + renders: list[str] = [] + api_calls: list[str] = [] + page.on("request", lambda req: renders.append(req.url) if "/render/" in req.url else None) + page.on("request", lambda req: api_calls.append(req.url) if "/api/" in req.url else None) + + # WHY a dispatched change instead of select_option: the statistical tab can + # lose active-ness mid-test (peek-panel / re-render timing), which makes the + # toolbar un-actionable even though the listener chain is intact. Setting the + # value and dispatching `change` exercises exactly the same production path: + # store -> bus -> moduleParams -> htmx -> /render with the new params. + page.evaluate( + "() => { const el = document.getElementById('stat-frequency');" + " el.value = 'W'; el.dispatchEvent(new Event('change', { bubbles: true })); }", + ) + page.wait_for_timeout(3_000) + + # The marketParams group feeds all three market tabs → all three re-run. + for kind in ("market_review", "statistical", "assessment"): + assert any(f"/render/{kind}" in url for url in renders), f"{kind} did not re-run: {renders}" + + # The new value must travel, together with the group's horizon. + assert all("frequency=W" in url for url in renders if "/render/statistical" in url), renders + assert all("from=" in url for url in renders), renders + + # …and nothing outside the group reacts to a market parameter. + assert not any("/render/option_chain" in url for url in renders), renders + assert not any("/api/option_chain" in url for url in api_calls), api_calls + + fatal = [e for e in js_errors if "favicon" not in e.lower()] + assert fatal == [], f"JS errors after a module param change: {fatal}" + + +def test_changing_the_option_filter_leaves_the_streaming_panes_alone( + page: Page, + live_server: str, + yf_stub: None, + seed_test_data: None, + open_tab, +) -> None: + """The chain filters are client-fired: no `/render` round trip at all.""" + _submit(page, live_server) + open_tab("tab-option-chain") + expect(page.locator(STATISTICAL_FRAGMENT)).to_be_attached(timeout=60_000) + page.wait_for_timeout(3_000) + + renders: list[str] = [] + page.on("request", lambda req: renders.append(req.url) if "/render/" in req.url else None) + + page.fill("#oc-max-dte", "60") + page.dispatch_event("#oc-max-dte", "change") + page.wait_for_timeout(3_000) + + assert renders == [], f"a chain-filter change re-ran the streaming panes: {renders}" + + +def test_the_rerun_survives_a_reload( + page: Page, + live_server: str, + yf_stub: None, + seed_test_data: None, + open_tab, +) -> None: + """Exit criterion: module parameter values survive a reload.""" + _submit(page, live_server) + open_tab("tab-statistical-analysis") + expect(page.locator(STATISTICAL_FRAGMENT)).to_be_visible(timeout=60_000) + page.wait_for_timeout(2_000) + + page.evaluate( + "() => { const el = document.getElementById('stat-frequency');" + " el.value = 'QE'; el.dispatchEvent(new Event('change', { bubbles: true })); }", + ) + page.wait_for_timeout(1_000) + + # The commit must have persisted the group before the reload. + assert json.loads(page.evaluate("() => localStorage.getItem('marketParams')"))["frequency"] == "QE" + + page.reload(wait_until="domcontentloaded") + page.wait_for_timeout(1_500) + + assert page.evaluate("() => appState.marketParams.get().frequency") == "QE" + # …and the hidden submit-only mirror follows the store. + assert page.evaluate("() => document.getElementById('start_time').value") != "" diff --git a/tests/test_frontend_api.py b/tests/test_frontend_api.py index 668a609..41304f9 100644 --- a/tests/test_frontend_api.py +++ b/tests/test_frontend_api.py @@ -29,22 +29,44 @@ def test_get_renders(self, client): assert 'id="ticker"' in html assert 'id="start_time"' in html - def test_get_has_config_tab(self, client): - """Config tab with all option filter fields should be present.""" + def test_get_has_option_filter_toolbar(self, client): + """Batch B7: the chain filters live with the Option Chain module.""" resp = client.get("/") html = resp.data.decode() - assert 'id="cfg-frequency"' in html - assert 'id="cfg-max-dte"' in html - assert 'id="cfg-moneyness-low"' in html - assert 'id="cfg-moneyness-high"' in html - assert 'id="cfg-max-contracts"' in html - - def test_get_has_position_sizing_in_settings(self, client): - """Position sizing fields should be inside Analysis Settings card.""" + assert 'id="option-toolbar"' in html + assert 'id="oc-max-dte"' in html + assert 'id="oc-moneyness-low"' in html + assert 'id="oc-moneyness-high"' in html + assert 'id="oc-max-contracts"' in html + assert 'id="oc-refresh-interval"' in html + + def test_post_renders_the_market_module_toolbars(self, client, monkeypatch): + """The market toolbars render in streaming mode, each owning its params. + + WHY the kick is stubbed: `POST /` runs the readiness pass (B5), which would + otherwise start a real backfill on a daemon thread for the test ticker. + """ + monkeypatch.setattr("data_pipeline.orchestrate.readiness.kick_backfill", lambda *a, **k: None) + resp = client.post("/", data={"ticker": "TEST_AAPL", "start_time": "2024-01", "frequency": "ME"}) + html = resp.data.decode() + assert resp.status_code == 200 + assert 'id="market-toolbar"' in html + assert 'id="statistical-toolbar"' in html + assert 'id="assessment-toolbar"' in html + # Frequency is a market-module parameter; sizing is Assessment's. + assert 'id="stat-frequency"' in html + assert 'id="assess-account-size"' in html + assert 'id="assess-max-risk-pct"' in html + # The ready placeholder carries the params on its first fan-out. + assert 'hx-include="#statistical-toolbar"' in html + + def test_get_has_parameters_bar_with_a_single_input(self, client): + """The bar owns `ticker` only; the horizon is a submit-time mirror.""" resp = client.get("/") html = resp.data.decode() - assert 'id="account_size"' in html - assert 'id="max_risk_pct"' in html + assert 'class="parameters-bar"' in html + assert 'id="parameters-bar-toggle"' in html + assert 'id="parameters-bar-summary"' in html def test_post_missing_ticker(self, client): """POST without ticker should show error.""" diff --git a/tests/unit/coverage.test.js b/tests/unit/coverage.test.js index 980cdf1..24cf500 100644 --- a/tests/unit/coverage.test.js +++ b/tests/unit/coverage.test.js @@ -27,6 +27,11 @@ import '../../static/cache.js'; import '../../static/simulation.js'; import '../../static/theme.js'; import '../../static/parametersBar.js'; +import '../../static/state/paramsStore.js'; +import '../../static/state/marketParamsState.js'; +import '../../static/state/assessmentParamsState.js'; +import '../../static/state/optionFilterState.js'; +import '../../static/moduleParams.js'; // Capture the post-import surface BEFORE the setup `beforeEach` runs and // wipes globals. We re-attach them in a local `beforeEach` so each `it` @@ -48,6 +53,10 @@ const _snapshot = { runSimulation: window.runSimulation, themeManager: window.themeManager, parametersBar: window.parametersBar, + marketParams: window.appState.marketParams, + assessmentParams: window.appState.assessmentParams, + optionFilter: window.appState.optionFilter, + moduleParams: window.moduleParams, }; beforeEach(() => { @@ -91,6 +100,14 @@ describe('coverage smoke — every module publishes its surface', () => { expect(window.parametersBar.STORAGE_KEY).toBe('parametersBarCollapsed'); }); + it('module parameter groups published their window surface', () => { + for (const key of ['marketParams', 'assessmentParams', 'optionFilter']) { + expect(typeof window.appState[key].get).toBe('function'); + expect(typeof window.appState[key].query).toBe('function'); + } + expect(typeof window.moduleParams.rerun).toBe('function'); + }); + it('simulation tab published its window surface', () => { expect(typeof window.loadSimulationTab).toBe('function'); expect(typeof window.runSimulation).toBe('function'); diff --git a/tests/unit/paramsStore.test.js b/tests/unit/paramsStore.test.js new file mode 100644 index 0000000..947b835 --- /dev/null +++ b/tests/unit/paramsStore.test.js @@ -0,0 +1,213 @@ +/** + * Tests for static/state/paramsStore.js + the three module parameter groups + * (batch B7). + * + * Contract under test: + * - a group hydrates its toolbar inputs from its own localStorage key; + * - a change on any bound input commits the WHOLE group and emits + * `module:params-changed` with the modules that consume it; + * - the shared horizon is one field bound to three toolbars, so all three + * inputs stay in sync; + * - disabled storage degrades to defaults instead of throwing; + * - `init()` returns the store (callers chain `.init()` onto the factory). + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { loadScript } from './_loadScript.js'; + +const MARKET_TOOLBAR_HTML = ` + + + + + + + + + +`; + +const ASSESS_TOOLBAR_HTML = ` + + + + +`; + +const OPTION_TOOLBAR_HTML = ` + + + + +`; + +function mount(html) { + document.body.innerHTML = html; +} + +beforeEach(() => { + window.localStorage.clear(); + delete window.appState; + delete window.createParamsStore; + delete window.__paramsDebug; + loadScript('static/eventBus.js'); + loadScript('static/state/store.js'); + loadScript('static/state/paramsStore.js'); +}); + +/** Mount the markup, THEN load the store scripts — the real page order: the + * toolbars are parsed before the scripts hydrate and bind them. */ +function loadStores() { + loadScript('static/state/marketParamsState.js'); + loadScript('static/state/assessmentParamsState.js'); + loadScript('static/state/optionFilterState.js'); +} + +describe('marketParams — hydration at parse time', () => { + it('restores the group from its own localStorage key', () => { + window.localStorage.setItem( + 'marketParams', + JSON.stringify({ from: '2024-01', to: '2024-03', frequency: 'W' }), + ); + mount(MARKET_TOOLBAR_HTML); + loadStores(); + + expect(window.appState.marketParams.get()).toEqual({ + from: '2024-01', + to: '2024-03', + frequency: 'W', + }); + expect(document.getElementById('stat-frequency').value).toBe('W'); + expect(document.getElementById('assess-frequency').value).toBe('W'); + }); + + it('mirrors the horizon into the bar\u2019s submit-only inputs', () => { + window.localStorage.setItem( + 'marketParams', + JSON.stringify({ from: '2024-01', to: '2024-03' }), + ); + mount(MARKET_TOOLBAR_HTML); + loadStores(); + + expect(document.getElementById('start_time').value).toBe('2024-01'); + expect(document.getElementById('end_time').value).toBe('2024-03'); + }); +}); + +describe('marketParams — commit + emit', () => { + it('commits a toolbar change and emits only the modules that consume it', () => { + mount(MARKET_TOOLBAR_HTML); + loadStores(); + const seen = []; + window.bus.on('module:params-changed', (payload) => seen.push(payload)); + + const select = document.getElementById('stat-frequency'); + select.value = 'W'; + select.dispatchEvent(new Event('change', { bubbles: true })); + + expect(window.appState.marketParams.get().frequency).toBe('W'); + expect(window.localStorage.getItem('marketParams')).toContain('"frequency":"W"'); + expect(seen).toHaveLength(1); + expect(seen[0].modules).toEqual(['market_review', 'statistical', 'assessment']); + }); + + it('keeps the three toolbars in sync through the shared horizon', () => { + mount(MARKET_TOOLBAR_HTML); + loadStores(); + const input = document.getElementById('assess-from'); + input.value = '2024-05'; + input.dispatchEvent(new Event('change', { bubbles: true })); + + expect(document.getElementById('mr-from').value).toBe('2024-05'); + expect(document.getElementById('stat-from').value).toBe('2024-05'); + expect(document.getElementById('start_time').value).toBe('2024-05'); + }); + + it('exposes a query fragment for the module URL builder', () => { + window.localStorage.setItem( + 'marketParams', + JSON.stringify({ from: '2024-01', to: '2024-03', frequency: 'W' }), + ); + mount(MARKET_TOOLBAR_HTML); + loadStores(); + + expect(window.appState.marketParams.query(['from', 'to', 'frequency'])).toBe( + 'from=2024-01&to=2024-03&frequency=W', + ); + }); +}); + +describe('assessmentParams', () => { + it('commits its own knobs', () => { + mount(ASSESS_TOOLBAR_HTML); + loadStores(); + const input = document.getElementById('assess-risk-threshold'); + input.value = '85'; + input.dispatchEvent(new Event('change', { bubbles: true })); + + expect(window.appState.assessmentParams.get().risk_threshold).toBe('85'); + expect(window.localStorage.getItem('assessmentParams')).toContain('"risk_threshold":"85"'); + }); +}); + +describe('optionFilter', () => { + it('hydrates and commits the chain filters', () => { + window.localStorage.setItem( + 'optionFilter', + JSON.stringify({ max_dte: '30', moneyness_low: '0.80', moneyness_high: '1.20' }), + ); + mount(OPTION_TOOLBAR_HTML); + loadStores(); + + expect(document.getElementById('oc-max-dte').value).toBe('30'); + + const input = document.getElementById('oc-refresh-interval'); + input.value = '120'; + input.dispatchEvent(new Event('change', { bubbles: true })); + + expect(window.appState.optionFilter.get().refresh_interval).toBe('120'); + }); + + it('does not ask the module rerunner for a /render round trip', () => { + mount(OPTION_TOOLBAR_HTML); + loadStores(); + expect(window.appState.optionFilter.MODULES).toEqual([]); + }); +}); + +describe('paramsStore — degraded storage', () => { + it('falls back to defaults when localStorage is denied', () => { + mount(MARKET_TOOLBAR_HTML); + loadStores(); + vi.spyOn(window.localStorage, 'getItem').mockImplementation(() => { + throw new Error('denied'); + }); + vi.spyOn(window.localStorage, 'setItem').mockImplementation(() => { + throw new Error('denied'); + }); + + expect(() => loadScript('static/state/marketParamsState.js')).not.toThrow(); + expect(() => window.appState.marketParams.set('frequency', 'Q')).not.toThrow(); + expect(window.appState.marketParams.get().frequency).toBe('Q'); + + vi.restoreAllMocks(); + }); +}); + +describe('paramsStore — init returns the store', () => { + it('does not clobber the published global', () => { + mount(MARKET_TOOLBAR_HTML); + loadStores(); + const published = window.appState.marketParams; + expect(published).toBeTruthy(); + expect(typeof published.init).toBe('function'); + expect(published.init()).toBe(published); + }); +}); From e026c670410f7229f1fd3904e8537338af085a86 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Fri, 11 Sep 2026 01:25:34 +0800 Subject: [PATCH 10/15] =?UTF-8?q?chore(ui):=20B8=20=E9=80=80=E5=BD=B9=20Co?= =?UTF-8?q?nfig=20=E9=A1=B5=E7=AD=BE=EF=BC=88=C2=A78=20Q2=20=3D=20?= =?UTF-8?q?=E5=88=A0=E9=99=A4=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 业务线重构计划书 §6 B8——整次重构的最后一个批次。 - 删除 templates/partials/tab_config.html、侧栏按钮与 include； 「System」分区标签随之移除（最后一个分区是 Portfolio） - 计划书预期的 switchTab config 特例并不存在（static/ 下 grep tab-config 为空）； 页签清单断言在三处：tests/test_pages_build.py、tests/e2e/test_smoke.py、 scripts/build_pages_site.py::build——后一处是本次遗漏后补的 - 风险利率仍硬编码在 static/sim/black_scholes.js 与 core/options/greeks 两处； 把它做成真正的全局设置是新特性而非清理，已记入 architecture_review.md §2 观察清单（不改一处不改另一处会使两套定价静默分叉） - 文档：计划书状态改为 LANDED、Q2 决议、B8 台账与 §8 注记、§9 引用路径、 frontend_architecture 页签表、l0 模板行、CODEBUDDY/CLAUDE 指针 验收：grep -rn tab_config（templates/ static/ tests/ site/）为空； pytest -m "not network" --ignore=tests/e2e → 505 passed / 5 skipped； pytest tests/e2e → 40 passed；npx vitest run → 198 passed / 16 files； doc_guard clean；arch_metrics --check ok；audit_tags 16 vs 16。 --- CLAUDE.md | 2 +- CODEBUDDY.md | 2 +- docs/architecture_review.md | 1 + docs/plans/business_line_reorg.md | 35 ++++++++++++++++++++++++------ scripts/build_pages_site.py | 1 - templates/index.html | 8 ------- templates/partials/tab_config.html | 30 ------------------------- tests/e2e/test_smoke.py | 1 - tests/test_pages_build.py | 1 - 9 files changed, 31 insertions(+), 50 deletions(-) delete mode 100644 templates/partials/tab_config.html diff --git a/CLAUDE.md b/CLAUDE.md index def6a9c..e069ca9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co > **⚠ Active reorg (2026-09) — [ADR 0011](docs/decisions/0011-pluggable-data-provider-seam.md) > + [0012](docs/decisions/0012-parameter-ownership-and-prefetch.md), both Accepted.** > Before touching `data_pipeline/`, the parameter surfaces -> (`templates/partials/parameters_bar.html`, `tab_config.html`, `static/main.js` +> (`templates/partials/parameters_bar.html`, the module toolbars in `tab_*.html`, `static/main.js` > `FormManager`) or `routes/core.py::index`, read > **[`docs/plans/business_line_reorg.md`](docs/plans/business_line_reorg.md) §0**: > work the numbered batches (B1–B8) in order, one batch per PR, update its ledger diff --git a/CODEBUDDY.md b/CODEBUDDY.md index def6a9c..e069ca9 100644 --- a/CODEBUDDY.md +++ b/CODEBUDDY.md @@ -12,7 +12,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co > **⚠ Active reorg (2026-09) — [ADR 0011](docs/decisions/0011-pluggable-data-provider-seam.md) > + [0012](docs/decisions/0012-parameter-ownership-and-prefetch.md), both Accepted.** > Before touching `data_pipeline/`, the parameter surfaces -> (`templates/partials/parameters_bar.html`, `tab_config.html`, `static/main.js` +> (`templates/partials/parameters_bar.html`, the module toolbars in `tab_*.html`, `static/main.js` > `FormManager`) or `routes/core.py::index`, read > **[`docs/plans/business_line_reorg.md`](docs/plans/business_line_reorg.md) §0**: > work the numbered batches (B1–B8) in order, one batch per PR, update its ledger diff --git a/docs/architecture_review.md b/docs/architecture_review.md index fc653a0..6453bd8 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -78,6 +78,7 @@ Rescoped in batch B1 of [ADR 0011](decisions/0011-pluggable-data-provider-seam.m |---|---|---| | `services/market/analysis/summary.py` (fan-in 0, tracked as `dead_code_candidates=1` in baseline) | `generate_summary_analysis` lost its caller when the streaming refactor removed the server-rendered `summary_data` template variable; the Summary tab button is gated off in `templates/index.html` and `summary_pending` in `routes/core.py` is vestigial | any request to ship the multi-ticker Summary tab ⇒ add a `summary` slice to `_RENDER_KIND_SLICES` (aggregates across the job's tickers, not per-ticker) ; otherwise delete the module + `partials/tab_summary.html` + the `summary_pending` flag in the same commit and reset the baseline | | ~~`data_pipeline/providers/yf_client.py` (391 lines, fan-in 11)~~ — **resolved 2026-09-10 (B1)** | it sat 9 lines below the 400-line god-file threshold | the option-chain section was extracted pre-emptively, as prescribed, into `providers/yf_snapshot.py`; `yf_client.py` is now a ~35-line shim. The pressure moved to `providers/yf_snapshot.py` (≈340 lines) and `providers/yfinance_provider.py` (≈290 lines) — watch them before adding endpoints | +| `static/sim/black_scholes.js` + `core/options/greeks` (risk-free rate) | the same constant is hard-coded in the client simulation **and** the server-side Greeks; changing one without the other makes the two pricings diverge silently | batch B8 retired the Config tab, so there is no global-setting surface to put it in; making it a parameter (store entry + form field + backend path) is a small feature — do it before anyone edits either constant | ## 3. Guardrails (how the score is kept) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index 0a476ee..b863202 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -4,9 +4,11 @@ **ADRs**: [0011](../decisions/0011-pluggable-data-provider-seam.md) (data-provider seam + canonical schema — **Accepted**), [0012](../decisions/0012-parameter-ownership-and-prefetch.md) (parameter ownership + readiness prefetch — **Accepted**) -> **Status: ACCEPTED TARGET, NOT YET IMPLEMENTED.** The shape below is the agreed -> destination. It ships as the batches in §6 — each one independently shippable -> and independently revertible. Track progress in the §0 ledger. +> **Status: LANDED (2026-09-10).** All eight §6 batches are implemented on branch +> `worktree-business-line-reorg`; the §0 ledger records what actually shipped, +> including the deliberate deviations from the shape below. The batches are still +> individually revertible — `git revert ` restores the previous +> behaviour without touching the others. --- @@ -45,7 +47,7 @@ | B5 — readiness plan + prefetch | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (**manifest**, params as `/render` query args); readiness plan on the job; cold-start hold fragment. Actuals + deferrals in §8 | | B6 — `ticker`-only Parameters bar | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q3 resolved (dedicated Portfolio tab); bar + collapse persisted; transitional settings group inside the bar's form until B7. Actuals in §8 | | B7 — module-scoped params | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (manifest). Backend query-arg contract + per-module allow-list; `state/*ParamsState.js`; module toolbars; bridge + hidden fields deleted; Config tab emptied (B8 decides its fate). See §8 B7 | -| B8 — retire / repurpose Config tab | ⬜ not started | — | — | gate: §8 Q2 | +| B8 — retire / repurpose Config tab | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q2 resolved (**deleted**). `grep tab_config` returns nothing; risk-free-rate follow-up on the watch list | States: `⬜ not started` → `🔨 in progress (PR #n)` → `✅ landed` → (`↩ reverted`). Keep the row order; edit the row in place. @@ -420,7 +422,7 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. | # | Question | Decision gate | Working lean | |---|---|---|---| | Q1 | **Submit contract** — does `POST /` carry a `modules` manifest with per-module params attached to each `/render` call, or do the streaming market tabs move fully to client-fired `/api/*` like Option Chain? Manifest keeps the streaming model; full client-fired is more uniform but a bigger diff. | ✅ resolved 2026-09-10 (B5) — **manifest**, per-module params as query args on each `/render` call (sub-option A1) | **manifest.** Decisive reasons: (1) ADR 0012's readiness pass needs the module list *at submit time* — with no POST manifest, B5 would need an extra `/api/ready` protocol; (2) the four streaming slices return server-rendered HTML + base64 PNG, so client-firing changes only the transport, not the product; (3) the diff and the revert surface stay one batch wide. Full client-fired would only win if the charts moved to client-side rendering (ADR 0006/0008 territory). Params travel as query args (not `hx-post` JSON) to match the existing `/api/option_chain?ticker=…` shape and stay bookmark-reproducible — recorded in the B5 note below | -| Q2 | **Config tab fate** — is there *any* genuine global setting to keep? Risk-free rate is the only candidate (hard-coded in `static/sim/` and again in `core/options/greeks`). Yes → tab shrinks to it; no → tab deleted. | before **B8** | keep risk-free rate, delete the rest | +| Q2 | **Config tab fate** — is there *any* genuine global setting to keep? Risk-free rate is the only candidate (hard-coded in `static/sim/` and again in `core/options/greeks`). Yes → tab shrinks to it; no → tab deleted. | ✅ resolved 2026-09-10 (B8) — **tab deleted** | **Deleted.** After B7 nothing on it was global: every field had moved to the module that consumes it, so keeping the shell meant keeping a page whose only content was "these settings moved". The risk-free rate is not a *setting* yet (hard-coded in two places) — wiring it up is a new feature, not a cleanup, so there was nothing to shrink the tab to. The divergence risk is now on the watch list (`architecture_review.md` §2) | | Q3 | **`positions` block** — Portfolio Analysis is its only consumer. Move into a dedicated "Portfolio" panel/tab, or keep as a section the bar's Run ignores? | ✅ resolved 2026-09-10 (B6) — **dedicated Portfolio tab** | **Dedicated Portfolio tab** (`tab-portfolio`). The positions table drives `POST /api/portfolio_analysis` (client-fired) and owns a full result surface (Greeks / P&L / theta / breakeven / VaR); keeping it inside the bar's form would re-couple that workflow to the streaming submit — exactly the coupling ADR 0012 removes — and a one-line bar has nowhere to put the results. A tab also makes the workflow discoverable instead of buried under "Parameters". `#positions-tbody` stays in the DOM on every load, so the existing global handlers are unchanged | | Q4 | **Table rename vs. reshape** — `raw_prices`→`raw_bars` with identical columns (minimal), or also move the yfinance-ism `adj_close` handling into the provider during the rename? | ✅ resolved 2026-09-10 (B2) | **minimal rename** — identical columns on both sides of each pair (structurally enforced: one column tuple per shape, used to create both names). The `adj_close` normalisation is already inside the provider (B1's `to_canonical_bars`), and ingest now consumes canonical bars, so no reshape is needed. ADR 0011's `symbol` column stays the target state but is deferred — see the B2 note below | | Q5 | **Second-provider protocol shape** — not in scope to *implement*, but `providers/base.py` (written in B1) must be sketched against *both* yfinance and `archive/futu_integration/field_mapping.md` so the protocol is not accidentally yfinance-shaped (IV unit, bid/ask availability, `inTheMoney` derivation all differ). | ✅ resolved 2026-09-10 (B1) — outcome table in ADR 0011 §"Protocol shape" | design review of `base.py` against both field maps: IV → decimal, bid/ask nullable, `inTheMoney` dropped (derivable), expiries ISO strings | @@ -685,13 +687,32 @@ full `pytest tests/e2e` → 41 passed; `npx vitest run` → 198 passed / 16 file `format --check` clean; `doc_guard.py` clean; `arch_metrics.py --check` ok (no baseline reset); `audit_tags.py` 16 vs baseline 16. +**B8 (landed 2026-09-10) — Config tab retired (§8 Q2 = delete).** + +- `templates/partials/tab_config.html` deleted with its sidebar button and include; the + `System` section label goes with it (the last section is Portfolio). The shell kept a + "these settings moved" note after B7; once every field had moved, a page whose only content + was that note had no reason to exist. +- **Nothing else changed**: no `switchTab` special case existed to remove (the plan anticipated + one; `grep tab-config` over `static/` is empty). The tab-list assertions lived in + `tests/{test_pages_build,e2e/test_smoke}.py` and `scripts/build_pages_site.py::build` — all three + updated. +- **Follow-up, deliberately not folded in** (rule 7): the risk-free rate is hard-coded in + `static/sim/black_scholes.js` and again in `core/options/greeks` — two places, one number. Making + it a real global setting means a store entry, a form field and a backend path; that is a feature, + and until then the divergence risk is on `architecture_review.md` §2's watch list. +- **Exit criteria**: `grep -rn tab_config` over `templates/ static/ tests/ site/` returns nothing; + `pytest -m "not network" --ignore=tests/e2e` → 505 passed / 5 skipped; `pytest tests/e2e` → 40 + passed; `npx vitest run` → 198 passed / 16 files; `doc_guard.py` clean; `arch_metrics.py --check` + ok; `audit_tags.py` 16 vs baseline 16. + --- ## 9. References - Current flow: `routes/core.py` → `services/market/dispatch.py` → `services/market/analysis/facade.py` -- Data layer: `data_pipeline/data_ops/{facade,_range,_query}.py`, `yf_client.py`, `downloader.py`, `db.py`, `repos.py` -- Frontend: `templates/partials/tab_parameter.html`, `tab_config.html`, `static/main.js`, `static/option-chain.js` +- Data layer: `data_pipeline/read/facade.py`, `orchestrate/{update,backfill}.py`, `providers/`, `store/{db,repos}.py`, `ingest/ohlcv.py` +- Frontend: `templates/partials/parameters_bar.html`, the module toolbars in `templates/partials/tab_*.html`, `static/main.js`, `static/moduleParams.js`, `static/option-chain.js` - `docs/decisions/0002-yfinance-as-sole-data-source.md`, `0004-no-iv-history-from-yfinance.md`, `0005-token-bucket-throttle.md` - `docs/constraints.md` §1–§6, `docs/frontend_architecture.md`, `docs/frontend_convergence.md` - `archive/futu_integration/field_mapping.md` diff --git a/scripts/build_pages_site.py b/scripts/build_pages_site.py index b9ae382..9d9960c 100644 --- a/scripts/build_pages_site.py +++ b/scripts/build_pages_site.py @@ -393,7 +393,6 @@ def assemble(out_dir: Path, ticker: str = DEMO_TICKER) -> Path: "tab-regime", "tab-simulation", "tab-option-pricing-matrix", - "tab-config", ): assert f'id="{tab_id}"' in html, f"missing tab body: {tab_id}" assert "./pages-shim.js" in html and "pages-demo-banner" in html diff --git a/templates/index.html b/templates/index.html index 19dd4e6..84878d2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -193,11 +193,6 @@

Market Dashboard

- -
System
- {% if tickers is defined and tickers|length > 1 %}
@@ -246,9 +241,6 @@

Market Dashboard

{% include 'partials/tab_option_pricing_matrix.html' %} - - {% include 'partials/tab_config.html' %} - diff --git a/templates/partials/tab_config.html b/templates/partials/tab_config.html deleted file mode 100644 index 542386a..0000000 --- a/templates/partials/tab_config.html +++ /dev/null @@ -1,30 +0,0 @@ -{# Config tab — emptied by batch B7. - - Everything this tab used to hold was mislabelled "global": the frequency and the - Assessment knobs belong to Assessment, the chain filters to Option Chain, the - refresh interval to the chain's auto-refresh. They now live in the module - toolbars, read through `static/state/*ParamsState.js` instead of the - localStorage -> hidden-input bridge. - - The tab shell survives (with a pointer) so existing deep links and the sidebar - keep working; batch B8 answers §8 Q2 — whether a genuine global setting (the - risk-free rate, currently hard-coded in two places) earns this tab or the tab - is deleted outright. #} -
-
-

Config

-

Nothing here yet.

-
- -
-
These settings moved
-
    -
  • Time horizon & frequency → the Market Review, Statistical and Assessment toolbars.
    • -
  • Side bias, risk threshold, rolling window, account size, max risk → the Assessment toolbar.
    • -
  • Max DTE, moneyness bounds, max contracts, refresh interval → the Option Chain toolbar (Payoff Ratio shares them).
    • -
-

- Each one is remembered per browser and re-runs only the module it belongs to. -

-
-
diff --git a/tests/e2e/test_smoke.py b/tests/e2e/test_smoke.py index 481e8f0..2854bdc 100644 --- a/tests/e2e/test_smoke.py +++ b/tests/e2e/test_smoke.py @@ -20,7 +20,6 @@ "tab-regime", "tab-simulation", "tab-option-pricing-matrix", - "tab-config", ] _ACTIVE_RE = re.compile(r"\bactive\b") diff --git a/tests/test_pages_build.py b/tests/test_pages_build.py index f3b15f8..8e95846 100644 --- a/tests/test_pages_build.py +++ b/tests/test_pages_build.py @@ -34,7 +34,6 @@ def test_assemble_matches_flask_partials(tmp_path): "tab-regime", "tab-simulation", "tab-option-pricing-matrix", - "tab-config", ): assert f'id="{tab_id}"' in html, tab_id From 969177627fcca5dcd4d110ba98c255bdd514f213 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Fri, 11 Sep 2026 01:38:25 +0800 Subject: [PATCH 11/15] =?UTF-8?q?docs(reorg):=20B1=E2=80=93B8=20=E9=AA=8C?= =?UTF-8?q?=E6=94=B6=E8=AF=84=E5=AE=A1=20+=20F6=20=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E9=99=88=E6=97=A7=E9=A1=B9=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 机械校验全绿（pytest 505/5skip、vitest 198、ruff/doc_guard/arch_metrics clean），8 批次退出标准与 §2 债务清零逐条核对通过。 计划书 §10 记录评审结论与 F1–F7 整改项，新增 B9 台账行： - F1（高，已复现）：job_cache.compute_or_get 按 (ticker,kind) 记忆， 未纳入 module_params；提交后 90s 内改模块参数会重发 /render 但返回 旧分片。B7 头号行为实际失效。 - F2/F3：参数栏「收起」在 B7 掏空 body 后形同虚设 + aria-controls 悬空 + FA 图标未加载 - F4：readiness 用 clean_bars 探针门控 feature_bars 覆盖 - F5：option_data 死路径 / GET 分支冗余模板变量 / header badge 恒显默认值 - F6：providers/* 文档字符串陈旧引用（downloader.py→ingest/ohlcv.py 等） + 台账 planning 行状态 —— 本提交已修 - F7：yf_client shim 无移除触发器；market_review_prices L5 仍在缝外 architecture_review.md §2 watch list 补两行（shim 移除触发器、L5）。 Co-Authored-By: Claude Sonnet 5 --- data_pipeline/providers/__init__.py | 2 +- data_pipeline/providers/base.py | 2 +- data_pipeline/providers/yf_client.py | 9 +++--- data_pipeline/providers/yfinance_provider.py | 8 +++--- docs/architecture_review.md | 2 ++ docs/plans/business_line_reorg.md | 29 +++++++++++++++++++- 6 files changed, 41 insertions(+), 11 deletions(-) diff --git a/data_pipeline/providers/__init__.py b/data_pipeline/providers/__init__.py index 396efe8..b975bef 100644 --- a/data_pipeline/providers/__init__.py +++ b/data_pipeline/providers/__init__.py @@ -6,7 +6,7 @@ mapped onto the canonical schema in ``providers/base.py``; processing and serving stay provider-agnostic. - ``yf_client.py`` is a compatibility shim over this package for one release; - ``downloader.py`` keeps only gap detection + DB upsert. + ``ingest/ohlcv.py`` keeps only gap detection + DB upsert. Contracts: - ``get_provider(name=None)``: resolve a ``MarketDataProvider`` implementation. - ``available_providers()``: registered provider names. diff --git a/data_pipeline/providers/base.py b/data_pipeline/providers/base.py index cc94b08..daf0a23 100644 --- a/data_pipeline/providers/base.py +++ b/data_pipeline/providers/base.py @@ -29,7 +29,7 @@ Dependencies UPWARD: - (none — no external SDK is imported here; implementations sit beside it) Dependencies DOWNWARD: - - providers/yfinance_provider.py, providers/yf_option_chain.py, + - providers/yfinance_provider.py, providers/yf_snapshot.py, providers/_registry.py """ diff --git a/data_pipeline/providers/yf_client.py b/data_pipeline/providers/yf_client.py index b132571..bf92f7a 100644 --- a/data_pipeline/providers/yf_client.py +++ b/data_pipeline/providers/yf_client.py @@ -4,9 +4,10 @@ Context: - Batch B1 (docs/plans/business_line_reorg.md §6) moved every yfinance call into ``data_pipeline/providers/``. This module is kept for one release so the - existing importers (``services/``, ``core/market/data_context.py``, - ``data_pipeline/read/``) do not have to change in the same PR as the - extraction. See ADR 0011. + existing importers (``services/``, ``data_pipeline/read/``) do not have to + change in the same PR as the extraction. See ADR 0011. (``core/`` used to + import it too; batch B4 removed that — ``core`` no longer touches + ``data_pipeline``.) - New code should import from ``data_pipeline.providers`` (canonical shapes) instead of here. Contracts: @@ -19,7 +20,7 @@ Dependencies UPWARD: - providers/yf_snapshot (live snapshots), providers/yfinance_provider (bars) Dependencies DOWNWARD: - - services/*, core/market/data_context.py, data_pipeline/read/* + - services/*, data_pipeline/read/* """ from __future__ import annotations diff --git a/data_pipeline/providers/yfinance_provider.py b/data_pipeline/providers/yfinance_provider.py index 973a791..88a4456 100644 --- a/data_pipeline/providers/yfinance_provider.py +++ b/data_pipeline/providers/yfinance_provider.py @@ -3,10 +3,10 @@ Domain: Data Pipeline — yfinance Provider Context: - This module (with ``yf_snapshot.py``) is the **only** place in the repo that - imports ``yfinance``; batch B1 moved these calls here out of - ``data_pipeline/yf_client.py`` and ``data_pipeline/downloader.py`` without - changing behaviour. See ADR 0002 (as amended by ADR 0011) and - docs/plans/business_line_reorg.md §6. + imports ``yfinance``; batch B1 moved these calls here out of the old + ``data_pipeline/yf_client.py`` and ``data_pipeline/downloader.py`` (now + ``ingest/ohlcv.py``) without changing behaviour. See ADR 0002 (as amended by + ADR 0011) and docs/plans/business_line_reorg.md §6. - ``YFinanceProvider`` is the canonical seam implementation (``history()`` / ``close_panel()`` / ``spot()`` / ``option_chain()``); it delegates the live snapshots to ``yf_snapshot.py``. The module-level ``fetch_*`` functions keep diff --git a/docs/architecture_review.md b/docs/architecture_review.md index 6453bd8..eeded21 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -79,6 +79,8 @@ Rescoped in batch B1 of [ADR 0011](decisions/0011-pluggable-data-provider-seam.m | `services/market/analysis/summary.py` (fan-in 0, tracked as `dead_code_candidates=1` in baseline) | `generate_summary_analysis` lost its caller when the streaming refactor removed the server-rendered `summary_data` template variable; the Summary tab button is gated off in `templates/index.html` and `summary_pending` in `routes/core.py` is vestigial | any request to ship the multi-ticker Summary tab ⇒ add a `summary` slice to `_RENDER_KIND_SLICES` (aggregates across the job's tickers, not per-ticker) ; otherwise delete the module + `partials/tab_summary.html` + the `summary_pending` flag in the same commit and reset the baseline | | ~~`data_pipeline/providers/yf_client.py` (391 lines, fan-in 11)~~ — **resolved 2026-09-10 (B1)** | it sat 9 lines below the 400-line god-file threshold | the option-chain section was extracted pre-emptively, as prescribed, into `providers/yf_snapshot.py`; `yf_client.py` is now a ~35-line shim. The pressure moved to `providers/yf_snapshot.py` (≈340 lines) and `providers/yfinance_provider.py` (≈290 lines) — watch them before adding endpoints | | `static/sim/black_scholes.js` + `core/options/greeks` (risk-free rate) | the same constant is hard-coded in the client simulation **and** the server-side Greeks; changing one without the other makes the two pricings diverge silently | batch B8 retired the Config tab, so there is no global-setting surface to put it in; making it a parameter (store entry + form field + backend path) is a small feature — do it before anyone edits either constant | +| `data_pipeline/providers/yf_client.py` (compat shim, ADR 0011 B1) | re-exports `fetch_spot` / `fetch_option_chain` / `fetch_close_panel` / `fetch_daily_ohlcv` with their old yfinance-shaped contracts; it was a **"one release"** bridge so importers did not have to change in the B1 PR | once `grep -rn "providers.yf_client\|yf_client import" services/ data_pipeline/read/` is empty (importers moved to the canonical `providers.get_provider()` shapes), delete `yf_client.py` and its re-exports from `providers/__init__.py` in one commit | +| `services/market_review/fetch.py` → `market_review_prices` (L5 in plan §4.1) | a second acquisition path outside the provider seam: its own L1/L2/L3 close-panel ladder writes a `market_review_prices` table that `data_pipeline/orchestrate/readiness.py` does **not** plan, so the benchmark panel still lazy-fetches on the Market Review slice | fold the ladder into `providers` + a canonical `bars` read (ADR 0011's L5 exit); until then, add `market_review` benchmark tickers to `KIND_DATASETS` so the readiness pass warms them | ## 3. Guardrails (how the score is kept) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index b863202..f519e9a 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -39,7 +39,7 @@ | Batch | State | PR | Landed (commit · date) | Notes | |---|---|---|---|---| -| — (planning + ADRs) | 🔨 in review (PR #7) | #7 | branch `worktree-business-line-reorg` · 2026-09-10 | plan, ADR 0011/0012 (Accepted), scaffolding (ledger, gates, AI-guide pointers, memory) | +| — (planning + ADRs) | ✅ landed | #7 + #8 | merged to `main` · 2026-09-10 | plan, ADR 0011/0012 (Accepted), scaffolding (ledger, gates, AI-guide pointers, memory) | | B1 — provider seam extraction | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | delivers the "pluggable API" seam on its own. Actual shape / deviations recorded in §8; `_ALLOWED_DEPS` promotion of `providers` deferred to B3 | | B2 — canonical raw store | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | gate §8 Q4 resolved (name-only rename). Actuals in §8; `symbol` column deferred (ADR 0011 amendment) | | B3 — package re-home | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | six stages + `_state.py`; first sub-layer guard table; `arch_baseline.json` **not** reset (no tracked drift). Actuals + deviations in §8 | @@ -48,6 +48,7 @@ | B6 — `ticker`-only Parameters bar | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q3 resolved (dedicated Portfolio tab); bar + collapse persisted; transitional settings group inside the bar's form until B7. Actuals in §8 | | B7 — module-scoped params | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (manifest). Backend query-arg contract + per-module allow-list; `state/*ParamsState.js`; module toolbars; bridge + hidden fields deleted; Config tab emptied (B8 decides its fate). See §8 B7 | | B8 — retire / repurpose Config tab | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q2 resolved (**deleted**). `grep tab_config` returns nothing; risk-free-rate follow-up on the watch list | +| B9 — acceptance-review remediation | ⬜ not started | — | — | F1–F7 from §10 (2026-09-11 review). F1 (module-param memo staleness) is the blocker; F6 done in the review commit | States: `⬜ not started` → `🔨 in progress (PR #n)` → `✅ landed` → (`↩ reverted`). Keep the row order; edit the row in place. @@ -716,3 +717,29 @@ full `pytest tests/e2e` → 41 passed; `npx vitest run` → 198 passed / 16 file - `docs/decisions/0002-yfinance-as-sole-data-source.md`, `0004-no-iv-history-from-yfinance.md`, `0005-token-bucket-throttle.md` - `docs/constraints.md` §1–§6, `docs/frontend_architecture.md`, `docs/frontend_convergence.md` - `archive/futu_integration/field_mapping.md` + +--- + +## 10. Acceptance review (2026-09-11) + +Post-B8 review of the landed branch. Mechanical gate is **green**: +`pytest -m "not network" --ignore=tests/e2e` exit 0 (505 / 5 skipped), +`npx vitest run` 198 / 16 files, `ruff check` + `format --check` clean (347 files), +`doc_guard.py` clean, `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / +dead 1 = pre-existing `summary.py`), `audit_tags.py` 16 vs 16. Every §6 exit +criterion and every §2 debt-row closure verified by grep. The three asks are +delivered — the backend acquire/process/serve separation in particular is clean +and enforceable. + +Findings, to be worked as **batch B9 (remediation)** under the §0 rules: + +| # | Sev | Finding | Fix | +|---|---|---|---| +| F1 | **high — correctness, CONFIRMED** | `job_cache.compute_or_get` memoises per `(ticker, kind)`; `dispatch.py::_compute` captures `module_params` from the query string but they are **not in the key** and nothing invalidates on change. Reproduced: two `/render/statistical` calls on one job, `frequency=ME` then `=W` → slice invoked once, both response bodies byte-identical. B7's headline behaviour ("changing a module control re-runs that module") re-fires the request and flashes "Updating…" but serves the stale fragment for up to `JOB_CACHE_TTL` (90 s). Params *do* work on the no-job direct-URL path (it bypasses the memo). `tests/e2e/test_module_params.py` missed it — it asserts request **URLs**, never that the fragment **content** changed. | Fold a stable digest of `module_params` into the memo key (4th arg to `compute_or_get`, or `f"{kind}|{sorted(module_params.items())}"`). Add a slice-level test asserting the **body** differs after a param change. | +| F2 | moderate — UX / a11y | The Parameters bar "collapse" is hollow after B7. B6 built it to hide `.parameters-bar-body` (the "Analysis settings" group); B7 moved that group to the tab toolbars and deleted the body but did not re-point the collapse. Collapsed now = `data-collapsed="true"` hides a non-existent element and reveals `▸ ^SPX` **beside the still-visible ticker input + label + Run** → ~zero visible effect. `aria-controls="parameters-bar-body"` is a dangling reference (no such id). The ask was 「常驻页面顶部，可收起」. | Make collapsed actually hide the label + input + validation, leaving toggle + `▸ ^SPX` (+ maybe Run) on one line; fix or remove `aria-controls`. | +| F3 | minor — UX | Collapse toggle icon is invisible: `parameters_bar.html` uses `` / `parametersBar.js` swaps to `fa-chevron-right`, but Font Awesome is not loaded (`index.html`: "Font Awesome removed for clean UI") and `.parameters-bar-toggle` has no CSS fallback. Toggle works (sr-only label + title) but renders an empty bordered box. | Draw the chevron in CSS (`::before` rotated by `[data-collapsed]`) or inline SVG like the theme toggle; drop the FA class. | +| F4 | minor — readiness gap | `readiness.check_and_kick` gates coverage on the `clean_bars` probe (`needs_backfill`). The "one pipeline run fills both" INVARIANT holds for a fresh backfill, but a DB with `clean_bars` for the range yet stale/missing `feature_bars` (new frequency, or processing failed after cleaning) → probe says "covered", no kick, `statistical`/`assessment` fall back to the per-slice `get_processed` → `manual_update` path. Fallback works, prefetch promise unmet. | `feature_bars`-aware probe, or a `WHY:` note in `readiness.py` accepting the gap. | +| F5 | minor — dead code / stale UI | (a) `form.py::parse_option_data` + `form_data["option_data"]`: FormManager no longer submits `#option_position` (B7) and no slice reads `option_data` — always `[]` on the POST path. (b) `routes/core.py` GET branch still passes `frequency`/`risk_threshold`/`rolling_window`/`side_bias` to `render_template`. (c) `index.html:112` header badge renders `{{ frequency_display or frequency }}, {{ side_bias }}` — after B7 these are always the POST-time defaults, so the badge shows "Monthly, Neutral" regardless of the toolbar selection. | Drop the dead form path or comment it dormant; remove the unused GET vars; fix or remove the header badge meta. | +| F6 | trivial — stale docstrings | `providers/base.py:32` "yf_option_chain.py" → `yf_snapshot.py`; `providers/yf_client.py:7,22` still lists `core/market/data_context.py` as an importer (B4 removed it); `providers/__init__.py:9` + `yfinance_provider.py:7` say `downloader.py` (it is `ingest/ohlcv.py` since B3); the §0 ledger's planning row still says "🔨 in review (PR #7)" (PR #7 + #8 both merged). | **Fixed in this review's commit.** | +| F7 | housekeeping | (a) `providers/yf_client.py` is a "one-release" shim with no tracked removal trigger. (b) `market_review_prices` (L5) is still a parallel acquisition path outside the seam — B5 acknowledged the deferral. Both should be `architecture_review.md` §2 watch-list rows. (c) Branch is 10 commits ahead of `origin/main`, unpushed, no PR — the batches have not landed on `main`. | Add the two §2 rows; push + PR the branch. | + From 2ab26cfa641028a613d06c99ac4375089d8aa5c2 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Fri, 11 Sep 2026 01:58:56 +0800 Subject: [PATCH 12/15] =?UTF-8?q?fix(reorg):=20B9=20=E6=95=B4=E6=94=B9=20F?= =?UTF-8?q?1=E2=80=93F3=20=E2=80=94=E2=80=94=20=E6=A8=A1=E5=9D=97=E5=8F=82?= =?UTF-8?q?=E6=95=B0=20memo=20/=20=E5=8F=82=E6=95=B0=E6=A0=8F=E6=94=B6?= =?UTF-8?q?=E8=B5=B7=20/=20=E5=9B=BE=E6=A0=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收评审（§10）发现的阻塞项与「可收起」诉求： F1（correctness，已复现）：job_cache.compute_or_get 按 (ticker,kind) 记忆，dispatch 的 module_params 来自 query string 却不在 key 里， 提交后 90s 内改 frequency/时间轴会重发 /render 但返回旧分片。 - compute_or_get 增加 keyword-only variant，key 改 (ticker,kind,variant) - dispatch._params_variant(module_params) 生成有序摘要 - 直链/旧调用方不传 variant → "" → 旧行为，无副作用 - 新测试：test_variant_computes_independently、 test_a_param_change_recomputes_within_the_same_job（原 e2e 只查 URL） F2（UX/a11y）：B7 掏空 bar body 后「收起」形同虚设、aria-controls 悬空 - label+输入+徽章包进
- 收起隐藏该 div + .ticker-validation，只留 toggle + ▸ ^SPX + Run - 删除失效的 .parameters-bar-body / -group-title CSS - 新 vitest 断言 aria-controls 目标存在且含 #ticker F3（UX）：收起图标用未加载的 Font Awesome - 改内联 SVG chevron，按 [data-collapsed] 旋转 -90° - parametersBar.js 去掉 icon class 交换，改 title 文案 docs：计划书 §10 表加 Status 列、§8 补 B9 batch note、台账 B9 行； frontend_architecture.md 参数栏段落更新到 B7/B9 现状。 pytest exit 0；vitest 199/16；e2e（bar 相关 20）pass； ruff/doc_guard/arch_metrics/audit_tags clean。 Co-Authored-By: Claude Sonnet 5 --- data_pipeline/orchestrate/job_cache.py | 30 ++++++++---- docs/frontend_architecture.md | 15 +++--- docs/plans/business_line_reorg.md | 63 +++++++++++++++++++++----- services/market/dispatch.py | 18 +++++++- static/parametersBar.js | 5 +- static/styles.css | 41 +++++++++-------- templates/partials/parameters_bar.html | 26 ++++++++--- tests/test_job_cache.py | 19 ++++++++ tests/test_module_params.py | 35 ++++++++++++++ tests/unit/parametersBar.test.js | 21 +++++++-- 10 files changed, 213 insertions(+), 60 deletions(-) diff --git a/data_pipeline/orchestrate/job_cache.py b/data_pipeline/orchestrate/job_cache.py index e33c0cc..4010c3d 100644 --- a/data_pipeline/orchestrate/job_cache.py +++ b/data_pipeline/orchestrate/job_cache.py @@ -73,18 +73,21 @@ def __init__(self, form_data: dict, tickers: list[str], plan: list | None = None # longer than the TTL must not lose its result to a mid-compute # eviction, and an active tab fan-out must not expire under load. self.last_access: float = self.created_at - # Memoised slice results, keyed by (ticker, kind). - self.results: dict[tuple[str, str], Any] = {} + # Memoised slice results, keyed by (ticker, kind, variant). `variant` is + # a digest of the module's own query-arg parameters (batch B7): without + # it a toolbar change (frequency, horizon, …) re-fires /render/ + # but this cache serves the first render for the whole job TTL. + self.results: dict[tuple[str, str, str], Any] = {} # Error-dict results get their own short TTL so a transient failure # (yfinance hiccup) is not sticky for the whole job lifetime. - self.error_results: dict[tuple[str, str], tuple[float, Any]] = {} + self.error_results: dict[tuple[str, str, str], tuple[float, Any]] = {} # Per-key locks so concurrent /render/* calls for the same slice # collapse into a single computation (single-flight). - self.key_locks: dict[tuple[str, str], threading.Lock] = {} + self.key_locks: dict[tuple[str, str, str], threading.Lock] = {} # Mutex for `key_locks` and `results` dict-level mutations. self._master_lock = threading.Lock() - def _lock_for(self, key: tuple[str, str]) -> threading.Lock: + def _lock_for(self, key: tuple[str, str, str]) -> threading.Lock: with self._master_lock: lock = self.key_locks.get(key) if lock is None: @@ -151,7 +154,7 @@ def get_job(job_id: str) -> _JobEntry | None: return entry -def _fresh_error(entry: _JobEntry, key: tuple[str, str], now: float) -> Any | None: +def _fresh_error(entry: _JobEntry, key: tuple[str, str, str], now: float) -> Any | None: """Return a still-fresh memoised error result, or None (and drop it if stale).""" hit = entry.error_results.get(key) if hit is None: @@ -163,8 +166,14 @@ def _fresh_error(entry: _JobEntry, key: tuple[str, str], now: float) -> Any | No return result -def compute_or_get(job_id: str, ticker: str, kind: str, compute_fn: Callable[[dict], Any]) -> Any: - """Memoised compute under a per-(ticker, kind) single-flight lock. +def compute_or_get(job_id: str, ticker: str, kind: str, compute_fn: Callable[[dict], Any], *, variant: str = "") -> Any: + """Memoised compute under a per-(ticker, kind, variant) single-flight lock. + + ``variant`` is a stable digest of any per-request parameters that change the + result (batch B7: a module toolbar sends its own ``?from=…&frequency=…`` on + each ``/render`` call). Two calls with the same ticker + kind but different + ``variant`` compute independently — otherwise the first render is served for + the whole job TTL and the toolbar change is silently ignored. Successful results are memoised for the full job TTL. Error-dict results (the slice methods' failure convention) are memoised only for @@ -178,7 +187,7 @@ def compute_or_get(job_id: str, ticker: str, kind: str, compute_fn: Callable[[di if entry is None: raise KeyError(f"unknown or expired job_id={job_id!r}") - key = (ticker, kind) + key = (ticker, kind, variant) # Fast path: already computed successfully. cached = entry.results.get(key) if cached is not None: @@ -224,10 +233,11 @@ def compute_or_get(job_id: str, ticker: str, kind: str, compute_fn: Callable[[di elapsed, ) logger.info( - "JobCache computed job=%s ticker=%s kind=%s in %.2fs", + "JobCache computed job=%s ticker=%s kind=%s%s in %.2fs", job_id[:8], ticker, kind, + f" variant={variant}" if variant else "", elapsed, ) return result diff --git a/docs/frontend_architecture.md b/docs/frontend_architecture.md index af3e6c7..ee56725 100644 --- a/docs/frontend_architecture.md +++ b/docs/frontend_architecture.md @@ -146,12 +146,15 @@ The application uses a **single-page template** (`index.html`) with tab-based na The **Parameters bar** (`templates/partials/parameters_bar.html`, batch B6) renders between the header and `.app-body`, is `position: sticky` under the header, and owns -exactly one input — `ticker` — plus the Run button and the ticker-validation badges. -It is **not** a tab: it survives tab switches, and collapsing it (persisted per -viewer under `localStorage['parametersBarCollapsed']`, guarded `try/catch`) leaves a -one-line summary (`▸ ^SPX`). Analysis settings that have not yet moved to their -module toolbars sit in a collapsible group inside the same `
`, so the POST -contract is unchanged until B7. +exactly one visible input — `ticker` — plus the Run button and the ticker-validation +badges. Every other parameter lives in its module's toolbar (batch B7); the bar +also carries two hidden `start_time`/`end_time` inputs that `POST /` validates and +uses to size the readiness prefetch, kept in sync by `state/marketParamsState.js`. +It is **not** a tab: it survives tab switches. Collapsing it (the chevron toggle; +state persisted per viewer under `localStorage['parametersBarCollapsed']`, guarded +`try/catch`) hides the `#parameters-bar-body` fields group and the validation line, +leaving the toggle, a one-line `▸ ^SPX` summary, and Run. The chevron is an inline +SVG (Font Awesome is not loaded on this page) rotated by `[data-collapsed]`. ### Peek Sidebar diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index f519e9a..699d1d9 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -48,7 +48,7 @@ | B6 — `ticker`-only Parameters bar | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q3 resolved (dedicated Portfolio tab); bar + collapse persisted; transitional settings group inside the bar's form until B7. Actuals in §8 | | B7 — module-scoped params | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (manifest). Backend query-arg contract + per-module allow-list; `state/*ParamsState.js`; module toolbars; bridge + hidden fields deleted; Config tab emptied (B8 decides its fate). See §8 B7 | | B8 — retire / repurpose Config tab | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q2 resolved (**deleted**). `grep tab_config` returns nothing; risk-free-rate follow-up on the watch list | -| B9 — acceptance-review remediation | ⬜ not started | — | — | F1–F7 from §10 (2026-09-11 review). F1 (module-param memo staleness) is the blocker; F6 done in the review commit | +| B9 — acceptance-review remediation | 🔨 F1–F3+F6 landed | — | branch `worktree-business-line-reorg` · 2026-09-11 | §10 review. **F1** (memo `variant` key), **F2** (real collapse target + hide fields), **F3** (inline-SVG chevron), **F6** (docstrings) done; F4/F5/F7c open | States: `⬜ not started` → `🔨 in progress (PR #n)` → `✅ landed` → (`↩ reverted`). Keep the row order; edit the row in place. @@ -707,6 +707,46 @@ full `pytest tests/e2e` → 41 passed; `npx vitest run` → 198 passed / 16 file passed; `npx vitest run` → 198 passed / 16 files; `doc_guard.py` clean; `arch_metrics.py --check` ok; `audit_tags.py` 16 vs baseline 16. +**B9 (2026-09-11) — acceptance-review remediation (F1–F3, F6).** + +Scope of this pass: the §10 findings that are correctness or the explicitly-asked +「可收起」 behaviour. F4 (readiness `feature_bars` gap), F5 (dead `option_data` path / +stale header badge) and F7c (merge to `main`) are left open — they are cleanups, +not blockers, and each is one small independent change. + +- **F1 — module-param memo staleness (correctness).** `job_cache.compute_or_get` + gained a keyword-only `variant: str = ""`; the memo/lock/error-cache key is now + `(ticker, kind, variant)`. `services/market/dispatch.py::_params_variant` + renders `module_params` as a sorted `k=v|k=v` digest and passes it. Legacy + callers and the direct-URL path pass no `variant` ⇒ `""` ⇒ the old key, so + nothing else changes. Reproduced before/after with a scratch script (slice + invoked once → twice). Guards: `test_job_cache.py::test_variant_computes_independently` + and `test_module_params.py::test_a_param_change_recomputes_within_the_same_job` + (the pre-existing e2e only checked request URLs, never fragment content). +- **F2 — hollow collapse.** `parameters_bar.html` now wraps the label + ticker + input + validation badges in `
` + — a real `aria-controls` target. Collapsed (`[data-collapsed="true"]`) hides + that div **and** `.ticker-validation`, leaving the toggle, the `▸ ^SPX` summary + and Run on one line. The dead `.parameters-bar-body` / `.parameters-bar-group-title` + rules (B7 had deleted their elements) are removed. New vitest case asserts the + `aria-controls` target exists and contains `#ticker`. +- **F3 — invisible chevron.** The Font Awesome `` is replaced with an inline + SVG (`.parameters-bar-chevron`) rotated `-90°` by `[data-collapsed="true"]`; + `parametersBar.js` drops the icon-class swap and updates `title` instead + (`Collapse` / `Expand parameters`). Matches how the theme toggle already ships + SVG on this FA-free page. +- **F6 — stale docstrings** (already in review commit `9691776`): `providers/base.py`, + `providers/yf_client.py`, `providers/__init__.py`, `providers/yfinance_provider.py`, + and the §0 ledger planning-row status. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → exit 0 + (+2 tests vs B8: `test_job_cache.py::test_variant_computes_independently`, + `test_module_params.py::test_a_param_change_recomputes_within_the_same_job`); + relevant `pytest tests/e2e` (module_params, smoke, localstorage, form_submit, + streaming) → 20 passed; `npx vitest run` → 199 passed / 16 files (+1 + aria-controls case); `ruff check` + `format --check` clean; `doc_guard.py` + clean; `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / dead 1); + `audit_tags.py` 16 vs 16. + --- ## 9. References @@ -731,15 +771,16 @@ criterion and every §2 debt-row closure verified by grep. The three asks are delivered — the backend acquire/process/serve separation in particular is clean and enforceable. -Findings, to be worked as **batch B9 (remediation)** under the §0 rules: +Findings worked as **batch B9 (remediation)** under the §0 rules. +**F1–F3 + F6 landed 2026-09-11** (this branch); F4/F5/F7 remain. -| # | Sev | Finding | Fix | -|---|---|---|---| -| F1 | **high — correctness, CONFIRMED** | `job_cache.compute_or_get` memoises per `(ticker, kind)`; `dispatch.py::_compute` captures `module_params` from the query string but they are **not in the key** and nothing invalidates on change. Reproduced: two `/render/statistical` calls on one job, `frequency=ME` then `=W` → slice invoked once, both response bodies byte-identical. B7's headline behaviour ("changing a module control re-runs that module") re-fires the request and flashes "Updating…" but serves the stale fragment for up to `JOB_CACHE_TTL` (90 s). Params *do* work on the no-job direct-URL path (it bypasses the memo). `tests/e2e/test_module_params.py` missed it — it asserts request **URLs**, never that the fragment **content** changed. | Fold a stable digest of `module_params` into the memo key (4th arg to `compute_or_get`, or `f"{kind}|{sorted(module_params.items())}"`). Add a slice-level test asserting the **body** differs after a param change. | -| F2 | moderate — UX / a11y | The Parameters bar "collapse" is hollow after B7. B6 built it to hide `.parameters-bar-body` (the "Analysis settings" group); B7 moved that group to the tab toolbars and deleted the body but did not re-point the collapse. Collapsed now = `data-collapsed="true"` hides a non-existent element and reveals `▸ ^SPX` **beside the still-visible ticker input + label + Run** → ~zero visible effect. `aria-controls="parameters-bar-body"` is a dangling reference (no such id). The ask was 「常驻页面顶部，可收起」. | Make collapsed actually hide the label + input + validation, leaving toggle + `▸ ^SPX` (+ maybe Run) on one line; fix or remove `aria-controls`. | -| F3 | minor — UX | Collapse toggle icon is invisible: `parameters_bar.html` uses `` / `parametersBar.js` swaps to `fa-chevron-right`, but Font Awesome is not loaded (`index.html`: "Font Awesome removed for clean UI") and `.parameters-bar-toggle` has no CSS fallback. Toggle works (sr-only label + title) but renders an empty bordered box. | Draw the chevron in CSS (`::before` rotated by `[data-collapsed]`) or inline SVG like the theme toggle; drop the FA class. | -| F4 | minor — readiness gap | `readiness.check_and_kick` gates coverage on the `clean_bars` probe (`needs_backfill`). The "one pipeline run fills both" INVARIANT holds for a fresh backfill, but a DB with `clean_bars` for the range yet stale/missing `feature_bars` (new frequency, or processing failed after cleaning) → probe says "covered", no kick, `statistical`/`assessment` fall back to the per-slice `get_processed` → `manual_update` path. Fallback works, prefetch promise unmet. | `feature_bars`-aware probe, or a `WHY:` note in `readiness.py` accepting the gap. | -| F5 | minor — dead code / stale UI | (a) `form.py::parse_option_data` + `form_data["option_data"]`: FormManager no longer submits `#option_position` (B7) and no slice reads `option_data` — always `[]` on the POST path. (b) `routes/core.py` GET branch still passes `frequency`/`risk_threshold`/`rolling_window`/`side_bias` to `render_template`. (c) `index.html:112` header badge renders `{{ frequency_display or frequency }}, {{ side_bias }}` — after B7 these are always the POST-time defaults, so the badge shows "Monthly, Neutral" regardless of the toolbar selection. | Drop the dead form path or comment it dormant; remove the unused GET vars; fix or remove the header badge meta. | -| F6 | trivial — stale docstrings | `providers/base.py:32` "yf_option_chain.py" → `yf_snapshot.py`; `providers/yf_client.py:7,22` still lists `core/market/data_context.py` as an importer (B4 removed it); `providers/__init__.py:9` + `yfinance_provider.py:7` say `downloader.py` (it is `ingest/ohlcv.py` since B3); the §0 ledger's planning row still says "🔨 in review (PR #7)" (PR #7 + #8 both merged). | **Fixed in this review's commit.** | -| F7 | housekeeping | (a) `providers/yf_client.py` is a "one-release" shim with no tracked removal trigger. (b) `market_review_prices` (L5) is still a parallel acquisition path outside the seam — B5 acknowledged the deferral. Both should be `architecture_review.md` §2 watch-list rows. (c) Branch is 10 commits ahead of `origin/main`, unpushed, no PR — the batches have not landed on `main`. | Add the two §2 rows; push + PR the branch. | +| # | Sev | Status | Finding | Fix | +|---|---|---|---|---| +| F1 | **high — correctness, CONFIRMED** | ✅ **fixed (B9)** | `job_cache.compute_or_get` memoised per `(ticker, kind)`; `dispatch.py::_compute` captured `module_params` from the query string but they were **not in the key** and nothing invalidated on change. Reproduced: two `/render/statistical` calls on one job, `frequency=ME` then `=W` → slice invoked once, both response bodies byte-identical. B7's headline behaviour ("changing a module control re-runs that module") re-fired the request and flashed "Updating…" but served the stale fragment for up to `JOB_CACHE_TTL` (90 s). `tests/e2e/test_module_params.py` missed it — it asserts request **URLs**, never fragment **content**. | `compute_or_get(..., *, variant="")` — key is now `(ticker, kind, variant)`; `dispatch._params_variant(module_params)` builds a sorted digest. New tests: `test_job_cache.py::test_variant_computes_independently`, `test_module_params.py::test_a_param_change_recomputes_within_the_same_job`. | +| F2 | moderate — UX / a11y | ✅ **fixed (B9)** | The Parameters bar "collapse" was hollow after B7: `data-collapsed="true"` hid a non-existent `.parameters-bar-body` and revealed `▸ ^SPX` **beside the still-visible ticker input + label + Run** → ~zero visible effect; `aria-controls="parameters-bar-body"` was a dangling reference. | The label + input + badges are now wrapped in `
` (real `aria-controls` target); collapsed hides that div **and** `.ticker-validation`, leaving toggle + `▸ ^SPX` + Run on one line. Dead `.parameters-bar-body` / `-group-title` CSS removed. New vitest: "has a real element behind aria-controls". | +| F3 | minor — UX | ✅ **fixed (B9)** | Collapse toggle icon was invisible — `` with Font Awesome not loaded and no CSS fallback. | Inline SVG chevron (`.parameters-bar-chevron`) rotated `-90°` by `.parameters-bar[data-collapsed="true"]`; `parametersBar.js` drops the `` class swap and updates `title` instead. | +| F4 | minor — readiness gap | ⬜ open | `readiness.check_and_kick` gates coverage on the `clean_bars` probe (`needs_backfill`). The "one pipeline run fills both" INVARIANT holds for a fresh backfill, but a DB with `clean_bars` for the range yet stale/missing `feature_bars` (new frequency, or processing failed after cleaning) → probe says "covered", no kick, `statistical`/`assessment` fall back to the per-slice `get_processed` → `manual_update` path. Fallback works, prefetch promise unmet. | `feature_bars`-aware probe, or a `WHY:` note in `readiness.py` accepting the gap. | +| F5 | minor — dead code / stale UI | ⬜ open | (a) `form.py::parse_option_data` + `form_data["option_data"]`: FormManager no longer submits `#option_position` (B7) and no slice reads `option_data` — always `[]` on the POST path. (b) `routes/core.py` GET branch still passes `frequency`/`risk_threshold`/`rolling_window`/`side_bias` to `render_template`. (c) `index.html:112` header badge renders `{{ frequency_display or frequency }}, {{ side_bias }}` — after B7 these are always the POST-time defaults, so the badge shows "Monthly, Neutral" regardless of the toolbar selection. | Drop the dead form path or comment it dormant; remove the unused GET vars; fix or remove the header badge meta. | +| F6 | trivial — stale docstrings | ✅ **fixed (review commit `9691776`)** | `providers/base.py` "yf_option_chain.py" → `yf_snapshot.py`; `providers/yf_client.py` listed `core/market/data_context.py` as an importer (B4 removed it); `providers/__init__.py` + `yfinance_provider.py` said `downloader.py` (it is `ingest/ohlcv.py` since B3); §0 ledger planning-row status. | done | +| F7 | housekeeping | 🔨 partial | (a) `providers/yf_client.py` "one-release" shim has no tracked removal trigger. (b) `market_review_prices` (L5) is still a parallel acquisition path outside the seam. (c) Branch is ahead of `origin/main`, unmerged. | (a)+(b) **added to `architecture_review.md` §2 watch list** (review commit); (c) push + merge still pending. | diff --git a/services/market/dispatch.py b/services/market/dispatch.py index 9fbddb1..f56fb7a 100644 --- a/services/market/dispatch.py +++ b/services/market/dispatch.py @@ -60,6 +60,20 @@ _RETRY_DELAY_SECONDS = 3 +def _params_variant(module_params: dict[str, Any]) -> str: + """Stable digest of a module's own params for the job-cache memo key. + + WHY: ``compute_or_get`` memoises per ``(ticker, kind)``; without folding the + toolbar params into the key, changing ``frequency`` / the horizon re-fires + ``/render/`` but the cache serves the first render for the job's TTL + (batch B9 / plan §10 F1). Empty params ⇒ empty digest ⇒ same key as the + direct-URL / legacy path. + """ + if not module_params: + return "" + return "|".join(f"{k}={module_params[k]}" for k in sorted(module_params)) + + def render_readiness_fragment(kind: str, job_id: str, ticker: str) -> tuple[str, int]: """Fragment that says "preparing data" and re-issues its own request. @@ -172,7 +186,9 @@ def _compute(form_data: dict[str, Any]) -> dict[str, Any]: # No job in cache — compute directly with the synthetic form. result = _compute(fallback_form) else: - result = compute_or_get(job_id, ticker, kind, _compute) + # Memo key folds in the toolbar params so a frequency/horizon + # change actually recomputes instead of replaying the first render. + result = compute_or_get(job_id, ticker, kind, _compute, variant=_params_variant(module_params)) except KeyError: return render_error_fragment(kind, "session expired", 200) except Exception as e: diff --git a/static/parametersBar.js b/static/parametersBar.js index 598084e..19dda8c 100644 --- a/static/parametersBar.js +++ b/static/parametersBar.js @@ -36,8 +36,9 @@ var toggle = document.getElementById('parameters-bar-toggle'); if (!toggle) return; toggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true'); - var icon = toggle.querySelector('i'); - if (icon) icon.className = collapsed ? 'fas fa-chevron-right' : 'fas fa-chevron-down'; + // The chevron is an inline SVG rotated by CSS via [data-collapsed]; only + // the label needs updating here. + toggle.title = collapsed ? 'Expand parameters' : 'Collapse parameters'; } function updateSummary(bar) { diff --git a/static/styles.css b/static/styles.css index f7d5fb8..340c311 100644 --- a/static/styles.css +++ b/static/styles.css @@ -3201,6 +3201,8 @@ textarea:focus-visible, } .parameters-bar-toggle { + display: inline-flex; + align-items: center; background: transparent; border: 1px solid var(--slate-200); border-radius: var(--radius-sm); @@ -3210,6 +3212,23 @@ textarea:focus-visible, line-height: 1; } +.parameters-bar-chevron { + display: block; + transition: transform .15s ease; +} + +.parameters-bar[data-collapsed="true"] .parameters-bar-chevron { + transform: rotate(-90deg); +} + +/* The collapsible region: label + ticker input + validation badges. */ +.parameters-bar-fields { + display: flex; + align-items: center; + gap: 10px; + min-width: 0; +} + .parameters-bar-label { color: var(--slate-700); font-size: 12px; @@ -3234,9 +3253,10 @@ textarea:focus-visible, font-variant-numeric: tabular-nums; } -/* Collapsed → hide the settings body; expanded → the input already shows the - ticker, so the one-line summary is redundant. */ -.parameters-bar[data-collapsed="true"] .parameters-bar-body { +/* Collapsed → only the toggle, the one-line summary and Run remain. Expanded → + the input already shows the ticker, so the summary is redundant. */ +.parameters-bar[data-collapsed="true"] .parameters-bar-fields, +.parameters-bar[data-collapsed="true"] .ticker-validation { display: none; } @@ -3244,21 +3264,6 @@ textarea:focus-visible, display: none; } -.parameters-bar-body { - border-top: 1px solid var(--slate-200); - padding-top: 10px; -} - -.parameters-bar-group-title { - display: block; - margin-bottom: 6px; - color: var(--slate-500); - font-size: 11px; - font-weight: 600; - letter-spacing: .04em; - text-transform: uppercase; -} - .parameters-bar-alert { margin: 0; } diff --git a/templates/partials/parameters_bar.html b/templates/partials/parameters_bar.html index 8d8b427..db2ad05 100644 --- a/templates/partials/parameters_bar.html +++ b/templates/partials/parameters_bar.html @@ -18,17 +18,29 @@ - - - {# Collapsed summary: the bar collapses to `▸ ^SPX` (see parametersBar.js) #} + {# Collapsed summary: the bar collapses to `▸ ^SPX` (see parametersBar.js). + Rendered before the fields so it takes their place when they hide. #} -
+ + {# The collapsible region (aria-controls target). Collapsed → hidden, + leaving only the toggle, the summary and Run. #} +
+ + +
+
diff --git a/tests/test_job_cache.py b/tests/test_job_cache.py index d036ae4..b7dc88c 100644 --- a/tests/test_job_cache.py +++ b/tests/test_job_cache.py @@ -67,6 +67,25 @@ def fn_b(_): assert jc.compute_or_get(job_id, "AAPL", "kind_b", fn_b) == "B" assert calls == {"a": 1, "b": 1} + def test_variant_computes_independently(self): + """Same ticker + kind, different `variant` (a digest of the module's + toolbar params) must not share a cache entry — otherwise a frequency / + horizon change replays the first render for the whole job TTL.""" + job_id = jc.create_job({}, ["AAPL"]) + calls = [] + + def fn(_): + calls.append(1) + return {"n": len(calls)} + + r1 = jc.compute_or_get(job_id, "AAPL", "stat", fn, variant="frequency=ME") + r2 = jc.compute_or_get(job_id, "AAPL", "stat", fn, variant="frequency=W") + r1_again = jc.compute_or_get(job_id, "AAPL", "stat", fn, variant="frequency=ME") + assert r1 == {"n": 1} + assert r2 == {"n": 2} + assert r1_again == {"n": 1} # cached per variant + assert len(calls) == 2 + def test_unknown_job_raises(self): with pytest.raises(KeyError): jc.compute_or_get("nope", "AAPL", "stat", lambda _: None) diff --git a/tests/test_module_params.py b/tests/test_module_params.py index 6700b18..e3d8c8a 100644 --- a/tests/test_module_params.py +++ b/tests/test_module_params.py @@ -141,3 +141,38 @@ def test_without_query_params_the_job_values_survive(self, monkeypatch): captured = self._render("statistical", "", monkeypatch) assert captured["frequency"] == "ME" assert captured["parsed_start_time"] == dt.date(2020, 1, 1) + + def test_a_param_change_recomputes_within_the_same_job(self, monkeypatch): + """Plan §10 F1: the job-cache slice memo is keyed by (ticker, kind); a + toolbar change re-fires ``/render/`` with new query args, so the + memo must fold them in or it replays the first render for the job TTL. + """ + from app import app + + from data_pipeline.orchestrate import job_cache + from services.market.analysis import AnalysisService + from services.market.dispatch import render_streaming_slice + + seen: list[str] = [] + + def _fake_slice(form_data): + seen.append(form_data.get("frequency")) + return {} + + monkeypatch.setattr(AnalysisService, "generate_statistical_slice", staticmethod(_fake_slice), raising=False) + + job_cache._reset() + job_id = job_cache.create_job( + {"ticker": "AAPL", "frequency": "ME", "parsed_start_time": dt.date(2020, 1, 1), "parsed_end_time": None}, + ["AAPL"], + ) + base = f"/render/statistical?job={job_id}&ticker=AAPL&from=2020-01&to=2024-01" + with app.test_request_context(f"{base}&frequency=ME"): + render_streaming_slice("statistical") + with app.test_request_context(f"{base}&frequency=W"): + render_streaming_slice("statistical") + # A repeat of an already-computed variant still hits the cache. + with app.test_request_context(f"{base}&frequency=ME"): + render_streaming_slice("statistical") + + assert seen == ["ME", "W"], seen diff --git a/tests/unit/parametersBar.test.js b/tests/unit/parametersBar.test.js index 714dccc..7fcd3ee 100644 --- a/tests/unit/parametersBar.test.js +++ b/tests/unit/parametersBar.test.js @@ -11,12 +11,15 @@ import { loadScript } from './_loadScript.js'; const BAR_HTML = ` - - -
+
+ + +
`; function bar() { @@ -48,7 +51,15 @@ describe('parametersBar — collapse persistence', () => { expect(bar().dataset.collapsed).toBe('true'); expect(window.localStorage.getItem('parametersBarCollapsed')).toBe('true'); expect(document.getElementById('parameters-bar-toggle').getAttribute('aria-expanded')).toBe('false'); - expect(document.querySelector('#parameters-bar-toggle i').className).toBe('fas fa-chevron-right'); + expect(document.getElementById('parameters-bar-toggle').title).toBe('Expand parameters'); + }); + + it('has a real element behind aria-controls (the collapsible fields)', () => { + mount(); + const toggle = document.getElementById('parameters-bar-toggle'); + const target = document.getElementById(toggle.getAttribute('aria-controls')); + expect(target).not.toBeNull(); + expect(target.contains(document.getElementById('ticker'))).toBe(true); }); it('restores the collapsed state on the next page load', () => { From e32e2b9164687ae8ded7aa5339857953e8ac2136 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Fri, 11 Sep 2026 02:00:54 +0800 Subject: [PATCH 13/15] =?UTF-8?q?docs(reorg):=20B9=20batch=20note=20?= =?UTF-8?q?=E2=80=94=E2=80=94=20e2e=20=E5=85=A8=E9=87=8F=20exit=200?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5 --- docs/plans/business_line_reorg.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index 699d1d9..54b2f77 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -741,8 +741,7 @@ not blockers, and each is one small independent change. - **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → exit 0 (+2 tests vs B8: `test_job_cache.py::test_variant_computes_independently`, `test_module_params.py::test_a_param_change_recomputes_within_the_same_job`); - relevant `pytest tests/e2e` (module_params, smoke, localstorage, form_submit, - streaming) → 20 passed; `npx vitest run` → 199 passed / 16 files (+1 + `pytest tests/e2e` → exit 0; `npx vitest run` → 199 passed / 16 files (+1 aria-controls case); `ruff check` + `format --check` clean; `doc_guard.py` clean; `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / dead 1); `audit_tags.py` 16 vs 16. From fea0e54698ea43a4c13352fbecf29ba36d653534 Mon Sep 17 00:00:00 2001 From: GradientDescent Date: Fri, 11 Sep 2026 02:21:26 +0800 Subject: [PATCH 14/15] =?UTF-8?q?fix(reorg):=20B9=20=E4=BA=8C=E8=BD=AE=20?= =?UTF-8?q?=E2=80=94=E2=80=94=20F4=20feature=5Fbars=20=E8=87=AA=E6=84=88?= =?UTF-8?q?=20+=20F5-b/c=20=E9=99=88=E6=97=A7=20UI=20=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F4（readiness feature_bars 缺口）: - backfill.py: _range_covers 抽公共 3 天容差；_feature_bars_behind 探针（frequency='D'，与 clean_bars 1:1） - needs_backfill: clean 覆盖但 feature 滞后时返回 True - _ensure_range_impl: clean 已覆盖的短路分支，feature 滞后则跑 process_frequencies（不下载）再 memo - read/_query.py::get_processed: 加 needs_backfill → kick + 宽限等待 自愈（同 get_cleaned_daily）；memo 守卫也拒绝 partial read - readiness.py docstring INVARIANT 更新 - 测试 TestFeatureBarsHeal（clean-only 探针为 True；ensure_range 零下载重建 feature） F5-b/c（B7 遗留陈旧 UI）: - routes/core.py GET 分支删除 B7 后无人读的 frequency/risk_threshold/ rolling_window/side_bias 模板变量与 import - index.html header badge 去掉 badge-meta（恒显 "Monthly, Neutral"），只留 ticker - market_review.html meta-bar 删掉 3 个显示常量的 chip，保留 ticker + 区间 - 4 处 fragment 空状态 + tab_simulation 占位符不再指向已删的 Parameter 页签 - form.py: option_data 加 DORMANT 注释（见 F5-a） F5-a 不是死代码，是 B7 回归（assessment.py 仍读 option_data 做 projection-vs-positions 叠加图 + sizing max-loss，B7 后永久断供、 静默降级）。retire vs re-feed 待定，已进 §2 watch list + 计划书 §10。 计划书 §10 补 B9 二轮 note + watch-list 权衡表；§2 watch list 加 F5-a 行。 full pytest exit 0；full e2e exit 0；ruff/doc_guard/arch_metrics clean。 Co-Authored-By: Claude Sonnet 5 --- data_pipeline/orchestrate/backfill.py | 71 +++++++++++++++---- data_pipeline/orchestrate/readiness.py | 7 +- data_pipeline/read/_query.py | 15 ++-- docs/architecture_review.md | 1 + docs/plans/business_line_reorg.md | 55 ++++++++++++-- routes/core.py | 15 ++-- services/market/form.py | 6 ++ templates/index.html | 4 +- templates/partials/fragments/assessment.html | 2 +- .../partials/fragments/market_review.html | 8 +-- .../partials/fragments/options_chain.html | 2 +- templates/partials/fragments/statistical.html | 2 +- templates/partials/tab_simulation.html | 2 +- tests/test_background_backfill.py | 60 ++++++++++++++++ 14 files changed, 203 insertions(+), 47 deletions(-) diff --git a/data_pipeline/orchestrate/backfill.py b/data_pipeline/orchestrate/backfill.py index 236be6f..5740160 100644 --- a/data_pipeline/orchestrate/backfill.py +++ b/data_pipeline/orchestrate/backfill.py @@ -20,13 +20,51 @@ _SENTINEL_MIN_DB_SPAN_DAYS = 365 -def needs_backfill(ticker: str, start: dt.date, end: dt.date) -> bool: - """Cheap probe: would ``ensure_range(ticker, start, end)`` need a download? +def _range_covers(cov, start: dt.date, end: dt.date) -> bool: + """True when a ``MIN(date)/MAX(date)/COUNT(*)`` coverage row spans [start, end]. + + Uses the same 3-day tail tolerance everywhere: the last few days may be a + weekend / not-yet-published, which is not a gap worth a download. + """ + if cov.empty or not cov.iloc[0]["n"]: + return False + try: + cmin = dt.date.fromisoformat(str(cov.iloc[0]["min_d"])) + cmax = dt.date.fromisoformat(str(cov.iloc[0]["max_d"])) + except (ValueError, TypeError): + return False + return cmin <= start and cmax >= end - dt.timedelta(days=3) + - Checks the memo and DB coverage only — never networks. Lets callers keep - wide-range backfills off the request thread. NOTE: the probe does not - model the sentinel short-circuit; a false positive there merely kicks a - background ``ensure_range`` that immediately short-circuits. +def _feature_bars_behind(ticker: str, start: dt.date, end: dt.date) -> bool: + """True when ``feature_bars`` does not cover [start, end] for *ticker*. + + WHY (plan §10 F4): ``ensure_range``'s clean-covered short-circuit and + ``needs_backfill`` used to probe ``clean_bars`` only, so a DB that has clean + rows but stale/missing ``feature_bars`` (a past ``process_frequencies`` + failure, or clean extended without a reprocess) was never healed — the + Statistical / Assessment slices read ``feature_bars`` and would render an + empty chart. ``process_frequencies`` writes D/W/ME/QE together and the D + series is 1:1 with ``clean_bars``, so the D-frequency span is the cheapest + honest probe. + """ + cov = _db.fetch_df( + "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n " + "FROM feature_bars WHERE ticker=? AND frequency='D'", + (ticker,), + ) + return not _range_covers(cov, start, end) + + +def needs_backfill(ticker: str, start: dt.date, end: dt.date) -> bool: + """Cheap probe: would ``ensure_range(ticker, start, end)`` need to do work? + + Checks the memo and DB coverage only — never networks. Returns True when + ``clean_bars`` is missing the span (needs a download) **or** ``feature_bars`` + lags behind clean (needs a reprocess only). Lets callers keep both kinds of + catch-up off the request thread. NOTE: the probe does not model the sentinel + short-circuit; a false positive there merely kicks a background + ``ensure_range`` that immediately short-circuits. """ now = time.monotonic() with _ensure_range_lock: @@ -39,14 +77,9 @@ def needs_backfill(ticker: str, start: dt.date, end: dt.date) -> bool: "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_bars WHERE ticker=?", (ticker,), ) - if cov.empty or not cov.iloc[0]["n"]: + if not _range_covers(cov, start, end): return True - try: - existing_min = dt.date.fromisoformat(str(cov.iloc[0]["min_d"])) - existing_max = dt.date.fromisoformat(str(cov.iloc[0]["max_d"])) - except (ValueError, TypeError): - return True - return not (existing_min <= start and existing_max >= end - dt.timedelta(days=3)) + return _feature_bars_behind(ticker, start, end) def ensure_range(ticker: str, start: dt.date, end: dt.date) -> bool: @@ -121,6 +154,18 @@ def _ensure_range_impl(ticker: str, start: dt.date, end: dt.date, now: float, wa existing_min = existing_max = None if existing_min is not None and existing_min <= start and existing_max >= end - dt.timedelta(days=3): + # clean_bars covers the span — no download. But features may still lag + # (a past processing failure, or clean extended without a reprocess); + # rebuild them here so the memo below is honest (plan §10 F4). + if _feature_bars_behind(ticker, start, end): + logger.info("ensure_range: %s clean covered but feature_bars behind — reprocessing", ticker) + pr = _pr.process_frequencies(ticker, start, end) + if not pr.ok: + logger.warning("ensure_range reprocess failed for %s: %s", ticker, pr.error) + return False + from data_pipeline import _state as _g + + _g._cache_invalidate(ticker) with _ensure_range_lock: _ensure_range_memo[ticker] = (now, start, end) return True diff --git a/data_pipeline/orchestrate/readiness.py b/data_pipeline/orchestrate/readiness.py index b74665c..364ff3d 100644 --- a/data_pipeline/orchestrate/readiness.py +++ b/data_pipeline/orchestrate/readiness.py @@ -16,9 +16,10 @@ on a daemon thread. ``check_and_kick`` therefore cannot block for more than a probe, so ``POST /`` still returns the skeleton in < 1 s. - INVARIANT: one pipeline run (download → clean → process) fills both - ``clean_bars`` and ``feature_bars``, so coverage is gated on the - ``clean_bars`` probe and a single kick per (ticker, range) covers every - dataset the plan asked for. + ``clean_bars`` and ``feature_bars``, so a single kick per (ticker, range) + covers every dataset the plan asked for. The probe (``backfill.needs_backfill``) + checks *both* families — clean coverage **and** a feature-bars lag — so a DB + with clean rows but stale features is still healed (plan §10 F4). Contracts: - ``plan_datasets(tickers, modules, *, start, end, today=None)`` - ``check_and_kick(plan, *, kick=None)`` diff --git a/data_pipeline/read/_query.py b/data_pipeline/read/_query.py index 05ac146..d824dc0 100644 --- a/data_pipeline/read/_query.py +++ b/data_pipeline/read/_query.py @@ -86,16 +86,21 @@ def get_processed( if cached is not None: return cached _u.manual_update(ticker, days=7) + if _bf.needs_backfill(ticker, start, end): + # A clean gap, or clean present but feature_bars lagging (plan §10 F4): + # heal off the request thread with a short grace wait, then read + # whatever coverage exists — same pattern as get_cleaned_daily. + _kick_backfill(ticker, start, end) + _wait_for_coverage(ticker, start, end, _BACKFILL_WAIT_SECONDS) init_db() df = fetch_df( "SELECT * FROM feature_bars WHERE ticker=? AND frequency=? AND date>=? AND date<=?", (ticker, frequency, start.isoformat(), end.isoformat()), ) - # Never memoise an empty read: a not-yet-generated frequency/range would - # otherwise be pinned for _QUERY_CACHE_TTL and hide the data once the - # processing pass completes (mirrors the partial-read guard in - # get_cleaned_daily above). - if not df.empty: + # Never memoise an empty or partial read: a not-yet-generated frequency/range + # would otherwise be pinned for _QUERY_CACHE_TTL and hide the data once the + # processing pass completes (mirrors the guard in get_cleaned_daily above). + if not df.empty and not _bf.needs_backfill(ticker, start, end): _g._cache_set(cache_key, df) return df diff --git a/docs/architecture_review.md b/docs/architecture_review.md index eeded21..2b0d373 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -81,6 +81,7 @@ Rescoped in batch B1 of [ADR 0011](decisions/0011-pluggable-data-provider-seam.m | `static/sim/black_scholes.js` + `core/options/greeks` (risk-free rate) | the same constant is hard-coded in the client simulation **and** the server-side Greeks; changing one without the other makes the two pricings diverge silently | batch B8 retired the Config tab, so there is no global-setting surface to put it in; making it a parameter (store entry + form field + backend path) is a small feature — do it before anyone edits either constant | | `data_pipeline/providers/yf_client.py` (compat shim, ADR 0011 B1) | re-exports `fetch_spot` / `fetch_option_chain` / `fetch_close_panel` / `fetch_daily_ohlcv` with their old yfinance-shaped contracts; it was a **"one release"** bridge so importers did not have to change in the B1 PR | once `grep -rn "providers.yf_client\|yf_client import" services/ data_pipeline/read/` is empty (importers moved to the canonical `providers.get_provider()` shapes), delete `yf_client.py` and its re-exports from `providers/__init__.py` in one commit | | `services/market_review/fetch.py` → `market_review_prices` (L5 in plan §4.1) | a second acquisition path outside the provider seam: its own L1/L2/L3 close-panel ladder writes a `market_review_prices` table that `data_pipeline/orchestrate/readiness.py` does **not** plan, so the benchmark panel still lazy-fetches on the Market Review slice | fold the ladder into `providers` + a canonical `bars` read (ADR 0011's L5 exit); until then, add `market_review` benchmark tickers to `KIND_DATASETS` so the readiness pass warms them | +| `services/market/analysis/assessment.py` option overlay (`form_data["option_data"]`) — **B7 regression, plan §10 F5-a** | batch B7 moved option positions to the Portfolio tab and dropped `option_position` from `POST /`, so the projection-vs-positions overlay chart and the sizing max-loss-per-contract are permanently unfed (they degrade silently — no error, chart absent, sizing assumes debit) | **retire** — delete the `option_data` branches in `assessment.py`, `FormService.parse_option_data`, and `core/market/{analyzer.analyze_options, option_pnl.py, charts/option_pnl.py}` if unused elsewhere (the Portfolio tab owns position P&L now) — **or** re-feed it via an Assessment-toolbar positions handle. Leaning retire; needs an explicit call | ## 3. Guardrails (how the score is kept) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index 54b2f77..fd44720 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -48,7 +48,7 @@ | B6 — `ticker`-only Parameters bar | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q3 resolved (dedicated Portfolio tab); bar + collapse persisted; transitional settings group inside the bar's form until B7. Actuals in §8 | | B7 — module-scoped params | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (manifest). Backend query-arg contract + per-module allow-list; `state/*ParamsState.js`; module toolbars; bridge + hidden fields deleted; Config tab emptied (B8 decides its fate). See §8 B7 | | B8 — retire / repurpose Config tab | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q2 resolved (**deleted**). `grep tab_config` returns nothing; risk-free-rate follow-up on the watch list | -| B9 — acceptance-review remediation | 🔨 F1–F3+F6 landed | — | branch `worktree-business-line-reorg` · 2026-09-11 | §10 review. **F1** (memo `variant` key), **F2** (real collapse target + hide fields), **F3** (inline-SVG chevron), **F6** (docstrings) done; F4/F5/F7c open | +| B9 — acceptance-review remediation | 🔨 F1–F4, F5-b/c, F6 landed | — | branch `worktree-business-line-reorg` · 2026-09-11 | §10 review. F1 (memo `variant` key), F2 (real collapse), F3 (SVG chevron), F4 (feature_bars self-heal), F5-b/c (stale UI), F6 (docstrings) done. **Open**: F5-a (retire-vs-refeed decision), F7c (merge to main). Watch-list: `summary.py` delete recommended, risk-free-rate + L5 deferred | States: `⬜ not started` → `🔨 in progress (PR #n)` → `✅ landed` → (`↩ reverted`). Keep the row order; edit the row in place. @@ -738,7 +738,7 @@ not blockers, and each is one small independent change. - **F6 — stale docstrings** (already in review commit `9691776`): `providers/base.py`, `providers/yf_client.py`, `providers/__init__.py`, `providers/yfinance_provider.py`, and the §0 ledger planning-row status. -- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → exit 0 +- **Exit criteria (F1–F3, F6)**: `pytest -m "not network" --ignore=tests/e2e` → exit 0 (+2 tests vs B8: `test_job_cache.py::test_variant_computes_independently`, `test_module_params.py::test_a_param_change_recomputes_within_the_same_job`); `pytest tests/e2e` → exit 0; `npx vitest run` → 199 passed / 16 files (+1 @@ -746,6 +746,53 @@ not blockers, and each is one small independent change. clean; `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / dead 1); `audit_tags.py` 16 vs 16. +**B9 — second pass (F4, F5-b/c), 2026-09-11.** + +- **F4 — feature_bars coverage gap.** `orchestrate/backfill.py`: new `_range_covers` + helper (shared 3-day tolerance) and `_feature_bars_behind(ticker, start, end)` + (probes `feature_bars` frequency='D', which is 1:1 with `clean_bars`). + `needs_backfill` now returns True when clean covers the span **but** features + lag; `_ensure_range_impl`'s clean-covered short-circuit runs + `process_frequencies` (no download) before memoising. `read/_query.py::get_processed` + gained the same `needs_backfill` → kick + grace-wait self-heal as + `get_cleaned_daily`, and its memo guard now also refuses a partial read. + `readiness.py` docstring INVARIANT updated. Tests: `TestFeatureBarsHeal` + (`needs_backfill` true on clean-only; `ensure_range` reprocesses with zero + downloads and clears the probe). +- **F5-b/c + stale refs.** `market_review.html` meta-bar drops the three chips + that showed constants after B7 (`frequency`, `Threshold`, `side_bias`) — keeps + ticker + horizon; "Required vars" comment trimmed. `index.html` header badge + drops `badge-meta` (no single frequency/side-bias exists per-page any more) — + ticker only. `routes/core.py` GET branch drops the now-unused + `frequency`/`risk_threshold`/`rolling_window`/`side_bias` template vars and + their imports. Four fragment empty-states + `tab_simulation.html`'s placeholder + stop pointing at the deleted "Parameter" tab. `form.py` gets a DORMANT note on + `option_data` (see F5-a below). +- **F5-a is NOT dead code — it is a B7 regression, left for a decision.** + `services/market/analysis/assessment.py` still reads `form_data["option_data"]` + for (i) the projection-vs-positions option overlay chart and (ii) the sizing + max-loss-per-contract. B7 moved positions to the Portfolio tab and stopped + `POST /` carrying `option_position`, so that overlay is now **permanently + unfed** (it degrades gracefully — no error, the chart is just absent) and + sizing always assumes a debit strategy. Decision needed: **retire** the overlay + (delete the `option_data` branches in `assessment.py`, `parse_option_data`, + `core/market/{analyzer.analyze_options, option_pnl, charts/option_pnl}` if + unused elsewhere) — the Portfolio tab is the home for position P&L now — or + **re-feed** it (give the Assessment toolbar a positions handle). Leaning + retire. Tracked on `architecture_review.md` §2 watch list. +- **Exit criteria (F4, F5)**: `pytest -m "not network" --ignore=tests/e2e` → exit + 0 (+2: `TestFeatureBarsHeal`); `pytest tests/e2e` → exit 0; `ruff` + + `doc_guard` + `arch_metrics --check` clean. + +### Watch-list follow-ups — 2026-09-11 assessment + +| Item | Call | Why | +|---|---|---| +| **Risk-free rate** hard-coded (`r = 0.05` default in `static/sim/{analyze,stats}.js`, `core/options/greeks/portfolio.py`, `grid.js` `r_pct=5`) → true global setting | **Defer the setting; do the cheap consolidation if wanted.** | A *user-facing* setting needs a surface — B8 deliberately deleted the Config tab, so this means re-introducing one for a number that moves ~quarterly. The silent-divergence risk is cheaply killed by one shared constant (`utils/constants.RISK_FREE_RATE` + a `static/sim/` mirror + a parity test) without any UI. Full setting = wait until someone actually wants to tweak it. | +| **`market_review_prices` (L5)** → fold into provider seam | **Defer to its own batch (B10).** | It is ADR 0011's stated L5 exit and it would let readiness prefetch the benchmark panel (closing the other half of F4's spirit), but it is a real refactor: benchmark tickers (SPY/QQQ/…) would route through `ensure_range`/`clean_bars` instead of the close-only ladder. ~1 day + tests. Not a bundle-in. | +| **ADR 0011 `symbol` column** (ticker→symbol rename) | **Agree — stay deferred.** | Pure churn with one provider (`ticker == symbol` for yfinance). Do it *with* the second provider, when the mapping actually has two shapes to reconcile. | +| **`services/market/analysis/summary.py`** (fan-in 0, `dead_code_candidates=1`) | **Delete now** (bundle into B9). | `summary_data` is never set anywhere; `tab_summary.html`, the correlation-heatmap JS, and the `summary_pending` flag are all vestigial. Building a real multi-ticker Summary tab is a feature nobody has asked for. Deleting the module + template + flag + sidebar button clears the standing `dead_code` finding. Pure removal, low risk. | + --- ## 9. References @@ -778,8 +825,8 @@ Findings worked as **batch B9 (remediation)** under the §0 rules. | F1 | **high — correctness, CONFIRMED** | ✅ **fixed (B9)** | `job_cache.compute_or_get` memoised per `(ticker, kind)`; `dispatch.py::_compute` captured `module_params` from the query string but they were **not in the key** and nothing invalidated on change. Reproduced: two `/render/statistical` calls on one job, `frequency=ME` then `=W` → slice invoked once, both response bodies byte-identical. B7's headline behaviour ("changing a module control re-runs that module") re-fired the request and flashed "Updating…" but served the stale fragment for up to `JOB_CACHE_TTL` (90 s). `tests/e2e/test_module_params.py` missed it — it asserts request **URLs**, never fragment **content**. | `compute_or_get(..., *, variant="")` — key is now `(ticker, kind, variant)`; `dispatch._params_variant(module_params)` builds a sorted digest. New tests: `test_job_cache.py::test_variant_computes_independently`, `test_module_params.py::test_a_param_change_recomputes_within_the_same_job`. | | F2 | moderate — UX / a11y | ✅ **fixed (B9)** | The Parameters bar "collapse" was hollow after B7: `data-collapsed="true"` hid a non-existent `.parameters-bar-body` and revealed `▸ ^SPX` **beside the still-visible ticker input + label + Run** → ~zero visible effect; `aria-controls="parameters-bar-body"` was a dangling reference. | The label + input + badges are now wrapped in `
` (real `aria-controls` target); collapsed hides that div **and** `.ticker-validation`, leaving toggle + `▸ ^SPX` + Run on one line. Dead `.parameters-bar-body` / `-group-title` CSS removed. New vitest: "has a real element behind aria-controls". | | F3 | minor — UX | ✅ **fixed (B9)** | Collapse toggle icon was invisible — `` with Font Awesome not loaded and no CSS fallback. | Inline SVG chevron (`.parameters-bar-chevron`) rotated `-90°` by `.parameters-bar[data-collapsed="true"]`; `parametersBar.js` drops the `` class swap and updates `title` instead. | -| F4 | minor — readiness gap | ⬜ open | `readiness.check_and_kick` gates coverage on the `clean_bars` probe (`needs_backfill`). The "one pipeline run fills both" INVARIANT holds for a fresh backfill, but a DB with `clean_bars` for the range yet stale/missing `feature_bars` (new frequency, or processing failed after cleaning) → probe says "covered", no kick, `statistical`/`assessment` fall back to the per-slice `get_processed` → `manual_update` path. Fallback works, prefetch promise unmet. | `feature_bars`-aware probe, or a `WHY:` note in `readiness.py` accepting the gap. | -| F5 | minor — dead code / stale UI | ⬜ open | (a) `form.py::parse_option_data` + `form_data["option_data"]`: FormManager no longer submits `#option_position` (B7) and no slice reads `option_data` — always `[]` on the POST path. (b) `routes/core.py` GET branch still passes `frequency`/`risk_threshold`/`rolling_window`/`side_bias` to `render_template`. (c) `index.html:112` header badge renders `{{ frequency_display or frequency }}, {{ side_bias }}` — after B7 these are always the POST-time defaults, so the badge shows "Monthly, Neutral" regardless of the toolbar selection. | Drop the dead form path or comment it dormant; remove the unused GET vars; fix or remove the header badge meta. | +| F4 | minor — readiness gap | ✅ **fixed (B9)** | `needs_backfill` probed `clean_bars` only, so a DB with clean rows but stale/missing `feature_bars` (a past `process_frequencies` failure, or clean extended without a reprocess) was never healed — Statistical/Assessment read `feature_bars`. `process_frequencies` writes D/W/ME/QE together so a *new* frequency is not the trigger; the partial-failure state is. | `backfill._feature_bars_behind` (probe frequency='D', 1:1 with clean); `needs_backfill` returns True on feature lag; `_ensure_range_impl` reprocesses (no download) on the clean-covered short-circuit; `get_processed` self-heals like `get_cleaned_daily`. Tests: `TestFeatureBarsHeal`. | +| F5 | mixed | 🔨 **b/c fixed (B9); a is a decision** | **(a) not dead code — a B7 regression:** `assessment.py` reads `form_data["option_data"]` (projection-vs-positions overlay + sizing max-loss); B7 stopped `POST /` carrying positions, so the overlay is permanently unfed (degrades gracefully). **(b/c) stale UI:** `routes/core.py` GET vars + `index.html` `badge-meta` + `market_review.html` meta-chips all showed POST-time defaults after B7 (`badge` always "Monthly, Neutral"). | **(a)** DORMANT note added; retire-vs-refeed decision open (§10 B9 note, §2 watch list) — leaning retire. **(b/c)** GET vars + imports removed; `badge-meta` dropped (ticker only); 3 stale `market_review` chips removed; 4 "Parameter tab" empty-state refs + `tab_simulation` placeholder fixed. | | F6 | trivial — stale docstrings | ✅ **fixed (review commit `9691776`)** | `providers/base.py` "yf_option_chain.py" → `yf_snapshot.py`; `providers/yf_client.py` listed `core/market/data_context.py` as an importer (B4 removed it); `providers/__init__.py` + `yfinance_provider.py` said `downloader.py` (it is `ingest/ohlcv.py` since B3); §0 ledger planning-row status. | done | | F7 | housekeeping | 🔨 partial | (a) `providers/yf_client.py` "one-release" shim has no tracked removal trigger. (b) `market_review_prices` (L5) is still a parallel acquisition path outside the seam. (c) Branch is ahead of `origin/main`, unmerged. | (a)+(b) **added to `architecture_review.md` §2 watch list** (review commit); (c) push + merge still pending. | diff --git a/routes/core.py b/routes/core.py index 395dd71..a5da849 100644 --- a/routes/core.py +++ b/routes/core.py @@ -24,13 +24,7 @@ from services.market.form import FormService from services.market.readiness import prepare_readiness from services.market.validation import ValidationService -from utils.constants import ( - DEFAULT_FREQUENCY, - DEFAULT_RISK_THRESHOLD, - DEFAULT_ROLLING_WINDOW, - DEFAULT_SIDE_BIAS, - DEFAULT_TICKER, -) +from utils.constants import DEFAULT_TICKER from utils.ticker_utils import normalize_ticker, parse_tickers logger = logging.getLogger(__name__) @@ -108,6 +102,9 @@ def index(): } return render_template("index.html", **template_data) + # The GET skeleton only needs the ticker + horizon the Parameters bar + # renders; every analysis knob is per-module (batch B7) and hydrated + # client-side from its own store. return render_template( "index.html", ticker=DEFAULT_TICKER, @@ -115,10 +112,6 @@ def index(): tickers_raw=DEFAULT_TICKER, start_time=(lambda today: f"{today.year - 5}-{today.month:02d}")(dt.date.today()), end_time="", - frequency=DEFAULT_FREQUENCY, - risk_threshold=DEFAULT_RISK_THRESHOLD, - rolling_window=DEFAULT_ROLLING_WINDOW, - side_bias=DEFAULT_SIDE_BIAS, ) except Exception as e: diff --git a/services/market/form.py b/services/market/form.py index c79e70f..88a1c59 100644 --- a/services/market/form.py +++ b/services/market/form.py @@ -155,6 +155,12 @@ def extract_form_data(request): rolling_window = DEFAULT_ROLLING_WINDOW side_bias = request.form.get("side_bias", DEFAULT_SIDE_BIAS) target_bias = None if side_bias == "Natural" else 0 + # DORMANT (plan §10 F5): batch B7 moved option positions to the Portfolio + # tab, so `POST /` no longer carries `option_position` and this is always + # []. `assessment.py` still *reads* `option_data` (the projection-vs- + # positions overlay + sizing max-loss); that overlay is currently unfed — + # retire vs. re-feed is an open decision. Kept wired so re-adding a POST + # positions field would just work. option_data = FormService.parse_option_data(request) # Position sizing (optional) diff --git a/templates/index.html b/templates/index.html index 84878d2..b8b4d84 100644 --- a/templates/index.html +++ b/templates/index.html @@ -107,9 +107,11 @@

Market Dashboard

{% if ticker %} + {# Analysis parameters are per-module now (batch B7): frequency / + horizon / side-bias differ per tab, so there is no single + value to show here. The badge is just the active ticker. #}
{{ ticker }} - {{ frequency_display or frequency }}, {{ side_bias or 'Natural' }}
{% endif %} - {% if summary_data is defined and summary_data %} - {% include 'partials/tab_summary.html' %} - {% endif %}
@@ -384,14 +374,6 @@

Market Dashboard

startAutoRefresh(); } })(); - - {% if summary_data is defined and summary_data and summary_data.correlation_matrix %} - // Render correlation heatmap from server data - (function() { - var corrData = {{ summary_data.correlation_matrix | tojson }}; - if (corrData) renderCorrelationHeatmap(corrData); - })(); - {% endif %} diff --git a/templates/partials/fragments/assessment.html b/templates/partials/fragments/assessment.html index d768b95..9e1351f 100644 --- a/templates/partials/fragments/assessment.html +++ b/templates/partials/fragments/assessment.html @@ -1,9 +1,9 @@ {# Fragment for /render/assessment. - Required vars: ticker, feat_projection_url, feat_projection_table, plot_url, + Required vars: ticker, feat_projection_url, feat_projection_table, position_sizing, assessment_error. #}
- {% if feat_projection_url is defined or plot_url is defined %} + {% if feat_projection_url is defined %}
{% if feat_projection_url %}
@@ -21,15 +21,6 @@
{% endif %} - {% if plot_url %} -
-
Options Portfolio P&L Analysis
-
- Options P&L Chart -
-
- {% endif %} - {% if position_sizing %}
Position Sizing
diff --git a/templates/partials/tab_summary.html b/templates/partials/tab_summary.html deleted file mode 100644 index 605fbb4..0000000 --- a/templates/partials/tab_summary.html +++ /dev/null @@ -1,52 +0,0 @@ - diff --git a/tests/e2e/test_smoke.py b/tests/e2e/test_smoke.py index 2854bdc..459da2e 100644 --- a/tests/e2e/test_smoke.py +++ b/tests/e2e/test_smoke.py @@ -1,4 +1,4 @@ -"""Smoke tests: page loads, no JS errors, all 11 tabs render and switch.""" +"""Smoke tests: page loads, no JS errors, all 10 tabs render and switch.""" from __future__ import annotations @@ -10,7 +10,6 @@ # All sidebar tab IDs (must match `data-tab` values in templates/index.html). TAB_IDS = [ "tab-portfolio", - "tab-summary", "tab-market-review", "tab-statistical-analysis", "tab-market-assessment", @@ -32,18 +31,14 @@ def test_index_loads_without_js_errors(page: Page, live_server: str, mock_apis, expect(page.locator("#analysis-form")).to_be_visible() expect(page.locator("#ticker")).to_have_count(1) - # tab-summary only renders for multi-ticker; skip in single-ticker default GET. - for tab_id in [tid for tid in TAB_IDS if tid != "tab-summary"]: + for tab_id in TAB_IDS: button = page.locator(f'.tab-btn[data-tab="{tab_id}"]') expect(button).to_have_count(1) assert js_errors == [], f"JS errors on initial load: {js_errors}" -@pytest.mark.parametrize( - "tab_id", - [tid for tid in TAB_IDS if tid != "tab-summary"], # summary is hidden when single-ticker -) +@pytest.mark.parametrize("tab_id", TAB_IDS) def test_tab_switch_activates_panel( page: Page, live_server: str, mock_apis, js_errors: list[str], open_tab, tab_id: str ) -> None: