diff --git a/.github/agents/pipeline-doctor.agent.md b/.github/agents/pipeline-doctor.agent.md index 1730d48..4dd140f 100644 --- a/.github/agents/pipeline-doctor.agent.md +++ b/.github/agents/pipeline-doctor.agent.md @@ -9,9 +9,9 @@ You are a data pipeline diagnostician for the OptionView project. Your job is to ## Architecture ``` -data_pipeline/downloader.py → raw_prices table -data_pipeline/cleaning.py → clean_prices table -data_pipeline/processing.py → processed_prices table +data_pipeline/ingest/ohlcv.py → raw_bars table +data_pipeline/transform/cleaning.py → clean_bars table +data_pipeline/transform/processing.py → feature_bars table core/price_dynamic.py → features DataFrame core/market_analyzer.py → chart generation services/market/analysis/facade.py → base64 images to frontend @@ -26,8 +26,8 @@ services/market/analysis/facade.py → base64 images to frontend ## Approach 1. **Clarify symptom**: What's the user seeing? Empty chart, wrong data, error message? -2. **Check DB tables** (raw_prices → clean_prices → processed_prices) for the target ticker -3. **Look for NaN-only rows**: `SELECT count(*) FROM raw_prices WHERE ticker=? AND open IS NULL AND close IS NULL` +2. **Check DB tables** (raw_bars → clean_bars → feature_bars) for the target ticker +3. **Look for NaN-only rows**: `SELECT count(*) FROM raw_bars WHERE ticker=? AND open IS NULL AND close IS NULL` 4. **Check logs**: Look for yfinance errors (429, timeout), "No new data", pipeline warnings 5. **Trace the failure**: Which stage first produced invalid data? Follow downstream 6. **Check connectivity**: If download is suspected, verify proxy and throttle state diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 5d8e7b1..d142f6d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -25,7 +25,7 @@ Flask-based market analysis dashboard with options strategy tools. ## Database -- SQLite via `data_pipeline/db.py` — always use `get_conn()` context manager +- SQLite via `data_pipeline/store/db.py` — always use `get_conn()` context manager - WAL mode enabled; `PRAGMA synchronous=NORMAL` - DB path from `MARKET_DB_PATH` env var, default `./market_data.sqlite` @@ -59,7 +59,7 @@ gunicorn app:app -b 0.0.0.0:5000 # production Before suggesting non-trivial changes, consult these: -- **⚠ [docs/plans/business_line_reorg.md](../docs/plans/business_line_reorg.md)** — **active reorg** of `data_pipeline/`, the parameter surfaces, and `routes/core.py::index`. ADRs [0011](../docs/decisions/0011-pluggable-data-provider-seam.md) / [0012](../docs/decisions/0012-parameter-ownership-and-prefetch.md) are **Accepted**. Read §0 first: work batches B1–B8 in order, one batch per PR, update the ledger in the same commit, don't re-litigate the Accepted ADRs. +- **[docs/plans/business_line_reorg.md](../docs/plans/business_line_reorg.md)** — the 2026-09 business-line reorg (provider seam + canonical schema, ticker-only Parameters bar, readiness prefetch). Batches **B1–B9 landed**; §10 lists the deferred follow-ups. ADRs [0011](../docs/decisions/0011-pluggable-data-provider-seam.md) / [0012](../docs/decisions/0012-parameter-ownership-and-prefetch.md) are **Accepted** — don't re-litigate. - **[docs/constraints.md](../docs/constraints.md)** — external/historical constraints (yfinance limits, SQLite choice, single-machine assumption, intentional "magic numbers"). Read this before flagging anything as tech debt. - **[docs/glossary.md](../docs/glossary.md)** — domain terms (IV vs HV, Greeks, regime, anomaly flags). Read this before assuming a term means what you think it means. - **[docs/decisions/](../docs/decisions/)** — Architecture Decision Records. Each ADR explains the context, options considered, and accepted trade-offs for a major design choice. diff --git a/.github/data/arch_baseline.json b/.github/data/arch_baseline.json index bc0f645..55c5cd0 100644 --- a/.github/data/arch_baseline.json +++ b/.github/data/arch_baseline.json @@ -2,5 +2,5 @@ "layer_violations": 0, "cycles": 0, "god_files": 0, - "dead_code_candidates": 1 + "dead_code_candidates": 0 } diff --git a/.github/data/failure-registry.yaml b/.github/data/failure-registry.yaml index 15ce8c5..f101e95 100644 --- a/.github/data/failure-registry.yaml +++ b/.github/data/failure-registry.yaml @@ -25,8 +25,8 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/cleaning.py - - data_pipeline/downloader.py + - data_pipeline/transform/cleaning.py + - data_pipeline/ingest/ohlcv.py - core/price_dynamic.py empty-dataframe: @@ -39,8 +39,8 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/downloader.py - - data_pipeline/processing.py + - data_pipeline/ingest/ohlcv.py + - data_pipeline/transform/processing.py - services/market/analysis/facade.py dtype-mismatch: @@ -53,7 +53,7 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/db.py + - data_pipeline/store/db.py - core/options_greeks.py - core/market_analyzer.py @@ -67,7 +67,7 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/downloader.py + - data_pipeline/ingest/ohlcv.py - utils/utils.py db-error: @@ -80,7 +80,7 @@ patterns: resolved: false resolution_note: null related_files: - - data_pipeline/db.py + - data_pipeline/store/db.py greeks-edge-case: hook_regex: "greeks|black.?scholes|delta|gamma|theta|vega.*nan" diff --git a/.github/data/tag_baseline.json b/.github/data/tag_baseline.json index cd9e9e8..9ca2ba0 100644 --- a/.github/data/tag_baseline.json +++ b/.github/data/tag_baseline.json @@ -1,23 +1,23 @@ { - "files_scanned": 136, + "files_scanned": 147, "tags_by_type": { - "WHY": 12, - "CONSTRAINT": 16, - "TRADEOFF": 2, - "INVARIANT": 3, + "WHY": 13, + "CONSTRAINT": 19, + "TRADEOFF": 3, + "INVARIANT": 9, "DOMAIN": 10, "HACK": 0, "WORKAROUND": 0 }, "uncovered_constants_count": 16, "uncovered_constants": [ - "data_pipeline/data_ops/_globals.py:14: _UPDATE_COOLDOWN=60", - "data_pipeline/data_ops/_globals.py:16: _QUERY_CACHE_TTL=60", - "data_pipeline/data_ops/_range.py:12: _ENSURE_RANGE_TTL=300", - "data_pipeline/data_ops/_range.py:19: _SENTINEL_GAP_THRESHOLD_DAYS=365", - "data_pipeline/data_ops/_range.py:20: _SENTINEL_MIN_DB_SPAN_DAYS=365", - "services/market/charts.py:30: _CACHE_MAX_ENTRIES=64", - "services/options/preload.py:32: CACHE_TTL_MINUTES=15", + "data_pipeline/_state.py:29: _UPDATE_COOLDOWN=60", + "data_pipeline/_state.py:31: _QUERY_CACHE_TTL=60", + "data_pipeline/orchestrate/backfill.py:12: _ENSURE_RANGE_TTL=300", + "data_pipeline/orchestrate/backfill.py:19: _SENTINEL_GAP_THRESHOLD_DAYS=365", + "data_pipeline/orchestrate/backfill.py:20: _SENTINEL_MIN_DB_SPAN_DAYS=365", + "services/market/charts.py:29: _CACHE_MAX_ENTRIES=64", + "services/options/preload.py:34: CACHE_TTL_MINUTES=15", "services/options/simulation.py:36: MAX_STRIKES=15", "services/options/simulation.py:37: MAX_EXPIRIES=6", "services/options/simulation.py:38: MAX_IVS=5", diff --git a/.github/instructions/data-pipeline.instructions.md b/.github/instructions/data-pipeline.instructions.md index b16800b..3894f74 100644 --- a/.github/instructions/data-pipeline.instructions.md +++ b/.github/instructions/data-pipeline.instructions.md @@ -6,7 +6,7 @@ applyTo: "data_pipeline/**" # Data Pipeline Rules ## DB Access -- Always use `get_conn()` context manager from `data_pipeline/db.py` — never raw `sqlite3.connect()` +- Always use `get_conn()` context manager from `data_pipeline/store/db.py` — never raw `sqlite3.connect()` - Use `fetch_df()` for reads, `upsert_many()` for writes - Convert DB-sourced columns with `pd.to_numeric(col, errors='coerce')` before any math — SQLite returns `object` dtype diff --git a/.github/prompts/diagnose.prompt.md b/.github/prompts/diagnose.prompt.md index 9474bec..bfb2885 100644 --- a/.github/prompts/diagnose.prompt.md +++ b/.github/prompts/diagnose.prompt.md @@ -8,7 +8,7 @@ argument-hint: "Ticker symbol or symptom (e.g., 'NVDA empty charts', 'stale TLT Diagnose why a specific ticker's data is missing, stale, or showing errors in the OptionView dashboard. Steps: -1. Check the DB for the ticker: query `raw_prices`, `clean_prices`, `processed_prices` for recent rows +1. Check the DB for the ticker: query `raw_bars`, `clean_bars`, `feature_bars` for recent rows 2. Look for NaN-only filler rows (root cause of empty charts) 3. Check yfinance download logs for errors (429, timeout) 4. Trace data flow through the 5-stage pipeline to find the failure point diff --git a/.github/prompts/new-test.prompt.md b/.github/prompts/new-test.prompt.md index 8bc2f71..f89c056 100644 --- a/.github/prompts/new-test.prompt.md +++ b/.github/prompts/new-test.prompt.md @@ -2,7 +2,7 @@ description: "Generate a test following OptionView project patterns for a specific module or function." agent: "agent" tools: [read, search, edit] -argument-hint: "Module or function to test (e.g., 'data_pipeline/cleaning.py clean_range')" +argument-hint: "Module or function to test (e.g., 'data_pipeline/transform/cleaning.py clean_range')" --- Generate a pytest test for the specified module/function following OptionView test conventions: diff --git a/.github/prompts/pipeline-status.prompt.md b/.github/prompts/pipeline-status.prompt.md index 92fa30a..69ccbf7 100644 --- a/.github/prompts/pipeline-status.prompt.md +++ b/.github/prompts/pipeline-status.prompt.md @@ -10,22 +10,22 @@ Check the current health of the OptionView data pipeline: 1. Query the SQLite database (`market_data.sqlite`) for: ```sql -- Row counts per table - SELECT 'raw_prices' as tbl, count(*) as rows FROM raw_prices - UNION SELECT 'clean_prices', count(*) FROM clean_prices - UNION SELECT 'processed_prices', count(*) FROM processed_prices; + SELECT 'raw_bars' as tbl, count(*) as rows FROM raw_bars + UNION SELECT 'clean_bars', count(*) FROM clean_bars + UNION SELECT 'feature_bars', count(*) FROM feature_bars; -- Latest data per ticker - SELECT ticker, MAX(date) as latest, COUNT(*) as rows FROM raw_prices GROUP BY ticker; + SELECT ticker, MAX(date) as latest, COUNT(*) as rows FROM raw_bars GROUP BY ticker; -- NaN-only filler rows (problematic) - SELECT ticker, count(*) as nan_rows FROM raw_prices + SELECT ticker, count(*) as nan_rows FROM raw_bars WHERE open IS NULL AND high IS NULL AND low IS NULL AND close IS NULL GROUP BY ticker HAVING nan_rows > 0; -- Data freshness (days since last update) SELECT ticker, MAX(date) as latest, julianday('now') - julianday(MAX(date)) as days_stale - FROM raw_prices GROUP BY ticker ORDER BY days_stale DESC; + FROM raw_bars GROUP BY ticker ORDER BY days_stale DESC; ``` 2. Report: diff --git a/.github/skills/debug-pipeline/SKILL.md b/.github/skills/debug-pipeline/SKILL.md index d7251f6..946d307 100644 --- a/.github/skills/debug-pipeline/SKILL.md +++ b/.github/skills/debug-pipeline/SKILL.md @@ -23,26 +23,26 @@ Classify the user's report: | Symptom | Likely Layer | |---------|-------------| | Empty chart panels | core/ (PriceDynamic) or data_pipeline/ (NaN filler rows) | -| "No data for TICKER" message | data_pipeline/downloader.py (download failed) | +| "No data for TICKER" message | data_pipeline/ingest/ohlcv.py (download failed) | | Stale prices (dates from days ago) | data_pipeline/data_service.py (cooldown blocking refresh) | | 429 / timeout errors | yfinance rate-limiting or proxy issue | -| Wrong values in analysis | data_pipeline/cleaning.py or processing.py | +| Wrong values in analysis | data_pipeline/transform/cleaning.py or processing.py | ### Step 2: Check DB State Query the database for the target ticker using the terminal: ```sql --- Check raw_prices for recent data -SELECT ticker, date, close FROM raw_prices WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; +-- Check raw_bars for recent data +SELECT ticker, date, close FROM raw_bars WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; -- Check for NaN-only filler rows (the root cause of empty charts) -SELECT count(*) FROM raw_prices WHERE ticker='{TICKER}' AND open IS NULL AND high IS NULL AND low IS NULL AND close IS NULL; +SELECT count(*) FROM raw_bars WHERE ticker='{TICKER}' AND open IS NULL AND high IS NULL AND low IS NULL AND close IS NULL; --- Check clean_prices status -SELECT ticker, date, missing_any, price_jump_flag FROM clean_prices WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; +-- Check clean_bars status +SELECT ticker, date, missing_any, price_jump_flag FROM clean_bars WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; --- Check processed_prices -SELECT ticker, date, frequency FROM processed_prices WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; +-- Check feature_bars +SELECT ticker, date, frequency FROM feature_bars WHERE ticker='{TICKER}' ORDER BY date DESC LIMIT 5; ``` ### Step 3: Check yfinance Connectivity @@ -66,10 +66,10 @@ print(df.tail() if not df.empty else "EMPTY - download failed") Follow the data through each stage, checking for where it breaks. See [pipeline stages reference](./references/pipeline-stages.md) for expected inputs/outputs at each stage. -1. **downloader.py** → `upsert_raw_prices()` → writes to `raw_prices` -2. **cleaning.py** → `clean_range()` → reads `raw_prices`, writes to `clean_prices` -3. **processing.py** → `build_features()` → reads `clean_prices`, writes to `processed_prices` -4. **core/price_dynamic.py** → `_fetch_daily_from_db()` → reads `processed_prices` +1. **downloader.py** → `upsert_raw_prices()` → writes to `raw_bars` +2. **cleaning.py** → `clean_range()` → reads `raw_bars`, writes to `clean_bars` +3. **processing.py** → `build_features()` → reads `clean_bars`, writes to `feature_bars` +4. **core/price_dynamic.py** → `_fetch_daily_from_db()` → reads `feature_bars` 5. **core/market_analyzer.py** → uses PriceDynamic features for charts 6. **services/market/analysis/facade.py** → calls chart methods, returns base64 images @@ -78,7 +78,7 @@ Follow the data through each stage, checking for where it breaks. See [pipeline Common root causes: | Root Cause | Evidence | Fix | |-----------|----------|-----| -| NaN-only filler rows from failed download | `raw_prices` has NULL in all price columns | Re-download with `yf_throttle()`, delete filler rows | +| NaN-only filler rows from failed download | `raw_bars` has NULL in all price columns | Re-download with `yf_throttle()`, delete filler rows | | 60s cooldown blocking retry | Download skipped, log says "No new data" | Wait 60s or reset cooldown in `DataService._ticker_locks` | | Proxy unreachable | `curl: (28) Operation timed out` | Check `YF_PROXY` in `.env`, verify proxy is running | | yfinance 429 rate limit | `YFRateLimitError` in logs | Wait 30s, ensure `yf_throttle()` is called everywhere | diff --git a/.github/skills/debug-pipeline/references/pipeline-stages.md b/.github/skills/debug-pipeline/references/pipeline-stages.md index ca3aa67..8147f7a 100644 --- a/.github/skills/debug-pipeline/references/pipeline-stages.md +++ b/.github/skills/debug-pipeline/references/pipeline-stages.md @@ -2,39 +2,39 @@ Data flows through 5 stages. A failure at any stage can propagate downstream as empty/NaN data. -## Stage 1: Download (`data_pipeline/downloader.py`) +## Stage 1: Download (`data_pipeline/ingest/ohlcv.py`) **Function**: `upsert_raw_prices(ticker, start, end)` **Input**: Ticker symbol, date range **Output**: `PipelineResult(ok=True, rows=N)` or `PipelineResult(ok=False, error="...")` -**Side effect**: Writes to `raw_prices` table +**Side effect**: Writes to `raw_bars` table **What can fail**: - yfinance returns empty DataFrame (rate-limit, invalid ticker, network error) - Proxy unreachable (curl_cffi timeout) - Staleness check incorrectly skips download -**Check**: `SELECT count(*) FROM raw_prices WHERE ticker=? AND date BETWEEN ? AND ?` +**Check**: `SELECT count(*) FROM raw_bars WHERE ticker=? AND date BETWEEN ? AND ?` -## Stage 2: Clean (`data_pipeline/cleaning.py`) +## Stage 2: Clean (`data_pipeline/transform/cleaning.py`) **Function**: `clean_range(ticker, start, end)` -**Input**: Reads from `raw_prices` table -**Output**: `PipelineResult` — writes to `clean_prices` table +**Input**: Reads from `raw_bars` table +**Output**: `PipelineResult` — writes to `clean_bars` table **Side effect**: Adds anomaly flags (price_jump_flag, vol_anom_flag, ohlc_inconsistent) **What can fail**: -- Source `raw_prices` has NaN-only filler rows → cleans "pass through" NaN +- Source `raw_bars` has NaN-only filler rows → cleans "pass through" NaN - `pd.to_numeric()` coerces strings to NaN silently - Anomaly flag thresholds are heuristic — may miss or over-flag -**Check**: `SELECT date, missing_any, price_jump_flag FROM clean_prices WHERE ticker=? ORDER BY date DESC LIMIT 10` +**Check**: `SELECT date, missing_any, price_jump_flag FROM clean_bars WHERE ticker=? ORDER BY date DESC LIMIT 10` -## Stage 3: Process (`data_pipeline/processing.py`) +## Stage 3: Process (`data_pipeline/transform/processing.py`) **Function**: `build_features(ticker, frequency)` -**Input**: Reads from `clean_prices` table -**Output**: `PipelineResult` — writes to `processed_prices` table +**Input**: Reads from `clean_bars` table +**Output**: `PipelineResult` — writes to `feature_bars` table **Side effect**: Computes MA, returns, volatility features **What can fail**: @@ -42,12 +42,12 @@ Data flows through 5 stages. A failure at any stage can propagate downstream as - Wrong frequency conversion (D→W→M) drops rows - `object` dtype from DB causes numpy math errors -**Check**: `SELECT date, frequency, ma_20, ma_50 FROM processed_prices WHERE ticker=? AND frequency=? ORDER BY date DESC LIMIT 5` +**Check**: `SELECT date, frequency, ma_20, ma_50 FROM feature_bars WHERE ticker=? AND frequency=? ORDER BY date DESC LIMIT 5` ## Stage 4: Core Analysis (`core/price_dynamic.py`, `core/market_analyzer.py`) **Function**: `PriceDynamic._fetch_daily_from_db()` → `MarketAnalyzer` methods -**Input**: Reads from `processed_prices` (or `clean_prices` for some features) +**Input**: Reads from `feature_bars` (or `clean_bars` for some features) **Output**: DataFrames for chart generation **What can fail**: diff --git a/.github/skills/fix-review/SKILL.md b/.github/skills/fix-review/SKILL.md index c6ec761..300dcb8 100644 --- a/.github/skills/fix-review/SKILL.md +++ b/.github/skills/fix-review/SKILL.md @@ -61,7 +61,7 @@ Verify the fix uses the correct error pattern for its layer: For each changed production file, check that a corresponding test exists: ```bash # Map production file to test file -# data_pipeline/downloader.py → tests/test_yf_download.py or tests/test_processing.py +# data_pipeline/ingest/ohlcv.py → tests/test_yf_download.py or tests/test_processing.py # core/market_analyzer.py → tests/test_market_review.py # services/market/validation.py → tests/test_validation.py ``` diff --git a/.github/skills/test-escalation/SKILL.md b/.github/skills/test-escalation/SKILL.md index 06c6dbd..50583a2 100644 --- a/.github/skills/test-escalation/SKILL.md +++ b/.github/skills/test-escalation/SKILL.md @@ -63,7 +63,7 @@ def test_download_empty(mock_dl): assert result.ok # Rows=0 is valid assert result.rows == 0 # Verify no NaN filler rows were created - df = fetch_df("SELECT * FROM raw_prices WHERE ticker='NVDA'") + df = fetch_df("SELECT * FROM raw_bars WHERE ticker='NVDA'") assert df.empty ``` @@ -78,7 +78,7 @@ def test_full_pipeline_with_nan_data(tmp_path, monkeypatch): # Seed NaN-only filler rows (simulates failed download) upsert_many( - "raw_prices", + "raw_bars", ["ticker", "date", "open", "high", "low", "close"], [("NVDA", "2026-03-28", None, None, None, None)], ) @@ -86,8 +86,8 @@ def test_full_pipeline_with_nan_data(tmp_path, monkeypatch): # Run cleaning — should NOT propagate NaN rows result = clean_range("NVDA", dt.date(2026, 3, 28), dt.date(2026, 3, 28)) - # Verify: clean_prices should be empty (NaN rows filtered) - df = fetch_df("SELECT * FROM clean_prices WHERE ticker='NVDA'") + # Verify: clean_bars should be empty (NaN rows filtered) + df = fetch_df("SELECT * FROM clean_bars WHERE ticker='NVDA'") assert df.empty or df["close"].notna().all() ``` diff --git a/.github/skills/test-escalation/references/escalation-levels.md b/.github/skills/test-escalation/references/escalation-levels.md index acaaf29..c1edde5 100644 --- a/.github/skills/test-escalation/references/escalation-levels.md +++ b/.github/skills/test-escalation/references/escalation-levels.md @@ -32,7 +32,7 @@ Is the bug reproducible with a simple unit test? - **Symptom**: Empty charts, 0 historical data points - **Root cause**: NaN-only filler rows from failed download survive cleaning - **Effective level**: Level 2 (integration — needs real DB to reproduce the chain) -- **Key assertion**: After pipeline, `processed_prices` has no NaN-only rows +- **Key assertion**: After pipeline, `feature_bars` has no NaN-only rows ### Pattern B: yfinance Silent Failure - **Symptom**: Data appears stale, "No new data" in logs diff --git a/CLAUDE.md b/CLAUDE.md index 4939417..169b86d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,14 +9,13 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co > Update `docs/` first, then mirror the summary here. `CODEBUDDY.md` and > `.github/copilot-instructions.md` are parallel AI-assistant guides kept in sync with this one. -> **⚠ Active reorg (2026-09) — [ADR 0011](docs/decisions/0011-pluggable-data-provider-seam.md) -> + [0012](docs/decisions/0012-parameter-ownership-and-prefetch.md), both Accepted.** -> Before touching `data_pipeline/`, the parameter surfaces -> (`templates/partials/tab_parameter.html`, `tab_config.html`, `static/main.js` -> `FormManager`) or `routes/core.py::index`, read -> **[`docs/plans/business_line_reorg.md`](docs/plans/business_line_reorg.md) §0**: -> work the numbered batches (B1–B8) in order, one batch per PR, update its ledger -> in the same commit, and do not re-litigate the Accepted ADRs. +> **Business-line reorg (2026-09) — [ADR 0011](docs/decisions/0011-pluggable-data-provider-seam.md) +> (provider seam + canonical schema) + [0012](docs/decisions/0012-parameter-ownership-and-prefetch.md) +> (ticker-only Parameters bar + readiness prefetch), both Accepted.** +> Batches B1–B9 **landed**; the architecture below reflects the end state. Only +> deferred follow-ups remain — see [`docs/plans/business_line_reorg.md`](docs/plans/business_line_reorg.md) +> §10 (risk-free-rate global setting, `market_review_prices` L5 → provider seam, +> ADR 0011 `symbol` column). Do not re-litigate the Accepted ADRs. ## Commands @@ -109,7 +108,9 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ are legal; only the *layer* direction is policed. - **`core/`** — pure computation: no Flask, no DB, no network. Data in → numbers/DataFrames out. `core/` and `data_pipeline/` must never import `services/`, `routes/`, or `app.py`; - `data_pipeline/` must never import `core/`. + `data_pipeline/` must never import `core/`. **`core/` must not import `data_pipeline/` either** + (closed in batch B4 — enforced by `doc_guard` `import-direction` and + `tests/test_architecture_purity.py`; acquisition belongs in `services/`). - **`data_pipeline/`** — owns **every** I/O boundary: yfinance, SQLite, the scheduler. - **`utils/`** — leaf helpers only. @@ -122,16 +123,21 @@ Always reference and import them package-qualified (`from core.options.greeks im ### The streaming / lazy-tab model (the key non-obvious flow) `POST /` computes **nothing**. `routes/core.py::index` normalises the form -(`FormService.extract_form_data` → `ValidationService.validate_input_data`), registers a job via -`data_pipeline/job_cache.py::create_job` (TTL default 90 s), and renders `templates/index.html` -with `streaming_mode=True`. Each tab shell emits an HTMX placeholder -(`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the browser fans out parallel requests. +(`FormService.extract_form_data` → `ValidationService.validate_input_data`), resolves the requested +modules (`FormService.extract_modules`), runs the **data-readiness pass** +(`services/market/readiness.py` → `data_pipeline/orchestrate/readiness.py`: plan the datasets the +modules need, probe the DB once, kick missing ranges on daemon threads, warm the live option-chain +preload), registers a job via `data_pipeline/orchestrate/job_cache.py::create_job` (TTL default 90 s, +carrying the readiness plan), and renders `templates/index.html` with `streaming_mode=True`. Each tab +shell emits an HTMX placeholder (`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the +browser fans out parallel requests. All `/render/` routes funnel into **`services/market/dispatch.py::render_streaming_slice`**, which: 1. auto-bootstraps a synthetic job with defaults when `job` is missing (direct URL / refresh / bookmark) instead of erroring; -2. dispatches via `_RENDER_KIND_SLICES` (kind → `(AnalysisService method name as a string, fragment template)`), late-bound with `getattr` so test monkey-patches are honoured; -3. memoises per `(job_id, ticker, kind)` through `compute_or_get`; -4. calls `close_thread_conn()` in `finally` to avoid leaking the thread-local SQLite connection. +2. consults the job's readiness plan and, on a **cold start** (no usable history yet *and* the backfill still running), returns `partials/fragments/readiness.html` — a self-re-firing "正在准备…" fragment — instead of an empty chart (bounded by `readiness.HOLD_SECONDS` and by thread liveness); +3. dispatches via `_RENDER_KIND_SLICES` (kind → `(AnalysisService method name as a string, fragment template)`), late-bound with `getattr` so test monkey-patches are honoured; +4. memoises per `(job_id, ticker, kind)` through `compute_or_get`; +5. calls `close_thread_conn()` in `finally` to avoid leaking the thread-local SQLite connection. Failures return `render_error_fragment` — usually **HTTP 200 on purpose** (expired job) so HTMX swaps a helpful message rather than a browser error toast. @@ -146,19 +152,37 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is ### `data_pipeline/` specifics -- **`data_ops/` — `DataService` (facade)** is the single read entry point. `ensure_range(ticker, - start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which stops concurrent UI - requests from stampeding Yahoo. `_query.py` calls `_update`/`_range` module functions directly - (never the facade) to avoid an import cycle. -- **`yf_client.py`** is the **only** module allowed to call yfinance (enforced by `doc_guard` - `single-yf-exit`; exceptions registered in `docs/architecture_review.md` §2). Every call goes - through `yf_throttle()` (token bucket, 5 req/s, burst 5). **Never** pass - `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails (ADR 0005). -- **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). - `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, - `busy_timeout=5000`) and does **not** close on exit. `repos.py` is the only place that builds SQL. -- **`cleaning.py` / `processing.py`** — align to business days, mark gaps NA with **no - interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. +Re-homed in batch B3 of ADR 0011 into six one-way stages; the authoritative layer table is +`docs/architecture_review.md` §3 and is enforced by `doc_guard` + `tests/test_architecture_purity.py`: + +- **`read/` — `DataService` (facade)** is the single read entry point above the package. + `ensure_range(ticker, start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which + stops concurrent UI requests from stampeding Yahoo. The facade and `read/_query.py` call the + `orchestrate` drivers directly (never each other's facade) — `read → orchestrate` is why + `orchestrate` must not import `read` (no cycle). +- **`providers/`** is the **only** package allowed to call yfinance (the chokepoint moved here from + `yf_client.py` in batch B1; enforced by `doc_guard` `single-yf-exit`, exceptions registered in + `docs/architecture_review.md` §2). It owns the mapping from the vendor's fields onto one canonical + schema (`providers/base.py`) — IV as a decimal, nullable bid/ask, no `inTheMoney`. Every call goes + through `yf_throttle()` (token bucket, 5 req/s, burst 5). `providers/yf_client.py` is a one-release + compatibility shim over the package; `_registry.py` is the `MARKET_DATA_PROVIDER` seam. + **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails + (ADR 0005). +- **`store/`** — `db.py` (`init_db()` uses `CREATE TABLE IF NOT EXISTS`; no migration framework), + `repos.py` (the only place that builds SQL) and `quality_log.py`. Tables are named canonically + (`raw_bars` / `clean_bars` / `feature_bars`); the pre-rename names (`raw_prices` / `clean_prices` / + `processed_prices`) are kept as shadows for one release — every `upsert_many` writes both families, + and `scripts/migrate_canonical_tables.py` backfills an existing DB. `get_conn()` yields a + **thread-local** WAL connection (`synchronous=NORMAL`, `busy_timeout=5000`) and does **not** close + on exit. +- **`ingest/ohlcv.py`** — business-day gap detection + `raw_bars` upsert; acquisition goes through + `providers.get_provider().history()`, so this module never names a vendor. +- **`transform/cleaning.py` / `transform/processing.py`** — align to business days, mark gaps NA with + **no interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. + Never imports `providers/` (asserted by a test). +- **`orchestrate/`** — `update.py` (incremental/full drivers), `backfill.py` (chunked coverage + repair), `job_cache.py` (streaming slice memo), `scheduler.py` (optional APScheduler). +- **`_state.py`** — process-local query cache + update locks, shared by `read` and `orchestrate`. - No option-chain history exists from yfinance — no IV rank/percentile/backtests; HV percentile is the deliberate substitute (ADR 0004). @@ -166,6 +190,8 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is `static/api.js` is the **only** `fetch` wrapper (owns aborting + `ApiError` normalisation) — components must not call `fetch` directly. `static/state/` holds tiny observable stores: +`marketParamsState.js` / `assessmentParamsState.js` / `optionFilterState.js` own the module-scoped +parameters (one `localStorage` key per group) and re-run exactly the modules that consume a change; `panelState.js` enforces the four-phase async contract (`idle → loading → loaded → empty|error`, no sixth state), `tabFlagsState.js` is the lazy-load guard, `abortRegistry.js` cancels in-flight requests on ticker switch. Charts are server PNGs except `static/market_review_chart.js` and @@ -204,8 +230,8 @@ Pages-only. Rendered `site/index.html` / `site/static/` are build artefacts — | Question | File | |---|---| | How does a request get served? | `routes/core.py` → `services/market/dispatch.py` | -| Where does data come from? | `data_pipeline/data_ops/facade.py`, `_range.py`, `yf_client.py` | -| Schema / SQL | `data_pipeline/db.py` (`init_db`), `repos.py` | +| Where does data come from? | `data_pipeline/read/facade.py`, `orchestrate/backfill.py`, `providers/` | +| Schema / SQL | `data_pipeline/store/db.py` (`init_db`), `store/repos.py` | | Chart / analysis maths | `core/market/analyzer.py`, `core/options/`, `core/strategies/` | | Frontend contract | `docs/frontend_architecture.md` | | Why is this weird? | `docs/constraints.md`, then `docs/decisions/` | diff --git a/CODEBUDDY.md b/CODEBUDDY.md index 4939417..169b86d 100644 --- a/CODEBUDDY.md +++ b/CODEBUDDY.md @@ -9,14 +9,13 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co > Update `docs/` first, then mirror the summary here. `CODEBUDDY.md` and > `.github/copilot-instructions.md` are parallel AI-assistant guides kept in sync with this one. -> **⚠ Active reorg (2026-09) — [ADR 0011](docs/decisions/0011-pluggable-data-provider-seam.md) -> + [0012](docs/decisions/0012-parameter-ownership-and-prefetch.md), both Accepted.** -> Before touching `data_pipeline/`, the parameter surfaces -> (`templates/partials/tab_parameter.html`, `tab_config.html`, `static/main.js` -> `FormManager`) or `routes/core.py::index`, read -> **[`docs/plans/business_line_reorg.md`](docs/plans/business_line_reorg.md) §0**: -> work the numbered batches (B1–B8) in order, one batch per PR, update its ledger -> in the same commit, and do not re-litigate the Accepted ADRs. +> **Business-line reorg (2026-09) — [ADR 0011](docs/decisions/0011-pluggable-data-provider-seam.md) +> (provider seam + canonical schema) + [0012](docs/decisions/0012-parameter-ownership-and-prefetch.md) +> (ticker-only Parameters bar + readiness prefetch), both Accepted.** +> Batches B1–B9 **landed**; the architecture below reflects the end state. Only +> deferred follow-ups remain — see [`docs/plans/business_line_reorg.md`](docs/plans/business_line_reorg.md) +> §10 (risk-free-rate global setting, `market_review_prices` L5 → provider seam, +> ADR 0011 `symbol` column). Do not re-litigate the Accepted ADRs. ## Commands @@ -109,7 +108,9 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ are legal; only the *layer* direction is policed. - **`core/`** — pure computation: no Flask, no DB, no network. Data in → numbers/DataFrames out. `core/` and `data_pipeline/` must never import `services/`, `routes/`, or `app.py`; - `data_pipeline/` must never import `core/`. + `data_pipeline/` must never import `core/`. **`core/` must not import `data_pipeline/` either** + (closed in batch B4 — enforced by `doc_guard` `import-direction` and + `tests/test_architecture_purity.py`; acquisition belongs in `services/`). - **`data_pipeline/`** — owns **every** I/O boundary: yfinance, SQLite, the scheduler. - **`utils/`** — leaf helpers only. @@ -122,16 +123,21 @@ Always reference and import them package-qualified (`from core.options.greeks im ### The streaming / lazy-tab model (the key non-obvious flow) `POST /` computes **nothing**. `routes/core.py::index` normalises the form -(`FormService.extract_form_data` → `ValidationService.validate_input_data`), registers a job via -`data_pipeline/job_cache.py::create_job` (TTL default 90 s), and renders `templates/index.html` -with `streaming_mode=True`. Each tab shell emits an HTMX placeholder -(`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the browser fans out parallel requests. +(`FormService.extract_form_data` → `ValidationService.validate_input_data`), resolves the requested +modules (`FormService.extract_modules`), runs the **data-readiness pass** +(`services/market/readiness.py` → `data_pipeline/orchestrate/readiness.py`: plan the datasets the +modules need, probe the DB once, kick missing ranges on daemon threads, warm the live option-chain +preload), registers a job via `data_pipeline/orchestrate/job_cache.py::create_job` (TTL default 90 s, +carrying the readiness plan), and renders `templates/index.html` with `streaming_mode=True`. Each tab +shell emits an HTMX placeholder (`hx-get="/render/?job=…&ticker=…" hx-trigger="load"`), and the +browser fans out parallel requests. All `/render/` routes funnel into **`services/market/dispatch.py::render_streaming_slice`**, which: 1. auto-bootstraps a synthetic job with defaults when `job` is missing (direct URL / refresh / bookmark) instead of erroring; -2. dispatches via `_RENDER_KIND_SLICES` (kind → `(AnalysisService method name as a string, fragment template)`), late-bound with `getattr` so test monkey-patches are honoured; -3. memoises per `(job_id, ticker, kind)` through `compute_or_get`; -4. calls `close_thread_conn()` in `finally` to avoid leaking the thread-local SQLite connection. +2. consults the job's readiness plan and, on a **cold start** (no usable history yet *and* the backfill still running), returns `partials/fragments/readiness.html` — a self-re-firing "正在准备…" fragment — instead of an empty chart (bounded by `readiness.HOLD_SECONDS` and by thread liveness); +3. dispatches via `_RENDER_KIND_SLICES` (kind → `(AnalysisService method name as a string, fragment template)`), late-bound with `getattr` so test monkey-patches are honoured; +4. memoises per `(job_id, ticker, kind)` through `compute_or_get`; +5. calls `close_thread_conn()` in `finally` to avoid leaking the thread-local SQLite connection. Failures return `render_error_fragment` — usually **HTTP 200 on purpose** (expired job) so HTMX swaps a helpful message rather than a browser error toast. @@ -146,19 +152,37 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is ### `data_pipeline/` specifics -- **`data_ops/` — `DataService` (facade)** is the single read entry point. `ensure_range(ticker, - start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which stops concurrent UI - requests from stampeding Yahoo. `_query.py` calls `_update`/`_range` module functions directly - (never the facade) to avoid an import cycle. -- **`yf_client.py`** is the **only** module allowed to call yfinance (enforced by `doc_guard` - `single-yf-exit`; exceptions registered in `docs/architecture_review.md` §2). Every call goes - through `yf_throttle()` (token bucket, 5 req/s, burst 5). **Never** pass - `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails (ADR 0005). -- **`db.py`** — `init_db()` uses `CREATE TABLE IF NOT EXISTS` (no migration framework). - `get_conn()` yields a **thread-local** WAL connection (`synchronous=NORMAL`, - `busy_timeout=5000`) and does **not** close on exit. `repos.py` is the only place that builds SQL. -- **`cleaning.py` / `processing.py`** — align to business days, mark gaps NA with **no - interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. +Re-homed in batch B3 of ADR 0011 into six one-way stages; the authoritative layer table is +`docs/architecture_review.md` §3 and is enforced by `doc_guard` + `tests/test_architecture_purity.py`: + +- **`read/` — `DataService` (facade)** is the single read entry point above the package. + `ensure_range(ticker, start, end)` is DB-first with a memo + in-flight de-duplication + TTL, which + stops concurrent UI requests from stampeding Yahoo. The facade and `read/_query.py` call the + `orchestrate` drivers directly (never each other's facade) — `read → orchestrate` is why + `orchestrate` must not import `read` (no cycle). +- **`providers/`** is the **only** package allowed to call yfinance (the chokepoint moved here from + `yf_client.py` in batch B1; enforced by `doc_guard` `single-yf-exit`, exceptions registered in + `docs/architecture_review.md` §2). It owns the mapping from the vendor's fields onto one canonical + schema (`providers/base.py`) — IV as a decimal, nullable bid/ask, no `inTheMoney`. Every call goes + through `yf_throttle()` (token bucket, 5 req/s, burst 5). `providers/yf_client.py` is a one-release + compatibility shim over the package; `_registry.py` is the `MARKET_DATA_PROVIDER` seam. + **Never** pass `session=requests.Session()` — yfinance ≥0.2.50 uses curl_cffi and silently fails + (ADR 0005). +- **`store/`** — `db.py` (`init_db()` uses `CREATE TABLE IF NOT EXISTS`; no migration framework), + `repos.py` (the only place that builds SQL) and `quality_log.py`. Tables are named canonically + (`raw_bars` / `clean_bars` / `feature_bars`); the pre-rename names (`raw_prices` / `clean_prices` / + `processed_prices`) are kept as shadows for one release — every `upsert_many` writes both families, + and `scripts/migrate_canonical_tables.py` backfills an existing DB. `get_conn()` yields a + **thread-local** WAL connection (`synchronous=NORMAL`, `busy_timeout=5000`) and does **not** close + on exit. +- **`ingest/ohlcv.py`** — business-day gap detection + `raw_bars` upsert; acquisition goes through + `providers.get_provider().history()`, so this module never names a vendor. +- **`transform/cleaning.py` / `transform/processing.py`** — align to business days, mark gaps NA with + **no interpolation** (invented prices are worse than missing ones), then engineer returns/MAs/HV. + Never imports `providers/` (asserted by a test). +- **`orchestrate/`** — `update.py` (incremental/full drivers), `backfill.py` (chunked coverage + repair), `job_cache.py` (streaming slice memo), `scheduler.py` (optional APScheduler). +- **`_state.py`** — process-local query cache + update locks, shared by `read` and `orchestrate`. - No option-chain history exists from yfinance — no IV rank/percentile/backtests; HV percentile is the deliberate substitute (ADR 0004). @@ -166,6 +190,8 @@ chart-level memo keyed by `(ticker, chart name, params)` because PNG encoding is `static/api.js` is the **only** `fetch` wrapper (owns aborting + `ApiError` normalisation) — components must not call `fetch` directly. `static/state/` holds tiny observable stores: +`marketParamsState.js` / `assessmentParamsState.js` / `optionFilterState.js` own the module-scoped +parameters (one `localStorage` key per group) and re-run exactly the modules that consume a change; `panelState.js` enforces the four-phase async contract (`idle → loading → loaded → empty|error`, no sixth state), `tabFlagsState.js` is the lazy-load guard, `abortRegistry.js` cancels in-flight requests on ticker switch. Charts are server PNGs except `static/market_review_chart.js` and @@ -204,8 +230,8 @@ Pages-only. Rendered `site/index.html` / `site/static/` are build artefacts — | Question | File | |---|---| | How does a request get served? | `routes/core.py` → `services/market/dispatch.py` | -| Where does data come from? | `data_pipeline/data_ops/facade.py`, `_range.py`, `yf_client.py` | -| Schema / SQL | `data_pipeline/db.py` (`init_db`), `repos.py` | +| Where does data come from? | `data_pipeline/read/facade.py`, `orchestrate/backfill.py`, `providers/` | +| Schema / SQL | `data_pipeline/store/db.py` (`init_db`), `store/repos.py` | | Chart / analysis maths | `core/market/analyzer.py`, `core/options/`, `core/strategies/` | | Frontend contract | `docs/frontend_architecture.md` | | Why is this weird? | `docs/constraints.md`, then `docs/decisions/` | diff --git a/README.md b/README.md index bff8620..3a7b5b7 100644 --- a/README.md +++ b/README.md @@ -82,17 +82,14 @@ app.py Flask entry point — registers blueprints, middlew ├── decision/ Put-selling candidate scoring pipeline ├── correlation_validator.py Rolling pairwise correlations └── _shared/ Plotting helpers, types, validators -└── data_pipeline/ Download · clean · process · persist - ├── data_ops/ DataService facade (DB-first cache, 60 s freshness) - ├── db.py SQLite context manager (WAL, synchronous=NORMAL) - ├── repos.py SQL builders (prices, regime, positions, …) - ├── yf_client.py Single chokepoint for yfinance (token-bucket throttle, proxy) - ├── downloader.py Raw bar / chain fetch with gap detection - ├── cleaning.py Time-series alignment; gaps marked NA (no interpolation) - ├── processing.py Feature engineering (returns, MAs, HV) - ├── scheduler.py APScheduler daily + monthly correlation refresh - ├── job_cache.py In-process TTL cache for /render/ payloads - └── quality_log.py Pipeline anomaly persistence +└── data_pipeline/ Acquire · process · serve (six one-way stages, ADR 0011) + ├── providers/ The ONLY `import yfinance`: vendor adapter + canonical schema + registry + ├── store/ Canonical schema, the only SQL, the failure log + ├── ingest/ Business-day gap detection + raw_bars upsert + ├── transform/ Alignment + anomaly flags + feature engineering (provider-agnostic) + ├── read/ DataService facade (DB-first cache, 60 s freshness) + ├── orchestrate/ Update/backfill drivers, job cache, optional scheduler + └── _state.py Process-local query cache + update locks └── utils/ Shared helpers (ticker normalisation, error envelopes, …) ├── constants.py Domain defaults (DEFAULT_TICKER, FREQUENCY_DISPLAY, …) ├── date_helpers.py parse_month_str, exclusive_month_end @@ -146,12 +143,12 @@ Packaged by business domain; each package exposes a `facade.py` entry point. | File | Role | Pulls from | |---|---|---| | [`services/market/facade.py`](services/market/facade.py) | Ticker validation + market-review summary for `/api/validate_*` and `/render/market_review`. | `core/market_review`, `core/market.data_context` | -| [`services/market/analysis/facade.py`](services/market/analysis/facade.py) | Top-level "run a full market analysis" facade for `/render/statistical` and `/render/assessment`. | `core/market.analyzer`, `core/market.correlation_validator`, `data_pipeline/data_ops` | -| [`services/market/charts.py`](services/market/charts.py) | Builds matplotlib figures and returns base64 PNGs; caches by `(ticker, kind, params)`. | `core/*`, `data_pipeline/data_ops` | -| [`services/market/signals.py`](services/market/signals.py) | Wraps `core/signals` over DB-cached daily bars for `/api/signals`. | `core/signals`, `data_pipeline/data_ops` | +| [`services/market/analysis/facade.py`](services/market/analysis/facade.py) | Top-level "run a full market analysis" facade for `/render/statistical` and `/render/assessment`. | `core/market.analyzer`, `core/market.correlation_validator`, `data_pipeline/read` | +| [`services/market/charts.py`](services/market/charts.py) | Builds matplotlib figures and returns base64 PNGs; caches by `(ticker, kind, params)`. | `core/*`, `data_pipeline/read` | +| [`services/market/signals.py`](services/market/signals.py) | Wraps `core/signals` over DB-cached daily bars for `/api/signals`. | `core/signals`, `data_pipeline/read` | | [`services/market/form.py`](services/market/form.py) | Extracts and normalises POST form fields, applying defaults from `utils/constants.py`. | `utils/constants`, `utils/date_helpers` | | [`services/market/validation.py`](services/market/validation.py) | Pure form-value validation rules (date ranges, frequency, …). | (none) | -| [`services/market/health.py`](services/market/health.py) | Aggregates DB freshness / row-count / NaN metrics for `/health/*`. | `data_pipeline/db`, `data_pipeline/repos` | +| [`services/market/health.py`](services/market/health.py) | Aggregates DB freshness / row-count / NaN metrics for `/health/*`. | `data_pipeline/store` | | [`services/market/dispatch.py`](services/market/dispatch.py) | Shared `/render/` handler: job lookup, memoisation, fragment render. | `services/market/analysis`, `services/options/chain` | **`services/options/`** @@ -168,15 +165,15 @@ Packaged by business domain; each package exposes a `facade.py` entry point. | File | Role | Pulls from | |---|---|---| -| [`services/portfolio/facade.py`](services/portfolio/facade.py) | CRUD for tracked positions in SQLite; computes live P&L via `data_pipeline/repos`. | `core/portfolio`, `data_pipeline/repos` | +| [`services/portfolio/facade.py`](services/portfolio/facade.py) | CRUD for tracked positions in SQLite; computes live P&L via `data_pipeline/store/repos`. | `core/portfolio`, `data_pipeline/store/repos` | | [`services/portfolio/analysis.py`](services/portfolio/analysis.py) | Stateless "analyse this basket of legs" endpoint backing `/api/portfolio_analysis`. | `core/options/greeks/portfolio`, `core/strategies` | **`services/regime/`** | File | Role | Pulls from | |---|---|---| -| [`services/regime/facade.py`](services/regime/facade.py) | Labels & persists market regimes; serves `/api/regime/*`. | `core/regime`, `data_pipeline/repos` | -| [`services/regime/ops/`](services/regime/ops/) | History bootstrap + `regime_log` read/write helpers. | `data_pipeline/db`, `data_pipeline/downloader` | +| [`services/regime/facade.py`](services/regime/facade.py) | Labels & persists market regimes; serves `/api/regime/*`. | `core/regime`, `data_pipeline/store/repos` | +| [`services/regime/ops/`](services/regime/ops/) | History bootstrap + `regime_log` read/write helpers. | `data_pipeline/store/db`, `data_pipeline/downloader` | ### `core/` — pure computation (no Flask, no I/O) @@ -201,16 +198,12 @@ Packaged by business domain; each package exposes a `facade.py` entry point. | File | Role | |---|---| -| [`data_pipeline/yf_client.py`](data_pipeline/yf_client.py) | Single chokepoint for `yfinance` calls: token-bucket throttle, proxy probe, error mapping. | -| [`data_pipeline/downloader.py`](data_pipeline/downloader.py) | Uses `yf_client` to fetch raw bars / option chains, with gap detection against the DB. | -| [`data_pipeline/cleaning.py`](data_pipeline/cleaning.py) | Aligns to business days and drops broken rows; missing gaps are marked NA — never interpolated. | -| [`data_pipeline/processing.py`](data_pipeline/processing.py) | Feature engineering (returns, MAs, HV) on cleaned bars. | -| [`data_pipeline/data_ops/`](data_pipeline/data_ops/) | `DataService` facade — DB-first cache with a 60 s freshness window, the single read entry-point. | -| [`data_pipeline/db.py`](data_pipeline/db.py) | `get_conn()` context manager, schema bootstrap, WAL pragmas, thread-local connection pooling. | -| [`data_pipeline/repos.py`](data_pipeline/repos.py) | Repository wrappers — the only modules that build SQL. | -| [`data_pipeline/scheduler.py`](data_pipeline/scheduler.py) | APScheduler wrapper: daily backfill + monthly correlation refresh, gated by a leader-lock file. APScheduler is imported lazily — it is optional and only needed when `AUTO_UPDATE_TICKERS` is set. | -| [`data_pipeline/job_cache.py`](data_pipeline/job_cache.py) | TTL'd in-process map keyed by `job_id`; lets `/render/` partials share the same form payload. | -| [`data_pipeline/quality_log.py`](data_pipeline/quality_log.py) | Persists pipeline anomalies for `/health/data`. | +| [`data_pipeline/providers/`](data_pipeline/providers/) | The single chokepoint for `yfinance` (adapter + canonical mapping + registry) and the token-bucket throttle / proxy probe. | +| [`data_pipeline/ingest/`](data_pipeline/ingest/) | Business-day gap detection and the `raw_bars` upsert, acquisition via `providers.get_provider()`. | +| [`data_pipeline/transform/`](data_pipeline/transform/) | Business-day alignment, anomaly flags (gaps NA — never interpolated) and feature engineering (returns, MAs, HV). | +| [`data_pipeline/read/`](data_pipeline/read/) | `DataService` facade — DB-first cache with a 60 s freshness window, the single read entry-point. | +| [`data_pipeline/store/`](data_pipeline/store/) | `db.py` (`get_conn()`, schema bootstrap, WAL pragmas, thread-local pooling), `repos.py` (the only SQL), `quality_log.py`. | +| [`data_pipeline/orchestrate/`](data_pipeline/orchestrate/) | Update/seed drivers, chunked backfill, the `/render/` TTL cache, and the optional APScheduler wrapper (lazy import; only needed when `AUTO_UPDATE_TICKERS` is set). | ### `utils/` @@ -369,7 +362,7 @@ See [`.env.example`](.env.example) for the full list. The most relevant ones: with a token bucket (default 5 req/s, burst 5) and uses a DB-first cache to avoid redundant downloads. Do not pass `session=requests.Session()` — yfinance uses `curl_cffi` and silently breaks otherwise. -- **DB layer**: always go through `data_pipeline/db.py::get_conn()`; it +- **DB layer**: always go through `data_pipeline/store/db.py::get_conn()`; it enables WAL mode, sets `synchronous=NORMAL`, and is safe to share across threads. - **Logging**: use `logging.getLogger(__name__)`; no `print()` in production diff --git a/app.py b/app.py index 2a9be9b..92c70c9 100644 --- a/app.py +++ b/app.py @@ -13,8 +13,8 @@ from dotenv import load_dotenv from flask import Flask -from data_pipeline.data_ops import DataService -from data_pipeline.scheduler import UpdateScheduler, acquire_scheduler_lock +from data_pipeline.orchestrate.scheduler import UpdateScheduler, acquire_scheduler_lock +from data_pipeline.read import DataService from utils.network import init_yf_proxy load_dotenv() diff --git a/core/market/__init__.py b/core/market/__init__.py index 4fb02f6..22a60b6 100644 --- a/core/market/__init__.py +++ b/core/market/__init__.py @@ -1,7 +1,7 @@ """Market Analysis Domain. Dependency graph (flows downward): - data_context # PriceDynamic — data fetching & resampling + data_context # pure data container + resampling (no I/O — ADR 0001) features/ # Pure numeric feature computation ├── osc.py ├── returns.py diff --git a/core/market/analyzer.py b/core/market/analyzer.py index 12752d5..9912fda 100644 --- a/core/market/analyzer.py +++ b/core/market/analyzer.py @@ -2,24 +2,26 @@ Domain: Market Analysis — Orchestration Context: - - Builds the data context and delegates all chart rendering to - ``core.market.charts.facade.MarketChartAssembly``. + - Wraps an already-fetched ``DataContext`` and delegates all chart rendering + to ``core.market.charts.facade.MarketChartAssembly``. + - INVARIANT (ADR 0001; batch B4): it does **not** fetch. Callers build the + context with ``services.market.data_context_fetch.fetch_data_context`` and + pass it in — see ``OptionsChainAnalyzer(snapshot=…)`` for the same pattern. - Kept intentionally thin: the chart-assembly fan-out (renderers + the feature/projection primitives that feed them) lives in the facade, so this module is no longer the repo's top change-magnet. Dependencies UPWARD: - core.market.data_context, core.market.charts.facade Dependencies DOWNWARD: - - services.market.analysis.facade, tests + - services.market.analysis.facade, services.market.analysis.statistical, tests """ from __future__ import annotations -import datetime as dt import logging from core.market.charts.facade import MarketChartAssembly -from core.market.data_context import build_data_context +from core.market.data_context import DataContext logger = logging.getLogger(__name__) @@ -27,14 +29,28 @@ class MarketAnalyzer: """High-level market analysis — thin orchestrator over core.market submodules.""" - def __init__(self, ticker: str, start_date: dt.date, frequency: str, end_date: dt.date | None = None): - self._ctx = build_data_context(ticker, start_date, frequency, end_date) - self.ticker = ticker - self.frequency = frequency - self.end_date = end_date + def __init__(self, data_context: DataContext): + """Wrap ``data_context``. + + WHY the context is injected rather than built here: constructing it means + reading the DB and possibly the provider, which core/ must not do (ADR + 0001). Services own that decision — see + ``services/market/data_context_fetch.py``. + """ + self._ctx = data_context + self.ticker = data_context.ticker + self.frequency = data_context.frequency + # Backward-compat: the old signature exposed the caller's ``end_date`` + # (None when the horizon end was implicit). + self.end_date = data_context.horizon.end if data_context.horizon.user_provided_end else None self.features_df = self._ctx.features_df self._charts = MarketChartAssembly(self._ctx, self.features_df, self.ticker, self.frequency) + @property + def data_context(self) -> DataContext: + """The wrapped context (public accessor for service-layer callers).""" + return self._ctx + def is_data_valid(self): return self._ctx.is_valid() @@ -61,6 +77,3 @@ def generate_volatility_dynamics(self): def generate_oscillation_projection(self, percentile=0.90, target_bias=None): return self._charts.generate_oscillation_projection(percentile, target_bias) - - def analyze_options(self, option_data): - return self._charts.analyze_options(option_data) diff --git a/core/market/charts/facade.py b/core/market/charts/facade.py index 6424f52..e31acf0 100644 --- a/core/market/charts/facade.py +++ b/core/market/charts/facade.py @@ -18,13 +18,11 @@ - generate_return_osc_high_low_chart(rolling_window=20, risk_threshold=90) -> str | None - generate_volatility_dynamics() -> str | None - generate_oscillation_projection(percentile=0.90, target_bias=None) -> (str|None, str|None) - - analyze_options(option_data) -> str | None Dependencies UPWARD: - - core.market.charts.{dynamics, projection, scatter_high_low, scatter_osc, volatility, option_pnl} + - core.market.charts.{dynamics, projection, scatter_high_low, scatter_osc, volatility} - core.market.charts._scales.format_projection_value - core.market.features.{osc, osc_high, osc_low, price_returns, _horizon, regime_segments, volatility} - core.market.projections.oscillation - - core.market.option_pnl Dependencies DOWNWARD: - core.market.analyzer """ @@ -35,7 +33,6 @@ from core.market.charts._scales import format_projection_value from core.market.charts.dynamics import render_dynamics -from core.market.charts.option_pnl import render_option_pnl from core.market.charts.projection import render_projection from core.market.charts.scatter_high_low import render_scatter_high_low from core.market.charts.scatter_osc import render_scatter_osc @@ -44,7 +41,6 @@ from core.market.features._horizon import apply_horizon from core.market.features.regime_segments import bull_bear_segments from core.market.features.volatility import calculate_volatility -from core.market.option_pnl import build_option_matrix from core.market.projections.oscillation import compute_oscillation_projection logger = logging.getLogger(__name__) @@ -194,18 +190,5 @@ def generate_oscillation_projection(self, percentile=0.90, target_bias=None): logger.error("Error generating oscillation projection: %s", e) return None, None - def analyze_options(self, option_data): - if not option_data: - return None - try: - current_price = self._ctx.current_price - if current_price is None: - return None - matrix_df = build_option_matrix(option_data, current_price) - return render_option_pnl(matrix_df, current_price, option_data) - except Exception as e: - logger.error("Error analyzing options: %s", e) - return None - __all__ = ["MarketChartAssembly"] diff --git a/core/market/charts/option_pnl.py b/core/market/charts/option_pnl.py deleted file mode 100644 index 28e1a90..0000000 --- a/core/market/charts/option_pnl.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Option portfolio P&L chart renderer. - -Domain: Market Analysis — Option P&L Chart -Contracts: - - render_option_pnl(matrix_df, current_price, option_data=None) -> str | None -Dependencies UPWARD: - - matplotlib, core._shared.plotting -Dependencies DOWNWARD: - - core.market.analyzer -""" - -from __future__ import annotations - -import logging - -from core._shared.plotting import encode_figure, new_figure -from core.market.option_pnl import find_breakeven_points - -logger = logging.getLogger(__name__) - - -def render_option_pnl(matrix_df, current_price, option_data=None) -> str | None: - """Render an option portfolio P&L chart as a base64 PNG string.""" - try: - with new_figure((12, 8)) as fig: - ax = fig.subplots() - ax.plot(matrix_df.index, matrix_df["PnL"], linewidth=3, color="blue") - ax.axhline(y=0, color="black", linestyle="-", alpha=0.8, linewidth=1) - ax.axvline( - x=current_price, - color="red", - linestyle="--", - alpha=0.8, - linewidth=2, - label=f"Current Price: ${current_price:.2f}", - ) - ax.fill_between( - matrix_df.index, - matrix_df["PnL"], - 0, - where=(matrix_df["PnL"] > 0), - color="green", - alpha=0.3, - label="Profit", - ) - ax.fill_between( - matrix_df.index, - matrix_df["PnL"], - 0, - where=(matrix_df["PnL"] < 0), - color="red", - alpha=0.3, - label="Loss", - ) - ax.set_xlabel("Stock Price ($)", fontsize=12) - ax.set_ylabel("P&L ($)", fontsize=12) - ax.set_title("Options Portfolio P&L Analysis", fontsize=14, fontweight="bold") - ax.grid(True, alpha=0.3) - ax.legend(fontsize=11) - - max_profit = matrix_df["PnL"].max() - max_loss = matrix_df["PnL"].min() - breakeven_points = find_breakeven_points(matrix_df) - stats_text = f"Max Profit: ${max_profit:.0f}\nMax Loss: ${max_loss:.0f}" - if breakeven_points: - stats_text += f"\nBreakeven: ${breakeven_points[0]:.0f}" - if len(breakeven_points) > 1: - stats_text += f", ${breakeven_points[1]:.0f}" - ax.text( - 0.02, - 0.98, - stats_text, - transform=ax.transAxes, - fontsize=12, - verticalalignment="top", - bbox=dict(boxstyle="round", facecolor="wheat", alpha=0.8), - ) - - # Greeks summary overlay - try: - if option_data: - from core.options.greeks.portfolio import portfolio_greeks_table - - positions = [] - for opt in option_data: - if opt.get("dte") and opt.get("iv"): - positions.append( - { - "type": opt["option_type"], - "strike": float(opt["strike"]), - "dte": int(opt["dte"]), - "iv": float(opt["iv"]), - "qty": int(opt["quantity"]), - "premium": float(opt["premium"]), - } - ) - if positions: - totals, _ = portfolio_greeks_table(positions, float(current_price)) - greeks_text = ( - f"Net Delta: {totals['delta']:+.3f}\n" - f"Net Gamma: {totals['gamma']:+.5f}\n" - f"Theta/day: {totals['theta']:+.2f}\n" - f"Vega/1%: {totals['vega']:+.2f}" - ) - ax.text( - 0.98, - 0.98, - greeks_text, - transform=ax.transAxes, - fontsize=10, - verticalalignment="top", - horizontalalignment="right", - family="monospace", - bbox=dict(boxstyle="round", facecolor="lightyellow", alpha=0.8), - ) - except Exception as e: - logger.debug("Greeks overlay skipped: %s", e) - - return encode_figure(fig) - except Exception as e: - logger.error("Error creating option P&L chart: %s", e) - return None diff --git a/core/market/correlation_validator.py b/core/market/correlation_validator.py index 11dfae1..04a9f42 100644 --- a/core/market/correlation_validator.py +++ b/core/market/correlation_validator.py @@ -2,6 +2,17 @@ Rolling correlation computation lives here (domain-specific logic); rendering is delegated to core.market.charts.correlation. + +Domain: Market Analysis — Correlation Validation +Context: + - INVARIANT (ADR 0001; batch B4): this module does **not** fetch. The bars + arrive via the injected ``price_data`` (a ``DataContext``, or anything + exposing ``bars``); callers use + ``services.market.data_context_fetch.fetch_data_context``. +Dependencies UPWARD: + - core.market.charts.correlation, core.market.features._horizon +Dependencies DOWNWARD: + - services.market.analysis.statistical """ from __future__ import annotations @@ -13,7 +24,6 @@ import pandas as pd from core.market.charts.correlation import render_correlation -from core.market.data_context import build_data_context from core.market.features._horizon import apply_horizon, compute_effective_end logger = logging.getLogger(__name__) @@ -36,16 +46,18 @@ def __init__( self.user_end_date = end_date or dt.date.today() self._user_provided_end = end_date is not None - # Duck-type support: injected object may be DataContext (has bars) - # or any object exposing a `_data` attribute that returns a DataFrame. - if price_data is not None: - self._raw_data = getattr(price_data, "_data", None) or getattr(price_data, "bars", None) - is_valid_fn = getattr(price_data, "is_valid", None) - self._is_valid = bool(is_valid_fn()) if is_valid_fn else self._raw_data is not None - else: - ctx = build_data_context(ticker, start_date, frequency, end_date) - self._raw_data = ctx.bars - self._is_valid = ctx.is_valid() + if price_data is None: + # WHY raise instead of fetching: the fetch used to live here, which + # made core/ depend on data_pipeline (architecture_review.md §2). + raise ValueError( + "CorrelationValidator requires price_data=; core/ must not fetch — " + "build it with services.market.data_context_fetch.fetch_data_context()" + ) + # Duck-type support: the injected object may be a DataContext (has + # `bars`) or any object exposing a `_data` attribute. + self._raw_data = getattr(price_data, "_data", None) or getattr(price_data, "bars", None) + is_valid_fn = getattr(price_data, "is_valid", None) + self._is_valid = bool(is_valid_fn()) if is_valid_fn else self._raw_data is not None self.data = self._build_data() diff --git a/core/market/data_context.py b/core/market/data_context.py index 44450e3..de9bec9 100644 --- a/core/market/data_context.py +++ b/core/market/data_context.py @@ -1,18 +1,26 @@ -"""Market data context — explicit data container with data-fetching logic. +"""Market data context — pure container + resampling. Domain: Market Analysis — Data Context Context: - - Encapsulates data-fetching/resampling logic previously inside PriceDynamic. - - Returns plain DataFrames so downstream features/charts are fully decoupled. + - The container market analysis works on: ``bars`` (at the requested + frequency) plus the ``daily_bars`` they were derived from. + - INVARIANT (ADR 0001; batch B4 of the reorg): this module performs **no + I/O**. It receives already-fetched bars and resamples them; acquisition + (DB-first read + provider fallback) lives in + ``services/market/data_context_fetch.py``. Closing that leak is what lets + ``tests/test_architecture_purity.py`` require ``core/`` to have zero + ``data_pipeline`` imports. - No feature calculation, no matplotlib, no business logic. Contracts: - - build_data_context(ticker, start_date, frequency, end_date) -> DataContext - - DataContext exposes bars, daily_bars, horizon, ticker, frequency, is_valid + - ``DataContext`` — the container (``bars``, ``daily_bars``, ``horizon``, + ``ticker``, ``frequency``, ``is_valid()``, ``features_df``, ``current_price``). + - ``refrequency(df, frequency)`` — daily bars → bars at ``D``/``W``/``ME``/``QE``. + - ``build_data_context(*, ticker, frequency, horizon, raw_data)`` — pure assembly. Dependencies UPWARD: - - core.market.features, core.market.charts, data_pipeline (I/O boundary, - see the doc-guard: allow=core-purity markers below) + - core.market.features, core.market.models, core._shared.types Dependencies DOWNWARD: - - core.market.analyzer, services.market.analysis.facade + - core.market.analyzer, core.market.correlation_validator, + core.market.charts.facade, services.market.data_context_fetch """ from __future__ import annotations @@ -27,161 +35,6 @@ logger = logging.getLogger(__name__) -# CONSTRAINT: bounded retries prevent transient yfinance failures from crashing the pipeline. -_YF_MAX_RETRIES = 2 - -# CONSTRAINT: sub-second retries hit Yahoo rate-limiting; 3 s is the minimum stable back-off. -_YF_RETRY_BASE_DELAY = 3 # seconds - - -# --------------------------------------------------------------------------- -# Internal helpers (extracted from former PriceDynamic) -# --------------------------------------------------------------------------- - - -def _normalize_ticker(ticker: str) -> str: - from utils.ticker_utils import normalize_ticker - - try: - yahoo_ticker, _ = normalize_ticker(ticker) - return yahoo_ticker or ticker - except (ValueError, ImportError): - return ticker - - -def _validate_inputs(ticker, start_date, frequency, end_date=None): - if not isinstance(ticker, str) or not ticker.strip(): - raise ValueError("Ticker must be a non-empty string") - if not isinstance(start_date, dt.date): - raise ValueError("start_date must be a datetime.date object") - if frequency not in ("D", "W", "ME", "QE"): - raise ValueError("frequency must be one of ['D', 'W', 'ME', 'QE']") - if end_date is not None and not isinstance(end_date, dt.date): - raise ValueError("end_date must be a datetime.date object or None") - if end_date is not None and end_date < start_date: - raise ValueError("end_date must be on or after start_date") - - -def _fetch_daily_from_db(ticker: str, download_start: dt.date): - from data_pipeline.data_ops import DataService # doc-guard: allow=core-purity - - try: - DataService.initialize() - except Exception: - pass - try: - df = DataService.get_cleaned_daily(ticker, download_start, dt.date.today()) - if df is None or df.empty: - return None - df = df.rename( - columns={ - "open": "Open", - "high": "High", - "low": "Low", - "close": "Close", - "adj_close": "Adj Close", - "volume": "Volume", - } - ) - for col in ("Open", "High", "Low", "Close", "Adj Close", "Volume"): - if col in df.columns: - df[col] = pd.to_numeric(df[col], errors="coerce") - price_cols = [c for c in ("Open", "High", "Low", "Close", "Adj Close") if c in df.columns] - if price_cols: - df = df.dropna(subset=price_cols, how="all") - return df if not df.empty else None - except Exception as e: - logger.warning("DB fetch failed for %s: %s", ticker, e) - return None - - -def _download_data(ticker: str, download_start: dt.date): - from data_pipeline.yf_client import fetch_daily_ohlcv # doc-guard: allow=core-purity - - yf_end = dt.date.today() + dt.timedelta(days=1) - df = fetch_daily_ohlcv( - ticker, - download_start, - yf_end, - auto_adjust=False, - max_retries=_YF_MAX_RETRIES, - retry_base_delay=_YF_RETRY_BASE_DELAY, - ) - if df.empty: - logger.warning("No data downloaded for %s", ticker) - return None - required_columns = ["Open", "High", "Low", "Close", "Adj Close", "Volume"] - missing_columns = [col for col in required_columns if col not in df.columns] - if missing_columns: - logger.error("Missing columns for %s: %s", ticker, missing_columns) - return None - return df[required_columns] - - -def _refrequency(df: pd.DataFrame | None, frequency: str) -> pd.DataFrame | None: - if df is None or df.empty: - return None - try: - if frequency == "D": - df = df.copy() - df["LastClose"] = df["Close"].shift(1) - df["LastAdjClose"] = df["Adj Close"].shift(1) - return df - resampled = ( - df.resample(frequency) - .agg( - { - "Open": "first", - "High": "max", - "Low": "min", - "Close": "last", - "Adj Close": "last", - "Volume": "sum", - } - ) - .dropna() - ) - resampled["LastClose"] = resampled["Close"].shift(1) - resampled["LastAdjClose"] = resampled["Adj Close"].shift(1) - date_agg = df.resample(frequency).agg( - { - "Open": lambda x: x.index[0] if len(x) > 0 else pd.NaT, - "High": lambda x: x.index[x.argmax()] if len(x) > 0 else pd.NaT, - "Low": lambda x: x.index[x.argmin()] if len(x) > 0 else pd.NaT, - "Close": lambda x: x.index[-1] if len(x) > 0 else pd.NaT, - } - ) - resampled["OpenDate"] = date_agg["Open"] - resampled["HighDate"] = date_agg["High"] - resampled["LowDate"] = date_agg["Low"] - resampled["CloseDate"] = date_agg["Close"] - return resampled - except Exception as e: - logger.error("Error resampling data: %s", e) - return None - - -def _fetch_raw_data(ticker: str, user_start_date: dt.date, frequency: str): - """L1: DB L2: yfinance fallback. Returns (daily_df, ticker).""" - download_start = dt.date(1900, 1, 1) - raw_data = _fetch_daily_from_db(ticker, download_start) - db_data = raw_data - db_min = raw_data.index.min().date() if raw_data is not None and not raw_data.empty else None - needs_yfinance = raw_data is None or raw_data.empty or (db_min is not None and db_min > user_start_date) - if needs_yfinance: - yf_data = _download_data(ticker, download_start) - if yf_data is not None and not yf_data.empty: - raw_data = yf_data - elif db_data is not None and not db_data.empty: - logger.warning("yfinance download failed for %s, using available DB data.", ticker) - raw_data = db_data - return raw_data, ticker - - -# --------------------------------------------------------------------------- -# DataContext -# --------------------------------------------------------------------------- - class DataContext: """Immutable-ish container for market data fetched for a given ticker/horizon.""" @@ -255,50 +108,83 @@ def bars_date_range(self) -> tuple[str, str] | None: return None +def refrequency(df: pd.DataFrame | None, frequency: Frequency) -> pd.DataFrame | None: + """Resample daily bars to ``frequency`` and add ``LastClose``/``LastAdjClose``.""" + if df is None or df.empty: + return None + try: + if frequency == "D": + df = df.copy() + df["LastClose"] = df["Close"].shift(1) + df["LastAdjClose"] = df["Adj Close"].shift(1) + return df + resampled = ( + df.resample(frequency) + .agg( + { + "Open": "first", + "High": "max", + "Low": "min", + "Close": "last", + "Adj Close": "last", + "Volume": "sum", + } + ) + .dropna() + ) + resampled["LastClose"] = resampled["Close"].shift(1) + resampled["LastAdjClose"] = resampled["Adj Close"].shift(1) + date_agg = df.resample(frequency).agg( + { + "Open": lambda x: x.index[0] if len(x) > 0 else pd.NaT, + "High": lambda x: x.index[x.argmax()] if len(x) > 0 else pd.NaT, + "Low": lambda x: x.index[x.argmin()] if len(x) > 0 else pd.NaT, + "Close": lambda x: x.index[-1] if len(x) > 0 else pd.NaT, + } + ) + resampled["OpenDate"] = date_agg["Open"] + resampled["HighDate"] = date_agg["High"] + resampled["LowDate"] = date_agg["Low"] + resampled["CloseDate"] = date_agg["Close"] + return resampled + except Exception as e: + logger.error("Error resampling data: %s", e) + return None + + def build_data_context( + *, ticker: str, - start_date: dt.date, - frequency: Frequency = "W", - end_date: dt.date | None = None, + frequency: Frequency, + horizon: Horizon, + raw_data: pd.DataFrame | None, ) -> DataContext: - """Build a DataContext by fetching and resampling market data. + """Assemble a ``DataContext`` from bars that were already fetched. - Data pipeline: - 1. Normalise ticker (futu-format -> yahoo-format). - 2. Validate inputs. - 3. Fetch from DB first; fall back to yfinance if DB coverage is insufficient. - 4. Resample to requested frequency. + WHY keyword-only with an explicit ``raw_data``: the caller (a service) owns + acquisition, so this function stays pure and cannot accidentally re-introduce + the core→data_pipeline edge that batch B4 removed. """ - try: - _validate_inputs(ticker, start_date, frequency, end_date) - norm_ticker = _normalize_ticker(ticker) - raw_data, final_ticker = _fetch_raw_data(norm_ticker, start_date, frequency) - bars = _refrequency(raw_data, frequency) - horizon = Horizon( - start=start_date, - end=end_date or dt.date.today(), - user_provided_end=end_date is not None, - frequency=frequency, - ) - return DataContext( - ticker=final_ticker, - frequency=frequency, - horizon=horizon, - bars=bars, - daily_bars=raw_data, - ) - except Exception as e: - logger.error("Failed to build DataContext for %s: %s", ticker, e) - horizon = Horizon( + return DataContext( + ticker=ticker, + frequency=frequency, + horizon=horizon, + bars=refrequency(raw_data, frequency), + daily_bars=raw_data, + ) + + +def empty_data_context(ticker: str, start_date: dt.date, frequency: Frequency, end_date: dt.date | None) -> DataContext: + """Return an invalid context for a failed acquisition (no bars).""" + return DataContext( + ticker=ticker, + frequency=frequency, + horizon=Horizon( start=start_date, end=end_date or dt.date.today(), user_provided_end=end_date is not None, frequency=frequency, - ) - return DataContext( - ticker=ticker, - frequency=frequency, - horizon=horizon, - bars=None, - daily_bars=None, - ) + ), + bars=None, + daily_bars=None, + ) diff --git a/core/market/option_pnl.py b/core/market/option_pnl.py deleted file mode 100644 index 533792a..0000000 --- a/core/market/option_pnl.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Option portfolio P&L matrix computation. - -Domain: Market Analysis — Option P&L -Context: - - Computes per-share P&L across a price grid for a basket of options. - - Input format matches the legacy MarketAnalyzer.analyze_options schema. -Contracts: - - build_option_matrix(option_data, current_price) -> pd.DataFrame | None - - single_option_pnl(prices, option_type, strike, quantity, premium) -> np.ndarray - - find_breakeven_points(matrix_df) -> list[float] -Dependencies UPWARD: - - numpy, pandas -Dependencies DOWNWARD: - - core.market.charts.option_pnl, services.market.analysis.facade -""" - -from __future__ import annotations - -import logging - -import numpy as np -import pandas as pd - -logger = logging.getLogger(__name__) - - -def build_option_matrix(option_data: list, current_price: float) -> pd.DataFrame | None: - """Build a price-grid P&L DataFrame for a basket of options. - - Parameters - ---------- - option_data: - List of dicts with keys option_type (SC|SP|LC|LP), strike, quantity, premium. - current_price: - Current underlying price used to center the price grid [0.7×, 1.3×]. - - Returns - ------- - DataFrame indexed by price with a single column "PnL" (per-share). - """ - if not option_data or current_price is None or current_price <= 0: - return None - try: - price_range = np.linspace(current_price * 0.7, current_price * 1.3, 301) - matrix_df = pd.DataFrame(index=price_range) - matrix_df["PnL"] = 0.0 - for option in option_data: - pnl = single_option_pnl( - price_range, - option["option_type"], - option["strike"], - option["quantity"], - option["premium"], - ) - matrix_df["PnL"] += pnl - return matrix_df - except Exception as e: - logger.error("Error calculating option matrix: %s", e) - return None - - -def single_option_pnl( - prices: np.ndarray, option_type: str, strike: float, quantity: float, premium: float -) -> np.ndarray: - """Per-share P&L for a single option leg.""" - if option_type == "SC": - return np.where(prices > strike, (premium - (prices - strike)) * quantity, premium * quantity) - elif option_type == "SP": - return np.where(prices < strike, (premium - (strike - prices)) * quantity, premium * quantity) - elif option_type == "LC": - return np.where(prices > strike, (prices - strike - premium) * quantity, -premium * quantity) - elif option_type == "LP": - return np.where(prices < strike, (strike - prices - premium) * quantity, -premium * quantity) - else: - return np.zeros_like(prices) - - -def find_breakeven_points(matrix_df: pd.DataFrame) -> list[float]: - """Return all prices where P&L crosses zero (linear interpolation).""" - try: - pnl_values = matrix_df["PnL"].values - prices = matrix_df.index.values - breakeven_points = [] - for i in range(len(pnl_values) - 1): - if (pnl_values[i] <= 0 <= pnl_values[i + 1]) or (pnl_values[i] >= 0 >= pnl_values[i + 1]): - if pnl_values[i + 1] != pnl_values[i]: - breakeven_price = prices[i] - pnl_values[i] * (prices[i + 1] - prices[i]) / ( - pnl_values[i + 1] - pnl_values[i] - ) - breakeven_points.append(breakeven_price) - return breakeven_points - except Exception as e: - logger.error("Error finding breakeven points: %s", e) - return [] diff --git a/core/options/chain/analyzer.py b/core/options/chain/analyzer.py index efe4d80..03b1147 100644 --- a/core/options/chain/analyzer.py +++ b/core/options/chain/analyzer.py @@ -113,7 +113,7 @@ class OptionsChainAnalyzer: """Analyses an option chain snapshot. INVARIANT: this class performs no I/O. Callers fetch the snapshot upstream - (``data_pipeline.yf_client.fetch_option_chain``) and inject it via + (``data_pipeline.providers.yf_client.fetch_option_chain``) and inject it via ``snapshot=``. WHY: keeping ``core/`` pure means the analyzer can be driven entirely by fixture data in tests, and every network call stays behind the single yfinance exit point where proxy setup and throttling are enforced. @@ -123,7 +123,7 @@ def __init__(self, ticker: str = "^SPX", *, snapshot: dict): if snapshot is None: raise ValueError( "OptionsChainAnalyzer requires snapshot=... — fetch it upstream via " - "data_pipeline.yf_client.fetch_option_chain (core/ must stay pure)" + "data_pipeline.providers.yf_client.fetch_option_chain (core/ must stay pure)" ) self.ticker = ticker self._init_from_snapshot(snapshot) diff --git a/core/options/simulation/expiry_calendar.py b/core/options/simulation/expiry_calendar.py index 0597dc0..1820580 100644 --- a/core/options/simulation/expiry_calendar.py +++ b/core/options/simulation/expiry_calendar.py @@ -9,7 +9,7 @@ day) series for the short end, plus a *weekly* listed series on every Friday for the longer maturities. When a Friday is an exchange holiday the expiration rolls back to the previous business day (usually Thursday). - - The project otherwise ignores exchange holidays (see data_pipeline/cleaning + - The project otherwise ignores exchange holidays (see data_pipeline/transform/cleaning for the "B" frequency), but the listed-expiration rules *require* them, so we ship a self-contained NYSE approximation here rather than depending on an external calendar package. diff --git a/data_pipeline/data_ops/_globals.py b/data_pipeline/_state.py similarity index 60% rename from data_pipeline/data_ops/_globals.py rename to data_pipeline/_state.py index 933a9ff..e65313f 100644 --- a/data_pipeline/data_ops/_globals.py +++ b/data_pipeline/_state.py @@ -1,8 +1,23 @@ -"""Shared globals for data operations (locks, caches, TTLs). +"""Process-local shared state for the read + orchestrate layers. -All heavy-lifting state (cooldown locks, query cache, TTL constants) is -co-located here so that ``data_ops.facade.DataService`` and tests operate -on the **same** underlying objects. +Domain: Data Pipeline — Shared State +Context: + - All heavy-lifting state (update cooldown locks, the query cache, TTL + constants) is co-located here so ``read`` and ``orchestrate`` — and the + tests that inspect them — operate on the **same** underlying objects. + - It lives at the ``data_pipeline/`` root rather than in ``read/`` or + ``orchestrate/`` because both layers need it and neither may import the + other's internals (``read`` imports ``orchestrate``, so ``orchestrate`` + must not import ``read``). See the layer table in + docs/architecture_review.md §3. +Why not in ``store/``: this is in-memory state, not persistence. +Contracts: + - ``_query_cache`` / ``_cache_get`` / ``_cache_set`` / ``_cache_invalidate`` + - ``_update_locks`` / ``_update_lock_mutex`` / ``_UPDATE_COOLDOWN`` / ``GAP_SCAN_DAYS`` +Dependencies UPWARD: + - (none — stdlib + pandas only) +Dependencies DOWNWARD: + - read/ (query cache), orchestrate/ (update locks), tests """ import os diff --git a/data_pipeline/data_ops/__init__.py b/data_pipeline/data_ops/__init__.py deleted file mode 100644 index b038b93..0000000 --- a/data_pipeline/data_ops/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Data operations — DataService facade and shared cache/lock globals. - -All heavy-lifting state (cooldown locks, query cache, TTL constants) is -co-located in ``_globals`` so that DataService and tests operate on the -same underlying objects. -""" - -from ._globals import ( - _QUERY_CACHE_TTL, - GAP_SCAN_DAYS, - _cache_get, - _cache_invalidate, - _cache_set, - _query_cache, - _query_cache_lock, - _update_lock_mutex, - _update_locks, -) -from .facade import DataService - -__all__ = [ - "DataService", - "_query_cache", - "_query_cache_lock", - "_update_lock_mutex", - "_update_locks", - "_cache_get", - "_cache_set", - "_cache_invalidate", - "_QUERY_CACHE_TTL", - "GAP_SCAN_DAYS", -] diff --git a/data_pipeline/ingest/__init__.py b/data_pipeline/ingest/__init__.py new file mode 100644 index 0000000..d12cb27 --- /dev/null +++ b/data_pipeline/ingest/__init__.py @@ -0,0 +1,20 @@ +"""INGEST — acquisition → store glue. + +Domain: Data Pipeline — Ingest +Context: + - ADR 0011: ``providers/`` touches the external API and returns canonical + data; this package is the glue that decides *what* to fetch (business-day + gap detection, the auto-backfill cap) and writes it to ``raw_bars``. + - Live snapshots (spot / option chain) have no ingest module on purpose: they + are never persisted (ADR 0004), so there is nothing to ingest — callers go + to ``providers`` directly. +Contracts: + - ``ohlcv.download_bars`` — canonical bars for a window, provider-agnostic. + - ``ohlcv.upsert_raw_prices`` — never raises; degrades through ``PipelineResult``. +Dependencies UPWARD: + - providers (acquisition), store (raw_bars), data_pipeline (PipelineResult) +Dependencies DOWNWARD: + - orchestrate (the "make it ready" driver) +""" + +from __future__ import annotations diff --git a/data_pipeline/downloader.py b/data_pipeline/ingest/ohlcv.py similarity index 67% rename from data_pipeline/downloader.py rename to data_pipeline/ingest/ohlcv.py index 9d86c32..0106393 100644 --- a/data_pipeline/downloader.py +++ b/data_pipeline/ingest/ohlcv.py @@ -1,4 +1,22 @@ -"""Market data downloader: fetches raw price data from external sources.""" +"""Market data downloader: business-day gap detection + raw OHLCV upsert. + +Domain: Data Pipeline — Ingest Glue +Context: + - Acquisition itself lives in ``data_pipeline/providers/``. This module keeps + only the DB-aware parts: business-day gap detection, the auto-backfill cap, + and the ``raw_bars`` upsert. Batch B1 (see + docs/plans/business_line_reorg.md §6) moved the ``yf.download`` call behind + ``providers.yfinance_provider.download_daily_frame``, so this module no + longer imports yfinance. +Contracts: + - ``upsert_raw_prices(ticker, start, end, days) -> PipelineResult`` — never + raises; degraded outcomes are reported through the ``PipelineResult``. + - ``find_missing_business_days(ticker, start, end) -> list[date]`` +Dependencies UPWARD: + - providers.yfinance_provider (download), .db (fetch_df / upsert_many) +Dependencies DOWNWARD: + - data_pipeline/orchestrate (update / backfill), services/regime/ops/_bootstrap.py +""" import datetime as dt import logging @@ -6,12 +24,12 @@ from pathlib import Path import pandas as pd -import yfinance as yf # doc-guard: allow=single-yf-exit -from utils.network import yf_throttle - -from . import PipelineResult -from .db import fetch_df, upsert_many +from data_pipeline import PipelineResult +from data_pipeline.providers import get_provider +from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS +from data_pipeline.providers.yfinance_provider import to_canonical_bars +from data_pipeline.store.db import fetch_df, upsert_many logger = logging.getLogger(__name__) @@ -37,10 +55,10 @@ def _last_business_day_on_or_before(d: dt.date) -> dt.date: def _load_test_fixture(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: """Synthesise OHLCV for ``TEST_*`` tickers without touching the network. - Matches the column shape produced by ``_download_yf`` so the rest of the - pipeline is fixture-agnostic. If a CSV exists at - ``tests/fixtures/yf/.csv`` it is used verbatim; otherwise a - deterministic synthetic series is generated. + Matches the yfinance column shape (Title Case + ``Adj_Close``) so the + canonical mapping is exercised identically for fixtures and real downloads. + If a CSV exists at ``tests/fixtures/yf/.csv`` it is used verbatim; + otherwise a deterministic synthetic series is generated. """ csv_path = _FIXTURE_DIR / f"{ticker}.csv" if csv_path.exists(): @@ -68,7 +86,7 @@ def _load_test_fixture(ticker: str, start: dt.date, end: dt.date) -> pd.DataFram def find_missing_business_days(ticker: str, start: dt.date, end: dt.date) -> list[dt.date]: - """Return business days in [start, end] (inclusive) that have no row in raw_prices. + """Return business days in [start, end] (inclusive) that have no row in raw_bars. Uses the same Mon-Fri business-day calendar as `cleaning._get_business_days` so gaps map 1:1 with cleaning's expected index. Holidays are intentionally @@ -78,7 +96,7 @@ def find_missing_business_days(ticker: str, start: dt.date, end: dt.date) -> lis if len(expected) == 0: return [] df = fetch_df( - "SELECT date FROM raw_prices WHERE ticker=? AND date>=? AND date<=?", + "SELECT date FROM raw_bars WHERE ticker=? AND date>=? AND date<=?", (ticker, start.isoformat(), end.isoformat()), ) have: set[dt.date] = set() @@ -94,32 +112,28 @@ def find_missing_business_days(ticker: str, start: dt.date, end: dt.date) -> lis return [ts.date() for ts in expected if ts.date() not in have] -def _download_yf(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: - # Test tickers never hit the network. Useful for unit tests + ad-hoc - # smoke tests under rate-limit conditions; see `_load_test_fixture`. +def download_bars(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: + """Acquire daily bars for ``[start, end]`` (inclusive) in the canonical schema. + + ``TEST_*`` tickers never hit the network (useful for unit tests + ad-hoc + smoke tests under rate-limit conditions; see ``_load_test_fixture``). + Everything else goes through the provider registry, which returns canonical + bars already (ADR 0011). + """ if ticker.startswith("TEST_"): logger.info("Loading fixture data for test ticker %s (%s..%s)", ticker, start, end) - return _load_test_fixture(ticker, start, end) - # yfinance 'end' is exclusive, so pass end + 1 day to include the requested end date - yf_end = end + dt.timedelta(days=1) - yf_throttle() - df = yf.download(ticker, start=start, end=yf_end, interval="1d", progress=False, auto_adjust=False) - if df is None or df.empty: - return pd.DataFrame() - if isinstance(df.columns, pd.MultiIndex): - df.columns = df.columns.droplevel(1) - cols = ["Open", "High", "Low", "Close", "Adj Close", "Volume"] - for c in cols: - if c not in df.columns: - df[c] = pd.NA - return df[cols].rename(columns={"Adj Close": "Adj_Close"}) + # WHY the yfinance mapper for fixtures: the fixture frames deliberately + # mimic `yf.download`'s Title-Case shape, so they need the same mapping + # the provider applies to a real download. + return to_canonical_bars(_load_test_fixture(ticker, start, end)) + return get_provider().history(ticker, start, end) def upsert_raw_prices( ticker: str, start: dt.date | None = None, end: dt.date | None = None, days: int = 7 ) -> PipelineResult: """ - Download OHLCV for [start, end) and upsert into raw_prices. + Download OHLCV for [start, end) and upsert into raw_bars (canonical store). If df for a day is entirely NA, skip and keep existing row. Returns a PipelineResult with row count and any warnings. """ @@ -136,7 +150,7 @@ def upsert_raw_prices( # ── Gap-aware coverage check ── # Skip the network only when every business day in [start, end] is already - # present in raw_prices. Otherwise expand the download range to cover the + # present in raw_bars. Otherwise expand the download range to cover the # earliest gap so back-fills happen automatically after an outage. missing = find_missing_business_days(ticker, start, end) if not missing: @@ -162,7 +176,7 @@ def upsert_raw_prices( start = effective_start try: - df_new = _download_yf(ticker, start, end) + df_new = download_bars(ticker, start, end) except Exception as e: logger.error(f"Download failed for {ticker}: {e}", exc_info=True) return PipelineResult(ok=False, error=f"download_failed: {e}") @@ -179,42 +193,33 @@ def upsert_raw_prices( df_new.index = idx.tz_localize(None) if idx.tz is None else idx.tz_convert(None) df_new["date"] = df_new.index.date + # INVARIANT: bars arrive canonical (ADR 0011), so the ingest stage never + # touches vendor column names — the mapping lives in the provider. + bar_cols = list(CANONICAL_BAR_COLUMNS) + provider_name = get_provider().name + rows = [] for _d, row in df_new.iterrows(): date_str = row["date"].isoformat() # If all new values are NA, retain old data (skip insert) and log - if row[["Open", "High", "Low", "Close", "Adj_Close", "Volume"]].isna().all(): + if row[bar_cols].isna().all(): msg = f"Blank data for {ticker} on {date_str}; retaining old data if exists" logger.warning(msg) result.warnings.append(msg) continue - tup = ( - ticker, - date_str, - float(row.get("Open", pd.NA)) if pd.notna(row.get("Open")) else None, - float(row.get("High", pd.NA)) if pd.notna(row.get("High")) else None, - float(row.get("Low", pd.NA)) if pd.notna(row.get("Low")) else None, - float(row.get("Close", pd.NA)) if pd.notna(row.get("Close")) else None, - float(row.get("Adj_Close", pd.NA)) if pd.notna(row.get("Adj_Close")) else None, - float(row.get("Volume", pd.NA)) if pd.notna(row.get("Volume")) else None, - "yfinance", + rows.append( + ( + ticker, + date_str, + *[float(row[col]) if pd.notna(row.get(col)) else None for col in bar_cols], + provider_name, + ) ) - rows.append(tup) if rows: upsert_many( - "raw_prices", - [ - "ticker", - "date", - "open", - "high", - "low", - "close", - "adj_close", - "volume", - "provider", - ], + "raw_bars", + ["ticker", "date", *bar_cols, "provider"], rows, ) result.rows = len(rows) diff --git a/data_pipeline/orchestrate/__init__.py b/data_pipeline/orchestrate/__init__.py new file mode 100644 index 0000000..eba4189 --- /dev/null +++ b/data_pipeline/orchestrate/__init__.py @@ -0,0 +1,22 @@ +"""ORCHESTRATE — the "make the data ready" layer. + +Domain: Data Pipeline — Orchestrate +Context: + - ADR 0011: this package owns everything that *sequences* the pipeline: + the incremental/full update drivers, the chunked backfill loop, the + streaming slice memo, and the optional cron scheduler. It is the only + layer allowed to call ingest + transform + store together. + - CONSTRAINT (docs/constraints.md §6): no job queue. Work either runs inside + a request or on a bounded daemon thread with an 8s grace window. +Contracts: + - ``update.manual_update`` / ``update.seed_history`` — the pipeline drivers. + - ``backfill.ensure_range`` / ``backfill.needs_backfill`` — chunked coverage repair. + - ``job_cache`` — TTL'd streaming slice memo (``create_job`` / ``compute_or_get``). + - ``scheduler`` — optional APScheduler entry point (leader-locked). +Dependencies UPWARD: + - ingest, transform, store, data_pipeline (PipelineResult + _state) +Dependencies DOWNWARD: + - services/, routes/, app.py +""" + +from __future__ import annotations diff --git a/data_pipeline/data_ops/_range.py b/data_pipeline/orchestrate/backfill.py similarity index 65% rename from data_pipeline/data_ops/_range.py rename to data_pipeline/orchestrate/backfill.py index 2cfe256..5740160 100644 --- a/data_pipeline/data_ops/_range.py +++ b/data_pipeline/orchestrate/backfill.py @@ -5,7 +5,7 @@ import threading import time -import data_pipeline.db as _db +import data_pipeline.store.db as _db logger = logging.getLogger(__name__) @@ -20,13 +20,51 @@ _SENTINEL_MIN_DB_SPAN_DAYS = 365 -def needs_backfill(ticker: str, start: dt.date, end: dt.date) -> bool: - """Cheap probe: would ``ensure_range(ticker, start, end)`` need a download? +def _range_covers(cov, start: dt.date, end: dt.date) -> bool: + """True when a ``MIN(date)/MAX(date)/COUNT(*)`` coverage row spans [start, end]. + + Uses the same 3-day tail tolerance everywhere: the last few days may be a + weekend / not-yet-published, which is not a gap worth a download. + """ + if cov.empty or not cov.iloc[0]["n"]: + return False + try: + cmin = dt.date.fromisoformat(str(cov.iloc[0]["min_d"])) + cmax = dt.date.fromisoformat(str(cov.iloc[0]["max_d"])) + except (ValueError, TypeError): + return False + return cmin <= start and cmax >= end - dt.timedelta(days=3) + - Checks the memo and DB coverage only — never networks. Lets callers keep - wide-range backfills off the request thread. NOTE: the probe does not - model the sentinel short-circuit; a false positive there merely kicks a - background ``ensure_range`` that immediately short-circuits. +def _feature_bars_behind(ticker: str, start: dt.date, end: dt.date) -> bool: + """True when ``feature_bars`` does not cover [start, end] for *ticker*. + + WHY (plan §10 F4): ``ensure_range``'s clean-covered short-circuit and + ``needs_backfill`` used to probe ``clean_bars`` only, so a DB that has clean + rows but stale/missing ``feature_bars`` (a past ``process_frequencies`` + failure, or clean extended without a reprocess) was never healed — the + Statistical / Assessment slices read ``feature_bars`` and would render an + empty chart. ``process_frequencies`` writes D/W/ME/QE together and the D + series is 1:1 with ``clean_bars``, so the D-frequency span is the cheapest + honest probe. + """ + cov = _db.fetch_df( + "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n " + "FROM feature_bars WHERE ticker=? AND frequency='D'", + (ticker,), + ) + return not _range_covers(cov, start, end) + + +def needs_backfill(ticker: str, start: dt.date, end: dt.date) -> bool: + """Cheap probe: would ``ensure_range(ticker, start, end)`` need to do work? + + Checks the memo and DB coverage only — never networks. Returns True when + ``clean_bars`` is missing the span (needs a download) **or** ``feature_bars`` + lags behind clean (needs a reprocess only). Lets callers keep both kinds of + catch-up off the request thread. NOTE: the probe does not model the sentinel + short-circuit; a false positive there merely kicks a background + ``ensure_range`` that immediately short-circuits. """ now = time.monotonic() with _ensure_range_lock: @@ -36,21 +74,16 @@ def needs_backfill(ticker: str, start: dt.date, end: dt.date) -> bool: if (now - last_ts) < _ENSURE_RANGE_TTL and last_start <= start and last_end >= end: return False cov = _db.fetch_df( - "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_prices WHERE ticker=?", + "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_bars WHERE ticker=?", (ticker,), ) - if cov.empty or not cov.iloc[0]["n"]: + if not _range_covers(cov, start, end): return True - try: - existing_min = dt.date.fromisoformat(str(cov.iloc[0]["min_d"])) - existing_max = dt.date.fromisoformat(str(cov.iloc[0]["max_d"])) - except (ValueError, TypeError): - return True - return not (existing_min <= start and existing_max >= end - dt.timedelta(days=3)) + return _feature_bars_behind(ticker, start, end) def ensure_range(ticker: str, start: dt.date, end: dt.date) -> bool: - """Ensure clean_prices covers [start, end]. + """Ensure clean_bars covers [start, end]. NOTE: only *successful* coverage is memoised. Memoising a failure would make every caller within the TTL believe the range is covered and silently @@ -102,13 +135,13 @@ def ensure_range(ticker: str, start: dt.date, end: dt.date) -> bool: def _ensure_range_impl(ticker: str, start: dt.date, end: dt.date, now: float, was_sentinel: bool = False) -> bool: """Internal: actual backfill. Caller must hold the in-flight slot.""" - import data_pipeline.cleaning as _cl - import data_pipeline.downloader as _dl - import data_pipeline.processing as _pr - from data_pipeline.downloader import MAX_AUTO_BACKFILL_DAYS + import data_pipeline.ingest.ohlcv as _dl + import data_pipeline.transform.cleaning as _cl + import data_pipeline.transform.processing as _pr + from data_pipeline.ingest.ohlcv import MAX_AUTO_BACKFILL_DAYS cov = _db.fetch_df( - "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_prices WHERE ticker=?", + "SELECT MIN(date) AS min_d, MAX(date) AS max_d, COUNT(*) AS n FROM clean_bars WHERE ticker=?", (ticker,), ) existing_min = None @@ -121,6 +154,18 @@ def _ensure_range_impl(ticker: str, start: dt.date, end: dt.date, now: float, wa existing_min = existing_max = None if existing_min is not None and existing_min <= start and existing_max >= end - dt.timedelta(days=3): + # clean_bars covers the span — no download. But features may still lag + # (a past processing failure, or clean extended without a reprocess); + # rebuild them here so the memo below is honest (plan §10 F4). + if _feature_bars_behind(ticker, start, end): + logger.info("ensure_range: %s clean covered but feature_bars behind — reprocessing", ticker) + pr = _pr.process_frequencies(ticker, start, end) + if not pr.ok: + logger.warning("ensure_range reprocess failed for %s: %s", ticker, pr.error) + return False + from data_pipeline import _state as _g + + _g._cache_invalidate(ticker) with _ensure_range_lock: _ensure_range_memo[ticker] = (now, start, end) return True @@ -177,7 +222,7 @@ def _ensure_range_impl(ticker: str, start: dt.date, end: dt.date, now: float, wa if not pr.ok: logger.warning("ensure_range processing failed for %s: %s", ticker, pr.error) return False - from . import _globals as _g + from data_pipeline import _state as _g _g._cache_invalidate(ticker) with _ensure_range_lock: diff --git a/data_pipeline/job_cache.py b/data_pipeline/orchestrate/job_cache.py similarity index 79% rename from data_pipeline/job_cache.py rename to data_pipeline/orchestrate/job_cache.py index b07d91a..4010c3d 100644 --- a/data_pipeline/job_cache.py +++ b/data_pipeline/orchestrate/job_cache.py @@ -4,8 +4,10 @@ ------------ The streaming render flow is: - POST / → JobCache.create_job(form_data, tickers) - returns job_id; render skeleton. + POST / → JobCache.create_job(form_data, tickers, plan) + returns job_id; render skeleton. `plan` is the + readiness status list (batch B5) so /render/* + knows whether its dataset is already covered. GET /render/?job=… → JobCache.compute_or_get(job_id, ticker, kind, fn) runs `fn` (the slice computation) under a per-(job, ticker, kind) lock; subsequent @@ -51,6 +53,7 @@ class _JobEntry: __slots__ = ( "form_data", "tickers", + "plan", "created_at", "last_access", "results", @@ -59,26 +62,32 @@ class _JobEntry: "_master_lock", ) - def __init__(self, form_data: dict, tickers: list[str]): + def __init__(self, form_data: dict, tickers: list[str], plan: list | None = None): self.form_data: dict = form_data self.tickers: list[str] = list(tickers) + # Readiness statuses computed at POST time (batch B5 / ADR 0012). Empty + # for jobs registered without a readiness pass (tests, legacy callers). + self.plan: list = list(plan or []) self.created_at: float = time.monotonic() # TTL counts from the LAST access, not creation: a slice that computes # longer than the TTL must not lose its result to a mid-compute # eviction, and an active tab fan-out must not expire under load. self.last_access: float = self.created_at - # Memoised slice results, keyed by (ticker, kind). - self.results: dict[tuple[str, str], Any] = {} + # Memoised slice results, keyed by (ticker, kind, variant). `variant` is + # a digest of the module's own query-arg parameters (batch B7): without + # it a toolbar change (frequency, horizon, …) re-fires /render/ + # but this cache serves the first render for the whole job TTL. + self.results: dict[tuple[str, str, str], Any] = {} # Error-dict results get their own short TTL so a transient failure # (yfinance hiccup) is not sticky for the whole job lifetime. - self.error_results: dict[tuple[str, str], tuple[float, Any]] = {} + self.error_results: dict[tuple[str, str, str], tuple[float, Any]] = {} # Per-key locks so concurrent /render/* calls for the same slice # collapse into a single computation (single-flight). - self.key_locks: dict[tuple[str, str], threading.Lock] = {} + self.key_locks: dict[tuple[str, str, str], threading.Lock] = {} # Mutex for `key_locks` and `results` dict-level mutations. self._master_lock = threading.Lock() - def _lock_for(self, key: tuple[str, str]) -> threading.Lock: + def _lock_for(self, key: tuple[str, str, str]) -> threading.Lock: with self._master_lock: lock = self.key_locks.get(key) if lock is None: @@ -110,15 +119,17 @@ def _evict_expired(now: float | None = None) -> None: logger.debug("JobCache evicted %d stale job(s)", len(stale)) -def create_job(form_data: dict, tickers: list[str]) -> str: +def create_job(form_data: dict, tickers: list[str], plan: list | None = None) -> str: """Register a new job and return its opaque id. `form_data` is shallow-copied so later mutations by the caller don't - leak into the cache. + leak into the cache. `plan` is the readiness status list from + ``orchestrate/readiness.py`` (optional — ``/render/*`` degrades to its + original behaviour when it is absent). """ _evict_expired() job_id = uuid.uuid4().hex - entry = _JobEntry(form_data=dict(form_data), tickers=list(tickers)) + entry = _JobEntry(form_data=dict(form_data), tickers=list(tickers), plan=plan) with _jobs_lock: _jobs[job_id] = entry logger.info("JobCache created job=%s tickers=%s", job_id[:8], tickers) @@ -143,7 +154,7 @@ def get_job(job_id: str) -> _JobEntry | None: return entry -def _fresh_error(entry: _JobEntry, key: tuple[str, str], now: float) -> Any | None: +def _fresh_error(entry: _JobEntry, key: tuple[str, str, str], now: float) -> Any | None: """Return a still-fresh memoised error result, or None (and drop it if stale).""" hit = entry.error_results.get(key) if hit is None: @@ -155,8 +166,14 @@ def _fresh_error(entry: _JobEntry, key: tuple[str, str], now: float) -> Any | No return result -def compute_or_get(job_id: str, ticker: str, kind: str, compute_fn: Callable[[dict], Any]) -> Any: - """Memoised compute under a per-(ticker, kind) single-flight lock. +def compute_or_get(job_id: str, ticker: str, kind: str, compute_fn: Callable[[dict], Any], *, variant: str = "") -> Any: + """Memoised compute under a per-(ticker, kind, variant) single-flight lock. + + ``variant`` is a stable digest of any per-request parameters that change the + result (batch B7: a module toolbar sends its own ``?from=…&frequency=…`` on + each ``/render`` call). Two calls with the same ticker + kind but different + ``variant`` compute independently — otherwise the first render is served for + the whole job TTL and the toolbar change is silently ignored. Successful results are memoised for the full job TTL. Error-dict results (the slice methods' failure convention) are memoised only for @@ -170,7 +187,7 @@ def compute_or_get(job_id: str, ticker: str, kind: str, compute_fn: Callable[[di if entry is None: raise KeyError(f"unknown or expired job_id={job_id!r}") - key = (ticker, kind) + key = (ticker, kind, variant) # Fast path: already computed successfully. cached = entry.results.get(key) if cached is not None: @@ -216,10 +233,11 @@ def compute_or_get(job_id: str, ticker: str, kind: str, compute_fn: Callable[[di elapsed, ) logger.info( - "JobCache computed job=%s ticker=%s kind=%s in %.2fs", + "JobCache computed job=%s ticker=%s kind=%s%s in %.2fs", job_id[:8], ticker, kind, + f" variant={variant}" if variant else "", elapsed, ) return result diff --git a/data_pipeline/orchestrate/readiness.py b/data_pipeline/orchestrate/readiness.py new file mode 100644 index 0000000..364ff3d --- /dev/null +++ b/data_pipeline/orchestrate/readiness.py @@ -0,0 +1,313 @@ +"""Data-readiness planning and prefetch (ADR 0012). + +Domain: Data Pipeline — Orchestrate (readiness) +Context: + - Before batch B5, ``POST /`` computed nothing and each ``/render/`` + discovered missing coverage on its own, so whichever tab the user opened + first paid for the whole backfill. This module turns that implicit per-slice + decision into an explicit plan: given the ticker(s) and the modules the user + asked for, work out which datasets they need, probe the DB once, and kick the + missing ranges immediately. + - The module → dataset map is static and lives here so the route never has to + know which slice touches which table. +Constraints: + - CONSTRAINT (docs/constraints.md §6): no job queue. Coverage probes are + DB-only — never network on the request thread — and a missing range is kicked + on a daemon thread. ``check_and_kick`` therefore cannot block for more than a + probe, so ``POST /`` still returns the skeleton in < 1 s. + - INVARIANT: one pipeline run (download → clean → process) fills both + ``clean_bars`` and ``feature_bars``, so a single kick per (ticker, range) + covers every dataset the plan asked for. The probe (``backfill.needs_backfill``) + checks *both* families — clean coverage **and** a feature-bars lag — so a DB + with clean rows but stale features is still healed (plan §10 F4). +Contracts: + - ``plan_datasets(tickers, modules, *, start, end, today=None)`` + - ``check_and_kick(plan, *, kick=None)`` + - ``dataset_for_module(module)`` / ``status_for(plan, ticker, module)`` + - ``kick_backfill(ticker, start, end)`` / ``join_backfills(timeout=None)`` +Dependencies UPWARD: + - data_pipeline.orchestrate.backfill, data_pipeline.store.db +Dependencies DOWNWARD: + - routes/core.py (through services/market/readiness.py), + data_pipeline/read/_query.py +""" + +from __future__ import annotations + +import datetime as dt +import logging +import threading +import time +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + +# INVARIANT: module token (the same token ``/render/`` and the sidebar use) +# → the datasets that module reads. A module with no DB dependency maps to (). +KIND_DATASETS: dict[str, tuple[str, ...]] = { + "market_review": ("clean_bars",), + "statistical": ("feature_bars",), + "assessment": ("feature_bars",), + # The volatility slice renders the HV context next to the live chain, so it + # needs daily bars as well as the live snapshot. + "options_chain": ("clean_bars",), + # Live-only modules: no stored dataset to prefetch (ADR 0004). + "payoff_ratio": (), + "regime": (), + "simulation": (), + "option_pricing_matrix": (), +} + +ALL_MODULES: tuple[str, ...] = tuple(KIND_DATASETS) + +# DOMAIN: default lookback when the caller does not pass a horizon. Mirrors the +# direct-URL bootstrap default in services/market/dispatch.py (2 years). +DEFAULT_LOOKBACK_DAYS = 365 * 2 + +# DOMAIN: how long /render/ will hold a tab in the "data is being +# prepared" state (cold start only) before falling back to computing with +# whatever coverage exists. Bounded so a stuck backfill degrades into the normal +# (graceful) slice error instead of an endless spinner. +HOLD_SECONDS = 30.0 + + +@dataclass(frozen=True) +class DatasetRequest: + """One (ticker, dataset) the requested modules will need.""" + + ticker: str + dataset: str + start: dt.date + end: dt.date + module: str + + +@dataclass(frozen=True) +class ReadinessStatus: + """Outcome of the coverage probe for one request. + + ``kicked_at`` is a ``time.monotonic()`` stamp (0.0 when nothing was kicked); + it lives on the status so ``hold_seconds_left`` needs no extra bookkeeping. + ``has_data`` distinguishes a **cold start** (no rows at all — nothing to show + yet, worth holding the tab for) from a **partial gap** (usable history exists, + so the tab should render now and let the backfill catch up behind it). + """ + + ticker: str + dataset: str + module: str + state: str # "covered" | "kicked" + kicked_at: float = field(default=0.0) + has_data: bool = True + # The (ticker, range) that was probed/kicked — used to ask whether the + # backfill is still running (see should_hold). + start: dt.date | None = None + end: dt.date | None = None + + +def dataset_for_module(module: str) -> str | None: + """Return the first dataset ``module`` reads, or None when it is live-only.""" + datasets = KIND_DATASETS.get(module) or () + return datasets[0] if datasets else None + + +def plan_datasets( + tickers: list[str], + modules: list[str], + *, + start: dt.date | None = None, + end: dt.date | None = None, + today: dt.date | None = None, +) -> list[DatasetRequest]: + """Union over the requested modules, one entry per (ticker, dataset). + + ORDER: preserves the module order the caller passed and, inside a module, the + ticker order — so the caller can prioritise ``tickers[0]`` downstream. + """ + today = today or dt.date.today() + end = end or today + start = start or (end - dt.timedelta(days=DEFAULT_LOOKBACK_DAYS)) + requests: list[DatasetRequest] = [] + seen: set[tuple[str, str]] = set() + for module in modules: + for dataset in KIND_DATASETS.get(module, ()): + for ticker in tickers: + key = (ticker, dataset) + if key in seen: + continue + seen.add(key) + requests.append(DatasetRequest(ticker=ticker, dataset=dataset, start=start, end=end, module=module)) + return requests + + +def check_and_kick(plan: list[DatasetRequest], *, kick=None) -> list[ReadinessStatus]: + """Probe coverage per (ticker, range) and kick the missing ones. + + ``kick`` is injectable so tests can assert the decision without starting + threads. Returns one status per request, in plan order. + """ + from data_pipeline.orchestrate import backfill as _bf + + kick = kick or kick_backfill + statuses: list[ReadinessStatus] = [] + for ticker, group in _by_ticker(plan): + start = min(req.start for req in group) + end = max(req.end for req in group) + try: + missing = _bf.needs_backfill(ticker, start, end) + except Exception as exc: # noqa: BLE001 — a probe must never break the POST + logger.warning("readiness probe failed for %s: %s", ticker, exc) + missing = False + kicked_at = 0.0 + has_data = True + if missing: + has_data = _has_any_prices(ticker) + kicked_at = time.monotonic() + kick(ticker, start, end) + state = "kicked" if missing else "covered" + statuses.extend( + ReadinessStatus( + ticker=req.ticker, + dataset=req.dataset, + module=req.module, + state=state, + kicked_at=kicked_at, + has_data=has_data, + start=start, + end=end, + ) + for req in group + ) + if statuses: + logger.info( + "readiness: %d request(s) — %d covered, %d kicked", + len(statuses), + sum(1 for s in statuses if s.state == "covered"), + sum(1 for s in statuses if s.state == "kicked"), + ) + return statuses + + +def status_for(plan: list[ReadinessStatus] | None, ticker: str, module: str) -> ReadinessStatus | None: + """Return the plan entry for ``(ticker, module)``, or None when not planned.""" + dataset = dataset_for_module(module) + if dataset is None or not plan: + return None + for status in plan: + if status.ticker == ticker and status.dataset == dataset: + return status + return None + + +def hold_seconds_left(status: ReadinessStatus | None, *, now: float | None = None) -> float | None: + """Seconds ``/render/*`` should keep holding this tab, or None to compute now. + + Holds only on a **cold start** (``has_data`` False): with usable history the tab + should paint immediately while the backfill fills the tail, which is what the + per-slice grace period already does. + + WHY bounded: an unbounded hold would spin forever if the backfill failed; after + HOLD_SECONDS the caller computes with whatever coverage exists and the slice's + own graceful-degradation path takes over. + """ + if status is None or status.state != "kicked" or status.has_data or not status.kicked_at: + return None + remaining = HOLD_SECONDS - ((now if now is not None else time.monotonic()) - status.kicked_at) + return remaining if remaining > 0 else None + + +def is_backfill_running(status: ReadinessStatus) -> bool: + """True when the daemon-thread backfill kicked for this entry is still alive.""" + if status.start is None or status.end is None: + return False + key = (status.ticker, str(status.start), str(status.end)) + with _backfill_lock: + thread = _backfill_threads.get(key) + return thread is not None and thread.is_alive() + + +def should_hold(status: ReadinessStatus | None, *, now: float | None = None) -> bool: + """Should ``/render/`` wait for data instead of computing now? + + Holds only while (a) this was a cold start, (b) the HOLD_SECONDS window has not + elapsed, and (c) the backfill thread is **still running**. (c) is what keeps a + failed download honest: once the thread dies the tab stops claiming to be + "preparing data" and the slice reports the real outcome (its own error). + """ + if hold_seconds_left(status, now=now) is None: + return False + return is_backfill_running(status) + + +def _has_any_prices(ticker: str) -> bool: + """True when the DB already holds at least one priced row for ``ticker``.""" + try: + from data_pipeline.store.repos import count_clean_rows + + return count_clean_rows(ticker) > 0 + except Exception as exc: # noqa: BLE001 — a probe must never break the POST + logger.debug("readiness: existing-data probe failed for %s: %s", ticker, exc) + return True + + +def _by_ticker(plan: list[DatasetRequest]) -> list[tuple[str, list[DatasetRequest]]]: + """Group a plan by ticker, preserving first-appearance order.""" + order: list[str] = [] + groups: dict[str, list[DatasetRequest]] = {} + for req in plan: + if req.ticker not in groups: + groups[req.ticker] = [] + order.append(req.ticker) + groups[req.ticker].append(req) + return [(t, groups[t]) for t in order] + + +# ── Daemon-thread backfill kicker ─────────────────────────────────────────── +# WHY it lives here and not in read/: `read` imports `orchestrate` (the read path +# triggers refreshes), so orchestrate may not import read. Both callers now share +# this one kicker instead of read keeping a private copy. +_backfill_lock = threading.Lock() +_backfill_threads: dict[tuple, threading.Thread] = {} + + +def kick_backfill(ticker: str, start: dt.date, end: dt.date) -> None: + """Start a daemon-thread backfill for ``[start, end]`` unless one is running. + + The in-flight map is the same de-duplication ``ensure_range`` applies + internally, hoisted to the thread level so a POST that fans out over N + tickers cannot start N copies of the same work. + """ + key = (ticker, str(start), str(end)) + with _backfill_lock: + existing = _backfill_threads.get(key) + if existing is not None and existing.is_alive(): + return + t = threading.Thread(target=_run_backfill, args=(ticker, start, end, key), daemon=True) + _backfill_threads[key] = t + t.start() + logger.info("background backfill kicked for %s [%s .. %s]", ticker, start, end) + + +def _run_backfill(ticker: str, start: dt.date, end: dt.date, key: tuple) -> None: + from data_pipeline.orchestrate import backfill as _bf + + try: + _bf.ensure_range(ticker, start, end) + except Exception as e: # noqa: BLE001 + logger.warning("background backfill failed for %s: %s", ticker, e) + finally: + # Daemon threads never run dispatch's finally-cleanups; drop this + # thread's SQLite connection so _all_conns doesn't grow per backfill. + from data_pipeline.store.db import close_thread_conn + + close_thread_conn() + with _backfill_lock: + _backfill_threads.pop(key, None) + + +def join_backfills(timeout: float | None = None) -> None: + """Test helper: wait for all in-flight background backfills.""" + with _backfill_lock: + threads = list(_backfill_threads.values()) + for t in threads: + t.join(timeout=timeout) diff --git a/data_pipeline/scheduler.py b/data_pipeline/orchestrate/scheduler.py similarity index 93% rename from data_pipeline/scheduler.py rename to data_pipeline/orchestrate/scheduler.py index 212b77c..23aaabe 100644 --- a/data_pipeline/scheduler.py +++ b/data_pipeline/orchestrate/scheduler.py @@ -11,7 +11,10 @@ import os from pathlib import Path -from .data_ops import DataService +# WHY (no DataService): the scheduler drives the pipeline, it does not read. +# Going through the read facade here would create read <-> orchestrate cycle +# (read/_query.py already imports this package to trigger refreshes). +from data_pipeline.orchestrate.update import manual_update logger = logging.getLogger(__name__) @@ -93,7 +96,7 @@ def start_daily_update(self, tickers: list[str]): def job(): for t in tickers: try: - DataService.manual_update(t, days=7) + manual_update(t, days=7) logger.info(f"Auto-updated {t}") except Exception as e: logger.exception(f"Auto-update failed for {t}: {e}") @@ -115,7 +118,7 @@ def correlation_job(): for t in tickers: try: # Trigger a full data update which includes correlation recalculation - DataService.manual_update(t, days=30) + manual_update(t, days=30) logger.info(f"Monthly correlation update completed for {t}") except Exception as e: logger.exception(f"Monthly correlation update failed for {t}: {e}") diff --git a/data_pipeline/data_ops/_update.py b/data_pipeline/orchestrate/update.py similarity index 87% rename from data_pipeline/data_ops/_update.py rename to data_pipeline/orchestrate/update.py index 15806a6..8f7ee59 100644 --- a/data_pipeline/data_ops/_update.py +++ b/data_pipeline/orchestrate/update.py @@ -4,10 +4,9 @@ import logging import time +from data_pipeline import _state as _g from utils.ticker_utils import is_valid_ticker_format -from . import _globals as _g - logger = logging.getLogger(__name__) @@ -38,7 +37,7 @@ def manual_update(ticker: str, days: int = 7) -> bool: start = end - dt.timedelta(days=days - 1) scan_start = end - dt.timedelta(days=_g.GAP_SCAN_DAYS) - from data_pipeline.downloader import find_missing_business_days + from data_pipeline.ingest.ohlcv import find_missing_business_days gaps = find_missing_business_days(ticker, scan_start, end) if gaps and min(gaps) < start: @@ -51,9 +50,9 @@ def manual_update(ticker: str, days: int = 7) -> bool: ) start = min(gaps) - import data_pipeline.cleaning as _cl - import data_pipeline.downloader as _dl - import data_pipeline.processing as _pr + import data_pipeline.ingest.ohlcv as _dl + import data_pipeline.transform.cleaning as _cl + import data_pipeline.transform.processing as _pr dl_result = _dl.upsert_raw_prices(ticker, start, end) if not dl_result.ok: @@ -81,9 +80,9 @@ def seed_history(ticker: str, years: int = 5) -> None: """One-time helper to seed multi-year history for a ticker into the DB.""" end = dt.date.today() start = end - dt.timedelta(days=years * 365) - import data_pipeline.cleaning as _cl - import data_pipeline.downloader as _dl - import data_pipeline.processing as _pr + import data_pipeline.ingest.ohlcv as _dl + import data_pipeline.transform.cleaning as _cl + import data_pipeline.transform.processing as _pr _dl.upsert_raw_prices(ticker, start, end) _cl.clean_range(ticker, start, end) diff --git a/data_pipeline/providers/__init__.py b/data_pipeline/providers/__init__.py new file mode 100644 index 0000000..b975bef --- /dev/null +++ b/data_pipeline/providers/__init__.py @@ -0,0 +1,44 @@ +"""Data-provider seam — the only package that touches an external market-data API. + +Domain: Data Pipeline — Providers +Context: + - ADR 0011. Every external acquisition call lives under this package and is + mapped onto the canonical schema in ``providers/base.py``; processing and + serving stay provider-agnostic. + - ``yf_client.py`` is a compatibility shim over this package for one release; + ``ingest/ohlcv.py`` keeps only gap detection + DB upsert. +Contracts: + - ``get_provider(name=None)``: resolve a ``MarketDataProvider`` implementation. + - ``available_providers()``: registered provider names. + - Canonical shapes: ``CanonicalBar`` / ``OptionLeg`` / ``OptionChainSnapshot``. +Dependencies UPWARD: + - (none) +Dependencies DOWNWARD: + - providers/base, providers/_registry, providers/yfinance_provider, + providers/yf_snapshot +""" + +from __future__ import annotations + +from data_pipeline.providers._registry import available_providers, get_provider +from data_pipeline.providers.base import ( + CANONICAL_BAR_COLUMNS, + CANONICAL_LEG_COLUMNS, + CanonicalBar, + MarketDataProvider, + OptionChainSnapshot, + OptionLeg, + bars_to_frame, +) + +__all__ = [ + "CANONICAL_BAR_COLUMNS", + "CANONICAL_LEG_COLUMNS", + "CanonicalBar", + "MarketDataProvider", + "OptionChainSnapshot", + "OptionLeg", + "available_providers", + "bars_to_frame", + "get_provider", +] diff --git a/data_pipeline/providers/_log.py b/data_pipeline/providers/_log.py new file mode 100644 index 0000000..682b824 --- /dev/null +++ b/data_pipeline/providers/_log.py @@ -0,0 +1,28 @@ +"""Provider-side hook into the pipeline's failure log. + +Domain: Data Pipeline — Providers (shared) +Context: + - Every acquisition failure worth surfacing in ``/health/data`` lands in + ``data_quality_log`` (see ``data_pipeline/store/quality_log.py``). Both option-chain + and general yfinance provider modules need to record failures, and neither + may import the other, so the best-effort wrapper lives here. +Why the ``source`` strings still read ``yf_client.*``: + - Batch B1 moved these calls without changing the stored ``data_quality_log`` + rows; renaming the source labels is a separate, observable change and is + deliberately deferred (see docs/plans/business_line_reorg.md §6 B1). +Dependencies UPWARD: + - data_pipeline.store.quality_log (imported lazily — keeps package import cheap and + avoids a cycle at import time) +""" + +from __future__ import annotations + + +def _log_dq(source: str, error_class: str, message: str, *, ticker: str | None = None) -> None: + """Best-effort write to ``data_quality_log``. Never raises.""" + try: + from data_pipeline.store.quality_log import log_failure + + log_failure(source, error_class, message, ticker=ticker) + except Exception: # noqa: BLE001 + pass diff --git a/data_pipeline/providers/_registry.py b/data_pipeline/providers/_registry.py new file mode 100644 index 0000000..0703494 --- /dev/null +++ b/data_pipeline/providers/_registry.py @@ -0,0 +1,55 @@ +"""Provider registry: name → MarketDataProvider instance. + +Domain: Data Pipeline — Provider Selection +Context: + - ADR 0011: acquisition is selected by name, so a second vendor is one new + provider module plus one line in ``_FACTORIES`` — no caller changes. + yfinance stays the only implementation until a second provider is actually + needed (the seam is the deliverable, not the vendor). + - Selection order: explicit ``name`` argument → ``MARKET_DATA_PROVIDER`` env + var → ``DEFAULT_PROVIDER``. +Contracts: + - ``get_provider(name=None) -> MarketDataProvider`` + - ``available_providers() -> tuple[str, ...]`` +Dependencies UPWARD: + - (none) +Dependencies DOWNWARD: + - providers/base, providers/yfinance_provider +""" + +from __future__ import annotations + +import os +from collections.abc import Callable + +from data_pipeline.providers.base import MarketDataProvider +from data_pipeline.providers.yfinance_provider import YFinanceProvider + +PROVIDER_ENV_VAR = "MARKET_DATA_PROVIDER" +DEFAULT_PROVIDER = "yfinance" + +_FACTORIES: dict[str, Callable[[], MarketDataProvider]] = { + DEFAULT_PROVIDER: YFinanceProvider, +} +# WHY (cache): providers are stateless, but re-reading env/config on every fetch +# is pointless. Instances are keyed by resolved name. +_INSTANCES: dict[str, MarketDataProvider] = {} + + +def available_providers() -> tuple[str, ...]: + """Return the registered provider names, sorted.""" + return tuple(sorted(_FACTORIES)) + + +def get_provider(name: str | None = None) -> MarketDataProvider: + """Return the (cached) provider instance for ``name``. + + Raises ``ValueError`` for an unknown name rather than silently falling back + to yfinance — a typo in ``MARKET_DATA_PROVIDER`` must fail loudly. + """ + resolved = name or os.environ.get(PROVIDER_ENV_VAR) or DEFAULT_PROVIDER + if resolved not in _FACTORIES: + raise ValueError(f"unknown data provider {resolved!r}; available: {list(available_providers())}") + if resolved not in _INSTANCES: + _INSTANCES[resolved] = _FACTORIES[resolved]() + return _INSTANCES[resolved] diff --git a/data_pipeline/providers/base.py b/data_pipeline/providers/base.py new file mode 100644 index 0000000..daf0a23 --- /dev/null +++ b/data_pipeline/providers/base.py @@ -0,0 +1,153 @@ +"""Provider seam: the canonical internal schema and the acquisition protocol. + +Domain: Data Pipeline — Acquisition Seam +Context: + - ADR 0011 splits acquisition from processing/serving: a *provider* owns every + call to an external market-data API and maps that API's fields onto one + canonical internal schema, so nothing downstream of acquisition sees a + vendor-specific shape. + - yfinance is the only implementation today (ADR 0002, as amended by 0011). + This protocol is deliberately sketched against *two* field maps — yfinance + and the archived futu integration + (``archive/futu_integration/field_mapping.md``) — so it does not bake in + yfinance-isms. That comparison is the §8 Q5 decision gate for batch B1 and is + recorded in ADR 0011. +Contracts: + - ``MarketDataProvider``: the minimal acquisition surface. + - ``CANONICAL_BAR_COLUMNS`` / ``CANONICAL_LEG_COLUMNS``: canonical frame columns. + - ``CanonicalBar`` / ``OptionLeg`` / ``OptionChainSnapshot``: canonical records. +Unit conventions — INVARIANT for every provider implementation: + - ``iv`` is a **decimal** (0.2436 means 24.36 %). futu reports percent, + yfinance reports decimal; the provider normalises at its own boundary. + - ``bid`` / ``ask`` may be ``None``. futu's ``get_stock_quote`` exposes no + bid/ask without an ORDER_BOOK subscription, so "absent" must be expressible — + a provider must never invent a quote. + - ``inTheMoney`` is intentionally **not** canonical: it is derivable from + ``(strike, spot)`` and futu has no equivalent column. + - OHLCV values are plain floats; ``volume`` / ``open_interest`` are + non-negative counts. +Dependencies UPWARD: + - (none — no external SDK is imported here; implementations sit beside it) +Dependencies DOWNWARD: + - providers/yfinance_provider.py, providers/yf_snapshot.py, + providers/_registry.py +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Protocol, runtime_checkable + +import pandas as pd + +# INVARIANT: the column names (and order) of the frame ``history()`` returns, +# indexed by a tz-naive DatetimeIndex of trading days. +CANONICAL_BAR_COLUMNS: tuple[str, ...] = ("open", "high", "low", "close", "adj_close", "volume") + +# INVARIANT: the fields every canonical option leg exposes. See the unit +# conventions in the module docstring for the semantics of ``iv`` / ``bid``. +CANONICAL_LEG_COLUMNS: tuple[str, ...] = ( + "strike", + "bid", + "ask", + "last", + "iv", + "open_interest", + "volume", +) + + +@dataclass(frozen=True) +class CanonicalBar: + """One daily OHLCV bar in the canonical schema.""" + + provider: str + symbol: str + date: dt.date + open: float | None = None + high: float | None = None + low: float | None = None + close: float | None = None + adj_close: float | None = None + volume: float | None = None + + +@dataclass(frozen=True) +class OptionLeg: + """One option contract quote in the canonical schema.""" + + strike: float + bid: float | None = None + ask: float | None = None + last: float | None = None + iv: float | None = None + open_interest: float | None = None + volume: float | None = None + + +@dataclass(frozen=True) +class OptionChainSnapshot: + """A live option-chain snapshot. Never persisted — see ADR 0004.""" + + provider: str + symbol: str + spot: float | None + expiries: tuple[str, ...] = () + chain: Mapping[str, Mapping[str, tuple[OptionLeg, ...]]] = field(default_factory=dict) + + def legs(self, expiry: str, side: str) -> tuple[OptionLeg, ...]: + """Return the legs for ``expiry`` and ``side`` (``calls`` | ``puts``).""" + return tuple(self.chain.get(expiry, {}).get(side, ())) + + +@runtime_checkable +class MarketDataProvider(Protocol): + """The acquisition surface a data provider must offer. + + An implementation owns (a) every call to its external API and (b) the mapping + from that API's fields onto the canonical schema above. Callers resolve an + implementation through ``providers.get_provider()`` rather than importing a + concrete provider, so adding a vendor is a localised change (ADR 0011). + """ + + @property + def name(self) -> str: + """Stable provider id, also stored in the ``provider`` column.""" + ... + + def history(self, symbol: str, start: dt.date, end: dt.date) -> pd.DataFrame: + """Daily bars for ``[start, end]`` as a ``CANONICAL_BAR_COLUMNS`` frame.""" + ... + + def close_panel( + self, + symbols: list[str], + *, + start: dt.date | str | None = None, + end: dt.date | str | None = None, + period: str | None = None, + ) -> pd.DataFrame: + """Wide Close-price frame: index = date, columns = symbols.""" + ... + + def spot(self, symbol: str) -> float | None: + """Latest traded price for ``symbol``, or ``None`` when unavailable.""" + ... + + def option_chain(self, symbol: str) -> OptionChainSnapshot: + """Live chain snapshot (spot + expiries + canonical legs).""" + ... + + +def bars_to_frame(bars: Iterable[CanonicalBar]) -> pd.DataFrame: + """Render ``CanonicalBar`` records as a ``CANONICAL_BAR_COLUMNS`` DataFrame.""" + rows = list(bars) + if not rows: + return pd.DataFrame(columns=list(CANONICAL_BAR_COLUMNS)) + frame = pd.DataFrame( + [{col: getattr(bar, col) for col in CANONICAL_BAR_COLUMNS} for bar in rows], + index=pd.DatetimeIndex([pd.Timestamp(bar.date) for bar in rows]), + ) + return frame.sort_index() diff --git a/data_pipeline/providers/yf_client.py b/data_pipeline/providers/yf_client.py new file mode 100644 index 0000000..bf92f7a --- /dev/null +++ b/data_pipeline/providers/yf_client.py @@ -0,0 +1,37 @@ +"""Compatibility shim over the yfinance provider seam. + +Domain: Data Pipeline — yfinance compatibility surface +Context: + - Batch B1 (docs/plans/business_line_reorg.md §6) moved every yfinance call + into ``data_pipeline/providers/``. This module is kept for one release so the + existing importers (``services/``, ``data_pipeline/read/``) do not have to + change in the same PR as the extraction. See ADR 0011. (``core/`` used to + import it too; batch B4 removed that — ``core`` no longer touches + ``data_pipeline``.) + - New code should import from ``data_pipeline.providers`` (canonical shapes) + instead of here. +Contracts: + - Re-exports the legacy function contracts unchanged: ``fetch_spot``, + ``fetch_spots_bulk``, ``fetch_option_chain``, ``fetch_close_panel``, + ``fetch_daily_ohlcv``. +Design rules: + - CONSTRAINT: this module must not import ``yfinance``; the single exit point + is ``data_pipeline/providers/`` (enforced by doc_guard ``single-yf-exit``). +Dependencies UPWARD: + - providers/yf_snapshot (live snapshots), providers/yfinance_provider (bars) +Dependencies DOWNWARD: + - services/*, data_pipeline/read/* +""" + +from __future__ import annotations + +from data_pipeline.providers.yf_snapshot import fetch_option_chain, fetch_spot, fetch_spots_bulk +from data_pipeline.providers.yfinance_provider import fetch_close_panel, fetch_daily_ohlcv + +__all__ = [ + "fetch_close_panel", + "fetch_daily_ohlcv", + "fetch_option_chain", + "fetch_spot", + "fetch_spots_bulk", +] diff --git a/data_pipeline/yf_client.py b/data_pipeline/providers/yf_snapshot.py similarity index 53% rename from data_pipeline/yf_client.py rename to data_pipeline/providers/yf_snapshot.py index ab8d0ea..bb5a698 100644 --- a/data_pipeline/yf_client.py +++ b/data_pipeline/providers/yf_snapshot.py @@ -1,52 +1,54 @@ -""" -Unified yfinance access layer. +"""yfinance live-snapshot acquisition: spot price + option chain. +Domain: Data Pipeline — yfinance Provider (Live Snapshots) Context: -- All direct yfinance calls (`yf.Ticker`, `yf.download`, `option_chain`, - `fast_info`) for live snapshot data go through this module. OHLCV historical - bulk downloads remain in ``downloader.py`` (which has DB-aware gap detection). -- Yahoo Finance has no SLA: rate limits, transient 5xx, and silently empty - payloads are routine. See docs/constraints.md §2 and ADR 0002 / 0005. - + - Live snapshots are the endpoints ADR 0004 says are never persisted: the + current spot and the current option chain. They are also the largest + yfinance surface, so they live in their own module and keep + ``providers/yfinance_provider.py`` under the 400-line god-file cap — a split + pre-registered in docs/architecture_review.md §2. + - Batch B1 moved this code verbatim out of ``data_pipeline/yf_client.py`` (no + behaviour change); only the canonical mapping at the bottom is new. + - INVARIANT (keeps the import graph acyclic): this module must never import + ``providers/yfinance_provider.py``. +Contracts: + - ``fetch_spot(ticker)`` / ``fetch_spots_bulk(tickers)`` -> ``float`` / ``dict``. + - ``fetch_option_chain(ticker)`` keeps the legacy payload contract + (``ticker`` / ``spot`` / ``expiries`` / ``chain{expiry:{calls,puts}}``) for + callers that still import it through the ``data_pipeline.providers.yf_client`` shim. + - ``to_option_chain_snapshot(payload)`` -> canonical ``OptionChainSnapshot``. Design rules: -- CONSTRAINT: every public function MUST call ``yf_throttle()`` before each - yfinance call. See docs/decisions/0005-token-bucket-throttle.md. -- WHY (no caching here): caching is the caller's concern (e.g. - ``app._option_chain_cache``, ``data_service`` 60s freshness window). -- WHY (never raise on transient failure): callers receive ``None`` / empty - dict and decide how to surface the error. Raising here would cascade into - unhandled 500s from many different routes. -- INVARIANT: returns plain Python types (float / dict / DataFrame), never - yfinance-specific objects — keeps the rest of the pipeline mockable. + - CONSTRAINT: every public function calls ``yf_throttle()`` before each + yfinance call — see docs/decisions/0005-token-bucket-throttle.md. + - WHY (never raise on transient failure): callers receive ``None`` / an empty + chain and decide how to surface the error. Raising here would cascade into + unhandled 500s from several routes. +Dependencies UPWARD: + - utils.network (throttle), data_pipeline.store.quality_log (via providers._log) +Dependencies DOWNWARD: + - providers/base, providers/_log """ from __future__ import annotations import logging +import math import os import threading -import time +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any import pandas as pd import yfinance as yf +from data_pipeline.providers._log import _log_dq +from data_pipeline.providers.base import OptionChainSnapshot, OptionLeg from utils.network import yf_throttle logger = logging.getLogger(__name__) -def _log_dq(source: str, error_class: str, message: str, *, ticker: str | None = None) -> None: - """Best-effort write to ``data_quality_log``. Never raises.""" - try: - from data_pipeline.quality_log import log_failure - - log_failure(source, error_class, message, ticker=ticker) - except Exception: # noqa: BLE001 - pass - - # Standard option-chain numeric columns we always coerce. _OPT_NUMERIC_COLS = ( "strike", @@ -194,19 +196,7 @@ def _fetch_one(exp: str): return exp, None with consecutive_empty_lock: consecutive_empty["n"] = 0 - calls = opt.calls.copy() - puts = opt.puts.copy() - for col in _OPT_NUMERIC_COLS: - if col in calls.columns: - calls[col] = pd.to_numeric(calls[col], errors="coerce") - if col in puts.columns: - puts[col] = pd.to_numeric(puts[col], errors="coerce") - for col in ("openInterest", "volume"): - if col in calls.columns: - calls[col] = calls[col].fillna(0) - if col in puts.columns: - puts[col] = puts[col].fillna(0) - return exp, {"calls": calls, "puts": puts} + return exp, _coerce_chain_side_payload(opt) with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = {executor.submit(_fetch_one, exp): exp for exp in expiries} @@ -221,6 +211,23 @@ def _fetch_one(exp: str): return {"ticker": ticker, "spot": spot, "expiries": list(ordered_chain.keys()), "chain": ordered_chain} +def _coerce_chain_side_payload(opt: Any) -> dict[str, pd.DataFrame]: + """Coerce one ``opt.calls`` / ``opt.puts`` pair: numeric columns, OI/volume NaN→0.""" + calls = opt.calls.copy() + puts = opt.puts.copy() + for col in _OPT_NUMERIC_COLS: + if col in calls.columns: + calls[col] = pd.to_numeric(calls[col], errors="coerce") + if col in puts.columns: + puts[col] = pd.to_numeric(puts[col], errors="coerce") + for col in ("openInterest", "volume"): + if col in calls.columns: + calls[col] = calls[col].fillna(0) + if col in puts.columns: + puts[col] = puts[col].fillna(0) + return {"calls": calls, "puts": puts} + + def _fetch_option_chain_serial( ticker: str, spot: float | None, expiries: list[str], empty_fail_fast: int ) -> dict[str, Any]: @@ -247,19 +254,7 @@ def _fetch_option_chain_serial( break continue consecutive_empty = 0 - calls = opt.calls.copy() - puts = opt.puts.copy() - for col in _OPT_NUMERIC_COLS: - if col in calls.columns: - calls[col] = pd.to_numeric(calls[col], errors="coerce") - if col in puts.columns: - puts[col] = pd.to_numeric(puts[col], errors="coerce") - for col in ("openInterest", "volume"): - if col in calls.columns: - calls[col] = calls[col].fillna(0) - if col in puts.columns: - puts[col] = puts[col].fillna(0) - chain[exp] = {"calls": calls, "puts": puts} + chain[exp] = _coerce_chain_side_payload(opt) except Exception as exc: # noqa: BLE001 logger.warning("fetch_option_chain: %s exp=%s failed: %s", ticker, exp, exc) continue @@ -268,131 +263,54 @@ def _fetch_option_chain_serial( # --------------------------------------------------------------------------- -# Close-only panel (used by correlation matrix and market review) +# Canonical mapping # --------------------------------------------------------------------------- -def fetch_close_panel( - tickers: list[str], - period: str | None = "90d", - *, - start: str | None = None, - end: str | None = None, - max_retries: int = 2, - retry_base_delay: float = 3.0, -) -> pd.DataFrame: - """Return a wide DataFrame of Close prices for ``tickers``. - - Either pass ``period`` (e.g. ``"90d"``, ``"400d"``) OR ``start``/``end`` - date strings — when ``start`` is provided it takes precedence. On failure - returns an empty DataFrame. One yfinance call total (yfinance natively - supports multi-ticker download). - - WHY (retry loop): Yahoo occasionally returns an empty payload on the first - call after a wake-from-sleep; one retry resolves it without escalating. - """ - if not tickers: - return pd.DataFrame() - if start is not None: - kwargs = {"start": start, "end": end} - else: - kwargs = {"period": period or "90d"} - last_err: Exception | None = None - for attempt in range(max_retries): - try: - yf_throttle() - data = yf.download(tickers, auto_adjust=False, progress=False, **kwargs) - if data is None or data.empty: - if attempt < max_retries - 1: - logger.warning( - "fetch_close_panel empty payload, retrying in %.1fs (attempt %d)", - retry_base_delay * (attempt + 1), - attempt + 1, - ) - time.sleep(retry_base_delay * (attempt + 1)) - continue - return pd.DataFrame() - if isinstance(data.columns, pd.MultiIndex): - if "Close" not in data.columns.get_level_values(0): - return pd.DataFrame() - close = data["Close"] - if isinstance(close.columns, pd.MultiIndex): - close.columns = close.columns.droplevel(1) - else: - if "Close" not in data.columns: - return pd.DataFrame() - close = data[["Close"]] - return close - except Exception as exc: # noqa: BLE001 - last_err = exc - is_rate_limit = "rate" in str(exc).lower() or "too many" in str(exc).lower() - if is_rate_limit and attempt < max_retries - 1: - time.sleep(retry_base_delay * (attempt + 1)) - continue - break - if last_err is not None: - logger.warning("fetch_close_panel failed for %s: %s", tickers, last_err) - is_rate_limit = "rate" in str(last_err).lower() or "too many" in str(last_err).lower() - _log_dq( - "yf_client.fetch_close_panel", - "rate_limited" if is_rate_limit else "download_error", - str(last_err), - ticker=",".join(tickers), - ) - return pd.DataFrame() - - -# --------------------------------------------------------------------------- -# Daily OHLCV (for core/market/data_context.py) -# --------------------------------------------------------------------------- -def fetch_daily_ohlcv( - ticker: str, - start, - end, - *, - auto_adjust: bool = False, - max_retries: int = 2, - retry_base_delay: float = 3.0, -) -> pd.DataFrame: - """Download daily OHLCV bars for ``ticker`` in ``[start, end)``. - - Returns a DataFrame indexed by Date with columns - ``[Open, High, Low, Close, Adj Close, Volume]``. Empty DataFrame on - failure. Includes simple retry loop for transient empty responses. +def _opt_float(value: Any) -> float | None: + """Coerce a yfinance cell to float, mapping NaN/inf/None to ``None``.""" + if value is None: + return None + try: + out = float(value) + except (TypeError, ValueError): + return None + return None if math.isnan(out) or math.isinf(out) else out + + +def _leg_from_row(row: pd.Series) -> OptionLeg: + """Map one yfinance chain row onto a canonical ``OptionLeg``.""" + return OptionLeg( + strike=_opt_float(row.get("strike")) or 0.0, + bid=_opt_float(row.get("bid")), + ask=_opt_float(row.get("ask")), + last=_opt_float(row.get("lastPrice")), + iv=_opt_float(row.get("impliedVolatility")), + open_interest=_opt_float(row.get("openInterest")), + volume=_opt_float(row.get("volume")), + ) + + +def to_option_chain_snapshot(payload: Mapping[str, Any], *, provider: str = "yfinance") -> OptionChainSnapshot: + """Map a legacy ``fetch_option_chain`` payload onto the canonical schema. + + INVARIANT: no ``inTheMoney`` column is carried over (derivable), and ``iv`` + stays a decimal — see ``providers/base.py`` for the unit conventions. """ - last_err: Exception | None = None - for attempt in range(max_retries): - try: - yf_throttle() - df = yf.download( - ticker, - start=start, - end=end, - interval="1d", - progress=False, - auto_adjust=auto_adjust, - ) - if df is None or df.empty: - if attempt < max_retries - 1: - time.sleep(retry_base_delay * (attempt + 1)) - continue - return pd.DataFrame() - if isinstance(df.columns, pd.MultiIndex): - df.columns = df.columns.droplevel(1) - df.index = pd.DatetimeIndex(df.index) - return df - except Exception as exc: # noqa: BLE001 - last_err = exc - is_rate_limit = "rate" in str(exc).lower() or "too many" in str(exc).lower() - if is_rate_limit and attempt < max_retries - 1: - time.sleep(retry_base_delay * (attempt + 1)) - continue - break - if last_err is not None: - logger.warning("fetch_daily_ohlcv failed for %s: %s", ticker, last_err) - is_rate_limit = "rate" in str(last_err).lower() or "too many" in str(last_err).lower() - _log_dq( - "yf_client.fetch_daily_ohlcv", - "rate_limited" if is_rate_limit else "download_error", - str(last_err), - ticker=ticker, - ) - return pd.DataFrame() + expiries = tuple(payload.get("expiries") or ()) + raw_chain = payload.get("chain") or {} + chain: dict[str, dict[str, tuple[OptionLeg, ...]]] = {} + for expiry in expiries: + sides = raw_chain.get(expiry) + if not sides: + continue + chain[expiry] = { + side: tuple(_leg_from_row(row) for _, row in sides[side].iterrows()) + for side in ("calls", "puts") + if side in sides + } + return OptionChainSnapshot( + provider=provider, + symbol=str(payload.get("ticker") or ""), + spot=payload.get("spot"), + expiries=expiries, + chain=chain, + ) diff --git a/data_pipeline/providers/yfinance_provider.py b/data_pipeline/providers/yfinance_provider.py new file mode 100644 index 0000000..88a4456 --- /dev/null +++ b/data_pipeline/providers/yfinance_provider.py @@ -0,0 +1,264 @@ +"""yfinance provider: historical bars, close panels, canonical mapping. + +Domain: Data Pipeline — yfinance Provider +Context: + - This module (with ``yf_snapshot.py``) is the **only** place in the repo that + imports ``yfinance``; batch B1 moved these calls here out of the old + ``data_pipeline/yf_client.py`` and ``data_pipeline/downloader.py`` (now + ``ingest/ohlcv.py``) without changing behaviour. See ADR 0002 (as amended by + ADR 0011) and docs/plans/business_line_reorg.md §6. + - ``YFinanceProvider`` is the canonical seam implementation (``history()`` / + ``close_panel()`` / ``spot()`` / ``option_chain()``); it delegates the live + snapshots to ``yf_snapshot.py``. The module-level ``fetch_*`` functions keep + their original yfinance-shaped contracts so the ~11 existing importers keep + working through the ``yf_client`` shim — new code should prefer the + canonical shapes (ADR 0011). +Design rules: + - CONSTRAINT: every public function calls ``yf_throttle()`` before each + yfinance call — see docs/decisions/0005-token-bucket-throttle.md. + - WHY (no caching here): caching is the caller's concern (e.g. ``app.py`` + option-chain cache, ``DataService`` 60s freshness window). + - WHY (never raise on transient failure): callers receive an empty DataFrame + and decide how to surface the error. Raising here would cascade into + unhandled 500s from many different routes. + - INVARIANT: returns plain Python types (float / DataFrame), never + yfinance-specific objects — keeps the rest of the pipeline mockable. + - CONSTRAINT: never pass ``session=requests.Session()`` — yfinance ≥0.2.50 uses + curl_cffi and silently fails (ADR 0005 / docs/constraints.md §2). +Dependencies UPWARD: + - utils.network (throttle), data_pipeline.store.quality_log (via providers._log) +Dependencies DOWNWARD: + - providers/base, providers/_log, providers/yf_snapshot +""" + +from __future__ import annotations + +import datetime as dt +import logging +import time + +import pandas as pd +import yfinance as yf + +from data_pipeline.providers._log import _log_dq +from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS, OptionChainSnapshot +from data_pipeline.providers.yf_snapshot import fetch_option_chain, fetch_spot, to_option_chain_snapshot +from utils.network import yf_throttle + +logger = logging.getLogger(__name__) + + +# WHY: yfinance's Title-Case columns (as returned by ``yf.download``) mapped onto +# the canonical lowercase schema. ``Adj Close`` arrives with a space from +# ``yf.download`` and as ``Adj_Close`` from the test fixtures / upsert path. +_YF_TO_CANONICAL = { + "Open": "open", + "High": "high", + "Low": "low", + "Close": "close", + "Adj Close": "adj_close", + "Adj_Close": "adj_close", + "Volume": "volume", +} + + +# --------------------------------------------------------------------------- +# Close-only panel (used by correlation matrix and market review) +# --------------------------------------------------------------------------- +def fetch_close_panel( + tickers: list[str], + period: str | None = "90d", + *, + start: str | None = None, + end: str | None = None, + max_retries: int = 2, + retry_base_delay: float = 3.0, +) -> pd.DataFrame: + """Return a wide DataFrame of Close prices for ``tickers``. + + Either pass ``period`` (e.g. ``"90d"``, ``"400d"``) OR ``start``/``end`` + date strings — when ``start`` is provided it takes precedence. On failure + returns an empty DataFrame. One yfinance call total (yfinance natively + supports multi-ticker download). + + WHY (retry loop): Yahoo occasionally returns an empty payload on the first + call after a wake-from-sleep; one retry resolves it without escalating. + """ + if not tickers: + return pd.DataFrame() + if start is not None: + kwargs = {"start": start, "end": end} + else: + kwargs = {"period": period or "90d"} + last_err: Exception | None = None + for attempt in range(max_retries): + try: + yf_throttle() + data = yf.download(tickers, auto_adjust=False, progress=False, **kwargs) + if data is None or data.empty: + if attempt < max_retries - 1: + logger.warning( + "fetch_close_panel empty payload, retrying in %.1fs (attempt %d)", + retry_base_delay * (attempt + 1), + attempt + 1, + ) + time.sleep(retry_base_delay * (attempt + 1)) + continue + return pd.DataFrame() + if isinstance(data.columns, pd.MultiIndex): + if "Close" not in data.columns.get_level_values(0): + return pd.DataFrame() + close = data["Close"] + if isinstance(close.columns, pd.MultiIndex): + close.columns = close.columns.droplevel(1) + else: + if "Close" not in data.columns: + return pd.DataFrame() + close = data[["Close"]] + return close + except Exception as exc: # noqa: BLE001 + last_err = exc + is_rate_limit = "rate" in str(exc).lower() or "too many" in str(exc).lower() + if is_rate_limit and attempt < max_retries - 1: + time.sleep(retry_base_delay * (attempt + 1)) + continue + break + if last_err is not None: + logger.warning("fetch_close_panel failed for %s: %s", tickers, last_err) + is_rate_limit = "rate" in str(last_err).lower() or "too many" in str(last_err).lower() + _log_dq( + "yf_client.fetch_close_panel", + "rate_limited" if is_rate_limit else "download_error", + str(last_err), + ticker=",".join(tickers), + ) + return pd.DataFrame() + + +# --------------------------------------------------------------------------- +# Daily OHLCV +# --------------------------------------------------------------------------- +def fetch_daily_ohlcv( + ticker: str, + start, + end, + *, + auto_adjust: bool = False, + max_retries: int = 2, + retry_base_delay: float = 3.0, +) -> pd.DataFrame: + """Download daily OHLCV bars for ``ticker`` in ``[start, end)``. + + Returns a DataFrame indexed by Date with columns + ``[Open, High, Low, Close, Adj Close, Volume]``. Empty DataFrame on + failure. Includes simple retry loop for transient empty responses. + """ + last_err: Exception | None = None + for attempt in range(max_retries): + try: + yf_throttle() + df = yf.download( + ticker, + start=start, + end=end, + interval="1d", + progress=False, + auto_adjust=auto_adjust, + ) + if df is None or df.empty: + if attempt < max_retries - 1: + time.sleep(retry_base_delay * (attempt + 1)) + continue + return pd.DataFrame() + if isinstance(df.columns, pd.MultiIndex): + df.columns = df.columns.droplevel(1) + df.index = pd.DatetimeIndex(df.index) + return df + except Exception as exc: # noqa: BLE001 + last_err = exc + is_rate_limit = "rate" in str(exc).lower() or "too many" in str(exc).lower() + if is_rate_limit and attempt < max_retries - 1: + time.sleep(retry_base_delay * (attempt + 1)) + continue + break + if last_err is not None: + logger.warning("fetch_daily_ohlcv failed for %s: %s", ticker, last_err) + is_rate_limit = "rate" in str(last_err).lower() or "too many" in str(last_err).lower() + _log_dq( + "yf_client.fetch_daily_ohlcv", + "rate_limited" if is_rate_limit else "download_error", + str(last_err), + ticker=ticker, + ) + return pd.DataFrame() + + +def download_daily_frame(ticker: str, start: dt.date, end: dt.date) -> pd.DataFrame: + """Download daily OHLCV for ``[start, end]`` (inclusive) from yfinance. + + Returns a frame with Title-Case columns plus ``Adj_Close`` — the shape + ``data_pipeline.ingest.ohlcv.upsert_raw_prices`` consumes. ``history()`` on + ``YFinanceProvider`` is the canonical equivalent. + """ + # yfinance 'end' is exclusive, so pass end + 1 day to include the requested end date + yf_end = end + dt.timedelta(days=1) + yf_throttle() + df = yf.download(ticker, start=start, end=yf_end, interval="1d", progress=False, auto_adjust=False) + if df is None or df.empty: + return pd.DataFrame() + if isinstance(df.columns, pd.MultiIndex): + df.columns = df.columns.droplevel(1) + cols = ["Open", "High", "Low", "Close", "Adj Close", "Volume"] + for c in cols: + if c not in df.columns: + df[c] = pd.NA + return df[cols].rename(columns={"Adj Close": "Adj_Close"}) + + +# --------------------------------------------------------------------------- +# Canonical mapping +# --------------------------------------------------------------------------- +def to_canonical_bars(frame: pd.DataFrame | None) -> pd.DataFrame: + """Map a yfinance OHLCV frame onto the canonical lower-case schema.""" + if frame is None or frame.empty: + return pd.DataFrame(columns=list(CANONICAL_BAR_COLUMNS)) + out = pd.DataFrame(index=pd.DatetimeIndex(frame.index)) + for source, target in _YF_TO_CANONICAL.items(): + if source in frame.columns: + out[target] = pd.to_numeric(frame[source], errors="coerce") + for col in CANONICAL_BAR_COLUMNS: + if col not in out.columns: + out[col] = pd.NA + return out[list(CANONICAL_BAR_COLUMNS)].sort_index() + + +# --------------------------------------------------------------------------- +# Provider implementation +# --------------------------------------------------------------------------- +class YFinanceProvider: + """yfinance implementation of ``MarketDataProvider`` (the only one today).""" + + name = "yfinance" + + def history(self, symbol: str, start: dt.date, end: dt.date) -> pd.DataFrame: + """Canonical daily bars for ``[start, end]`` (inclusive).""" + return to_canonical_bars(download_daily_frame(symbol, start, end)) + + def close_panel( + self, + symbols: list[str], + *, + start: dt.date | str | None = None, + end: dt.date | str | None = None, + period: str | None = None, + ) -> pd.DataFrame: + """Wide Close-price panel for ``symbols``.""" + return fetch_close_panel(symbols, period or "90d", start=start, end=end) + + def spot(self, symbol: str) -> float | None: + """Latest traded price for ``symbol``.""" + return fetch_spot(symbol) + + def option_chain(self, symbol: str) -> OptionChainSnapshot: + """Canonical live option-chain snapshot for ``symbol``.""" + return to_option_chain_snapshot(fetch_option_chain(symbol), provider=self.name) diff --git a/data_pipeline/read/__init__.py b/data_pipeline/read/__init__.py new file mode 100644 index 0000000..2228e99 --- /dev/null +++ b/data_pipeline/read/__init__.py @@ -0,0 +1,22 @@ +"""READ — the DB-first read API services call. + +Domain: Data Pipeline — Read +Context: + - ADR 0011: ``DataService`` is the single entry point above ``data_pipeline/``. + It is DB-first and, when coverage is missing, triggers the orchestration + layer rather than downloading inline (that is why this package may import + ``orchestrate`` — see the layer table in docs/architecture_review.md §3). +Contracts: + - ``DataService`` — the facade used by services/, routes/ and app.py. + - ``_query`` — memoised reads with in-flight de-duplication. +Dependencies UPWARD: + - store, orchestrate (refresh triggers), providers (spot fallback) +Dependencies DOWNWARD: + - services/, routes/ +""" + +from __future__ import annotations + +from data_pipeline.read.facade import DataService + +__all__ = ["DataService"] diff --git a/data_pipeline/data_ops/_query.py b/data_pipeline/read/_query.py similarity index 64% rename from data_pipeline/data_ops/_query.py rename to data_pipeline/read/_query.py index 51e5d9f..d824dc0 100644 --- a/data_pipeline/data_ops/_query.py +++ b/data_pipeline/read/_query.py @@ -3,16 +3,15 @@ import datetime as dt import logging import os -import threading import time import pandas as pd -from data_pipeline.db import fetch_df, init_db - -from . import _globals as _g -from . import _range as _r -from . import _update as _u +from data_pipeline import _state as _g +from data_pipeline.orchestrate import backfill as _bf +from data_pipeline.orchestrate import update as _u +from data_pipeline.orchestrate.readiness import kick_backfill as _kick_backfill +from data_pipeline.store.db import fetch_df, init_db logger = logging.getLogger(__name__) @@ -24,52 +23,20 @@ # kicks), the request waits a short grace period so the common "one chunk # missing" case still returns full data, then reads whatever coverage exists. _BACKFILL_WAIT_SECONDS = float(os.environ.get("BACKFILL_WAIT_SECONDS", "8")) -_backfill_lock = threading.Lock() -_backfill_threads: dict[tuple, threading.Thread] = {} - - -def _kick_backfill(ticker: str, start, end) -> None: - key = (ticker, str(start), str(end)) - with _backfill_lock: - existing = _backfill_threads.get(key) - if existing is not None and existing.is_alive(): - return - t = threading.Thread(target=_run_backfill, args=(ticker, start, end, key), daemon=True) - _backfill_threads[key] = t - t.start() - logger.info("background backfill kicked for %s [%s .. %s]", ticker, start, end) - - -def _run_backfill(ticker, start, end, key) -> None: - try: - _r.ensure_range(ticker, start, end) - except Exception as e: - logger.warning("background backfill failed for %s: %s", ticker, e) - finally: - # Daemon threads never re-run dispatch's finally-cleanups; drop this - # thread's SQLite connection so _all_conns doesn't grow per backfill. - from data_pipeline.db import close_thread_conn - - close_thread_conn() - with _backfill_lock: - _backfill_threads.pop(key, None) - -def _join_backfills(timeout: float | None = None) -> None: - """Test helper: wait for all in-flight background backfills.""" - with _backfill_lock: - threads = list(_backfill_threads.values()) - for t in threads: - t.join(timeout=timeout) +# NOTE: the kicker itself lives in ``orchestrate/readiness.py`` — batch B5 made it +# shared between this per-slice path and the readiness pass on POST /. It is +# re-exported (as ``_kick_backfill`` / ``_join_backfills``) at the top of this +# module so existing callers and tests keep working. def _wait_for_coverage(ticker, start, end, timeout: float) -> bool: deadline = time.monotonic() + timeout while time.monotonic() < deadline: - if not _r.needs_backfill(ticker, start, end): + if not _bf.needs_backfill(ticker, start, end): return True time.sleep(0.25) - return not _r.needs_backfill(ticker, start, end) + return not _bf.needs_backfill(ticker, start, end) def get_cleaned_daily(ticker: str, start: dt.date | None = None, end: dt.date | None = None) -> pd.DataFrame: @@ -87,24 +54,24 @@ def get_cleaned_daily(ticker: str, start: dt.date | None = None, end: dt.date | # ensure_range path. return cached _u.manual_update(ticker, days=7) - if _r.needs_backfill(ticker, start, end): + if _bf.needs_backfill(ticker, start, end): # Missing span ⇒ keep the heavy download off the request thread; give # it a short grace period so "one chunk missing" still returns full # data, then fall through to whatever coverage the DB has now. _kick_backfill(ticker, start, end) _wait_for_coverage(ticker, start, end, _BACKFILL_WAIT_SECONDS) else: - _r.ensure_range(ticker, start, end) + _bf.ensure_range(ticker, start, end) init_db() df = fetch_df( - "SELECT date, open, high, low, close, adj_close, volume FROM clean_prices WHERE ticker=? AND date>=? AND date<=?", + "SELECT date, open, high, low, close, adj_close, volume FROM clean_bars WHERE ticker=? AND date>=? AND date<=?", (ticker, start.isoformat(), end.isoformat()), ) # Never memoise a partial read: while the background backfill is running # this df may lack the requested span. ensure_range invalidates the # ticker's cache entries on success, so the completed data becomes visible # on the next request. - if not _r.needs_backfill(ticker, start, end): + if not _bf.needs_backfill(ticker, start, end): _g._cache_set(cache_key, df) return df @@ -119,16 +86,21 @@ def get_processed( if cached is not None: return cached _u.manual_update(ticker, days=7) + if _bf.needs_backfill(ticker, start, end): + # A clean gap, or clean present but feature_bars lagging (plan §10 F4): + # heal off the request thread with a short grace wait, then read + # whatever coverage exists — same pattern as get_cleaned_daily. + _kick_backfill(ticker, start, end) + _wait_for_coverage(ticker, start, end, _BACKFILL_WAIT_SECONDS) init_db() df = fetch_df( - "SELECT * FROM processed_prices WHERE ticker=? AND frequency=? AND date>=? AND date<=?", + "SELECT * FROM feature_bars WHERE ticker=? AND frequency=? AND date>=? AND date<=?", (ticker, frequency, start.isoformat(), end.isoformat()), ) - # Never memoise an empty read: a not-yet-generated frequency/range would - # otherwise be pinned for _QUERY_CACHE_TTL and hide the data once the - # processing pass completes (mirrors the partial-read guard in - # get_cleaned_daily above). - if not df.empty: + # Never memoise an empty or partial read: a not-yet-generated frequency/range + # would otherwise be pinned for _QUERY_CACHE_TTL and hide the data once the + # processing pass completes (mirrors the guard in get_cleaned_daily above). + if not df.empty and not _bf.needs_backfill(ticker, start, end): _g._cache_set(cache_key, df) return df @@ -143,10 +115,10 @@ def get_processed_data(ticker: str, start: dt.date, end: dt.date, frequency: str def get_latest_spot(ticker: str) -> float | None: - """Return latest close price for *ticker* from clean_prices (Yahoo-sourced).""" + """Return latest close price for *ticker* from clean_bars (provider-sourced).""" init_db() df = fetch_df( - "SELECT close FROM clean_prices WHERE ticker=? AND close IS NOT NULL ORDER BY date DESC LIMIT 1", + "SELECT close FROM clean_bars WHERE ticker=? AND close IS NOT NULL ORDER BY date DESC LIMIT 1", (ticker,), ) if not df.empty: @@ -156,7 +128,7 @@ def get_latest_spot(ticker: str) -> float | None: except (TypeError, ValueError): pass - from data_pipeline.yf_client import fetch_spot + from data_pipeline.providers.yf_client import fetch_spot try: price = fetch_spot(ticker) diff --git a/data_pipeline/data_ops/facade.py b/data_pipeline/read/facade.py similarity index 50% rename from data_pipeline/data_ops/facade.py rename to data_pipeline/read/facade.py index 69feaba..657e9ea 100644 --- a/data_pipeline/data_ops/facade.py +++ b/data_pipeline/read/facade.py @@ -1,24 +1,40 @@ -"""DataService facade — thin orchestrator over data_ops submodules.""" - -from data_pipeline.db import init_db +"""DataService facade — the DB-first read entry point (ADR 0011). + +Domain: Data Pipeline — Read +Context: + - This is the only ``data_pipeline`` surface above the package: services/, + routes/ and app.py talk to ``DataService`` and never to store/ingest/ + transform directly. Reads come from ``read/_query.py``; anything that has to + *make data ready* is delegated to ``orchestrate``. +Contracts: + - ``DataService`` staticmethods — initialize, manual_update, seed_history, + has_data_for_date, ensure_range, get_cleaned_daily, get_processed, + get_processed_data, get_latest_spot. +Dependencies UPWARD: + - store (db), orchestrate (backfill / update) +Dependencies DOWNWARD: + - services/, routes/, app.py +""" + +from data_pipeline.orchestrate import backfill as _bf +from data_pipeline.orchestrate import update as _u +from data_pipeline.store.db import init_db from . import _query as _q -from . import _range as _r -from . import _update as _u class DataService: """Facade for data operations.""" # Re-export class-level attributes for backward compat - _ENSURE_RANGE_TTL = _r._ENSURE_RANGE_TTL - _ensure_range_memo = _r._ensure_range_memo - _ensure_range_lock = _r._ensure_range_lock - _ensure_range_inflight = _r._ensure_range_inflight - _ensure_range_inflight_lock = _r._ensure_range_inflight_lock - _BACKFILL_MIN_DATE = _r._BACKFILL_MIN_DATE - _SENTINEL_GAP_THRESHOLD_DAYS = _r._SENTINEL_GAP_THRESHOLD_DAYS - _SENTINEL_MIN_DB_SPAN_DAYS = _r._SENTINEL_MIN_DB_SPAN_DAYS + _ENSURE_RANGE_TTL = _bf._ENSURE_RANGE_TTL + _ensure_range_memo = _bf._ensure_range_memo + _ensure_range_lock = _bf._ensure_range_lock + _ensure_range_inflight = _bf._ensure_range_inflight + _ensure_range_inflight_lock = _bf._ensure_range_inflight_lock + _BACKFILL_MIN_DATE = _bf._BACKFILL_MIN_DATE + _SENTINEL_GAP_THRESHOLD_DAYS = _bf._SENTINEL_GAP_THRESHOLD_DAYS + _SENTINEL_MIN_DB_SPAN_DAYS = _bf._SENTINEL_MIN_DB_SPAN_DAYS @staticmethod def initialize(): @@ -35,23 +51,23 @@ def seed_history(ticker: str, years: int = 5): @staticmethod def has_data_for_date(ticker: str, date) -> bool: init_db() - from data_pipeline.db import fetch_df + from data_pipeline.store.db import fetch_df df = fetch_df( - "SELECT * FROM clean_prices WHERE ticker=? AND date=?", + "SELECT * FROM clean_bars WHERE ticker=? AND date=?", (ticker, date.isoformat()), ) if not df.empty: return True df2 = fetch_df( - "SELECT * FROM raw_prices WHERE ticker=? AND date=?", + "SELECT * FROM raw_bars WHERE ticker=? AND date=?", (ticker, date.isoformat()), ) return not df2.empty @staticmethod def ensure_range(ticker: str, start, end) -> bool: - return _r.ensure_range(ticker, start, end) + return _bf.ensure_range(ticker, start, end) @staticmethod def get_cleaned_daily(ticker: str, start=None, end=None): @@ -76,5 +92,5 @@ def clear_ensure_range_memo(ticker: str) -> None: Call this before ``seed_history`` when you want to bypass a previously cached failure and force a fresh backfill. """ - with _r._ensure_range_lock: - _r._ensure_range_memo.pop(ticker, None) + with _bf._ensure_range_lock: + _bf._ensure_range_memo.pop(ticker, None) diff --git a/data_pipeline/store/__init__.py b/data_pipeline/store/__init__.py new file mode 100644 index 0000000..b2934c7 --- /dev/null +++ b/data_pipeline/store/__init__.py @@ -0,0 +1,18 @@ +"""STORE — schema, SQL, and the failure log. + +Domain: Data Pipeline — Store +Context: + - ADR 0011 split ``data_pipeline/`` into stages. This package owns the SQLite + boundary: the schema, the only SQL builder, and the ``data_quality_log`` + writer. Nothing here knows about vendors, pandas transforms, or Flask. +Contracts: + - ``db`` — connections (thread-local WAL), ``init_db``, ``upsert_many``, ``fetch_df``. + - ``repos`` — the only place that builds SQL. + - ``quality_log`` — the append-only fetch-failure log. +Dependencies UPWARD: + - (none — stdlib + pandas only) +Dependencies DOWNWARD: + - everything above: providers, ingest, transform, read, orchestrate +""" + +from __future__ import annotations diff --git a/data_pipeline/db.py b/data_pipeline/store/db.py similarity index 62% rename from data_pipeline/db.py rename to data_pipeline/store/db.py index ec78c6e..0ec52ae 100644 --- a/data_pipeline/db.py +++ b/data_pipeline/store/db.py @@ -12,6 +12,14 @@ Do NOT add: a migration framework, an ORM, connection pooling beyond the thread-local cache. Each was considered and rejected as overkill for this project's scale. + +Canonical table names (ADR 0011, batch B2): +- The pipeline reads and writes ``raw_bars`` / ``clean_bars`` / ``feature_bars``. + The pre-rename names (``raw_prices`` / ``clean_prices`` / ``processed_prices``) + are kept for one release as shadow tables: ``upsert_many`` writes both, so a + ``git revert`` of batch B2 loses no rows. Column sets are deliberately + identical (decision gate Q4 = minimal rename); ``tests/test_canonical_tables.py`` + asserts that and ``scripts/migrate_canonical_tables.py`` backfills old DBs. """ import logging @@ -104,86 +112,104 @@ def close_all_conns() -> None: _thread_local.conns.clear() +# ── Schema: one column tuple per table shape ──────────────────────── +# INVARIANT: a canonical table and its pre-rename shadow are created from the +# SAME tuple, so their column sets cannot drift while both exist (the rename is +# a pure name change — decision gate §8 Q4). +# INVARIANT: ``frequency`` is one of D / W / ME / QE (see +# ``core/_shared/types.Frequency``). +_BARS_COLUMNS: tuple[str, ...] = ( + "ticker TEXT NOT NULL", + "date TEXT NOT NULL", + "open REAL", + "high REAL", + "low REAL", + "close REAL", + "adj_close REAL", + "volume REAL", + "provider TEXT DEFAULT 'yfinance'", + "PRIMARY KEY (ticker, date)", +) + +_CLEAN_BARS_COLUMNS: tuple[str, ...] = ( + "ticker TEXT NOT NULL", + "date TEXT NOT NULL", + "open REAL", + "high REAL", + "low REAL", + "close REAL", + "adj_close REAL", + "volume REAL", + "is_trading_day INTEGER DEFAULT 1", + "missing_any INTEGER DEFAULT 0", + "price_jump_flag INTEGER DEFAULT 0", + "vol_anom_flag INTEGER DEFAULT 0", + "ohlc_inconsistent INTEGER DEFAULT 0", + "PRIMARY KEY (ticker, date)", +) + +_FEATURE_BARS_COLUMNS: tuple[str, ...] = ( + "ticker TEXT NOT NULL", + "date TEXT NOT NULL", + "frequency TEXT NOT NULL", + "open REAL", + "high REAL", + "low REAL", + "close REAL", + "adj_close REAL", + "volume REAL", + "last_close REAL", + "log_return REAL", + "amplitude REAL", + "log_hl_spread REAL", + "parkinson_var REAL", + "gk_var REAL", + "log_vol_delta REAL", + "vol_zscore REAL", + "ma_5 REAL", + "ma_10 REAL", + "ma_20 REAL", + "ma_60 REAL", + "ma_120 REAL", + "ma_250 REAL", + "mom_10 REAL", + "mom_20 REAL", + "mom_60 REAL", + "osc_high REAL", + "osc_low REAL", + "osc REAL", + "PRIMARY KEY (ticker, date, frequency)", +) + + +def _create_table(cur, name: str, columns: tuple[str, ...]) -> None: + """Create ``name`` if absent, from ``columns``. + + CONSTRAINT: ``name`` is interpolated into SQL; it is only ever a literal from + this module (never caller input) — mirrors ``upsert_many``'s table validation. + """ + body = ",\n ".join(columns) + cur.execute(f"CREATE TABLE IF NOT EXISTS {name} (\n {body}\n)") + + def init_db(db_path: str | None = None): path = db_path or DB_PATH Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True) conn = _get_or_create_conn(path) cur = conn.cursor() - # Raw OHLCV data - cur.execute( - """ - CREATE TABLE IF NOT EXISTS raw_prices ( - ticker TEXT NOT NULL, - date TEXT NOT NULL, - open REAL, - high REAL, - low REAL, - close REAL, - adj_close REAL, - volume REAL, - provider TEXT DEFAULT 'yfinance', - PRIMARY KEY (ticker, date) - ) - """ - ) - # Cleaned daily OHLCV with flags - cur.execute( - """ - CREATE TABLE IF NOT EXISTS clean_prices ( - ticker TEXT NOT NULL, - date TEXT NOT NULL, - open REAL, - high REAL, - low REAL, - close REAL, - adj_close REAL, - volume REAL, - is_trading_day INTEGER DEFAULT 1, - missing_any INTEGER DEFAULT 0, - price_jump_flag INTEGER DEFAULT 0, - vol_anom_flag INTEGER DEFAULT 0, - ohlc_inconsistent INTEGER DEFAULT 0, - PRIMARY KEY (ticker, date) - ) - """ - ) - # Processed features per frequency - cur.execute( - """ - CREATE TABLE IF NOT EXISTS processed_prices ( - ticker TEXT NOT NULL, - date TEXT NOT NULL, - frequency TEXT NOT NULL, -- D/W/M - open REAL, - high REAL, - low REAL, - close REAL, - adj_close REAL, - volume REAL, - last_close REAL, - log_return REAL, - amplitude REAL, - log_hl_spread REAL, - parkinson_var REAL, - gk_var REAL, - log_vol_delta REAL, - vol_zscore REAL, - ma_5 REAL, - ma_10 REAL, - ma_20 REAL, - ma_60 REAL, - ma_120 REAL, - ma_250 REAL, - mom_10 REAL, - mom_20 REAL, - mom_60 REAL, - osc_high REAL, - osc_low REAL, - osc REAL, - PRIMARY KEY (ticker, date, frequency) - ) - """ - ) + # Canonical store (ADR 0011) — the pipeline reads and writes these names. + _create_table(cur, "raw_bars", _BARS_COLUMNS) + _create_table(cur, "clean_bars", _CLEAN_BARS_COLUMNS) + _create_table(cur, "feature_bars", _FEATURE_BARS_COLUMNS) + # ── Compatibility shadows (transitional — one release) ────────────── + # TRADEOFF: the pre-rename names are kept, created from the same column + # tuples, so reverting batch B2 loses no rows and a DB written before the + # rename keeps working until scripts/migrate_canonical_tables.py has run + # (and afterwards: `upsert_many` writes both families). Drop these three + # lines + `_TABLE_SHADOWS` one release after the rename. + _create_table(cur, "raw_prices", _BARS_COLUMNS) + _create_table(cur, "clean_prices", _CLEAN_BARS_COLUMNS) + _create_table(cur, "processed_prices", _FEATURE_BARS_COLUMNS) # Market review benchmark close prices cur.execute( """ @@ -274,6 +300,11 @@ def get_conn(db_path: str | None = None): _UPSERTABLE_TABLES = frozenset( { + # canonical (ADR 0011) + "raw_bars", + "clean_bars", + "feature_bars", + # compatibility shadows — removable one release after the rename "raw_prices", "clean_prices", "processed_prices", @@ -284,14 +315,41 @@ def get_conn(db_path: str | None = None): } ) +# ── Canonical ↔ legacy table naming (transitional) ────────────────── +# INVARIANT: each pair has an identical column set — enforced by +# tests/test_canonical_tables.py, because the two families must stay +# interchangeable for the compatibility window to be safe. +CANONICAL_TABLES: dict[str, str] = { + "raw_prices": "raw_bars", + "clean_prices": "clean_bars", + "processed_prices": "feature_bars", +} + +# WHY bidirectional: writes may arrive under either name during the window (old +# call sites, tests seeding fixtures directly). Mirroring both ways keeps the two +# families identical no matter which name a caller uses. +_TABLE_SHADOWS: dict[str, str] = { + **CANONICAL_TABLES, + **{canonical: legacy for legacy, canonical in CANONICAL_TABLES.items()}, +} + + +def canonical_table(name: str) -> str: + """Return the canonical table name for a legacy name (identity otherwise).""" + return CANONICAL_TABLES.get(name, name) + def upsert_many(table: str, columns: Iterable[str], rows: Iterable[Iterable], db_path: str | None = None): - """Bulk-upsert *rows* into *table*. + """Bulk-upsert *rows* into *table*, plus its shadow table if it has one. CONSTRAINT: the table name is interpolated into SQL (values are still parameterised), so it is validated against the known schema tables — a typo or caller-supplied name fails fast instead of building a malformed (or, with hostile input, malicious) statement. + TRADEOFF (transitional — one release): the canonical and pre-rename table + families are written together (see ``_TABLE_SHADOWS``) so that either name + can be read during the rename. The extra write is one more executemany over + the same small batches; drop it with the shadow tables. """ if table not in _UPSERTABLE_TABLES: raise ValueError(f"upsert_many: unknown table {table!r}; expected one of {sorted(_UPSERTABLE_TABLES)}") @@ -301,11 +359,16 @@ def upsert_many(table: str, columns: Iterable[str], rows: Iterable[Iterable], db cols = list(columns) placeholders = ",".join(["?"] * len(cols)) updates = ",".join([f"{c}=excluded.{c}" for c in cols if c not in ("ticker", "date", "frequency")]) - sql = f"INSERT INTO {table} ({','.join(cols)}) VALUES ({placeholders}) ON CONFLICT DO UPDATE SET {updates}" + targets = [table, *((_TABLE_SHADOWS[table],) if table in _TABLE_SHADOWS else ())] + statements = [ + f"INSERT INTO {target} ({','.join(cols)}) VALUES ({placeholders}) ON CONFLICT DO UPDATE SET {updates}" + for target in targets + ] conn = _get_or_create_conn(db_path or DB_PATH) try: conn.execute("BEGIN") - conn.executemany(sql, rows) + for sql in statements: + conn.executemany(sql, rows) conn.execute("COMMIT") except Exception: try: diff --git a/data_pipeline/quality_log.py b/data_pipeline/store/quality_log.py similarity index 98% rename from data_pipeline/quality_log.py rename to data_pipeline/store/quality_log.py index 15feee1..750ba9e 100644 --- a/data_pipeline/quality_log.py +++ b/data_pipeline/store/quality_log.py @@ -15,7 +15,7 @@ from datetime import UTC, datetime from typing import Any -from data_pipeline.db import get_conn +from data_pipeline.store.db import get_conn _logger = logging.getLogger(__name__) diff --git a/data_pipeline/repos.py b/data_pipeline/store/repos.py similarity index 94% rename from data_pipeline/repos.py rename to data_pipeline/store/repos.py index 941b58e..83e03f3 100644 --- a/data_pipeline/repos.py +++ b/data_pipeline/store/repos.py @@ -1,11 +1,11 @@ """Repository layer — the only place that builds SQL. INVARIANT (doc_guard `db-access`): upper layers (routes/services) must import -these functions instead of touching ``data_pipeline.db`` connection primitives +these functions instead of touching ``data_pipeline.store.db`` connection primitives directly, so WAL pragmas and the query cache apply uniformly (ADR 0003). This module is part of ``data_pipeline`` (an I/O layer); importing -``data_pipeline.db`` here is the intended single exception and is not flagged +``data_pipeline.store.db`` here is the intended single exception and is not flagged by the guardrail. """ @@ -17,12 +17,12 @@ import pandas as pd -from data_pipeline.db import fetch_df, get_conn, init_db, upsert_many +from data_pipeline.store.db import fetch_df, get_conn, init_db, upsert_many # ── Health / data-quality inventory ───────────────────────────────── def fetch_ticker_inventory() -> list[tuple[Any, ...]]: - """Return one row per ticker from ``raw_prices`` with row counts + NaN tallies. + """Return one row per ticker from ``raw_bars`` with row counts + NaN tallies. Columns: ``(ticker, rows, latest_date, earliest_date, null_close, null_volume)`` ordered by ticker. @@ -35,7 +35,7 @@ def fetch_ticker_inventory() -> list[tuple[Any, ...]]: MIN(date) AS earliest_date, SUM(CASE WHEN close IS NULL THEN 1 ELSE 0 END) AS null_close, SUM(CASE WHEN volume IS NULL THEN 1 ELSE 0 END) AS null_volume - FROM raw_prices + FROM raw_bars GROUP BY ticker ORDER BY ticker """ @@ -172,7 +172,7 @@ def count_clean_rows(ticker: str) -> int: """Return how many priced rows the DB holds for ``ticker``.""" ensure_schema() df = fetch_df( - "SELECT COUNT(*) AS n FROM clean_prices WHERE ticker=? AND close IS NOT NULL", + "SELECT COUNT(*) AS n FROM clean_bars WHERE ticker=? AND close IS NOT NULL", (ticker,), ) if df.empty: diff --git a/data_pipeline/transform/__init__.py b/data_pipeline/transform/__init__.py new file mode 100644 index 0000000..e7fd076 --- /dev/null +++ b/data_pipeline/transform/__init__.py @@ -0,0 +1,19 @@ +"""TRANSFORM — raw → clean → features. + +Domain: Data Pipeline — Transform +Context: + - ADR 0011: the processing stage is provider-agnostic — it reads canonical + ``raw_bars`` / ``clean_bars`` frames and never imports ``providers/``. + - Domain rules live here: business-day alignment with **no interpolation** + (invented prices are worse than missing ones — docs/constraints.md §4), + anomaly flagging, then per-frequency feature engineering. +Contracts: + - ``cleaning.clean_range`` — raw_bars → clean_bars. + - ``processing.process_frequencies`` — clean_bars → feature_bars (D/W/ME/QE). +Dependencies UPWARD: + - store (canonical tables), data_pipeline (PipelineResult) +Dependencies DOWNWARD: + - orchestrate +""" + +from __future__ import annotations diff --git a/data_pipeline/cleaning.py b/data_pipeline/transform/cleaning.py similarity index 88% rename from data_pipeline/cleaning.py rename to data_pipeline/transform/cleaning.py index d492f60..ba544c7 100644 --- a/data_pipeline/cleaning.py +++ b/data_pipeline/transform/cleaning.py @@ -1,4 +1,18 @@ -"""Data cleaning utilities for raw market price data.""" +"""Data cleaning: raw_prices → clean_bars. + +Domain: Data Pipeline — Transform (cleaning) +Context: + - Aligns raw bars to business days, flags anomalies, and marks missing days as + NA. INVARIANT: **no interpolation** — inventing prices that never traded + would corrupt every downstream indicator (HV, MA, regime). See + docs/constraints.md §4. +Contracts: + - ``clean_range(ticker, start, end) -> PipelineResult`` — raw_bars → clean_bars. +Dependencies UPWARD: + - store (db), data_pipeline (PipelineResult) +Dependencies DOWNWARD: + - orchestrate/update.py, orchestrate/backfill.py +""" import datetime as dt import logging @@ -6,8 +20,8 @@ import numpy as np import pandas as pd -from . import PipelineResult -from .db import fetch_df, upsert_many +from data_pipeline import PipelineResult +from data_pipeline.store.db import fetch_df, upsert_many logger = logging.getLogger(__name__) @@ -52,7 +66,7 @@ def _flag_anomalies(df: pd.DataFrame) -> pd.DataFrame: def clean_range(ticker: str, start: dt.date | None = None, end: dt.date | None = None) -> PipelineResult: """ Clean data for [start, end). Align to business days, mark missing days as NA - (no interpolation for full missing days), flag anomalies, and upsert to clean_prices. + (no interpolation for full missing days), flag anomalies, and upsert to clean_bars. Returns a PipelineResult with row count and any warnings. INVARIANT: missing trading days remain NA. We do NOT interpolate prices @@ -65,27 +79,27 @@ def clean_range(ticker: str, start: dt.date | None = None, end: dt.date | None = # Inclusive end date query df = fetch_df( - "SELECT * FROM raw_prices WHERE ticker=? AND date>=? AND date<=?", + "SELECT * FROM raw_bars WHERE ticker=? AND date>=? AND date<=?", (ticker, start.isoformat(), end.isoformat()), ) - # WHY: If raw_prices has zero rows for this ticker (e.g. the ticker was + # WHY: If raw_bars has zero rows for this ticker (e.g. the ticker was # invalid and yfinance returned nothing), do NOT generate a business-day - # aligned all-NaN frame and upsert it. Doing so pollutes clean_prices + # aligned all-NaN frame and upsert it. Doing so pollutes clean_bars # with phantom rows for arbitrary user input — including XSS payloads — # and turns a read-only "validate_ticker" call into a DB writer. if df.empty: - # Sanity check: only short-circuit when the ticker has no clean_prices + # Sanity check: only short-circuit when the ticker has no clean_bars # history at all. Established tickers might legitimately have a quiet # period (e.g. exchange holiday week) where the requested raw range # is empty; in that case fall through and align as before so existing # downstream guarantees about business-day alignment are preserved. existing = fetch_df( - "SELECT 1 FROM clean_prices WHERE ticker=? LIMIT 1", + "SELECT 1 FROM clean_bars WHERE ticker=? LIMIT 1", (ticker,), ) if existing.empty: logger.info( - "clean_range: skipping upsert for %s — no raw rows and no existing clean_prices", + "clean_range: skipping upsert for %s — no raw rows and no existing clean_bars", ticker, ) return PipelineResult(ok=True, rows=0, warnings=["no_data_for_ticker"]) @@ -180,7 +194,7 @@ def clean_range(ticker: str, start: dt.date | None = None, end: dt.date | None = ) if rows: upsert_many( - "clean_prices", + "clean_bars", [ "ticker", "date", diff --git a/data_pipeline/processing.py b/data_pipeline/transform/processing.py similarity index 93% rename from data_pipeline/processing.py rename to data_pipeline/transform/processing.py index b808e8f..b3b9273 100644 --- a/data_pipeline/processing.py +++ b/data_pipeline/transform/processing.py @@ -1,9 +1,11 @@ """Feature engineering for cleaned daily price series. Context: -- Reads from ``clean_prices`` and emits resampled bars + indicator columns to - ``processed_prices``. Pure pandas; no I/O outside the DB helpers in - ``data_pipeline.db``. +- Reads from ``clean_bars`` and emits resampled bars + indicator columns to + ``feature_bars``. Pure pandas; no I/O outside the DB helpers in + ``data_pipeline.store.db``. INVARIANT (ADR 0011 §5.2): never imports + ``providers/`` — the processing stage only ever sees canonical tables. + ``data_pipeline.store.db``. """ import datetime as dt @@ -12,8 +14,8 @@ import numpy as np import pandas as pd -from . import PipelineResult -from .db import fetch_df, upsert_many +from data_pipeline import PipelineResult +from data_pipeline.store.db import fetch_df, upsert_many logger = logging.getLogger(__name__) @@ -73,7 +75,7 @@ def process_frequencies(ticker: str, start: dt.date | None = None, end: dt.date start = start or (end - dt.timedelta(days=90)) daily = fetch_df( - "SELECT date, open, high, low, close, adj_close, volume FROM clean_prices WHERE ticker=? AND date>=? AND date<=?", + "SELECT date, open, high, low, close, adj_close, volume FROM clean_bars WHERE ticker=? AND date>=? AND date<=?", (ticker, start.isoformat(), end.isoformat()), ) if daily.empty: @@ -131,7 +133,7 @@ def process_frequencies(ticker: str, start: dt.date | None = None, end: dt.date ) if rows: upsert_many( - "processed_prices", + "feature_bars", [ "ticker", "date", diff --git a/docs/architecture_review.md b/docs/architecture_review.md index ce6f1ae..6621899 100644 --- a/docs/architecture_review.md +++ b/docs/architecture_review.md @@ -42,11 +42,15 @@ count can only go down without an explicit baseline update. Run `grep -rn "doc-guard: allow" --include='*.py' core services data_pipeline` for the live list. State at registration: -### core-purity (core must not import data_pipeline) — 2 markers +### core-purity (core must not import data_pipeline) — 0 markers + +**All registered core-purity debt is closed (batch B4, 2026-09-10).** `core/` has +zero `data_pipeline` imports; `tests/test_architecture_purity.py` now refuses the +suppression marker outright, and the layer table (§3) only allows `core → utils`. | Location | Why it exists | Exit condition | |---|---|---| -| `core/market/data_context.py` (DataService, fetch_daily_ohlcv) | `build_data_context` *is* the DB-first read path; extracting it means inverting who constructs `DataContext` | `DataContext` becomes data-in/data-out; the fetch moves into a service factory | +| `core/market/data_context.py` (DataService, fetch_daily_ohlcv) — **resolved 2026-09-10 (B4)** | `build_data_context` *was* the DB-first read path, so core did its own I/O | split: `core/market/data_context.py` keeps a pure `DataContext` + `refrequency` + the data-in/data-out `build_data_context(*, ticker, frequency, horizon, raw_data)`; acquisition moved to `services/market/data_context_fetch.py::fetch_data_context`. `MarketAnalyzer` / `CorrelationValidator` now take the context instead of building it (same pattern as `OptionsChainAnalyzer(snapshot=…)`) | | `core/market_review/fetch.py`, `core/market_review/__init__.py` (fetch_close_panel, get_conn) — **resolved 2026-09-03** | L1/L2/L3 cache ladder lived beside the computation it feeds | ladder moved to `services/market_review` (`fetch.py` + `facade.py`); `core/market_review` now receives panels via pure `build_review` / `build_timeseries` | | `core/options/chain/analyzer.py` — **resolved 2026-09-03** | former ticker-only constructor fetched yfinance internally | constructor now requires `snapshot=`; fetch lives in `services/options/chain._build_analyzer` | @@ -54,40 +58,55 @@ for the live list. State at registration: | Location | Why it exists | Exit condition | |---|---|---| -| `services/market/health.py` — **resolved 2026-09-03**, `services/portfolio/facade.py` — **resolved 2026-09-03** (`get_conn`) | ad-hoc health/inventory SQL predates `repos.py` coverage | queries moved into `data_pipeline/repos.py` | -| `services/regime/facade.py`, `services/regime/ops/_bootstrap.py`, `services/regime/ops/_persistence.py` (`fetch_df`, `init_db`, `upsert_many`) — **resolved 2026-09-03** | regime log writes were split across service and ops modules | consolidated behind `data_pipeline/repos.py` (regime-log + clean-row ops) | +| `services/market/health.py` — **resolved 2026-09-03**, `services/portfolio/facade.py` — **resolved 2026-09-03** (`get_conn`) | ad-hoc health/inventory SQL predates `repos.py` coverage | queries moved into `data_pipeline/store/repos.py` | +| `services/regime/facade.py`, `services/regime/ops/_bootstrap.py`, `services/regime/ops/_persistence.py` (`fetch_df`, `init_db`, `upsert_many`) — **resolved 2026-09-03** | regime log writes were split across service and ops modules | consolidated behind `data_pipeline/store/repos.py` (regime-log + clean-row ops) | + +### single-yf-exit (only `data_pipeline/providers/` may import yfinance) — 0 markers -### single-yf-exit (only `yf_client.py` may import yfinance) — 1 marker +Rescoped in batch B1 of [ADR 0011](decisions/0011-pluggable-data-provider-seam.md) +(2026-09-10): the chokepoint moved from `yf_client.py` into the provider package, and +`yf_client.py` is now a compatibility shim that no longer imports yfinance. | Location | Why it exists | Exit condition | |---|---|---| -| `data_pipeline/downloader.py` | DB-aware gap-detection bulk downloads; documented chokepoint alongside `yf_client` (see `yf_client` module docstring) | fold the gap logic into `yf_client` | -| `data_pipeline/data_ops/_query.py::get_latest_spot` — **resolved 2026-09-03** | former spot fast-path fetched yfinance internally | now routes through `yf_client.fetch_spot` | +| `data_pipeline/ingest/ohlcv.py` — **resolved 2026-09-10 (B1)** | DB-aware gap-detection bulk downloads; it used to call `yf.download` directly as a registered second exit point | the download call moved to `providers/yfinance_provider.py::download_daily_frame`; `downloader.py` keeps only gap detection + `raw_bars` upsert, so it no longer imports yfinance | +| `data_pipeline/read/_query.py::get_latest_spot` — **resolved 2026-09-03** | former spot fast-path fetched yfinance internally | now routes through `fetch_spot` (provider, re-exported by `providers/yf_client.py`) | ### Watch list (pre-debt, no marker yet) | Location | Concern | Trigger to act | |---|---|---| -| `services/market/analysis/summary.py` (fan-in 0, tracked as `dead_code_candidates=1` in baseline) | `generate_summary_analysis` lost its caller when the streaming refactor removed the server-rendered `summary_data` template variable; the Summary tab button is gated off in `templates/index.html` and `summary_pending` in `routes/core.py` is vestigial | any request to ship the multi-ticker Summary tab ⇒ add a `summary` slice to `_RENDER_KIND_SLICES` (aggregates across the job's tickers, not per-ticker) ; otherwise delete the module + `partials/tab_summary.html` + the `summary_pending` flag in the same commit and reset the baseline | -| `data_pipeline/yf_client.py` (391 lines, fan-in 11) | 9 lines below the 400-line god-file threshold; the throttle wrapper itself already lives in `utils/network.py::yf_throttle`, but each new yfinance endpoint (option greeks feeds, dividends/splits, etc.) grows the file | any edit that pushes it past 400 lines ⇒ extract the option-chain section (~150 lines, `fetch_option_chain` + `_fetch_option_chain_serial` + `_OPT_NUMERIC_COLS`) into `data_pipeline/yf_option_chain.py` in the same commit | +| ~~`services/market/analysis/summary.py` (fan-in 0, `dead_code_candidates=1`)~~ — **deleted 2026-09-11 (B9)** | `generate_summary_analysis` lost its caller in the streaming refactor; `summary_data` was never set, so `tab_summary.html`, the sidebar button, the correlation-heatmap JS and the `summary_pending` flag were all vestigial. A real multi-ticker Summary tab is a feature nobody requested | removed the module + `partials/tab_summary.html` + the sidebar button + `summary_pending` + `renderCorrelationHeatmap`/`corrToColor` in `market_review_chart.js`; `arch_baseline.json` `dead_code_candidates` reset 1 → 0 | +| ~~`data_pipeline/providers/yf_client.py` (391 lines, fan-in 11)~~ — **resolved 2026-09-10 (B1)** | it sat 9 lines below the 400-line god-file threshold | the option-chain section was extracted pre-emptively, as prescribed, into `providers/yf_snapshot.py`; `yf_client.py` is now a ~35-line shim. The pressure moved to `providers/yf_snapshot.py` (≈340 lines) and `providers/yfinance_provider.py` (≈290 lines) — watch them before adding endpoints | +| `static/sim/black_scholes.js` + `core/options/greeks` (risk-free rate) | the same constant is hard-coded in the client simulation **and** the server-side Greeks; changing one without the other makes the two pricings diverge silently | batch B8 retired the Config tab, so there is no global-setting surface to put it in; making it a parameter (store entry + form field + backend path) is a small feature — do it before anyone edits either constant | +| `data_pipeline/providers/yf_client.py` (compat shim, ADR 0011 B1) | re-exports `fetch_spot` / `fetch_option_chain` / `fetch_close_panel` / `fetch_daily_ohlcv` with their old yfinance-shaped contracts; it was a **"one release"** bridge so importers did not have to change in the B1 PR | once `grep -rn "providers.yf_client\|yf_client import" services/ data_pipeline/read/` is empty (importers moved to the canonical `providers.get_provider()` shapes), delete `yf_client.py` and its re-exports from `providers/__init__.py` in one commit | +| `services/market_review/fetch.py` → `market_review_prices` (L5 in plan §4.1) | a second acquisition path outside the provider seam: its own L1/L2/L3 close-panel ladder writes a `market_review_prices` table that `data_pipeline/orchestrate/readiness.py` does **not** plan, so the benchmark panel still lazy-fetches on the Market Review slice | fold the ladder into `providers` + a canonical `bars` read (ADR 0011's L5 exit); until then, add `market_review` benchmark tickers to `KIND_DATASETS` so the readiness pass warms them | +| ~~`services/market/analysis/assessment.py` option overlay~~ — **retired 2026-09-11 (B9, plan §10 F5-a)** | B7 left the projection-vs-positions overlay + the sizing max-loss unfed (positions moved to the Portfolio tab) | removed: the `option_data` branches in `assessment.py`, `FormService.parse_option_data`, `MarketAnalyzer.analyze_options` / `MarketChartAssembly.analyze_options`, and `core/market/option_pnl.py` + `core/market/charts/option_pnl.py` (whole files). Assessment sizing is now debit-only; position P&L lives in the Portfolio tab | ## 3. Guardrails (how the score is kept) | Tool | Role | Run where | |---|---|---| -| `scripts/doc_guard.py` | blocks violating edits: `import-direction`, `core-purity`, `db-access`, `single-yf-exit`, `sqlite-bypass`, `yfinance-throttle`, `yfinance-session-kwarg`, `tag-syntax`, `module-docstring`, ADR rules | pre-commit + CI, per changed file | +| `scripts/doc_guard.py` | blocks violating edits: `import-direction` (sub-package aware since B3), `core-purity`, `db-access`, `single-yf-exit`, `sqlite-bypass`, `yfinance-throttle`, `yfinance-session-kwarg`, `tag-syntax`, `module-docstring`, ADR rules | pre-commit + CI, per changed file | | `scripts/arch_metrics.py` | trend metrics: layer-edge violations, import cycles (Tarjan), god files, dead-code candidates, fan-in/out Top-5; `--check` fails CI on regression vs `.github/data/arch_baseline.json` | CI, whole repo | -| `tests/test_architecture_purity.py` | contract test re-asserting core purity at the test layer so suppressed markers stay visible in the test report | pytest | +| `tests/test_architecture_purity.py` | contract tests at the test layer: core purity, the `data_pipeline/` layer graph, transform↛providers, and that the two copies of the layer table agree | pytest | **Layer allow-list** (single source of truth: `doc_guard.py::_ALLOWED_DEPS`, -mirrored in `arch_metrics.py`): +mirrored in `arch_metrics.py`; asserted equal by +`tests/test_architecture_purity.py::test_layer_tables_are_in_sync`): ``` -app → routes, services, core, data_pipeline, utils -routes → services, data_pipeline, utils (never core directly) -services → core, data_pipeline, utils -core → utils (data_pipeline only via §2 markers) -data_pipeline → utils +app → routes, services, core, data_pipeline, utils, read, orchestrate +routes → services, data_pipeline, utils, store, read, orchestrate (never core directly) +services → core, data_pipeline, utils, providers, store, ingest, transform, read, orchestrate +core → utils (zero data_pipeline imports — closed in B4) +data_pipeline → utils # root: PipelineResult (types) + _state.py + store → (nothing upward) + providers → store, utils # store = quality_log; see plan §8 B3 + ingest → data_pipeline, providers, store, utils + transform → data_pipeline, store, utils # never providers (asserted by test) + read → data_pipeline, orchestrate, providers, store, utils + orchestrate → data_pipeline, ingest, store, transform, utils utils → (leaf: nothing upward) ``` @@ -105,7 +124,8 @@ utils → (leaf: nothing upward) 5. Import cycle `data_ops/_query.py <-> facade.py` broken (query calls sibling modules, never the facade). 6. Renames for D5: `market_analysis/{_service,_statistical,_assessment, - _sizing,_summary}.py` → `{facade,statistical,assessment,sizing,summary}.py`; + _sizing,_summary}.py` → `{facade,statistical,assessment,sizing,summary}.py` + (`summary.py` later deleted — B9 F5-a); same for `data_ops/_service.py` → `facade.py`. Dead code `core/_shared/validators.py` deleted; `correlation_validator.py` moved into `core/market/`. @@ -115,7 +135,8 @@ utils → (leaf: nothing upward) from the chart assembly. All chart-producing methods (`generate_scatter_plots`, `generate_high_low_scatter`, `generate_return_osc_high_low_chart`, `generate_volatility_dynamics`, `generate_oscillation_projection`, - `analyze_options`, plus the feature/projection primitives that feed them) + `analyze_options` (later removed — B9 F5-a), plus the feature/projection + primitives that feed them) moved into `core/market/charts/facade.py::MarketChartAssembly`. `MarketAnalyzer` is now a thin orchestrator that builds the `DataContext` and delegates rendering, mirroring the options-side facade. The chart-assembly fan-out (14) now lives in @@ -133,7 +154,8 @@ utils → (leaf: nothing upward) 1. **§2 debt paydown** (easiest first): `_query.get_latest_spot` → `yf_client` — **done 2026-09-03**; `health`/`portfolio` SQL → `repos.py` — **done 2026-09-03**; `market_review` cache ladder → `services/market_review` — **done 2026-09-03**; `regime` SQL consolidation → - `repos.py` — **done 2026-09-03**. All registered §2 debt is now resolved. + `repos.py` — **done 2026-09-03**. All registered §2 debt is now resolved, including the last + `core-purity` markers (`core/market/data_context.py` — **done 2026-09-10, batch B4**). 3. **Frontend consolidation** (P3): move the eight loose root-level scripts in `static/` (`option-chain.js`, `position.js`, `regime.js`, `simulation.js`, `market_review.js`, …) into `static/features/`. diff --git a/docs/automation.md b/docs/automation.md index 5341d48..5ebe2fa 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -28,9 +28,9 @@ What it checks (each rule produces a non-zero exit code on violation): | Rule | What it catches | Why | | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | | `tag-syntax` | `WHY:`/`CONSTRAINT:`/etc. used outside the canonical vocabulary, or with malformed forms (lowercase, missing colon). | Keeps the tag set stable so AI grep is reliable. | -| `yfinance-throttle` | Any new `yf.download` / `yf.Ticker(...)` call site not preceded by `yf_throttle()` or routed through `data_pipeline/yf_client.py`. | Hard architectural invariant from ADR 0005. | +| `yfinance-throttle` | Any new `yf.download` / `yf.Ticker(...)` call site not preceded by `yf_throttle()` or routed through `data_pipeline/providers/` (the chokepoint since ADR 0011 / batch B1). | Hard architectural invariant from ADR 0005. | | `yfinance-session-kwarg` | Any call passing `session=` to a yfinance API. | Silent failure mode (curl_cffi). See `docs/constraints.md` §2. | -| `sqlite-bypass` | New `sqlite3.connect(` outside `data_pipeline/db.py`. | Bypasses WAL pragmas (ADR 0003). | +| `sqlite-bypass` | New `sqlite3.connect(` outside `data_pipeline/store/db.py`. | Bypasses WAL pragmas (ADR 0003). | | `import-direction` | Imports from `services/` inside `core/` or `data_pipeline/`; from `core/` inside `data_pipeline/`. | ADR 0001 — already enforced by an existing hook; doc-guard is the safety net. | | `adr-link-integrity` | Markdown links from `docs/decisions/` / `docs/constraints.md` / `docs/glossary.md` that point at non-existent files or anchors. | ADRs must stay reachable. | | `adr-index-fresh` | `docs/decisions/README.md` index does not match the actual ADR files in the folder. | Auto-fixable; CI fails if not regenerated. | diff --git a/docs/constraints.md b/docs/constraints.md index 2c1d664..100ac42 100644 --- a/docs/constraints.md +++ b/docs/constraints.md @@ -12,6 +12,11 @@ is usually a workaround for one of the items below. ## 1. yfinance is the only data source - **Constraint**: This is a personal-use research tool. We do not pay for Bloomberg / Polygon / Tradier. +- **Amended by [ADR 0011](decisions/0011-pluggable-data-provider-seam.md) (2026-09-10)**: yfinance is + the only data-source **implementation**, not the only possible one. It lives behind the provider + seam in `data_pipeline/providers/` and maps its fields onto one canonical internal schema, so a + second vendor is one provider module + one registry line. The option-history caveats below are + unchanged — a second vendor does not conjure option history that no free API publishes. - **Implications**: - No SLA, no support, no stable schema. yfinance can break on any release. - Aggressive rate limiting (HTTP 429) — see §2. @@ -24,22 +29,22 @@ is usually a workaround for one of the items below. - **Do NOT pass `session=requests.Session()`** to any yfinance call — it silently fails to make the request. - Proxy must be set via `HTTP_PROXY` / `HTTPS_PROXY` env vars; we read `YF_PROXY` and propagate to both. See [utils/network.py](../utils/network.py) `init_yf_proxy`. - **Dead-proxy poisoning**: an unreachable proxy makes curl_cffi hang. We TCP-probe before activating; falls back to direct connect. -- **Global throttle**: token-bucket limiter (default 5 req/s, burst 5) in [utils/network.py](../utils/network.py)::`yf_throttle`. Every yfinance call MUST be routed through `data_pipeline/yf_client.py` (the single chokepoint) rather than calling `yf_throttle()` directly at each call site — see ADR 0005. -- **DB-first pattern**: never re-download data already in `clean_prices`. The 60-second cooldown in `DataService` exists to prevent thundering herd from concurrent UI requests. -- **Single yfinance exit point**: only `data_pipeline/yf_client.py` may `import yfinance`. Any other module needs a `# doc-guard: allow=single-yf-exit` marker, which is tracked as architecture debt (see [architecture_review.md](architecture_review.md) §2). Enforced by `scripts/doc_guard.py` rules `single-yf-exit`, `import-direction`, `core-purity` and `db-access`; trend-gated in CI by `scripts/arch_metrics.py --check`. +- **Global throttle**: token-bucket limiter (default 5 req/s, burst 5) in [utils/network.py](../utils/network.py)::`yf_throttle`. Every yfinance call MUST be routed through `data_pipeline/providers/` (the single chokepoint since batch B1 of ADR 0011) rather than calling `yf_throttle()` directly at each call site — see ADR 0005. +- **DB-first pattern**: never re-download data already in `clean_bars`. The 60-second cooldown in `DataService` exists to prevent thundering herd from concurrent UI requests. +- **Single yfinance exit point**: only `data_pipeline/providers/` may `import yfinance`. The chokepoint moved there from `yf_client.py` in batch B1; `yf_client.py` is now a one-release compatibility shim over the provider package. Any other module needs a `# doc-guard: allow=single-yf-exit` marker (tracked as architecture debt — see [architecture_review.md](architecture_review.md) §2). `tests/` and `scripts/` are exempt by design (test doubles patch `yfinance.download` on the module object). Enforced by `scripts/doc_guard.py` rules `single-yf-exit`, `import-direction`, `core-purity` and `db-access`; trend-gated in CI by `scripts/arch_metrics.py --check`. ## 3. SQLite, single-machine deployment - **Constraint**: this app runs on one developer machine, occasionally a small VPS. Postgres is overkill. - **WAL mode + `synchronous=NORMAL`**: chosen for read concurrency. Do not switch to `FULL` (latency) or remove WAL (locks block reads during scheduler writes). -- **Thread-local connections** (`data_pipeline/db.py`): SQLite connections are not thread-safe to share, but per-query reconnects are wasteful. We cache one connection per (thread, path) and apply PRAGMAs once. +- **Thread-local connections** (`data_pipeline/store/db.py`): SQLite connections are not thread-safe to share, but per-query reconnects are wasteful. We cache one connection per (thread, path) and apply PRAGMAs once. - **No migration framework**: schemas are created via `CREATE TABLE IF NOT EXISTS`. Breaking changes require manual `.sqlite` migration scripts in `scripts/`. ## 4. The machine is not 24/7 - Snapshot cadence (scheduler) **will have gaps**: laptop sleeps, weekends off, network outages. - Any feature that consumes time-series data must tolerate **sparse, non-contiguous days**. Do NOT assume daily continuity. -- `data_pipeline/cleaning.py` aligns to business days and marks missing days as NA — **no interpolation**, by design. Filling gaps would invent prices that didn't trade. +- `data_pipeline/transform/cleaning.py` aligns to business days and marks missing days as NA — **no interpolation**, by design. Filling gaps would invent prices that didn't trade. ## 5. Financial domain "magic numbers" are intentional @@ -55,7 +60,7 @@ These are NOT magic numbers — they encode domain knowledge. Do not "DRY" them ## 6. Computation must finish in one HTTP request - **No background job queue** (no Celery, no RQ). The Flask process serves the UI and runs the scheduler in-thread. -- **APScheduler is optional and lazily imported.** The scheduler only starts when `AUTO_UPDATE_TICKERS` is set, and `data_pipeline/scheduler.py` imports APScheduler inside `UpdateScheduler.__init__` (plus a lazy `CronTrigger` import) so the rest of the app — and `acquire_scheduler_lock`'s unit tests — run without the package installed. **Do not move that import back to module scope**: an optional feature must not become a hard startup dependency. +- **APScheduler is optional and lazily imported.** The scheduler only starts when `AUTO_UPDATE_TICKERS` is set, and `data_pipeline/orchestrate/scheduler.py` imports APScheduler inside `UpdateScheduler.__init__` (plus a lazy `CronTrigger` import) so the rest of the app — and `acquire_scheduler_lock`'s unit tests — run without the package installed. **Do not move that import back to module scope**: an optional feature must not become a hard startup dependency. - Long-running computations either: - Run inside a request and respond synchronously (fine for <2s), or - Are pre-computed by the scheduler and read from DB. diff --git a/docs/decisions/0011-pluggable-data-provider-seam.md b/docs/decisions/0011-pluggable-data-provider-seam.md index 2458d45..ad13c74 100644 --- a/docs/decisions/0011-pluggable-data-provider-seam.md +++ b/docs/decisions/0011-pluggable-data-provider-seam.md @@ -100,6 +100,39 @@ objects. `transform/`, `read/`, `services/` or `core/`." The option-history caveats (no IV rank / percentile / backtests — ADR 0004) are unchanged. +### Protocol shape — decision gate §8 Q5 (resolved 2026-09-10) + +`providers/base.py` was reviewed against **two** field maps before being written, so the +protocol is not accidentally yfinance-shaped +(`archive/futu_integration/field_mapping.md` is the second map): + +| Concern | yfinance | futu | Protocol decision | +|---|---|---|---| +| IV unit | decimal (`0.2436`) | percent (`24.359`) | canonical `iv` is a **decimal**; the provider normalises at its own boundary | +| bid / ask | present per expiry | absent without an `ORDER_BOOK` subscription | canonical `bid`/`ask` are **nullable**; a provider must not invent a quote | +| `inTheMoney` | present | absent (must be derived from strike vs spot) | **dropped** from the canonical leg — derivable, so not a contract | +| Expiries | `tk.options` list | `get_option_expiration_date()` frame | canonical = `expiries: tuple[str, ...]` (ISO dates) | +| Greeks | absent | native (`delta`/`gamma`/…) | out of scope for B1; the protocol has no Greek fields yet | +| History | `yf.download` frame | `get_stock_quote` / history API | canonical `history()` returns `CANONICAL_BAR_COLUMNS` (`open…adj_close, volume`) | + +A provider id is also a first-class value (`MarketDataProvider.name`), because a second +vendor means two providers coexist rather than one replacing the other. + +**Implementation status**: B1 (seam extraction, no behaviour change) landed 2026-09-10 — +`data_pipeline/providers/{base,_log,_registry,yfinance_provider,yf_snapshot}.py`; +`yf_client.py` is a compatibility shim and `downloader.py` no longer imports yfinance. +Batch ledger: [`docs/plans/business_line_reorg.md`](../plans/business_line_reorg.md) §0. + +**Amendment (batch B2, 2026-09-10) — canonical tables are a name-only rename.** B2 landed +`raw_bars` / `clean_bars` / `feature_bars` carrying *exactly* the column sets of the tables they +replace, including the `ticker` column. Renaming `ticker` → `symbol` (and introducing a +`symbol_map`) is deferred until a second provider actually needs a provider-native identifier: +today it is a cross-cutting rename through `repos.py`, `data_ops/`, `services/` and the test +fixtures with no consumer, which batch scope forbids riding along with. The target-state column +lists in the Decision section above stay the reference for when that provider lands. +Compatibility: the pre-rename names remain as shadow tables for one release (`upsert_many` +writes both families) and `scripts/migrate_canonical_tables.py` backfills an existing DB. + ## Consequences - Positive: a second provider is one file + one registry line + a field-map test. diff --git a/docs/decisions/0012-parameter-ownership-and-prefetch.md b/docs/decisions/0012-parameter-ownership-and-prefetch.md index 966de5b..8f7dfb8 100644 --- a/docs/decisions/0012-parameter-ownership-and-prefetch.md +++ b/docs/decisions/0012-parameter-ownership-and-prefetch.md @@ -111,6 +111,37 @@ The **submit contract** — whether `POST /` carries a `modules` manifest with per-module params attached to each `/render` call, or the streaming tabs move fully to client-fired `/api/*` — is deferred to the implementation batch. +> **Resolved (batch B5, 2026-09-10): manifest.** `POST /` carries the module tokens; each module's +> parameters travel as query args on its own `/render` call, mirroring +> `/api/option_chain?ticker=…`. Full client-fired was rejected because (a) this prefetch pass needs +> the module list *at submit time*, (b) the four streaming slices return server-rendered HTML + +> base64 PNG, so changing the transport would not change the product, and (c) the diff/revert surface +> would span four templates plus four loaders. It would only win if the charts moved to client-side +> rendering (ADR 0006 / 0008). See the plan §8 gate table and the B5 note. + +**Implementation status (B5)**: `data_pipeline/orchestrate/readiness.py` plans datasets per module, +probes coverage (DB-only) and kicks missing ranges on a daemon thread; +`services/market/readiness.py` adds the live-preload warm; the plan is stored on the job and +`/render/` holds a cold-start tab with a self-re-firing readiness fragment (bounded by +`HOLD_SECONDS` and by backfill-thread liveness). The per-module toolbars and the removal of +`syncConfigToForm` remain B7's work. + +**Implementation status (B6)**: the Parameters tab is gone. A persistent, collapsible +Parameters bar (`templates/partials/parameters_bar.html` + `static/parametersBar.js`, +`localStorage` key `parametersBarCollapsed`) renders above `.main-panel`, is `position: +sticky` under the header, and owns exactly one input — `ticker` — plus the Run button and the +validation badges. Analysis settings that B7 will move to module toolbars stay in a collapsible +group inside the same `
`, so the submit contract is untouched. The `positions` block moved +to its own **Portfolio** tab (decision gate §8 Q3). + +**Implementation status (B7)**: the per-module toolbars are live and the bridge is gone. Each +module's parameters travel as query args on its own `/render` call (or, for the client-fired chain, +in the request `static/option-chain.js` builds from the `optionFilter` store); a change re-runs +exactly the modules that consume that group. Three bugs were caught by the new tests while landing +this — a shared-field clobber between toolbars, an `init()` that clobbered its own published global, +and an `input`+`change` double-emit race — and are pinned by `tests/unit/paramsStore.test.js` and +`tests/e2e/test_module_params.py`. + ## Consequences - Positive: the always-visible surface is one field; module parameters are diff --git a/docs/frontend_architecture.md b/docs/frontend_architecture.md index b0048e6..ee56725 100644 --- a/docs/frontend_architecture.md +++ b/docs/frontend_architecture.md @@ -60,10 +60,13 @@ templates/ Heavy analysis no longer runs synchronously inside `POST /`. The flow is: ``` -Browser ── POST / (form data) ─────────────────► Flask +Browser ── POST / (form data + module tokens) ──► Flask │ -Flask creates a JobCache entry (job_id) and immediately renders -`index.html` with `streaming_mode=True`. Each tab partial emits an +Flask resolves the modules, runs the **readiness pass** (ADR 0012): +plan the datasets those modules need, probe the DB once, kick anything +missing on a daemon thread, warm the live option-chain preload — then +creates a JobCache entry (job_id, carrying that plan) and immediately +renders `index.html` with `streaming_mode=True`. Each tab partial emits an HTMX placeholder:
in parallel for each visible ticker: /render/assessment /render/options_chain +Flask ── consults the job's readiness plan ─────► cold start? hold Flask ── compute_or_get(job_id, ticker, kind) ──► AnalysisService.*_slice └─ memoised per (job, ticker, kind) @@ -86,8 +90,10 @@ Flask ── HTML fragment ───────────────── ``` Key files: -- `data_pipeline/job_cache.py` — in-process JobCache (TTL 90 s). -- `app.py::_render_streaming_slice` — shared `/render/` handler. +- `data_pipeline/orchestrate/job_cache.py` — in-process JobCache (TTL 90 s), carries the plan. +- `data_pipeline/orchestrate/readiness.py` — dataset plan, coverage probe, backfill kicker. +- `services/market/readiness.py` — services half (preload warm), called from `routes/core.py::index`. +- `services/market/dispatch.py::render_streaming_slice` — shared `/render/` handler. - `services/market/analysis/facade.py::generate_*_slice` — per-tab compute. - `templates/partials/fragments/*.html` — rendered fragments. @@ -95,6 +101,19 @@ The browser-side HTMX library replaces each placeholder when its fragment arrives, so users see tabs populate as their data is ready instead of waiting for the slowest tab. +**Cold start** (batch B5): if the readiness pass kicked this module's dataset, the ticker has no +usable history yet *and* the backfill is still running, `/render/` returns +`partials/fragments/readiness.html` — a self-re-firing "正在准备…" fragment (`hx-trigger="load +delay:3s"`) — instead of rendering an empty chart. The hold is bounded by +`readiness.HOLD_SECONDS` (30 s) **and** by backfill-thread liveness, so a failed download quickly +falls through to the slice's own error rather than a permanent spinner. + +**Parameter ownership** (batch B7, per §8 Q1): the module tokens above (`market_review`, +`statistical`, `assessment`, `options_chain`, `payoff_ratio`, `regime`, `simulation`, +`option_pricing_matrix`) are the same vocabulary the readiness plan uses. Each module's parameters +travel as **query args on its own `/render` call**, mirroring `/api/option_chain?ticker=…`; the +persistent Parameters bar owns only `ticker`. + --- ## Page Architecture @@ -109,12 +128,14 @@ The application uses a **single-page template** (`index.html`) with tab-based na │ ├── Brand (icon + title + subtitle) │ │ └── Ticker Badge (when analysis active) │ ├─────────────────────────────────────────────────────────┤ +│ Parameters bar [ticker] [Run] (collapse ▸/▾) │ ← sticky, not a tab +├─────────────────────────────────────────────────────────┤ │ Sidebar │ Main Panel │ │ (tab-nav) │ (tab-content) │ │ │ │ -│ • Parameter │ Tab 1: Parameter Form │ -│ • Market │ Tab 2: Market Review Table/Chart │ -│ • Statistics │ Tab 3: Statistical Analysis Charts │ +│ • Market │ Tab 1: Market Review Table/Chart │ +│ • Statistics │ Tab 2: Statistical Analysis Charts │ +│ • Portfolio │ Tab 3: Positions / Portfolio Analysis │ │ • Assessment │ Tab 4: Assessment & Projections │ │ • Chain │ Tab 5: Option Chain T-View │ │ • Volatility │ Tab 6: Volatility Analysis │ @@ -123,6 +144,18 @@ The application uses a **single-page template** (`index.html`) with tab-based na └───────────────┴─────────────────────────────────────────┘ ``` +The **Parameters bar** (`templates/partials/parameters_bar.html`, batch B6) renders +between the header and `.app-body`, is `position: sticky` under the header, and owns +exactly one visible input — `ticker` — plus the Run button and the ticker-validation +badges. Every other parameter lives in its module's toolbar (batch B7); the bar +also carries two hidden `start_time`/`end_time` inputs that `POST /` validates and +uses to size the readiness prefetch, kept in sync by `state/marketParamsState.js`. +It is **not** a tab: it survives tab switches. Collapsing it (the chevron toggle; +state persisted per viewer under `localStorage['parametersBarCollapsed']`, guarded +`try/catch`) hides the `#parameters-bar-body` fields group and the validation line, +leaving the toggle, a one-line `▸ ^SPX` summary, and Run. The chevron is an inline +SVG (Font Awesome is not loaded on this page) rotated by `[data-collapsed]`. + ### Peek Sidebar At rest the sidebar is only the rail — a 2px vertical line in `var(--blue)` @@ -149,7 +182,7 @@ inline as before, since a hover-peek panel is unusable at that width. | Tab ID | Label | Data Source | | -------------------------- | --------------------------- | -------------------------------- | -| `tab-parameter` | Parameter | Static form | +| `tab-portfolio` | Portfolio | `POST /api/portfolio_analysis` | | `tab-summary` | 综合 (Multi-ticker summary) | `results.__综合__` | | `tab-market-review` | Market Review | `market_review_table` | | `tab-statistical-analysis` | Statistical Analysis | `scatter_*`, `dynamics_*` charts | diff --git a/docs/frontend_convergence.md b/docs/frontend_convergence.md index b395cc6..4fa32b3 100644 --- a/docs/frontend_convergence.md +++ b/docs/frontend_convergence.md @@ -57,6 +57,15 @@ static/ * Did not introduce a build step. Plain ES modules served as static files remain the deployment unit. +## Landed Since This Pass + +* **Batch B7 of the reorg (2026-09-10)**: parameters are owned by their modules — + `state/{market,assessment,optionFilter}ParamsState.js` (one localStorage key per + group) hydrate the module toolbars and re-run exactly the modules that consume + them; the `localStorage -> hidden input` bridge and `syncConfigToForm` are gone. + This is the "read/write through `state/`" direction this document asked for, + applied to the last place that still held module-local state. + ## Next Concrete Steps (when time permits) 1. Delete `static/eventBus.js` if `state/store.js` covers all listeners. diff --git a/docs/glossary.md b/docs/glossary.md index 2764a44..bfb0939 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -64,11 +64,16 @@ User-supplied directional preference (Bull / Bear / Neutral) used to filter stra ## Data Pipeline -### `raw_prices` -Untouched OHLCV pulled from yfinance. Indexed by `(ticker, date)`. +### `raw_bars` +Untouched OHLCV pulled from a provider (yfinance today) and mapped onto the canonical schema. Indexed by `(ticker, date)`. -### `clean_prices` -`raw_prices` aligned to business days, anomalies flagged, missing days = NA. **NO interpolation** — see [constraints.md §4](constraints.md#4-the-machine-is-not-247). +### `clean_bars` +`raw_bars` aligned to business days, anomalies flagged, missing days = NA. **NO interpolation** — see [constraints.md §4](constraints.md#4-the-machine-is-not-247). + +### `feature_bars` +`clean_bars` resampled per frequency (D/W/ME/QE) with engineered features (returns, MA, HV, oscillation). Indexed by `(ticker, date, frequency)`. + +> **Compatibility (one release)**: the pre-rename names `raw_prices` / `clean_prices` / `processed_prices` still exist as shadow tables — every write goes to both families (see `data_pipeline/store/db.py`) — so an un-migrated DB and a `git revert` of the rename keep working. See [ADR 0011](decisions/0011-pluggable-data-provider-seam.md). ### Anomaly Flags - `price_jump_flag`: |log return| > 5σ. diff --git a/docs/guides/README.md b/docs/guides/README.md index 4167468..e3724f1 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -50,7 +50,7 @@ services/ # 请求编排层（按业务域分包） facade.py # 状态标注与持久化 ops/ # 历史回填 + regime_log 写入 data_pipeline/ # 数据管道（下载 → 清洗 → 加工 → 服务） - downloader.py # 通过 yfinance 下载 OHLCV 并写入 raw_prices + downloader.py # 通过 yfinance 下载 OHLCV 并写入 raw_bars cleaning.py # 对齐交易日、标记异常（5σ 波动、成交量异常）、前向填充 processing.py # 日/周/月级聚合及衍生指标（收益率、振幅、Parkinson/GK 方差、动量等） data_service.py # 数据门面：初始化 DB，按需 7 日增量刷新，60s 并发节流 @@ -64,7 +64,7 @@ tests/ # 回归测试 ## 数据管道 ``` -Yahoo Finance ──▶ downloader (upsert raw_prices) +Yahoo Finance ──▶ downloader (upsert raw_bars) │ ▼ cleaning (对齐交易日, 异常标记, 前向填充) diff --git a/docs/guides/USER_GUIDE.md b/docs/guides/USER_GUIDE.md index 3eda2a4..2ae045e 100644 --- a/docs/guides/USER_GUIDE.md +++ b/docs/guides/USER_GUIDE.md @@ -615,7 +615,7 @@ The combined regime is the cartesian product (e.g. *"High vol / Down"*). The her | `GET /api/regime/history` | Full labelled time-series | | `POST /api/regime/backfill` | Recompute and persist the series | -Computation lives in the `core/regime/` package (`classify.py`, `series.py`, `models.py`); persistence goes through `data_pipeline/repos.py`. +Computation lives in the `core/regime/` package (`classify.py`, `series.py`, `models.py`); persistence goes through `data_pipeline/store/repos.py`. --- diff --git a/docs/l0_architecture.md b/docs/l0_architecture.md index f9e9cbd..b0f3db7 100644 --- a/docs/l0_architecture.md +++ b/docs/l0_architecture.md @@ -38,10 +38,10 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ | `routes/` | 909 lines · 8 files | 7 blueprints + `__init__.py` aggregate export; no business logic | good | | `services/` | 3 540 lines · 5 domain packages | `market` (incl. `analysis/` slice factory), `market_review`, `options`, `portfolio`, `regime` | good | | `core/` | 6 372 lines · 8 sub-packages + `_shared` | Pure computation — no Flask, no DB, no network | good | -| `data_pipeline/` | 2 645 lines · 12 files | The only I/O boundary: `yf_client`, `db`/`repos`, `data_ops`, `scheduler`, `job_cache` | good | +| `data_pipeline/` | 3 618 lines · 27 files | The only I/O boundary, re-homed into six one-way stages (ADR 0011, batch B3): `providers/` · `store/` · `ingest/` · `transform/` · `read/` · `orchestrate/` (+ `_state.py`) | good | | `utils/` | 756 lines · 7 files | Leaf layer; highest fan-in (`ticker_utils.py` = 11) | good | -| `templates/` | 1 546 lines · 17 files | `index.html` skeleton + `partials/fragments/*` (HTMX swap targets) | good | -| `static/` | 5 473 lines · 31 JS/CSS | `state/` · `sim/` · `components/` · `features/` + tab entry files | fair (see §4 P3-1) | +| `templates/` | 1 676 lines · 19 files | `index.html` skeleton + `partials/fragments/*` (HTMX swap targets) | good | +| `static/` | 6 287 lines · 35 JS + 1 CSS | `state/` (now incl. the module parameter groups) · `sim/` · `components/` · `features/` + tab entry files | fair (see §4 P3-1) | ### B. Dependencies & configuration @@ -68,7 +68,7 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ | Item | State | Note | |---|---|---| -| `market_data.sqlite` (11.7 MB) | git-ignored, still in repo root | code default is now `data/market_data.sqlite` (`data_pipeline/db.py:27`); the local `.env` overrides it back to the root file — move the file into `data/` whenever convenient | +| `market_data.sqlite` (11.7 MB) | git-ignored, still in repo root | code default is now `data/market_data.sqlite` (`data_pipeline/store/db.py:27`); the local `.env` overrides it back to the root file — move the file into `data/` whenever convenient. Schema: canonical `raw_bars` / `clean_bars` / `feature_bars`, plus the one-release shadows `raw_prices` / `clean_prices` / `processed_prices` — see [ADR 0011](decisions/0011-pluggable-data-provider-seam.md) | | `site/` | **inputs committed (9 files) · build output ignored** | tracked: `fixtures/` (7) · `snapshot/snapshot.json` · `pages-shim.js`; ignored: `index.html`, 5 feature + 6 showcase redirects, `static/**` (42 generated files) — see §5 P1-1 | | `archive/` (8 files · 1 070 lines) | committed | retired code still in tree — P3-3 | | `test.ipynb` (58 lines) | git-ignored | leftover scratch file — P2-2 | @@ -78,17 +78,19 @@ app.py → routes/ → services/ → core/ → data_pipeline/ → utils/ ## 2. Measured shape (`scripts/arch_metrics.py`) +_Refreshed 2026-09-10 after batches B1 (provider seam), B2 (canonical table names), B3 (data_pipeline re-home), B4 (core purity), B5 (readiness), B6 (Parameters bar) and B7 (module-scoped params); the L1 inventory in §1 above is otherwise the 2026-09-08 snapshot._ + ``` -modules=146 import_edges=300 +modules=159 import_edges=324 Layer-edge violations : (none) Import cycles : 0 God files (>400 lines): (none) -Top fan-out : core/market/charts/facade.py(14) · core/market/data_context.py(7) - routes/__init__.py(7) · routes/core.py(7) · app.py(6) -Top fan-in : core/_shared/plotting.py(13) · data_pipeline/yf_client.py(11) - utils/ticker_utils.py(11) · data_pipeline/db.py(9) -Dead code : services/market/analysis/summary.py (only one; already on the - watch list in docs/architecture_review.md §2) +Top fan-out : core/market/charts/facade.py(14) · routes/core.py(8) + routes/__init__.py(7) · services/market/analysis/facade.py(7) + services/market/dispatch.py(7) +Top fan-in : core/_shared/plotting.py(13) · data_pipeline/providers/yf_client.py(11) + utils/ticker_utils.py(11) · data_pipeline/store/db.py(10) +Dead code : (none — summary.py deleted 2026-09-11, B9 F5-a) ``` Static checks at snapshot time: `ruff check .` clean · `ruff format --check .` @@ -113,6 +115,31 @@ OptionLab/ └─ archive/ # mark read-only or move out ``` +### Inside `data_pipeline/` — the six stages (ADR 0011, achieved in B3) + +``` +data_pipeline/ + providers/ ACQUIRE yfinance adapter + canonical schema + registry — the ONLY `import yfinance` + store/ SERVE schema, the only SQL, the failure log + ingest/ GLUE business-day gap detection + raw_bars upsert + transform/ PROCESS raw_bars → clean_bars → feature_bars (never imports providers) + read/ SERVE DataService facade + memoised queries + orchestrate/ DRIVERS manual/seed update, chunked backfill, readiness, job cache, scheduler + _state.py process-local shared state (query cache, update locks) +``` + +Call direction (same allow-list in `scripts/doc_guard.py` and +`scripts/arch_metrics.py`): + +``` +services → read → {store, orchestrate, providers} +services → orchestrate → {ingest, transform, store} +ingest → {providers, store} +transform→ store +providers→ {store, utils} # store = the failure log, see plan §8 B3 +read → orchestrate # the read path triggers refreshes +``` + --- ## 4. Open findings (summary; details in the 2026-09-08 review) diff --git a/docs/plans/business_line_reorg.md b/docs/plans/business_line_reorg.md index e056727..c55d175 100644 --- a/docs/plans/business_line_reorg.md +++ b/docs/plans/business_line_reorg.md @@ -4,9 +4,12 @@ **ADRs**: [0011](../decisions/0011-pluggable-data-provider-seam.md) (data-provider seam + canonical schema — **Accepted**), [0012](../decisions/0012-parameter-ownership-and-prefetch.md) (parameter ownership + readiness prefetch — **Accepted**) -> **Status: ACCEPTED TARGET, NOT YET IMPLEMENTED.** The shape below is the agreed -> destination. It ships as the batches in §6 — each one independently shippable -> and independently revertible. Track progress in the §0 ledger. +> **Status: LANDED (2026-09-10) + acceptance review B9 (2026-09-11).** All eight +> §6 batches plus the B9 review-remediation shipped on branch +> `worktree-business-line-reorg`; the §0 ledger records what actually shipped and +> §10 the review findings. Only the deferred follow-ups in §10 remain (risk-free +> global setting, `market_review_prices` L5, ADR 0011 `symbol` column). Batches +> are individually revertible — `git revert `. --- @@ -37,15 +40,16 @@ | Batch | State | PR | Landed (commit · date) | Notes | |---|---|---|---|---| -| — (planning + ADRs) | 🔨 in review (PR #7) | #7 | branch `worktree-business-line-reorg` · 2026-09-10 | plan, ADR 0011/0012 (Accepted), scaffolding (ledger, gates, AI-guide pointers, memory) | -| B1 — provider seam extraction | ⬜ not started | — | — | delivers the "pluggable API" seam on its own | -| B2 — canonical raw store | ⬜ not started | — | — | gate: §8 Q4 | -| B3 — package re-home | ⬜ not started | — | — | resets `arch_baseline.json` | -| B4 — close L1 (`core-purity`) | ⬜ not started | — | — | — | -| B5 — readiness plan + prefetch | ⬜ not started | — | — | gate: §8 Q1 | -| B6 — `ticker`-only Parameters bar | ⬜ not started | — | — | gate: §8 Q3; depends on B5 | -| B7 — module-scoped params | ⬜ not started | — | — | gate: §8 Q1; depends on B6 | -| B8 — retire / repurpose Config tab | ⬜ not started | — | — | gate: §8 Q2 | +| — (planning + ADRs) | ✅ landed | #7 + #8 | merged to `main` · 2026-09-10 | plan, ADR 0011/0012 (Accepted), scaffolding (ledger, gates, AI-guide pointers, memory) | +| B1 — provider seam extraction | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | delivers the "pluggable API" seam on its own. Actual shape / deviations recorded in §8; `_ALLOWED_DEPS` promotion of `providers` deferred to B3 | +| B2 — canonical raw store | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | gate §8 Q4 resolved (name-only rename). Actuals in §8; `symbol` column deferred (ADR 0011 amendment) | +| B3 — package re-home | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | six stages + `_state.py`; first sub-layer guard table; `arch_baseline.json` **not** reset (no tracked drift). Actuals + deviations in §8 | +| B4 — close L1 (`core-purity`) | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | zero core→data_pipeline edges; markers deleted and refused by test; `core` layer tightened to `{utils}` | +| B5 — readiness plan + prefetch | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (**manifest**, params as `/render` query args); readiness plan on the job; cold-start hold fragment. Actuals + deferrals in §8 | +| B6 — `ticker`-only Parameters bar | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q3 resolved (dedicated Portfolio tab); bar + collapse persisted; transitional settings group inside the bar's form until B7. Actuals in §8 | +| B7 — module-scoped params | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q1 resolved (manifest). Backend query-arg contract + per-module allow-list; `state/*ParamsState.js`; module toolbars; bridge + hidden fields deleted; Config tab emptied (B8 decides its fate). See §8 B7 | +| B8 — retire / repurpose Config tab | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-10 | Q2 resolved (**deleted**). `grep tab_config` returns nothing; risk-free-rate follow-up on the watch list | +| B9 — acceptance-review remediation | ✅ landed | — | branch `worktree-business-line-reorg` · 2026-09-11 | §10 review: F1 (memo `variant` key), F2 (real collapse), F3 (SVG chevron), F4 (feature_bars self-heal), F5 a+b+c (a=retire option overlay per owner; b/c=stale UI), F6 (docstrings). Watch-list: `summary.py` deleted; risk-free-rate + L5 + `symbol` column deferred. **Open**: F7c (merge to main). | States: `⬜ not started` → `🔨 in progress (PR #n)` → `✅ landed` → (`↩ reverted`). Keep the row order; edit the row in place. @@ -419,19 +423,419 @@ batch starts coding (§0 rule 5). Until then the batch stays `⬜ not started`. | # | Question | Decision gate | Working lean | |---|---|---|---| -| Q1 | **Submit contract** — does `POST /` carry a `modules` manifest with per-module params attached to each `/render` call, or do the streaming market tabs move fully to client-fired `/api/*` like Option Chain? Manifest keeps the streaming model; full client-fired is more uniform but a bigger diff. | before **B5** (locks how B7 wires params) | manifest | -| Q2 | **Config tab fate** — is there *any* genuine global setting to keep? Risk-free rate is the only candidate (hard-coded in `static/sim/` and again in `core/options/greeks`). Yes → tab shrinks to it; no → tab deleted. | before **B8** | keep risk-free rate, delete the rest | -| Q3 | **`positions` block** — Portfolio Analysis is its only consumer. Move into a dedicated "Portfolio" panel/tab, or keep as a section the bar's Run ignores? | before **B6** | dedicated Portfolio panel | -| Q4 | **Table rename vs. reshape** — `raw_prices`→`raw_bars` with identical columns (minimal), or also move the yfinance-ism `adj_close` handling into the provider during the rename? | before **B2** | minimal rename | -| Q5 | **Second-provider protocol shape** — not in scope to *implement*, but `providers/base.py` (written in B1) must be sketched against *both* yfinance and `archive/futu_integration/field_mapping.md` so the protocol is not accidentally yfinance-shaped (IV unit, bid/ask availability, `inTheMoney` derivation all differ). | before **B1** | design review of `base.py` against both field maps | +| Q1 | **Submit contract** — does `POST /` carry a `modules` manifest with per-module params attached to each `/render` call, or do the streaming market tabs move fully to client-fired `/api/*` like Option Chain? Manifest keeps the streaming model; full client-fired is more uniform but a bigger diff. | ✅ resolved 2026-09-10 (B5) — **manifest**, per-module params as query args on each `/render` call (sub-option A1) | **manifest.** Decisive reasons: (1) ADR 0012's readiness pass needs the module list *at submit time* — with no POST manifest, B5 would need an extra `/api/ready` protocol; (2) the four streaming slices return server-rendered HTML + base64 PNG, so client-firing changes only the transport, not the product; (3) the diff and the revert surface stay one batch wide. Full client-fired would only win if the charts moved to client-side rendering (ADR 0006/0008 territory). Params travel as query args (not `hx-post` JSON) to match the existing `/api/option_chain?ticker=…` shape and stay bookmark-reproducible — recorded in the B5 note below | +| Q2 | **Config tab fate** — is there *any* genuine global setting to keep? Risk-free rate is the only candidate (hard-coded in `static/sim/` and again in `core/options/greeks`). Yes → tab shrinks to it; no → tab deleted. | ✅ resolved 2026-09-10 (B8) — **tab deleted** | **Deleted.** After B7 nothing on it was global: every field had moved to the module that consumes it, so keeping the shell meant keeping a page whose only content was "these settings moved". The risk-free rate is not a *setting* yet (hard-coded in two places) — wiring it up is a new feature, not a cleanup, so there was nothing to shrink the tab to. The divergence risk is now on the watch list (`architecture_review.md` §2) | +| Q3 | **`positions` block** — Portfolio Analysis is its only consumer. Move into a dedicated "Portfolio" panel/tab, or keep as a section the bar's Run ignores? | ✅ resolved 2026-09-10 (B6) — **dedicated Portfolio tab** | **Dedicated Portfolio tab** (`tab-portfolio`). The positions table drives `POST /api/portfolio_analysis` (client-fired) and owns a full result surface (Greeks / P&L / theta / breakeven / VaR); keeping it inside the bar's form would re-couple that workflow to the streaming submit — exactly the coupling ADR 0012 removes — and a one-line bar has nowhere to put the results. A tab also makes the workflow discoverable instead of buried under "Parameters". `#positions-tbody` stays in the DOM on every load, so the existing global handlers are unchanged | +| Q4 | **Table rename vs. reshape** — `raw_prices`→`raw_bars` with identical columns (minimal), or also move the yfinance-ism `adj_close` handling into the provider during the rename? | ✅ resolved 2026-09-10 (B2) | **minimal rename** — identical columns on both sides of each pair (structurally enforced: one column tuple per shape, used to create both names). The `adj_close` normalisation is already inside the provider (B1's `to_canonical_bars`), and ingest now consumes canonical bars, so no reshape is needed. ADR 0011's `symbol` column stays the target state but is deferred — see the B2 note below | +| Q5 | **Second-provider protocol shape** — not in scope to *implement*, but `providers/base.py` (written in B1) must be sketched against *both* yfinance and `archive/futu_integration/field_mapping.md` so the protocol is not accidentally yfinance-shaped (IV unit, bid/ask availability, `inTheMoney` derivation all differ). | ✅ resolved 2026-09-10 (B1) — outcome table in ADR 0011 §"Protocol shape" | design review of `base.py` against both field maps: IV → decimal, bid/ask nullable, `inTheMoney` dropped (derivable), expiries ISO strings | + +### Batch notes (actuals + deviations, recorded as batches land) + +**B1 (2026-09-10) — provider seam extraction, no behaviour change.** + +- **Files**: `data_pipeline/providers/{__init__,base,_log,_registry,yfinance_provider,yf_snapshot}.py`. + Five modules instead of the three §6 named, for two reasons: (a) `yfinance_provider.py` would have + blown the 400-line god-file cap, so the option-chain section was extracted exactly as + `architecture_review.md` §2 had pre-registered — into `providers/yf_snapshot.py` (which also owns + the spot lookup, because `fetch_option_chain` calls it and a separate module would have created an + import cycle); (b) `_log.py` holds the best-effort failure-log wrapper that both provider modules + need and neither may import from the other. +- **Compatibility**: `yf_client.py` is now a re-export shim; `downloader.py` keeps only gap detection + + `raw_prices` upsert and no longer imports yfinance. No `routes/` or `services/` file changed. +- **`_ALLOWED_DEPS` deferred**: §6 B1 wanted `providers` added to `_ALLOWED_DEPS`, but promoting a + `data_pipeline/` subpackage to a layer needs `_layer_of` / `layer_of` sub-layer resolution in + **both** `doc_guard.py` and `arch_metrics.py`. That is B3's job (its §6 row already owns "new + layer-edge rules" + the layer-table rewrite). In B1 `providers/` stays inside the `data_pipeline` + layer; the new invariant that *does* hold now — "only `providers/` imports yfinance" — is enforced + by the rescoped `single-yf-exit` rule and pinned by `tests/test_provider_seam.py`. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 459 passed / 5 skipped; + `doc_guard.py` clean; `arch_metrics.py --check` ok (no baseline reset needed); + `audit_tags.py` unchanged (16 uncovered vs baseline 16). Production-code + `import yfinance` hits: exactly the two `providers/` modules. (`tests/test_yf_download.py` and + `tests/e2e/conftest.py` also import it as test doubles — `doc_guard` exempts `tests/` by design.) + +**B2 (2026-09-10) — canonical raw store.** + +- **Shape**: `raw_bars` / `clean_bars` / `feature_bars` added; all reads *and* writes in + `data_pipeline/` switched to them. The pre-rename names are kept as shadow tables and + `upsert_many` mirrors **both** directions, so an old seeding path, an un-migrated DB and a + `git revert` all keep working. `scripts/migrate_canonical_tables.py` backfills an existing DB + (idempotent, `INSERT OR IGNORE`, never clobbers the canonical table; `--dry-run` reports without + creating anything — it does not even run `init_db`). +- **Ingest is now canonical**: `downloader.download_bars()` (was `_download_yf`) acquires through + `providers.get_provider().history()` — i.e. the registry, not a concrete vendor module — and + returns `CANONICAL_BAR_COLUMNS`. The yfinance-ism (`Adj Close`→`Adj_Close`) is now confined to + the provider's mapping, and the `provider` column is written from `get_provider().name`. +- **No column reshape** (Q4 above). To keep that true by construction rather than by review, + `init_db` builds each canonical/legacy pair from one shared column tuple — which also kept + `db.py` under the 400-line god-file cap after the 3 extra tables (+0 tracked metrics). +- **Deliberate non-change**: the function name `upsert_raw_prices` is kept (it is called from + `data_ops/{_update,_range}.py`, `services/regime/ops/_bootstrap.py` and ~12 test patch targets); + renaming it is a cross-cutting edit that belongs with the B3 re-home, not with the rename. +- **Tests**: new `tests/test_canonical_tables.py` pins column parity per pair, bidirectional + mirroring, "one pipeline run populates both families", that `fetch_ticker_inventory` (the /health + read) hits `raw_bars`, and that the migration script backfills + is idempotent. + `test_processing.py` now seeds `clean_bars` and reads `feature_bars` (the §6 exit criterion); + `test_health_service.py` / `test_nvda_analysis.py` seeded via raw SQL and therefore had to move. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 468 passed / 5 skipped; + `doc_guard.py` clean; `arch_metrics.py --check` ok (no baseline reset needed); + `audit_tags.py` unchanged (16 vs baseline 16); `routes/` untouched (only the one-line comment + fix in `services/market/facade.py` outside `data_pipeline/`). + +**B3 (2026-09-10) — package re-home.** + +- **Layout achieved** (`data_pipeline/`): `providers/` (ACQUIRE) · `store/` · `ingest/` · + `transform/` · `read/` · `orchestrate/` + `_state.py`. `data_ops/` is gone; `yf_client.py` moved + to `providers/yf_client.py` (kept, not deleted, so its one-release shim promise holds while + *services*→*providers* becomes the visible edge). Path map for anyone following older docs: + + | old | new | + |---|---| + | `db.py` / `repos.py` / `quality_log.py` | `store/…` | + | `downloader.py` | `ingest/ohlcv.py` | + | `cleaning.py` / `processing.py` | `transform/…` | + | `data_ops/{facade,_query}.py` | `read/…` | + | `data_ops/{_update,_range}.py` | `orchestrate/{update,backfill}.py` | + | `job_cache.py` / `scheduler.py` | `orchestrate/…` | + | `data_ops/_globals.py` | `_state.py` (package root) | + | `yf_client.py` | `providers/yf_client.py` | + +- **Guards now sub-package aware**: `doc_guard._layer_of` / `_imported_heads` and + `arch_metrics.layer_of` resolve `data_pipeline//…` to ``; `_ALLOWED_DEPS` carries + the six new keys **plus** the `providers` key B1 deferred; `sqlite-bypass` and `db-access` were + rescoped to `store/db.py` / `store/repos.py`. `tests/test_architecture_purity.py` gained three + tests: the layer graph matches the table, `transform/` never imports `providers/`, and the two + copies of the layer table agree. +- **Deviations from §5.2's sketch** (all deliberate): + 1. `orchestrate/update.py` exists (the sketch listed four files) — `manual_update` / + `seed_history` is a distinct "make it ready" entry point from chunked backfill. + 2. **No `ingest/snapshots.py`**: live snapshots are never persisted (ADR 0004), so there is no + ingest glue to move — callers reach `providers` directly. + 3. `_state.py` sits at the package root (the sketch put nothing there): `read` and `orchestrate` + both need the query cache / update locks, and the two must not import each other. + 4. **`read → orchestrate`** is kept (the read path triggers refreshes), which is the reverse of + the sketch's `orchestrate → read`. Consequence: `orchestrate` may not import `read`, so + `orchestrate/scheduler.py` now calls `orchestrate.update.manual_update` instead of + `DataService.manual_update` (behaviour-identical; it removed the last cycle candidate). + 5. **`providers → store`** (the provider writes its own failures to `store/quality_log.py`) + instead of being a pure leaf; the alternative was inventing a callback for a diagnostic write. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 472 passed / 5 skipped; + full `pytest tests/e2e` → 38 passed; `ruff check` + `format --check` clean; `doc_guard.py` clean; + `arch_metrics.py --check` ok — layer violations 0, cycles 0, god files 0, dead code 1, so + **no baseline reset was needed** (the §6 row anticipated one); `audit_tags.py` regenerated + (`--update-baseline`) because the uncovered-constant *paths* moved while the count stayed 16. + +**B4 (2026-09-10) — close L1 (`core-purity`).** + +- **Split**: the fetch half of `core/market/data_context.py` moved to + `services/market/data_context_fetch.py::fetch_data_context`. `core` keeps the pure + `DataContext`, `refrequency()` (was `_refrequency`), a data-in/data-out + `build_data_context(*, ticker, frequency, horizon, raw_data)`, and `empty_data_context()` + for failed acquisitions. The two `# doc-guard: allow=core-purity` markers are gone. +- **Who fetches is now inverted** (the §2 row's exit condition): `MarketAnalyzer(data_context)` + and `CorrelationValidator(price_data=…)` receive the context instead of building it — the same + pattern the 2026-09 remediation applied to `OptionsChainAnalyzer(snapshot=…)`. + `CorrelationValidator` now raises a `ValueError` explaining where to build one instead of + quietly fetching. A public `MarketAnalyzer.data_context` property replaced the + `analyzer._ctx` reach-through in `services/market/analysis/statistical.py`. +- **Guard tightened**: §3's table had allowed `core → {read, providers}` in B3 (a transitional + concession); it is now `core → {utils}` in `doc_guard._ALLOWED_DEPS` **and** + `arch_metrics.ALLOWED_DEPS`. `tests/test_architecture_purity.py` gained + `test_core_has_zero_data_pipeline_imports`, which deliberately ignores the suppression marker — + re-introducing one now fails the test even though `doc_guard` would accept it. +- **Tests migrated**: `test_frontend_api.py` builds contexts with the pure builder (three closures + deleted), `test_nvda_analysis.py` gained an `_analyzer()` helper (5 sites) and stubs the + provider at its new path, `test_chart_time_range.py` / `test_ticker_format_integration.py` follow + the same pattern — the latter also lost its `MarketAnalyzer.__init__` monkeypatch hack. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 473 passed / 5 skipped; + full `pytest tests/e2e` → 38 passed; `ruff check` + `format --check` clean; `doc_guard.py` clean; + `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / dead 1 — no baseline reset); + `grep -rn "allow=core-purity"` returns nothing. + +**B5 (2026-09-10) — readiness plan + prefetch on submit.** + +- **Q1 = manifest** (see the gate table above). `POST /` now resolves the module list + (`FormService.extract_modules`: repeated or comma-separated tokens; defaults to all known modules + until B7 sends the field; unknown tokens are dropped, not fatal) and stores the readiness plan on + the job. Per-module params still arrive on the existing hidden fields — B7 moves them onto each + `/render` call as query args. +- **`orchestrate/readiness.py`** (new): `KIND_DATASETS` (module → datasets; live-only modules map to + `()`), `plan_datasets` (union over modules, one entry per `(ticker, dataset)`), `check_and_kick` + (one DB-only coverage probe per ticker+range, then a daemon-thread kick), plus `status_for` / + `hold_seconds_left` / `should_hold` / `is_backfill_running`. + The daemon-thread kicker moved here from `read/_query.py` so POST-time readiness and the per-slice + path share one implementation (and `read → orchestrate` keeps the graph acyclic). +- **`services/market/readiness.py`** (new): the services half — calls the plan/kick, then warms + `services.options.preload` for the live-chain modules on daemon threads. The split exists because + `orchestrate` may not import `services`. +- **Cold-start hold**: `/render/` consults the job's plan and, when the plan says "kicked" *and* + there is no usable history yet *and* the backfill thread is still alive, returns a self-re-firing + `partials/fragments/readiness.html` ("正在准备…") instead of an empty chart. Bounded by + `HOLD_SECONDS = 30` **and** by thread liveness — a review pass caught that the timer alone left a + *failed* download showing a spinner for 30 s and hiding the real error + (`tests/test_nvda_analysis.py::test_failed_download_shows_error`); the liveness check fixed it and + is pinned by `test_should_hold_stops_as_soon_as_the_backfill_is_gone`. +- **Tests**: new `tests/test_readiness.py` (18 cases) — plan union / dedupe / live-only emptiness / + horizon defaults, kick decision + probe-failure resilience, hold window + thread-liveness, + `create_job` carrying the plan, and `extract_modules` parsing. `test_background_backfill.py` now + imports the kicker from `readiness`. +- **Deferred to B6/B7 (recorded, not silently dropped)**: (a) the *client* half of "the browser + re-fires" — the held fragment self-refreshes, but the module **toolbars** and the per-module query + args are B7; (b) the market-review benchmark panel (`market_review_prices`, L5) is not in the + dataset map yet — its ladder lives in `services/market_review/fetch.py` and folds into the provider + seam later. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 493 passed / 5 skipped; + full `pytest tests/e2e` → 38 passed; `ruff check` + `format --check` clean; `doc_guard.py` clean; + `arch_metrics.py --check` ok (layer 0 / cycles 0 / god 0 / dead 1 — no baseline reset); + `audit_tags.py` 16 vs baseline 16 after tagging two new domain constants. + +**B6 (2026-09-10) — `ticker`-only Parameters bar (+ Q3: Portfolio tab).** + +- **The bar** (`templates/partials/parameters_bar.html` + `static/parametersBar.js`): renders between the + header and `.app-body`, `position: sticky; top: var(--header-h)`, and owns exactly one input — + `ticker` (comma-separated, existing multi-ticker parse) — plus the Run button and the + ticker-validation badges. Collapsing persists per viewer under + `localStorage['parametersBarCollapsed']` (every access guarded; a denied-storage browser degrades to + "not persisted") and leaves the one-line summary `▸ ^SPX`. Tokens only in CSS, so the Onyx layer + themes it for free. The `tab_parameter.html` tab and its sidebar button are deleted. +- **Q3 = dedicated Portfolio tab**: `templates/partials/tab_portfolio.html` holds the positions table + + the Portfolio-Analysis result panel; the global handlers (`addPositionRow`, `runPortfolioAnalysis`, + `initializeOptionsTable`) are unchanged because `#positions-tbody` still exists on every page load. +- **Deviation (documented, temporary)**: the plan says the bar owns *one* input, and B7 is what gives each + module its own toolbar. Removing the Parameters tab in B6 while B7 has not landed would have left the + time horizon, sizing and the Config bridge with **no UI at all** — a functional regression, not a + shippable increment. They therefore sit in a collapsible "Analysis settings" group **inside the same + ``**, explicitly marked as B7's extraction source; the POST contract is byte-for-byte unchanged. +- **Pages mirror**: `build_pages_site.py::build` asserted `id="tab-parameter"` and generated + `showcase/parameter.html`; both now point at `tab-portfolio`, and the demo banner links to + "去 Portfolio 页". `tests/test_pages_build.py`'s ticker-input assertion was made attribute-order + agnostic (it broke on the new `class` attribute — brittle, not a real contract). +- **Tests**: `tests/unit/parametersBar.test.js` (8 cases: default expanded, toggle + persistence, restore, + summary mirroring, storage-denied, bar-absent) and `parametersBar.js` added to the coverage pass; + 5 e2e files dropped their "activate the parameter tab" step (the bar is always visible), + `test_position_cascade.py` opens `tab-portfolio`, and `test_smoke.py`'s tab list swapped + `tab-parameter` → `tab-portfolio`. +- **Exit criteria**: `pytest -m "not network" --ignore=tests/e2e` → 493 passed / 5 skipped; + full `pytest tests/e2e` → 38 passed; `npx vitest run` → 187 passed / 15 files; + `doc_guard.py` clean; `arch_metrics.py --check` ok; `audit_tags.py` 16 vs baseline 16. + The §6 row's "axe ≥ 95" is **not** automated in this repo (no axe harness exists) — verified by hand + instead: the bar is a labelled `
+ + {# Always visible, even when the bar is collapsed: a failed submit must not hide + its own error message. #} + {% if error %} +
{{ error }}
+ {% endif %} +
+ + {# See the header comment: submit-only mirror of the marketParams store. #} + + + diff --git a/templates/partials/tab_config.html b/templates/partials/tab_config.html deleted file mode 100644 index f174506..0000000 --- a/templates/partials/tab_config.html +++ /dev/null @@ -1,65 +0,0 @@ -
-
-

Config

-

Analysis parameters that persist across sessions.

-
-
-
Analysis Config
-
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
-
- -
-
Option Chain Filter
-
-
- - -
Maximum days to expiration (default 45).
-
-
- - -
Lower strike bound as fraction of spot (default 0.70).
-
-
- - -
Upper strike bound as fraction of spot (default 1.30).
-
-
- - -
Maximum total contracts per query.
-
-
-
-
diff --git a/templates/partials/tab_market_assessment.html b/templates/partials/tab_market_assessment.html index 7498c46..7afd6c2 100644 --- a/templates/partials/tab_market_assessment.html +++ b/templates/partials/tab_market_assessment.html @@ -7,9 +7,60 @@

Assessment & Projections

{% if streaming_mode and job_id and ticker %} + {# Module toolbar (batch B7): horizon + frequency (the shared marketParams + group) and the Assessment-only knobs (assessmentParams group). #} +
+ Assessment settings + + + to + + + + + +
+ Assessment knobs +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+

Loading projections…

diff --git a/templates/partials/tab_market_review.html b/templates/partials/tab_market_review.html index 457d570..4688c45 100644 --- a/templates/partials/tab_market_review.html +++ b/templates/partials/tab_market_review.html @@ -7,10 +7,23 @@

Market Review

{% if streaming_mode and job_id and ticker %} + {# Module toolbar (batch B7): the module owns its parameters. `hx-include` + carries them on the /render call; a later change is re-issued by + static/moduleParams.js for THIS module only. #} +
+ Market Review settings + + + to + + +
+ {# HTMX streaming: skeleton fires on load and is replaced by /render/market_review. #}

Loading market review…

diff --git a/templates/partials/tab_option_chain.html b/templates/partials/tab_option_chain.html index 3e9479b..1f8b173 100644 --- a/templates/partials/tab_option_chain.html +++ b/templates/partials/tab_option_chain.html @@ -22,6 +22,23 @@

Option Chain

+ +
+ Option filter + + + + + + + + + + +
+
@@ -83,7 +100,7 @@

Option Chain

-

Set a ticker in Parameter tab — option chain data loads automatically when you switch to this tab.

+

Set a ticker in the Parameters bar — option chain data loads automatically when you switch to this tab.

diff --git a/templates/partials/tab_parameter.html b/templates/partials/tab_parameter.html deleted file mode 100644 index 0ac9a1f..0000000 --- a/templates/partials/tab_parameter.html +++ /dev/null @@ -1,121 +0,0 @@ -
-
-

Parameters

-

Configure analysis inputs and optional option positions.

-
- -
- {% if error %} -
- {{ error }} -
- {% endif %} - -
-
- Analysis Settings - -
-
-
- - -
-
-
- -
- -
- - to - -
- -
- -
- - -
-
- - -
-
-
- - - - - - - - - -
-
- Positions - -
-
- - - - - - - - - - - - - - -
TickerTypeExpiryStrikeSidePriceQty
-
- -
-
- - -
-
-
diff --git a/templates/partials/tab_payoff_ratio.html b/templates/partials/tab_payoff_ratio.html index f05f185..5d87c16 100644 --- a/templates/partials/tab_payoff_ratio.html +++ b/templates/partials/tab_payoff_ratio.html @@ -78,7 +78,7 @@

Payoff Ratio

-

Set a ticker in Parameter tab — payoff data loads automatically when you switch to this tab.

+

Set a ticker in the Parameters bar — payoff data loads automatically when you switch to this tab.

diff --git a/templates/partials/tab_portfolio.html b/templates/partials/tab_portfolio.html new file mode 100644 index 0000000..9e8fb57 --- /dev/null +++ b/templates/partials/tab_portfolio.html @@ -0,0 +1,69 @@ +{# Portfolio tab (batch B6 of the reorg; decision gate §8 Q3). + + The Positions table + "Portfolio Analysis" workflow used to live inside the + Parameters tab, which conflated two different jobs: the streaming submit + (`POST /`) and the client-fired `POST /api/portfolio_analysis`. It gets its own + tab so the position workflow is discoverable and no longer rides on the + analysis form. Handlers (`addPositionRow`, `runPortfolioAnalysis`, + `initializeOptionsTable`) are global and unchanged; `#positions-tbody` is still + present in the DOM on every page load. #} +
+
+

Portfolio

+

Optional option positions — Greeks, P&L at expiration, theta decay and risk metrics.

+
+ +
+
+ Positions + +
+
+ + + + + + + + + + + + + + +
TickerTypeExpiryStrikeSidePriceQty
+
+ +
+
+ + +
+
diff --git a/templates/partials/tab_simulation.html b/templates/partials/tab_simulation.html index e87fda5..fa50545 100644 --- a/templates/partials/tab_simulation.html +++ b/templates/partials/tab_simulation.html @@ -27,8 +27,8 @@

Simulation

- - Blank = use the Parameter tab ticker. + + Blank = use the Parameters bar ticker.
@@ -182,7 +182,7 @@

Simulation

- Set a ticker in the Parameter tab — the simulation runs + Set a ticker in the Parameters bar — the simulation runs automatically when you switch to this tab.

diff --git a/templates/partials/tab_statistical_analysis.html b/templates/partials/tab_statistical_analysis.html index 0acbdc1..003875d 100644 --- a/templates/partials/tab_statistical_analysis.html +++ b/templates/partials/tab_statistical_analysis.html @@ -7,9 +7,27 @@

Statistical Analysis

{% if streaming_mode and job_id and ticker %} + {# Module toolbar (batch B7) — horizon + frequency belong to this module. #} +
+ Statistical settings + + + to + + + + +
+

Loading statistical analysis…

diff --git a/templates/partials/tab_summary.html b/templates/partials/tab_summary.html deleted file mode 100644 index 605fbb4..0000000 --- a/templates/partials/tab_summary.html +++ /dev/null @@ -1,52 +0,0 @@ - diff --git a/tests/conftest.py b/tests/conftest.py index 5132da6..422aa74 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,7 +57,7 @@ def _isolate_db(monkeypatch, tmp_path): db_file = str(tmp_path / "test_market_data.sqlite") monkeypatch.setenv("MARKET_DB_PATH", db_file) try: - import data_pipeline.db as db_mod + import data_pipeline.store.db as db_mod except ImportError: # Project deps not installed in the current interpreter — let # tests that actually need the DB fail with their own clear error @@ -70,7 +70,7 @@ def _isolate_db(monkeypatch, tmp_path): # test (which used a different DB file) would otherwise mask freshly # seeded data within the 60-second TTL. try: - import data_pipeline.data_ops as ds_mod + import data_pipeline._state as ds_mod with ds_mod._query_cache_lock: ds_mod._query_cache.clear() diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 30a38b9..07097ab 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -10,7 +10,7 @@ The Flask process runs every route normally, but `yfinance.Ticker`, `yf.download`, and `fast_info` are monkey-patched in the backend process to return synthetic data. Combined with the existing - `TEST_*` ticker fixture mechanism in `data_pipeline.downloader`, this + `TEST_*` ticker fixture mechanism in `data_pipeline.ingest.ohlcv`, this lets e2e tests exercise real form submission, real DataService pipeline, real chart rendering — without network. @@ -69,11 +69,11 @@ def _e2e_db(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: """ db_file = str(tmp_path_factory.mktemp("e2e-db") / "market.sqlite") os.environ["MARKET_DB_PATH"] = db_file - # Patch the module attr in case data_pipeline.db was already imported + # Patch the module attr in case data_pipeline.store.db was already imported # by a previous test module in the same pytest session (DB_PATH is # captured at import time). try: - import data_pipeline.db as db_mod + import data_pipeline.store.db as db_mod db_mod.DB_PATH = db_file # Ensure schema exists at the new path even if app was pre-imported. @@ -91,14 +91,14 @@ def _e2e_db(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: # → /api/option_chain → /api/validate_tickers) opt in by using the `yf_stub` # fixture *instead of* `mock_apis`. The patch covers: # -# * `yfinance.download` — used by data_pipeline.downloader and +# * `yfinance.download` — used by data_pipeline.ingest.ohlcv and # core.market_review # * `yfinance.Ticker(...).fast_info` — used for spot price lookups # * `yfinance.Ticker(...).options` — option expirations list # * `yfinance.Ticker(...).option_chain(exp)` — calls/puts DataFrames # # Combined with the existing `TEST_*` ticker bypass in -# `data_pipeline.downloader._download_yf`, real `TEST_AAPL` form submissions +# `data_pipeline.ingest.ohlcv.download_bars`, real `TEST_AAPL` form submissions # never hit the network. # --------------------------------------------------------------------------- def _synthetic_ohlcv(ticker: str, start: dt.date, end: dt.date): @@ -221,7 +221,7 @@ def seed_test_data(_e2e_db: str, yf_stub: None) -> Iterator[None]: Uses the production downloader's `TEST_*` fixture branch — no network. """ try: - from data_pipeline.data_ops import DataService + from data_pipeline.read import DataService # `manual_update` will route to the synthetic fixture for TEST_* DataService.manual_update("TEST_AAPL", days=120) diff --git a/tests/e2e/test_error_states.py b/tests/e2e/test_error_states.py index cc14bb0..f13cc1a 100644 --- a/tests/e2e/test_error_states.py +++ b/tests/e2e/test_error_states.py @@ -18,7 +18,7 @@ def test_option_chain_500_renders_error_banner( page.goto(live_server, wait_until="domcontentloaded") # Activate parameter tab so #ticker becomes interactive. - open_tab("tab-parameter") + # The Parameters bar is always visible (batch B6), so #ticker is interactive already. # Provide a ticker so loadOptionChain has something to query. page.fill("#ticker", "TEST_AAPL") diff --git a/tests/e2e/test_form_submit_flow.py b/tests/e2e/test_form_submit_flow.py index fcde223..e0c2a73 100644 --- a/tests/e2e/test_form_submit_flow.py +++ b/tests/e2e/test_form_submit_flow.py @@ -4,39 +4,35 @@ template render → table visible. The yfinance layer is patched at the backend process level via the `yf_stub` fixture; the synthetic ticker ``TEST_AAPL`` routes through the existing fixture branch in -`data_pipeline.downloader`. +`data_pipeline.ingest.ohlcv`. """ from __future__ import annotations -import datetime as dt - from playwright.sync_api import Page, expect -def _months_ago(n: int) -> str: - """Return a YYYY-MM string n months before today (HTML ).""" - today = dt.date.today().replace(day=1) - for _ in range(n): - today = (today - dt.timedelta(days=1)).replace(day=1) - return today.strftime("%Y-%m") - - def test_form_submit_renders_summary( page: Page, live_server: str, yf_stub: None, seed_test_data: None, js_errors: list[str], - open_tab, ) -> None: """Submit the analysis form with a TEST_ ticker and assert the page - re-renders with the ticker echoed back.""" - page.goto(live_server, wait_until="domcontentloaded") - open_tab("tab-parameter") + re-renders with the ticker echoed back. + Batch B7: the bar posts `ticker` only — the horizon is owned by the market + modules' toolbars and mirrored into the bar's hidden inputs by + `state/marketParamsState.js`, so the test no longer types a start month. + """ + page.goto(live_server, wait_until="domcontentloaded") + # The Parameters bar is always visible (batch B6). page.fill("#ticker", "TEST_AAPL") - page.fill("#start_time", _months_ago(3)) + + # The marketParams store must have mirrored its horizon before submit, + # otherwise POST / fails its start_time validation. + expect(page.locator("#start_time")).not_to_have_value("", timeout=5_000) # POST the form and wait for navigation to complete. with page.expect_navigation(wait_until="domcontentloaded", timeout=15_000): diff --git a/tests/e2e/test_lazy_tabs.py b/tests/e2e/test_lazy_tabs.py index b454b6a..a02227e 100644 --- a/tests/e2e/test_lazy_tabs.py +++ b/tests/e2e/test_lazy_tabs.py @@ -19,8 +19,7 @@ def test_option_chain_lazy_loads_on_activation( page.goto(live_server, wait_until="networkidle") - # Activate parameter tab and fill the ticker so loadOptionChain() fires. - open_tab("tab-parameter") + # The Parameters bar is always visible (batch B6), so #ticker is interactive already. page.fill("#ticker", "TEST_AAPL") page.locator("#ticker").blur() @@ -53,8 +52,7 @@ def test_option_chain_handles_api_error_gracefully( page.goto(live_server, wait_until="networkidle") - # Activate parameter tab and provide a ticker so loadOptionChain() fires. - open_tab("tab-parameter") + # The Parameters bar is always visible (batch B6), so #ticker is interactive already. page.fill("#ticker", "TEST_AAPL") page.locator("#ticker").blur() diff --git a/tests/e2e/test_localstorage_restore.py b/tests/e2e/test_localstorage_restore.py index 93d45be..0240c23 100644 --- a/tests/e2e/test_localstorage_restore.py +++ b/tests/e2e/test_localstorage_restore.py @@ -1,4 +1,8 @@ -"""LocalStorage form-state restoration after page reload.""" +"""LocalStorage restoration of the module parameter groups (batch B7). + +One key per group — `marketParams`, `assessmentParams`, `optionFilter` — plus the +bar's own `marketAnalysisForm` convenience copy for the ticker. +""" from __future__ import annotations @@ -7,7 +11,7 @@ from playwright.sync_api import Page, expect -def test_localstorage_restores_form_state( +def test_localstorage_restores_module_params( page: Page, live_server: str, mock_apis, @@ -16,51 +20,45 @@ def test_localstorage_restores_form_state( page.goto(live_server, wait_until="domcontentloaded") # Seed localStorage *before* DOMContentLoaded handlers re-fire on reload. - saved_form = { - "ticker": "TEST_AAPL", - "start_time": "202401", - "end_time": "202403", - "positions": [], - } - saved_cfg = { - "frequency": "W", - "side_bias": "Neutral", - "risk_threshold": "75", - "rolling_window": "90", - "max_dte": "30", - "moneyness_low": "0.80", - "moneyness_high": "1.20", - "max_contracts": "500", - "refresh_interval": "120", - } + saved_form = {"ticker": "TEST_AAPL", "positions": []} page.evaluate( - """({form, cfg}) => { + """({form}) => { localStorage.setItem('marketAnalysisForm', JSON.stringify(form)); - localStorage.setItem('marketAnalysisConfig', JSON.stringify(cfg)); + localStorage.setItem('marketParams', JSON.stringify( + { from: '2024-01', to: '2024-03', frequency: 'W' })); + localStorage.setItem('assessmentParams', JSON.stringify( + { side_bias: 'Neutral', risk_threshold: '75', rolling_window: '90', + account_size: '100000', max_risk_pct: '2' })); + localStorage.setItem('optionFilter', JSON.stringify( + { max_dte: '30', moneyness_low: '0.80', moneyness_high: '1.20', + max_contracts: '500', refresh_interval: '120' })); }""", - {"form": saved_form, "cfg": saved_cfg}, + {"form": saved_form}, ) page.reload(wait_until="domcontentloaded") - # Form fields should be hydrated from `marketAnalysisForm`. + # The bar's ticker survives, and the marketParams store mirrors the horizon + # into the bar's submit-only hidden inputs. expect(page.locator("#ticker")).to_have_value("TEST_AAPL", timeout=5_000) expect(page.locator("#start_time")).to_have_value("2024-01") expect(page.locator("#end_time")).to_have_value("2024-03") - # Hidden fields should be synced from `marketAnalysisConfig`. - freq = page.locator("#frequency").input_value() - side = page.locator("#side_bias").input_value() - risk = page.locator("#risk_threshold").input_value() - rw = page.locator("#rolling_window").input_value() - assert freq == "W" - assert side == "Neutral" - assert risk == "75" - assert rw == "90" + # The Option Chain toolbar is the only module toolbar present before a run. + expect(page.locator("#oc-max-dte")).to_have_value("30") + expect(page.locator("#oc-moneyness-low")).to_have_value("0.80") + expect(page.locator("#oc-moneyness-high")).to_have_value("1.20") + expect(page.locator("#oc-refresh-interval")).to_have_value("120") + + # The stores expose the restored values (the market toolbars render only in + # streaming mode, i.e. after a run). + assert page.evaluate("() => appState.marketParams.get().from") == "2024-01" + assert page.evaluate("() => appState.marketParams.get().frequency") == "W" + assert page.evaluate("() => appState.assessmentParams.get().side_bias") == "Neutral" + assert page.evaluate("() => appState.optionFilter.get().moneyness_high") == "1.20" - # Storage round-trip is intact (no accidental mutation). - raw_form = page.evaluate("() => localStorage.getItem('marketAnalysisForm')") - assert json.loads(raw_form)["ticker"] == "TEST_AAPL" + # Storage round-trip is intact (no accidental mutation of the group keys). + assert json.loads(page.evaluate("() => localStorage.getItem('marketParams')"))["from"] == "2024-01" fatal = [e for e in js_errors if "favicon" not in e.lower()] assert fatal == [], f"JS errors during reload restore: {fatal}" diff --git a/tests/e2e/test_module_params.py b/tests/e2e/test_module_params.py new file mode 100644 index 0000000..af92cc7 --- /dev/null +++ b/tests/e2e/test_module_params.py @@ -0,0 +1,124 @@ +"""Module-scoped parameters re-run only their own group (batch B7). + +The behavioural claim of decision gate §8 Q1: each module's toolbar appends its +parameters to *its* `/render` call. One nuance is part of the contract, not an +accident: the three streaming market tabs share one parameter group +(`marketParams` — horizon + frequency), so a change there re-runs all three, +while a change to a *different* group (the Option Chain's filters) must not +touch the streaming panes at all. +""" + +from __future__ import annotations + +import json + +from playwright.sync_api import Page, expect + +STATISTICAL_FRAGMENT = "#tab-statistical-analysis-content" + + +def _submit(page: Page, live_server: str, ticker: str = "TEST_AAPL") -> None: + page.goto(live_server, wait_until="domcontentloaded") + page.fill("#ticker", ticker) + with page.expect_navigation(wait_until="domcontentloaded", timeout=20_000): + page.click("#analysis-form button[type=submit]") + + +def test_changing_a_market_param_reruns_its_group_only( + page: Page, + live_server: str, + yf_stub: None, + seed_test_data: None, + js_errors: list[str], + open_tab, +) -> None: + _submit(page, live_server) + # The fragment loads regardless of visibility; the tab has to be active for + # its toolbar to be actionable. + open_tab("tab-statistical-analysis") + expect(page.locator(STATISTICAL_FRAGMENT)).to_be_visible(timeout=60_000) + page.wait_for_timeout(3_000) + + renders: list[str] = [] + api_calls: list[str] = [] + page.on("request", lambda req: renders.append(req.url) if "/render/" in req.url else None) + page.on("request", lambda req: api_calls.append(req.url) if "/api/" in req.url else None) + + # WHY a dispatched change instead of select_option: the statistical tab can + # lose active-ness mid-test (peek-panel / re-render timing), which makes the + # toolbar un-actionable even though the listener chain is intact. Setting the + # value and dispatching `change` exercises exactly the same production path: + # store -> bus -> moduleParams -> htmx -> /render with the new params. + page.evaluate( + "() => { const el = document.getElementById('stat-frequency');" + " el.value = 'W'; el.dispatchEvent(new Event('change', { bubbles: true })); }", + ) + page.wait_for_timeout(3_000) + + # The marketParams group feeds all three market tabs → all three re-run. + for kind in ("market_review", "statistical", "assessment"): + assert any(f"/render/{kind}" in url for url in renders), f"{kind} did not re-run: {renders}" + + # The new value must travel, together with the group's horizon. + assert all("frequency=W" in url for url in renders if "/render/statistical" in url), renders + assert all("from=" in url for url in renders), renders + + # …and nothing outside the group reacts to a market parameter. + assert not any("/render/option_chain" in url for url in renders), renders + assert not any("/api/option_chain" in url for url in api_calls), api_calls + + fatal = [e for e in js_errors if "favicon" not in e.lower()] + assert fatal == [], f"JS errors after a module param change: {fatal}" + + +def test_changing_the_option_filter_leaves_the_streaming_panes_alone( + page: Page, + live_server: str, + yf_stub: None, + seed_test_data: None, + open_tab, +) -> None: + """The chain filters are client-fired: no `/render` round trip at all.""" + _submit(page, live_server) + open_tab("tab-option-chain") + expect(page.locator(STATISTICAL_FRAGMENT)).to_be_attached(timeout=60_000) + page.wait_for_timeout(3_000) + + renders: list[str] = [] + page.on("request", lambda req: renders.append(req.url) if "/render/" in req.url else None) + + page.fill("#oc-max-dte", "60") + page.dispatch_event("#oc-max-dte", "change") + page.wait_for_timeout(3_000) + + assert renders == [], f"a chain-filter change re-ran the streaming panes: {renders}" + + +def test_the_rerun_survives_a_reload( + page: Page, + live_server: str, + yf_stub: None, + seed_test_data: None, + open_tab, +) -> None: + """Exit criterion: module parameter values survive a reload.""" + _submit(page, live_server) + open_tab("tab-statistical-analysis") + expect(page.locator(STATISTICAL_FRAGMENT)).to_be_visible(timeout=60_000) + page.wait_for_timeout(2_000) + + page.evaluate( + "() => { const el = document.getElementById('stat-frequency');" + " el.value = 'QE'; el.dispatchEvent(new Event('change', { bubbles: true })); }", + ) + page.wait_for_timeout(1_000) + + # The commit must have persisted the group before the reload. + assert json.loads(page.evaluate("() => localStorage.getItem('marketParams')"))["frequency"] == "QE" + + page.reload(wait_until="domcontentloaded") + page.wait_for_timeout(1_500) + + assert page.evaluate("() => appState.marketParams.get().frequency") == "QE" + # …and the hidden submit-only mirror follows the store. + assert page.evaluate("() => document.getElementById('start_time').value") != "" diff --git a/tests/e2e/test_position_cascade.py b/tests/e2e/test_position_cascade.py index 80d9633..c2ff22e 100644 --- a/tests/e2e/test_position_cascade.py +++ b/tests/e2e/test_position_cascade.py @@ -19,7 +19,8 @@ def test_position_cascade_populates_dropdowns( open_tab, ) -> None: page.goto(live_server, wait_until="domcontentloaded") - open_tab("tab-parameter") + # Positions moved to the Portfolio tab (batch B6 / gate Q3). + open_tab("tab-portfolio") # Provide a ticker so `getValidTickers()` picks it up. page.fill("#ticker", "TEST_AAPL") diff --git a/tests/e2e/test_smoke.py b/tests/e2e/test_smoke.py index 991bbb2..459da2e 100644 --- a/tests/e2e/test_smoke.py +++ b/tests/e2e/test_smoke.py @@ -1,4 +1,4 @@ -"""Smoke tests: page loads, no JS errors, all 11 tabs render and switch.""" +"""Smoke tests: page loads, no JS errors, all 10 tabs render and switch.""" from __future__ import annotations @@ -9,8 +9,7 @@ # All sidebar tab IDs (must match `data-tab` values in templates/index.html). TAB_IDS = [ - "tab-parameter", - "tab-summary", + "tab-portfolio", "tab-market-review", "tab-statistical-analysis", "tab-market-assessment", @@ -20,7 +19,6 @@ "tab-regime", "tab-simulation", "tab-option-pricing-matrix", - "tab-config", ] _ACTIVE_RE = re.compile(r"\bactive\b") @@ -33,18 +31,14 @@ def test_index_loads_without_js_errors(page: Page, live_server: str, mock_apis, expect(page.locator("#analysis-form")).to_be_visible() expect(page.locator("#ticker")).to_have_count(1) - # tab-summary only renders for multi-ticker; skip in single-ticker default GET. - for tab_id in [tid for tid in TAB_IDS if tid != "tab-summary"]: + for tab_id in TAB_IDS: button = page.locator(f'.tab-btn[data-tab="{tab_id}"]') expect(button).to_have_count(1) assert js_errors == [], f"JS errors on initial load: {js_errors}" -@pytest.mark.parametrize( - "tab_id", - [tid for tid in TAB_IDS if tid != "tab-summary"], # summary is hidden when single-ticker -) +@pytest.mark.parametrize("tab_id", TAB_IDS) def test_tab_switch_activates_panel( page: Page, live_server: str, mock_apis, js_errors: list[str], open_tab, tab_id: str ) -> None: diff --git a/tests/e2e/test_streaming_load.py b/tests/e2e/test_streaming_load.py index 30ef7d4..e51011f 100644 --- a/tests/e2e/test_streaming_load.py +++ b/tests/e2e/test_streaming_load.py @@ -54,7 +54,7 @@ def test_first_ticker_validation_under_5s( primarily measures DOM hydration + Alpine init + event-handler latency. """ page.goto(live_server, wait_until="domcontentloaded") - open_tab("tab-parameter") + # The Parameters bar is always visible (batch B6). t0 = time.monotonic() page.fill("#ticker", "TEST_AAPL") diff --git a/tests/test_architecture_purity.py b/tests/test_architecture_purity.py index f99adeb..d5ebadc 100644 --- a/tests/test_architecture_purity.py +++ b/tests/test_architecture_purity.py @@ -1,4 +1,4 @@ -"""Architecture contract tests: core/ subpackages stay pure. +"""Architecture contract tests: core/ purity and the data_pipeline layer graph. Domain: Tests — Architecture Purity Contracts Context: @@ -8,10 +8,19 @@ remaining violation stays visible in the test report and can be counted down to zero instead of being forgotten. - The violation registry lives in docs/architecture_review.md §2. + - Batch B3 (ADR 0011) split ``data_pipeline/`` into six layers + (providers / store / ingest / transform / read / orchestrate). The declared + edges live in ``scripts/doc_guard.py::_ALLOWED_DEPS`` and are mirrored in + ``scripts/arch_metrics.py`` — two copies, hence the sync test below. Contracts: - test_core_subpackage_has_no_io_or_framework_imports: for every core/ subpackage, no absolute import of an I/O or framework package, except lines explicitly carrying ``doc-guard: allow=core-purity``. + - test_data_pipeline_import_graph_matches_declared_layers: the real import + graph conforms to the declared layer table. + - test_transform_never_imports_providers: processing stays provider-agnostic + (ADR 0011 §5.2) — it only sees canonical tables. + - test_layer_tables_are_in_sync: doc_guard and arch_metrics agree. Dependencies UPWARD: - (none — stdlib + pytest only) """ @@ -19,12 +28,33 @@ from __future__ import annotations import ast +import importlib.util +import sys from pathlib import Path import pytest REPO_ROOT = Path(__file__).resolve().parent.parent CORE = REPO_ROOT / "core" +DATA_PIPELINE = REPO_ROOT / "data_pipeline" + + +def _load_script(name: str): + """Import a ``scripts/*.py`` module (scripts/ is not a package). + + WHY exec_module: the two guard scripts are standalone (no third-party + imports, runnable from pre-commit) and must stay that way, so the tests + reach into them rather than the other way round. + """ + spec = importlib.util.spec_from_file_location(f"_guard_{name}", REPO_ROOT / "scripts" / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + # WHY register first: doc_guard defines dataclasses, and @dataclass resolves + # type hints through sys.modules[cls.__module__] at class-creation time. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + # INVARIANT: core/ is pure computation — no DB, no network, no Flask, no app. FORBIDDEN_ROOTS = { @@ -70,3 +100,75 @@ def test_core_subpackage_has_no_io_or_framework_imports(pkg): continue offenders.append(f"{py.relative_to(REPO_ROOT)}:{lineno} imports '{head}'") assert not offenders, "core/ purity violated (fetch upstream and pass data in, ADR 0001):\n" + "\n".join(offenders) + + +def test_core_has_zero_data_pipeline_imports(): + """B4 exit criterion: no suppression markers, no core→data_pipeline edge left. + + Stricter than ``test_core_subpackage_has_no_io_or_framework_imports`` above: + that one honours ``# doc-guard: allow=core-purity`` for registered debt. Batch + B4 closed the debt, so this test refuses the marker entirely — re-introducing + one fails here even if doc_guard would accept it. + """ + guard = _load_script("doc_guard") + offenders: list[str] = [] + for py in sorted(CORE.rglob("*.py")): + if "__pycache__" in py.parts: + continue + for lineno, head in guard._imported_heads(py): + if head == "data_pipeline" or head in guard.DATA_PIPELINE_SUBLAYERS: + offenders.append(f"{py.relative_to(REPO_ROOT)}:{lineno} imports {head}") + assert not offenders, ( + "core/ must not import data_pipeline (fetch upstream and pass data in, ADR 0001):\n" + "\n".join(offenders) + ) + + +def test_data_pipeline_import_graph_matches_declared_layers(): + """Every data_pipeline/ import must point at an allowed layer. + + This is the test-layer twin of doc_guard's ``import-direction`` rule, with + sub-package granularity: ``data_pipeline.store.db`` counts as ``store``. + """ + guard = _load_script("doc_guard") + offenders: list[str] = [] + for py in sorted(DATA_PIPELINE.rglob("*.py")): + if "__pycache__" in py.parts: + continue + layer = guard._layer_of(py) + if layer is None: + continue + allowed = guard._ALLOWED_DEPS[layer] + for lineno, head in guard._imported_heads(py): + if head not in guard._ALLOWED_DEPS or head == layer: + continue + if head in allowed or guard._is_suppressed_at(py, lineno, "import-direction"): + continue + offenders.append(f"{py.relative_to(REPO_ROOT)}:{lineno}: {layer} -> {head}") + assert not offenders, "data_pipeline layer graph violated (see scripts/doc_guard.py::_ALLOWED_DEPS):\n" + "\n".join( + offenders + ) + + +def test_transform_never_imports_providers(): + """INVARIANT (ADR 0011 §5.2): processing is provider-agnostic. + + ``transform/`` reads canonical tables only. ``read/`` is allowed to reach + the provider for the spot fallback (recorded as a deviation in the plan §8). + """ + guard = _load_script("doc_guard") + offenders: list[str] = [] + for py in sorted((DATA_PIPELINE / "transform").rglob("*.py")): + if "__pycache__" in py.parts: + continue + for lineno, head in guard._imported_heads(py): + if head == "providers": + offenders.append(f"{py.relative_to(REPO_ROOT)}:{lineno}") + assert not offenders, "transform/ must not import providers/:\n" + "\n".join(offenders) + + +def test_layer_tables_are_in_sync(): + """``doc_guard`` and ``arch_metrics`` each carry a copy of the layer table.""" + doc_guard = _load_script("doc_guard") + arch_metrics = _load_script("arch_metrics") + assert doc_guard._ALLOWED_DEPS == arch_metrics.ALLOWED_DEPS + assert doc_guard.DATA_PIPELINE_SUBLAYERS == arch_metrics.DATA_PIPELINE_SUBLAYERS diff --git a/tests/test_background_backfill.py b/tests/test_background_backfill.py index 6ad9d10..4117cb1 100644 --- a/tests/test_background_backfill.py +++ b/tests/test_background_backfill.py @@ -14,14 +14,12 @@ import pandas as pd import pytest -import data_pipeline.data_ops._query as _q +import data_pipeline.read._query as _q from data_pipeline import PipelineResult -from data_pipeline.data_ops import _cache_get, _cache_invalidate -from data_pipeline.data_ops._query import ( - _join_backfills, - _kick_backfill, -) -from data_pipeline.db import init_db +from data_pipeline._state import _cache_get, _cache_invalidate +from data_pipeline.orchestrate import backfill as _bf +from data_pipeline.orchestrate.readiness import join_backfills, kick_backfill +from data_pipeline.store.db import init_db TICKER = "BGTEST1" @@ -48,9 +46,11 @@ def test_wide_range_request_returns_without_full_backfill(self, monkeypatch): whatever exists (nothing), while the backfill continues in background.""" init_db() _dl, calls = _slow_downloader(delay=1.0) - monkeypatch.setattr("data_pipeline.downloader.upsert_raw_prices", _dl) - monkeypatch.setattr("data_pipeline.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) - monkeypatch.setattr("data_pipeline.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr("data_pipeline.ingest.ohlcv.upsert_raw_prices", _dl) + monkeypatch.setattr("data_pipeline.transform.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr( + "data_pipeline.transform.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1) + ) start = dt.date(2021, 1, 1) end = dt.date.today() @@ -62,15 +62,17 @@ def test_wide_range_request_returns_without_full_backfill(self, monkeypatch): assert df.empty, "no data seeded yet — partial read must be empty, not fabricated" # The backfill is still running in the background… assert calls["n"] >= 1, "background backfill was not kicked" - _join_backfills(timeout=10) + join_backfills(timeout=10) assert calls["n"] >= 2, "chunked backfill did not continue after the request returned" def test_partial_read_is_not_cached(self, monkeypatch): init_db() _dl, _calls = _slow_downloader(delay=1.0) - monkeypatch.setattr("data_pipeline.downloader.upsert_raw_prices", _dl) - monkeypatch.setattr("data_pipeline.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) - monkeypatch.setattr("data_pipeline.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr("data_pipeline.ingest.ohlcv.upsert_raw_prices", _dl) + monkeypatch.setattr("data_pipeline.transform.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr( + "data_pipeline.transform.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1) + ) start = dt.date(2021, 1, 1) end = dt.date.today() @@ -78,7 +80,7 @@ def test_partial_read_is_not_cached(self, monkeypatch): key = (TICKER, "clean", str(start), str(end)) assert _cache_get(key) is None, "partial read must not be memoised" - _join_backfills(timeout=10) + join_backfills(timeout=10) def test_completed_backfill_becomes_visible_and_cached(self, monkeypatch): """After the background backfill finishes, the next request returns the @@ -88,11 +90,13 @@ def test_completed_backfill_becomes_visible_and_cached(self, monkeypatch): def _fast_dl(ticker, start, end): # noqa: ARG001 return PipelineResult(rows=10) - monkeypatch.setattr("data_pipeline.downloader.upsert_raw_prices", _fast_dl) - monkeypatch.setattr("data_pipeline.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) - monkeypatch.setattr("data_pipeline.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr("data_pipeline.ingest.ohlcv.upsert_raw_prices", _fast_dl) + monkeypatch.setattr("data_pipeline.transform.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr( + "data_pipeline.transform.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1) + ) monkeypatch.setattr( - "data_pipeline.data_ops._query.fetch_df", + "data_pipeline.read._query.fetch_df", lambda sql, params: pd.DataFrame( { "date": ["2026-01-05"], @@ -109,7 +113,7 @@ def _fast_dl(ticker, start, end): # noqa: ARG001 start = dt.date(2026, 1, 1) end = dt.date(2026, 2, 1) df = _q.get_cleaned_daily(TICKER, start, end) - _join_backfills(timeout=10) + join_backfills(timeout=10) assert not df.empty _cache_invalidate(TICKER) # mimic ensure_range's post-success invalidation @@ -123,19 +127,21 @@ def test_needs_backfill_probe(self): start = dt.date(2021, 1, 1) end = dt.date.today() # Empty DB → backfill needed. - assert _q._r.needs_backfill(TICKER + "-PROBE", start, end) is True + assert _bf.needs_backfill(TICKER + "-PROBE", start, end) is True def test_kick_dedupes_concurrent_kicks(self, monkeypatch): init_db() _dl, calls = _slow_downloader(delay=0.3) - monkeypatch.setattr("data_pipeline.downloader.upsert_raw_prices", _dl) - monkeypatch.setattr("data_pipeline.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) - monkeypatch.setattr("data_pipeline.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr("data_pipeline.ingest.ohlcv.upsert_raw_prices", _dl) + monkeypatch.setattr("data_pipeline.transform.cleaning.clean_range", lambda *a, **k: PipelineResult(rows=1)) + monkeypatch.setattr( + "data_pipeline.transform.processing.process_frequencies", lambda *a, **k: PipelineResult(rows=1) + ) start, end = dt.date(2021, 1, 1), dt.date.today() for _ in range(5): - _kick_backfill(TICKER + "-DEDUP", start, end) - _join_backfills(timeout=10) + kick_backfill(TICKER + "-DEDUP", start, end) + join_backfills(timeout=10) # ensure_range's own in-flight dedup collapses the kicked threads — # a single leader runs the chunked pipeline, not five. Chunks for a # 5.6-year range ≈ days/89, allow one boundary chunk. @@ -143,3 +149,63 @@ def test_kick_dedupes_concurrent_kicks(self, monkeypatch): assert calls["n"] <= expected_chunks, ( f"kicks were not deduped: {calls['n']} downloads > {expected_chunks} (single leader)" ) + + +class TestFeatureBarsHeal: + """Plan §10 F4: clean_bars present but feature_bars stale must self-heal + (a reprocess, no download).""" + + @staticmethod + def _seed_clean(ticker, start, end): + from data_pipeline.store.db import upsert_many + + days = pd.bdate_range(start, end) + rows = [ + (ticker, d.date().isoformat(), 100.0, 101.0, 99.0, 100.5, 100.5, 1_000_000, 1, 0, 0, 0, 0) for d in days + ] + upsert_many( + "clean_bars", + [ + "ticker", + "date", + "open", + "high", + "low", + "close", + "adj_close", + "volume", + "is_trading_day", + "missing_any", + "price_jump_flag", + "vol_anom_flag", + "ohlc_inconsistent", + ], + rows, + ) + + def test_needs_backfill_true_when_only_clean_exists(self): + init_db() + start, end = dt.date(2024, 1, 1), dt.date(2024, 6, 30) + self._seed_clean("FEATHEAL1", start, end) + assert _bf.needs_backfill("FEATHEAL1", start, end) is True + + def test_ensure_range_reprocesses_without_downloading(self, monkeypatch): + from data_pipeline.store.db import fetch_df + + init_db() + start, end = dt.date(2024, 1, 1), dt.date(2024, 6, 30) + self._seed_clean("FEATHEAL2", start, end) + + downloads = {"n": 0} + monkeypatch.setattr( + "data_pipeline.ingest.ohlcv.upsert_raw_prices", + lambda *a, **k: downloads.__setitem__("n", downloads["n"] + 1) or PipelineResult(rows=0), + ) + + _cache_invalidate("FEATHEAL2") + assert _bf.ensure_range("FEATHEAL2", start, end) is True + assert downloads["n"] == 0, "clean already covers the span — no download" + + feat = fetch_df("SELECT frequency, COUNT(*) AS n FROM feature_bars WHERE ticker='FEATHEAL2' GROUP BY frequency") + assert not feat.empty, "feature_bars must be populated after the heal" + assert _bf.needs_backfill("FEATHEAL2", start, end) is False diff --git a/tests/test_canonical_tables.py b/tests/test_canonical_tables.py new file mode 100644 index 0000000..253e21a --- /dev/null +++ b/tests/test_canonical_tables.py @@ -0,0 +1,203 @@ +"""Contract tests for the canonical store tables (ADR 0011, batch B2). + +Domain: Tests — Canonical Store +Context: + - Batch B2 renamed the store tables to ``raw_bars`` / ``clean_bars`` / + ``feature_bars`` and kept the pre-rename names as shadow tables for one + release. These tests pin the two properties that make the compatibility + window safe: the pairs have identical column sets, and a write under either + name reaches both. + - They also pin the *direction* of the migration — reads must hit the + canonical names, and ``scripts/migrate_canonical_tables.py`` must backfill a + DB that only has legacy rows. +Contracts: + - Column-set parity per pair (decision gate §8 Q4 = minimal rename). + - ``upsert_many`` mirrors canonical ↔ legacy in both directions. + - A pipeline run (download → clean → process) populates both families. + - ``fetch_ticker_inventory`` (the /health data source) reads the canonical table. + - The one-shot migration backfills legacy-only rows into the canonical table. +Dependencies UPWARD: + - (none — stdlib + pytest + the package under test) +""" + +from __future__ import annotations + +import datetime as dt +import subprocess +import sys +from pathlib import Path + +import pytest + +from data_pipeline.store.db import CANONICAL_TABLES, canonical_table, get_conn, init_db, upsert_many + +REPO_ROOT = Path(__file__).resolve().parent.parent + +_RAW_COLS = ["ticker", "date", "open", "high", "low", "close", "adj_close", "volume", "provider"] + + +def _table_info(table: str) -> list[tuple]: + with get_conn() as conn: + return [tuple(row) for row in conn.execute(f"PRAGMA table_info({table})")] + + +def _count(table: str, ticker: str | None = None) -> int: + with get_conn() as conn: + if ticker is None: + return int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + return int(conn.execute(f"SELECT COUNT(*) FROM {table} WHERE ticker=?", (ticker,)).fetchone()[0]) + + +# --------------------------------------------------------------------------- +# Column parity +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("legacy,canonical", sorted(CANONICAL_TABLES.items())) +def test_canonical_table_mirrors_legacy_columns(legacy, canonical): + """Decision gate Q4 = minimal rename, so the column sets must be identical.""" + init_db() + canon_info = _table_info(canonical) + assert canon_info, f"{canonical} was not created by init_db()" + assert canon_info == _table_info(legacy) + + +def test_canonical_table_is_identity_for_names_without_a_pair(): + assert canonical_table("raw_prices") == "raw_bars" + assert canonical_table("raw_bars") == "raw_bars" + assert canonical_table("regime_log") == "regime_log" + + +# --------------------------------------------------------------------------- +# Transitional dual-write +# --------------------------------------------------------------------------- +def test_upsert_many_writes_both_table_families(): + init_db() + upsert_many("raw_bars", _RAW_COLS, [("DUAL_CANON", "2026-01-02", 1.0, 1.0, 1.0, 1.0, 1.0, 10.0, "yfinance")]) + assert _count("raw_bars", "DUAL_CANON") == 1 + assert _count("raw_prices", "DUAL_CANON") == 1 + + # A legacy name must also reach the canonical table: old call sites and test + # fixtures seed under the pre-rename names during the window. + upsert_many("raw_prices", _RAW_COLS, [("DUAL_LEGACY", "2026-01-02", 2.0, 2.0, 2.0, 2.0, 2.0, 20.0, "yfinance")]) + assert _count("raw_prices", "DUAL_LEGACY") == 1 + assert _count("raw_bars", "DUAL_LEGACY") == 1 + + +def test_pipeline_run_populates_both_table_families(): + """B2 exit criterion: one pipeline run leaves both families populated.""" + from data_pipeline.ingest.ohlcv import upsert_raw_prices + from data_pipeline.transform.cleaning import clean_range + from data_pipeline.transform.processing import process_frequencies + + ticker = "TEST_CANON" + end = dt.date.today() + start = end - dt.timedelta(days=45) + + init_db() + assert upsert_raw_prices(ticker, start, end).ok + assert clean_range(ticker, start, end).ok + assert process_frequencies(ticker, start, end).ok + + for legacy, canonical in CANONICAL_TABLES.items(): + canonical_rows = _count(canonical, ticker) + assert canonical_rows > 0, f"{canonical} was not written for {ticker}" + assert canonical_rows == _count(legacy, ticker), f"{legacy} / {canonical} diverged" + + +# --------------------------------------------------------------------------- +# Reads target the canonical tables +# --------------------------------------------------------------------------- +def test_health_inventory_reads_canonical_table(): + """A row that exists only in ``raw_bars`` must be visible to the health read.""" + from data_pipeline.store.repos import fetch_ticker_inventory + + init_db() + with get_conn() as conn: + conn.execute( + "INSERT OR REPLACE INTO raw_bars (ticker,date,open,high,low,close,adj_close,volume) " + "VALUES (?,?,?,?,?,?,?,?)", + ("CANON_ONLY", "2026-01-06", 7.0, 7.0, 7.0, 7.0, 7.0, 70.0), + ) + conn.execute("DELETE FROM raw_prices WHERE ticker=?", ("CANON_ONLY",)) + conn.commit() + + rows = [r for r in fetch_ticker_inventory() if r[0] == "CANON_ONLY"] + assert rows, "fetch_ticker_inventory did not read raw_bars" + assert rows[0][1] == 1 + + +# --------------------------------------------------------------------------- +# One-shot backfill script +# --------------------------------------------------------------------------- +def test_migration_script_backfills_legacy_only_rows(): + init_db() + with get_conn() as conn: + conn.execute( + "INSERT OR REPLACE INTO raw_prices (ticker,date,close) VALUES (?,?,?)", + ("LEGACY_ONLY", "2026-01-05", 42.0), + ) + conn.execute("DELETE FROM raw_bars WHERE ticker=?", ("LEGACY_ONLY",)) + conn.commit() + + result = subprocess.run( + [sys.executable, str(REPO_ROOT / "scripts" / "migrate_canonical_tables.py")], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + with get_conn() as conn: + row = conn.execute("SELECT close FROM raw_bars WHERE ticker=?", ("LEGACY_ONLY",)).fetchone() + assert row is not None and row[0] == pytest.approx(42.0) + + +def test_migration_dry_run_has_no_side_effects(): + """`--dry-run` must report without creating or copying anything.""" + import os + + db_file = os.environ["MARKET_DB_PATH"] + init_db(db_file) + with get_conn() as conn: + conn.execute("DROP TABLE IF EXISTS raw_bars") + conn.execute( + "INSERT OR REPLACE INTO raw_prices (ticker,date,close) VALUES (?,?,?)", + ("DRY_RUN", "2026-01-05", 3.0), + ) + conn.commit() + + result = subprocess.run( + [sys.executable, str(REPO_ROOT / "scripts" / "migrate_canonical_tables.py"), "--db", db_file, "--dry-run"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "nothing written" in result.stdout + + with get_conn() as conn: + created = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='raw_bars'").fetchone()[ + 0 + ] + assert created == 0, "dry run created the canonical table" + + +def test_migration_script_is_idempotent(): + """Re-running the backfill must not duplicate or clobber canonical rows.""" + init_db() + with get_conn() as conn: + conn.execute( + "INSERT OR REPLACE INTO raw_prices (ticker,date,close) VALUES (?,?,?)", + ("IDEMPOTENT", "2026-01-05", 1.0), + ) + conn.commit() + + for _ in range(2): + result = subprocess.run( + [sys.executable, str(REPO_ROOT / "scripts" / "migrate_canonical_tables.py")], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + assert _count("raw_bars", "IDEMPOTENT") == 1 diff --git a/tests/test_chart_time_range.py b/tests/test_chart_time_range.py index 7a63afc..0fe011f 100644 --- a/tests/test_chart_time_range.py +++ b/tests/test_chart_time_range.py @@ -48,11 +48,10 @@ def _run_single_case(test_case: dict) -> None: """Assert that a single test case produces sufficient data points and charts.""" + from services.market.data_context_fetch import fetch_data_context + analyzer = MarketAnalyzer( - ticker=test_case["ticker"], - start_date=test_case["start"], - frequency=test_case["frequency"], - end_date=test_case["end"], + fetch_data_context(test_case["ticker"], test_case["start"], test_case["frequency"], test_case["end"]) ) assert analyzer.is_data_valid(), f"{test_case['description']}: No valid data returned" diff --git a/tests/test_cleaning.py b/tests/test_cleaning.py index de18559..1191ced 100644 --- a/tests/test_cleaning.py +++ b/tests/test_cleaning.py @@ -1,11 +1,11 @@ -"""Tests for data_pipeline/cleaning.py — anomaly flags and business-day alignment.""" +"""Tests for data_pipeline/transform/cleaning.py — anomaly flags and business-day alignment.""" import datetime as dt import numpy as np import pandas as pd -from data_pipeline.cleaning import _flag_anomalies, _get_business_days +from data_pipeline.transform.cleaning import _flag_anomalies, _get_business_days class TestGetBusinessDays: diff --git a/tests/test_concurrency.py b/tests/test_concurrency.py index 6bb7cc7..a1bddea 100644 --- a/tests/test_concurrency.py +++ b/tests/test_concurrency.py @@ -8,9 +8,8 @@ import pytest from data_pipeline import PipelineResult -from data_pipeline.data_ops import ( +from data_pipeline._state import ( _QUERY_CACHE_TTL, - DataService, _cache_get, _cache_invalidate, _cache_set, @@ -19,7 +18,8 @@ _update_lock_mutex, _update_locks, ) -from data_pipeline.db import init_db +from data_pipeline.read import DataService +from data_pipeline.store.db import init_db @pytest.fixture(autouse=True) @@ -40,18 +40,18 @@ def _reset_state(): class TestCooldown: - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_first_call_runs_pipeline(self, mock_dl, mock_cl, mock_pr): init_db() result = DataService.manual_update("COOL1") assert result is True mock_dl.assert_called_once() - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_second_call_within_cooldown_skips(self, mock_dl, mock_cl, mock_pr): init_db() DataService.manual_update("COOL2") @@ -59,9 +59,9 @@ def test_second_call_within_cooldown_skips(self, mock_dl, mock_cl, mock_pr): assert result is False assert mock_dl.call_count == 1 # only first call - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_different_tickers_not_blocked(self, mock_dl, mock_cl, mock_pr): init_db() DataService.manual_update("TCKR_A") @@ -69,7 +69,9 @@ def test_different_tickers_not_blocked(self, mock_dl, mock_cl, mock_pr): assert result is True assert mock_dl.call_count == 2 - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(ok=False, error="download_failed")) + @patch( + "data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(ok=False, error="download_failed") + ) def test_failed_pipeline_clears_cooldown(self, mock_dl): """If download fails, cooldown should NOT prevent retry (since we return False, not raise).""" init_db() @@ -84,9 +86,9 @@ def test_failed_pipeline_clears_cooldown(self, mock_dl): class TestConcurrentUpdates: - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_concurrent_same_ticker_only_one_runs(self, mock_dl, mock_cl, mock_pr): """Two threads updating same ticker: only first should actually run.""" init_db() @@ -105,9 +107,9 @@ def update(): # One True (ran), one False (cooldown) assert sorted(results) == [False, True] - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) def test_concurrent_different_tickers_both_run(self, mock_dl, mock_cl, mock_pr): """Two threads updating different tickers: both should run.""" init_db() @@ -188,10 +190,10 @@ def setup_method(self): DataService._ensure_range_memo.clear() DataService._ensure_range_inflight.clear() - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.db.fetch_df") + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.store.db.fetch_df") def test_concurrent_calls_run_backfill_only_once(self, mock_fetch_df, mock_dl, mock_cl, mock_pr): import datetime as dt @@ -239,8 +241,8 @@ def test_sentinel_start_skips_backfill_when_db_has_coverage(self): DataService._ensure_range_memo.clear() with ( - patch("data_pipeline.db.fetch_df") as mock_fetch_df, - patch("data_pipeline.downloader.upsert_raw_prices") as mock_dl, + patch("data_pipeline.store.db.fetch_df") as mock_fetch_df, + patch("data_pipeline.ingest.ohlcv.upsert_raw_prices") as mock_dl, ): # DB has 2021-01-01 .. today coverage already. mock_fetch_df.return_value = pd.DataFrame( @@ -254,10 +256,10 @@ def test_sentinel_start_skips_backfill_when_db_has_coverage(self): assert ok is True assert mock_dl.call_count == 0, "must NOT walk yfinance back to 1990" - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.db.fetch_df") + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.store.db.fetch_df") def test_explicit_multiyear_request_does_backfill(self, mock_fetch_df, mock_dl, mock_cl, mock_pr): """Regression for the 'NVDA only has 30 days, user asked for 5 years, sentinel short-circuit silently lied' bug. A user-explicit @@ -281,10 +283,10 @@ def test_explicit_multiyear_request_does_backfill(self, mock_fetch_df, mock_dl, "user-explicit 5-year range must trigger backfill — sentinel short-circuit must NOT apply here" ) - @patch("data_pipeline.processing.process_frequencies", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.cleaning.clean_range", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.downloader.upsert_raw_prices", return_value=PipelineResult(rows=5)) - @patch("data_pipeline.db.fetch_df") + @patch("data_pipeline.transform.processing.process_frequencies", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.transform.cleaning.clean_range", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices", return_value=PipelineResult(rows=5)) + @patch("data_pipeline.store.db.fetch_df") def test_sentinel_with_thin_db_still_backfills(self, mock_fetch_df, mock_dl, mock_cl, mock_pr): """Regression: sentinel start (PriceDynamic uses 1900-01-01 always) but DB has only ~1 month of recent data MUST backfill. The sentinel diff --git a/tests/test_db.py b/tests/test_db.py index bc6babf..045f16b 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,11 +1,11 @@ -"""Tests for data_pipeline/db.py — init, get_conn, upsert, fetch.""" +"""Tests for data_pipeline/store/db.py — init, get_conn, upsert, fetch.""" import sqlite3 import threading import pytest -from data_pipeline.db import close_thread_conn, fetch_df, get_conn, init_db, upsert_many +from data_pipeline.store.db import close_thread_conn, fetch_df, get_conn, init_db, upsert_many class TestInitDb: @@ -14,6 +14,11 @@ def test_creates_tables(self, tmp_path): init_db(db) with sqlite3.connect(db) as conn: tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()} + # canonical store (ADR 0011) + assert "raw_bars" in tables + assert "clean_bars" in tables + assert "feature_bars" in tables + # compatibility shadows — removable one release after the rename assert "raw_prices" in tables assert "clean_prices" in tables assert "processed_prices" in tables diff --git a/tests/test_db_errors.py b/tests/test_db_errors.py index c80c2fd..27eced1 100644 --- a/tests/test_db_errors.py +++ b/tests/test_db_errors.py @@ -1,4 +1,4 @@ -"""Tests for data_pipeline.db — error scenarios and edge cases.""" +"""Tests for data_pipeline.store.db — error scenarios and edge cases.""" import os import sqlite3 @@ -6,7 +6,7 @@ import pandas as pd import pytest -from data_pipeline.db import fetch_df, get_conn, init_db, upsert_many +from data_pipeline.store.db import fetch_df, get_conn, init_db, upsert_many class TestInitDb: @@ -15,6 +15,11 @@ def test_creates_tables(self, tmp_path): init_db(db_path) with sqlite3.connect(db_path) as conn: tables = [r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()] + # canonical store (ADR 0011) + assert "raw_bars" in tables + assert "clean_bars" in tables + assert "feature_bars" in tables + # compatibility shadows — removable one release after the rename assert "raw_prices" in tables assert "clean_prices" in tables assert "processed_prices" in tables diff --git a/tests/test_downloader_gap.py b/tests/test_downloader_gap.py index e945573..f9f173f 100644 --- a/tests/test_downloader_gap.py +++ b/tests/test_downloader_gap.py @@ -1,4 +1,4 @@ -"""Tests for the gap-aware downloader logic in `data_pipeline/downloader.py`. +"""Tests for the gap-aware downloader logic in `data_pipeline/ingest/ohlcv.py`. These lock in the behavior fix for the NVDA-style outage: when historical business days are missing from `raw_prices` (e.g. after a yfinance rate-limit @@ -14,15 +14,10 @@ import pandas as pd import pytest -from data_pipeline.data_ops import ( - DataService, - _query_cache, - _query_cache_lock, - _update_lock_mutex, - _update_locks, -) -from data_pipeline.db import init_db, upsert_many -from data_pipeline.downloader import find_missing_business_days, upsert_raw_prices +from data_pipeline._state import _query_cache, _query_cache_lock, _update_lock_mutex, _update_locks +from data_pipeline.ingest.ohlcv import find_missing_business_days, upsert_raw_prices +from data_pipeline.read import DataService +from data_pipeline.store.db import init_db, upsert_many @pytest.fixture(autouse=True) @@ -108,8 +103,8 @@ def test_weekends_are_not_missing(self): class TestUpsertRawPricesGapAware: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_interior_gap_triggers_download(self, mock_throttle, mock_dl): """The NVDA regression: existing rows on edges + interior hole → must download.""" start = dt.date(2024, 1, 1) @@ -132,8 +127,8 @@ def test_interior_gap_triggers_download(self, mock_throttle, mock_dl): # No remaining business-day gap after upsert. assert find_missing_business_days("NVDA_REGRESSION", start, end) == [] - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_full_coverage_skips_download(self, mock_throttle, mock_dl): start = dt.date(2024, 1, 1) end = dt.date(2024, 1, 5) @@ -146,8 +141,8 @@ def test_full_coverage_skips_download(self, mock_throttle, mock_dl): mock_dl.assert_not_called() mock_throttle.assert_not_called() - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_download_start_expanded_to_earliest_gap_within_request(self, mock_throttle, mock_dl): """When the requested window contains an earlier gap, the actual download `start` is widened to that gap (so we don't waste a round-trip on the tail).""" @@ -172,8 +167,8 @@ def test_download_start_expanded_to_earliest_gap_within_request(self, mock_throt class TestManualUpdateGapScan: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_old_gap_within_scan_window_triggers_download(self, mock_throttle, mock_dl): """`manual_update(days=7)` must still back-fill a gap older than 7 days when it falls within `GAP_SCAN_DAYS`.""" @@ -200,8 +195,8 @@ def test_old_gap_within_scan_window_triggers_download(self, mock_throttle, mock_ called_start = mock_dl.call_args.kwargs.get("start") assert called_start <= old_gap - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_no_gaps_no_download(self, mock_throttle, mock_dl): """When the gap-scan window is fully covered, `manual_update` skips the network.""" end = dt.date.today() diff --git a/tests/test_frontend_api.py b/tests/test_frontend_api.py index d96e9c2..41304f9 100644 --- a/tests/test_frontend_api.py +++ b/tests/test_frontend_api.py @@ -6,7 +6,7 @@ - The /api/option_chain endpoint handles valid/invalid inputs - Config-driven filter parameters (DTE, moneyness) are respected - MarketAnalyzer features_df is non-empty with adequate data - - build_data_context normalizes futu-format tickers to yahoo format + - fetch_data_context normalizes futu-format tickers to yahoo format """ import datetime as dt @@ -29,22 +29,44 @@ def test_get_renders(self, client): assert 'id="ticker"' in html assert 'id="start_time"' in html - def test_get_has_config_tab(self, client): - """Config tab with all option filter fields should be present.""" + def test_get_has_option_filter_toolbar(self, client): + """Batch B7: the chain filters live with the Option Chain module.""" resp = client.get("/") html = resp.data.decode() - assert 'id="cfg-frequency"' in html - assert 'id="cfg-max-dte"' in html - assert 'id="cfg-moneyness-low"' in html - assert 'id="cfg-moneyness-high"' in html - assert 'id="cfg-max-contracts"' in html - - def test_get_has_position_sizing_in_settings(self, client): - """Position sizing fields should be inside Analysis Settings card.""" + assert 'id="option-toolbar"' in html + assert 'id="oc-max-dte"' in html + assert 'id="oc-moneyness-low"' in html + assert 'id="oc-moneyness-high"' in html + assert 'id="oc-max-contracts"' in html + assert 'id="oc-refresh-interval"' in html + + def test_post_renders_the_market_module_toolbars(self, client, monkeypatch): + """The market toolbars render in streaming mode, each owning its params. + + WHY the kick is stubbed: `POST /` runs the readiness pass (B5), which would + otherwise start a real backfill on a daemon thread for the test ticker. + """ + monkeypatch.setattr("data_pipeline.orchestrate.readiness.kick_backfill", lambda *a, **k: None) + resp = client.post("/", data={"ticker": "TEST_AAPL", "start_time": "2024-01", "frequency": "ME"}) + html = resp.data.decode() + assert resp.status_code == 200 + assert 'id="market-toolbar"' in html + assert 'id="statistical-toolbar"' in html + assert 'id="assessment-toolbar"' in html + # Frequency is a market-module parameter; sizing is Assessment's. + assert 'id="stat-frequency"' in html + assert 'id="assess-account-size"' in html + assert 'id="assess-max-risk-pct"' in html + # The ready placeholder carries the params on its first fan-out. + assert 'hx-include="#statistical-toolbar"' in html + + def test_get_has_parameters_bar_with_a_single_input(self, client): + """The bar owns `ticker` only; the horizon is a submit-time mirror.""" resp = client.get("/") html = resp.data.decode() - assert 'id="account_size"' in html - assert 'id="max_risk_pct"' in html + assert 'class="parameters-bar"' in html + assert 'id="parameters-bar-toggle"' in html + assert 'id="parameters-bar-summary"' in html def test_post_missing_ticker(self, client): """POST without ticker should show error.""" @@ -185,43 +207,33 @@ def _make_price_df(n_rows=60): index=dates, ) - def test_features_df_nonempty_with_synthetic_data(self, monkeypatch): + def test_features_df_nonempty_with_synthetic_data(self): """features_df should have rows when data context has adequate data.""" from core.market.analyzer import MarketAnalyzer - from core.market.data_context import DataContext, Horizon + from core.market.data_context import Horizon, build_data_context fake_df = self._make_price_df(60) start = fake_df.index[0].date() end = fake_df.index[-1].date() - def _mock_build(ticker, start_date, frequency="D", end_date=None): - df = fake_df.copy() - df["LastClose"] = df["Close"].shift(1) - df["LastAdjClose"] = df["Adj Close"].shift(1) - return DataContext( - ticker=ticker, - frequency=frequency, - horizon=Horizon( - start=start_date, - end=end_date or end, - user_provided_end=end_date is not None, - frequency=frequency, - ), - bars=df, - daily_bars=fake_df.copy(), - ) - - monkeypatch.setattr("core.market.analyzer.build_data_context", _mock_build) - - analyzer = MarketAnalyzer("TEST", start, "D", end_date=end) + # WHY the pure builder: core/ must not fetch (ADR 0001 / batch B4), so + # the context is assembled here from synthetic bars and injected. + ctx = build_data_context( + ticker="TEST", + frequency="D", + horizon=Horizon(start=start, end=end, user_provided_end=True, frequency="D"), + raw_data=fake_df, + ) + + analyzer = MarketAnalyzer(ctx) assert not analyzer.features_df.empty, f"features_df should not be empty, shape={analyzer.features_df.shape}" assert len(analyzer.features_df) >= 50, f"Expected >=50 rows, got {len(analyzer.features_df)}" assert set(analyzer.features_df.columns) == {"Oscillation", "Osc_high", "Osc_low", "Returns", "Difference"} - def test_features_df_tolerates_partial_nan(self, monkeypatch): + def test_features_df_tolerates_partial_nan(self): """features_df should retain rows even when one column has NaN at a few spots.""" from core.market.analyzer import MarketAnalyzer - from core.market.data_context import DataContext, Horizon + from core.market.data_context import Horizon, build_data_context fake_df = self._make_price_df(60) # Introduce NaN in High for a few rows (osc_high will be NaN there) @@ -230,35 +242,23 @@ def test_features_df_tolerates_partial_nan(self, monkeypatch): start = fake_df.index[0].date() end = fake_df.index[-1].date() - def _mock_build(ticker, start_date, frequency="D", end_date=None): - df = fake_df.copy() - df["LastClose"] = df["Close"].shift(1) - df["LastAdjClose"] = df["Adj Close"].shift(1) - return DataContext( - ticker=ticker, - frequency=frequency, - horizon=Horizon( - start=start_date, - end=end_date or end, - user_provided_end=end_date is not None, - frequency=frequency, - ), - bars=df, - daily_bars=fake_df.copy(), - ) - - monkeypatch.setattr("core.market.analyzer.build_data_context", _mock_build) - - analyzer = MarketAnalyzer("TEST", start, "D", end_date=end) + ctx = build_data_context( + ticker="TEST", + frequency="D", + horizon=Horizon(start=start, end=end, user_provided_end=True, frequency="D"), + raw_data=fake_df, + ) + + analyzer = MarketAnalyzer(ctx) # With dropna(how='all'), rows with partial NaN are kept assert not analyzer.features_df.empty # At least most rows should survive — only first row (shift NaN) removed assert len(analyzer.features_df) >= 50 - def test_features_df_empty_when_all_nan(self, monkeypatch): + def test_features_df_empty_when_all_nan(self): """features_df should have 0 rows when all data is NaN.""" from core.market.analyzer import MarketAnalyzer - from core.market.data_context import DataContext, Horizon + from core.market.data_context import Horizon, build_data_context fake_df = self._make_price_df(10) fake_df["Adj Close"] = np.nan @@ -268,61 +268,48 @@ def test_features_df_empty_when_all_nan(self, monkeypatch): start = fake_df.index[0].date() end = fake_df.index[-1].date() - def _mock_build(ticker, start_date, frequency="D", end_date=None): - df = fake_df.copy() - df["LastClose"] = df["Close"].shift(1) - df["LastAdjClose"] = df["Adj Close"].shift(1) - return DataContext( - ticker=ticker, - frequency=frequency, - horizon=Horizon( - start=start_date, - end=end_date or end, - user_provided_end=end_date is not None, - frequency=frequency, - ), - bars=df, - daily_bars=fake_df.copy(), - ) - - monkeypatch.setattr("core.market.analyzer.build_data_context", _mock_build) - - analyzer = MarketAnalyzer("TEST", start, "D", end_date=end) + ctx = build_data_context( + ticker="TEST", + frequency="D", + horizon=Horizon(start=start, end=end, user_provided_end=True, frequency="D"), + raw_data=fake_df, + ) + + analyzer = MarketAnalyzer(ctx) assert analyzer.features_df.empty # ═══════════════════════════════════════════════════════════════════════════ -# 5. build_data_context ticker normalization +# 5. fetch_data_context ticker normalization # ═══════════════════════════════════════════════════════════════════════════ class TestDataContextTickerNorm: - def test_futu_format_normalized(self, monkeypatch): - """build_data_context('US.NVDA', ...) should normalize to 'NVDA'.""" - monkeypatch.setattr("core.market.data_context._fetch_raw_data", lambda ticker, start, freq: (None, ticker)) - - from core.market.data_context import build_data_context + """Normalisation lives in services/market/data_context_fetch.py (batch B4).""" + + def _ctx_for(self, monkeypatch, ticker: str): + # WHY the fetch stub: normalisation happens before the DB/provider hop, + # so stubbing the raw fetch keeps these tests offline and fast. + monkeypatch.setattr( + "services.market.data_context_fetch._fetch_raw_data", + lambda t, start, freq: (None, t), + ) - ctx = build_data_context("US.NVDA", dt.date(2024, 1, 1)) - assert ctx.ticker == "NVDA" + from services.market.data_context_fetch import fetch_data_context - def test_yahoo_format_unchanged(self, monkeypatch): - """build_data_context('NVDA', ...) should keep ticker as 'NVDA'.""" - monkeypatch.setattr("core.market.data_context._fetch_raw_data", lambda ticker, start, freq: (None, ticker)) + return fetch_data_context(ticker, dt.date(2024, 1, 1)) - from core.market.data_context import build_data_context + def test_futu_format_normalized(self, monkeypatch): + """fetch_data_context('US.NVDA', ...) should normalize to 'NVDA'.""" + assert self._ctx_for(monkeypatch, "US.NVDA").ticker == "NVDA" - ctx = build_data_context("NVDA", dt.date(2024, 1, 1)) - assert ctx.ticker == "NVDA" + def test_yahoo_format_unchanged(self, monkeypatch): + """fetch_data_context('NVDA', ...) should keep ticker as 'NVDA'.""" + assert self._ctx_for(monkeypatch, "NVDA").ticker == "NVDA" def test_hk_format_normalized(self, monkeypatch): - """build_data_context('HK.00700', ...) should normalize to '0700.HK'.""" - monkeypatch.setattr("core.market.data_context._fetch_raw_data", lambda ticker, start, freq: (None, ticker)) - - from core.market.data_context import build_data_context - - ctx = build_data_context("HK.00700", dt.date(2024, 1, 1)) - assert ctx.ticker == "0700.HK" + """fetch_data_context('HK.00700', ...) should normalize to '0700.HK'.""" + assert self._ctx_for(monkeypatch, "HK.00700").ticker == "0700.HK" # ═══════════════════════════════════════════════════════════════════════════ diff --git a/tests/test_health_service.py b/tests/test_health_service.py index 36793bb..ed831be 100644 --- a/tests/test_health_service.py +++ b/tests/test_health_service.py @@ -4,17 +4,17 @@ import pandas as pd -from data_pipeline.db import get_conn +from data_pipeline.store.db import get_conn from services.market.health import overall_summary, per_ticker_summary def _seed(ticker: str, dates: list[str], close_vals: list[float | None]) -> None: - from data_pipeline.db import init_db + from data_pipeline.store.db import init_db init_db() with get_conn() as conn: conn.executemany( - "INSERT OR REPLACE INTO raw_prices (ticker,date,open,high,low,close,adj_close,volume) " + "INSERT OR REPLACE INTO raw_bars (ticker,date,open,high,low,close,adj_close,volume) " "VALUES (?,?,?,?,?,?,?,?)", [(ticker, d, 1.0, 1.0, 1.0, c, c, 100.0) for d, c in zip(dates, close_vals, strict=True)], ) diff --git a/tests/test_job_cache.py b/tests/test_job_cache.py index f14ffcf..b7dc88c 100644 --- a/tests/test_job_cache.py +++ b/tests/test_job_cache.py @@ -1,11 +1,11 @@ -"""Tests for data_pipeline/job_cache.py.""" +"""Tests for data_pipeline/orchestrate/job_cache.py.""" import threading import time import pytest -from data_pipeline import job_cache as jc +from data_pipeline.orchestrate import job_cache as jc @pytest.fixture(autouse=True) @@ -67,6 +67,25 @@ def fn_b(_): assert jc.compute_or_get(job_id, "AAPL", "kind_b", fn_b) == "B" assert calls == {"a": 1, "b": 1} + def test_variant_computes_independently(self): + """Same ticker + kind, different `variant` (a digest of the module's + toolbar params) must not share a cache entry — otherwise a frequency / + horizon change replays the first render for the whole job TTL.""" + job_id = jc.create_job({}, ["AAPL"]) + calls = [] + + def fn(_): + calls.append(1) + return {"n": len(calls)} + + r1 = jc.compute_or_get(job_id, "AAPL", "stat", fn, variant="frequency=ME") + r2 = jc.compute_or_get(job_id, "AAPL", "stat", fn, variant="frequency=W") + r1_again = jc.compute_or_get(job_id, "AAPL", "stat", fn, variant="frequency=ME") + assert r1 == {"n": 1} + assert r2 == {"n": 2} + assert r1_again == {"n": 1} # cached per variant + assert len(calls) == 2 + def test_unknown_job_raises(self): with pytest.raises(KeyError): jc.compute_or_get("nope", "AAPL", "stat", lambda _: None) diff --git a/tests/test_market_review.py b/tests/test_market_review.py index fdda1e3..7c9fa53 100644 --- a/tests/test_market_review.py +++ b/tests/test_market_review.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -from data_pipeline.db import get_conn, init_db +from data_pipeline.store.db import get_conn, init_db # ── Helpers ─────────────────────────────────────────────────────── diff --git a/tests/test_module_params.py b/tests/test_module_params.py new file mode 100644 index 0000000..e3d8c8a --- /dev/null +++ b/tests/test_module_params.py @@ -0,0 +1,178 @@ +"""Module-scoped parameters travel as query args on each `/render` call (B7). + +Domain: Tests — Module Parameter Contract +Context: + - Decision gate §8 Q1 chose the **manifest** shape: `POST /` carries the module + list, and each module's own toolbar appends its parameters to its `/render` + call (`?from=…&to=…&frequency=…`). This file pins the backend half of that + contract; the toolbars and the `state/*ParamsState.js` stores land with the + rest of B7. +Contracts: + - ``FormService.extract_module_params`` reads **only** the per-module allow-list, + so a hand-crafted query string cannot inject arbitrary ``form_data`` keys. + - ``/render/`` lets the query args override the values recorded on the job + at POST time, falling back to the job when they are absent (direct URL). +Dependencies UPWARD: + - (none — stdlib + pytest + the packages under test) +""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from services.market.form import FormService + + +class TestExtractModuleParams: + def test_horizon_is_parsed_into_form_data_dates(self): + out = FormService.extract_module_params("statistical", {"from": "2024-03", "to": "2024-09"}) + assert out["start_time"] == "2024-03" + assert out["end_time"] == "2024-09" + assert out["parsed_start_time"] == dt.date(2024, 3, 1) + assert out["parsed_end_time"] == dt.date(2024, 9, 1) + + def test_market_review_accepts_only_the_horizon(self): + """market_review declares no frequency, so a stray one is ignored.""" + out = FormService.extract_module_params("market_review", {"from": "2024-03", "frequency": "W"}) + assert set(out) == {"start_time", "parsed_start_time"} + + def test_options_chain_declares_no_parameters(self): + assert FormService.extract_module_params("options_chain", {"from": "2024-03"}) == {} + + def test_unknown_module_is_inert(self): + assert FormService.extract_module_params("nope", {"from": "2024-03"}) == {} + + def test_undeclared_keys_are_never_read(self): + """A query string must not be able to smuggle keys into form_data.""" + out = FormService.extract_module_params( + "statistical", + {"from": "2024-03", "option_position": "[]", "ticker": "EVIL", "positions": "x"}, + ) + assert "option_position" not in out + assert "option_data" not in out + assert "ticker" not in out + assert "positions" not in out + + def test_blank_and_malformed_values_are_skipped_not_defaulted(self): + """Skipping lets the job's POST-time value remain the fallback.""" + assert FormService.extract_module_params("statistical", {"from": "", "frequency": ""}) == {} + assert FormService.extract_module_params("statistical", {"frequency": "YEARLY"}) == {} + assert FormService.extract_module_params("assessment", {"risk_threshold": "abc"}) == {} + + def test_assessment_group_parses_its_own_knobs(self): + out = FormService.extract_module_params( + "assessment", + { + "frequency": "W", + "side_bias": "Neutral", + "risk_threshold": "85", + "rolling_window": "90", + "account_size": "250000", + "max_risk_pct": "1.5", + }, + ) + assert out["frequency"] == "W" + assert out["side_bias"] == "Neutral" + assert out["target_bias"] == 0 + assert out["risk_threshold"] == 85 + assert out["rolling_window"] == 90 + assert out["account_size"] == 250000.0 + assert out["max_risk_pct"] == 1.5 + + def test_natural_side_bias_keeps_a_null_target_bias(self): + out = FormService.extract_module_params("assessment", {"side_bias": "Natural"}) + assert out["side_bias"] == "Natural" + assert out["target_bias"] is None + + +class TestRenderSliceParameterPrecedence: + """The query args win over the job; the job is the direct-URL fallback.""" + + @staticmethod + def _render(kind: str, query: str, monkeypatch) -> dict: + from app import app + + from data_pipeline.orchestrate import job_cache + from services.market.analysis import AnalysisService + from services.market.dispatch import render_streaming_slice + + captured: dict = {} + + def _fake_slice(form_data): + captured.clear() + captured.update(form_data) + return {} + + monkeypatch.setattr(AnalysisService, f"generate_{kind}_slice", staticmethod(_fake_slice), raising=False) + + job_cache._reset() + job_id = job_cache.create_job( + { + "ticker": "AAPL", + "frequency": "ME", + "start_time": "2020-01", + "parsed_start_time": dt.date(2020, 1, 1), + "parsed_end_time": None, + }, + ["AAPL"], + ) + with app.test_request_context(f"/render/{kind}?job={job_id}&ticker=AAPL{query}"): + render_streaming_slice(kind) + return captured + + @pytest.mark.parametrize( + ("kind", "slice_name"), + [("statistical", "generate_statistical_slice"), ("market_review", "generate_market_review_slice")], + ) + def test_query_params_override_the_job(self, kind, slice_name, monkeypatch): + captured = self._render(kind, "&from=2024-03&to=2024-09&frequency=W", monkeypatch) + assert captured["parsed_start_time"] == dt.date(2024, 3, 1) + assert captured["parsed_end_time"] == dt.date(2024, 9, 1) + assert captured["ticker"] == "AAPL" + if kind == "statistical": + assert captured["frequency"] == "W" + else: + # market_review declares no frequency: the job's value survives. + assert captured["frequency"] == "ME" + + def test_without_query_params_the_job_values_survive(self, monkeypatch): + captured = self._render("statistical", "", monkeypatch) + assert captured["frequency"] == "ME" + assert captured["parsed_start_time"] == dt.date(2020, 1, 1) + + def test_a_param_change_recomputes_within_the_same_job(self, monkeypatch): + """Plan §10 F1: the job-cache slice memo is keyed by (ticker, kind); a + toolbar change re-fires ``/render/`` with new query args, so the + memo must fold them in or it replays the first render for the job TTL. + """ + from app import app + + from data_pipeline.orchestrate import job_cache + from services.market.analysis import AnalysisService + from services.market.dispatch import render_streaming_slice + + seen: list[str] = [] + + def _fake_slice(form_data): + seen.append(form_data.get("frequency")) + return {} + + monkeypatch.setattr(AnalysisService, "generate_statistical_slice", staticmethod(_fake_slice), raising=False) + + job_cache._reset() + job_id = job_cache.create_job( + {"ticker": "AAPL", "frequency": "ME", "parsed_start_time": dt.date(2020, 1, 1), "parsed_end_time": None}, + ["AAPL"], + ) + base = f"/render/statistical?job={job_id}&ticker=AAPL&from=2020-01&to=2024-01" + with app.test_request_context(f"{base}&frequency=ME"): + render_streaming_slice("statistical") + with app.test_request_context(f"{base}&frequency=W"): + render_streaming_slice("statistical") + # A repeat of an already-computed variant still hits the cache. + with app.test_request_context(f"{base}&frequency=ME"): + render_streaming_slice("statistical") + + assert seen == ["ME", "W"], seen diff --git a/tests/test_nvda_analysis.py b/tests/test_nvda_analysis.py index 63aafb5..eb123ea 100644 --- a/tests/test_nvda_analysis.py +++ b/tests/test_nvda_analysis.py @@ -14,7 +14,7 @@ import pandas as pd import pytest -from data_pipeline.db import get_conn, init_db +from data_pipeline.store.db import get_conn, init_db _JOB_ID_RE = re.compile(r'STREAMING_JOB_ID\s*=\s*"([^"]+)"') @@ -30,8 +30,8 @@ def _extract_job_id(html: str) -> str: # --------------------------------------------------------------------------- -def _seed_clean_prices(ticker: str, n_rows: int = 30, *, nan_only: bool = False): - """Insert synthetic price rows into clean_prices. +def _seed_clean_bars(ticker: str, n_rows: int = 30, *, nan_only: bool = False): + """Insert synthetic price rows into clean_bars. Wipes any previous rows for `ticker` first so the seeded distribution is deterministic regardless of test ordering, and invalidates the in-memory @@ -39,25 +39,25 @@ def _seed_clean_prices(ticker: str, n_rows: int = 30, *, nan_only: bool = False) """ init_db() # Drop the cross-test query cache that DataService maintains (TTL 60s). - from data_pipeline.data_ops import _cache_invalidate + from data_pipeline._state import _cache_invalidate _cache_invalidate(ticker) dates = pd.bdate_range(end=dt.date.today(), periods=n_rows) np.random.seed(42) close = 120.0 + np.cumsum(np.random.randn(n_rows) * 0.5) with get_conn() as conn: - conn.execute("DELETE FROM clean_prices WHERE ticker = ?", (ticker,)) + conn.execute("DELETE FROM clean_bars WHERE ticker = ?", (ticker,)) for i, d in enumerate(dates): date_str = d.strftime("%Y-%m-%d") if nan_only: conn.execute( - "INSERT OR REPLACE INTO clean_prices (ticker, date, is_trading_day, missing_any) VALUES (?,?,?,?)", + "INSERT OR REPLACE INTO clean_bars (ticker, date, is_trading_day, missing_any) VALUES (?,?,?,?)", (ticker, date_str, 0, 1), ) else: c = float(close[i]) conn.execute( - "INSERT OR REPLACE INTO clean_prices " + "INSERT OR REPLACE INTO clean_bars " "(ticker, date, open, high, low, close, adj_close, volume) " "VALUES (?,?,?,?,?,?,?,?)", (ticker, date_str, c - 0.5, c + 1.0, c - 1.0, c, c, 1_000_000), @@ -65,22 +65,34 @@ def _seed_clean_prices(ticker: str, n_rows: int = 30, *, nan_only: bool = False) conn.commit() +def _analyzer(ticker: str, start: dt.date, frequency: str = "D"): + """Build a MarketAnalyzer the way production does: services fetch, core wraps. + + WHY: since batch B4 ``core/`` never fetches (ADR 0001), so the DataContext + has to be built in the service layer and injected. + """ + from core.market.analyzer import MarketAnalyzer + from services.market.data_context_fetch import fetch_data_context + + return MarketAnalyzer(fetch_data_context(ticker, start, frequency)) + + @pytest.fixture() def _patch_downloads(monkeypatch): """Disable all real yfinance download paths for unit tests.""" - from data_pipeline.data_ops import DataService + from data_pipeline.read import DataService # Block the manual_update → pipeline path. Patch BOTH the DataService # facade and the module-level function: _query.py calls the _update module # directly because facade imports _query (the reverse edge would be an # import cycle), so patching only the class would be bypassed. monkeypatch.setattr(DataService, "manual_update", staticmethod(lambda *a, **kw: None)) - monkeypatch.setattr("data_pipeline.data_ops._update.manual_update", lambda *a, **kw: None) + monkeypatch.setattr("data_pipeline.orchestrate.update.manual_update", lambda *a, **kw: None) # Block the ensure_range → chunked backfill path (same dual patching). monkeypatch.setattr(DataService, "ensure_range", staticmethod(lambda *a, **kw: True)) - monkeypatch.setattr("data_pipeline.data_ops._range.ensure_range", lambda *a, **kw: True) - # Block the data_context fallback to yfinance - monkeypatch.setattr("core.market.data_context._download_data", lambda *a, **kw: None) + monkeypatch.setattr("data_pipeline.orchestrate.backfill.ensure_range", lambda *a, **kw: True) + # Block the data_context fallback to the provider + monkeypatch.setattr("services.market.data_context_fetch._download_data", lambda *a, **kw: None) # --------------------------------------------------------------------------- @@ -93,29 +105,23 @@ class TestFeaturesDF: def test_good_data_produces_nonempty_features(self, _patch_downloads): """With 30 rows of price data, features_df should have ~29 rows.""" - _seed_clean_prices("NVDA", 30) - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + _seed_clean_bars("NVDA", 30) + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) assert analyzer.is_data_valid() assert analyzer.features_df.shape[0] >= 20 assert set(analyzer.features_df.columns) == {"Oscillation", "Osc_high", "Osc_low", "Returns", "Difference"} def test_nan_only_filler_rows_produce_empty_features(self, _patch_downloads): """NaN-only filler rows from clean_range should not fool is_valid.""" - _seed_clean_prices("NVDA", 5, nan_only=True) - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + _seed_clean_bars("NVDA", 5, nan_only=True) + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) assert not analyzer.is_data_valid() assert analyzer.features_df.empty def test_empty_db_no_download(self, _patch_downloads): """Empty DB + failed download → proper error, no crash.""" init_db() - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) assert not analyzer.is_data_valid() assert analyzer.features_df.empty @@ -131,41 +137,37 @@ def test_mixed_real_and_nan_rows(self, _patch_downloads): if i < 7: # 7 real rows c = float(close[i]) conn.execute( - "INSERT OR REPLACE INTO clean_prices " + "INSERT OR REPLACE INTO clean_bars " "(ticker, date, open, high, low, close, adj_close, volume) " "VALUES (?,?,?,?,?,?,?,?)", ("NVDA", date_str, c - 0.5, c + 1.0, c - 1.0, c, c, 1_000_000), ) else: # 3 NaN filler rows conn.execute( - "INSERT OR REPLACE INTO clean_prices " + "INSERT OR REPLACE INTO clean_bars " "(ticker, date, is_trading_day, missing_any) VALUES (?,?,?,?)", ("NVDA", date_str, 0, 1), ) conn.commit() - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) assert analyzer.is_data_valid() # 7 real rows → shift(1) eats 1 → 6 feature rows assert analyzer.features_df.shape[0] == 6 def test_single_row_produces_empty_features(self, _patch_downloads): """Only 1 row of data → shift(1) creates NaN → no valid features.""" - _seed_clean_prices("NVDA", 1) - from core.market.analyzer import MarketAnalyzer - - analyzer = MarketAnalyzer("NVDA", dt.date(2026, 1, 1), "D") + _seed_clean_bars("NVDA", 1) + analyzer = _analyzer("NVDA", dt.date(2026, 1, 1)) # 1 row is valid data, but after shift(1) → 0 feature rows assert analyzer.features_df.shape[0] == 0 def test_futu_format_ticker_normalized(self, _patch_downloads): - """build_data_context normalizes US.NVDA → NVDA for DB lookup.""" - _seed_clean_prices("NVDA", 10) - from core.market.data_context import build_data_context + """fetch_data_context normalizes US.NVDA → NVDA for DB lookup.""" + _seed_clean_bars("NVDA", 10) + from services.market.data_context_fetch import fetch_data_context - ctx = build_data_context("US.NVDA", dt.date(2026, 1, 1), "D") + ctx = fetch_data_context("US.NVDA", dt.date(2026, 1, 1), "D") assert ctx.ticker == "NVDA" assert ctx.is_valid() @@ -180,8 +182,8 @@ def client(_patch_downloads): """Create Flask test client with isolated DB.""" import app as flask_app - from data_pipeline import data_ops as _ds - from data_pipeline import job_cache as _jc + from data_pipeline import _state as _ds + from data_pipeline.orchestrate import job_cache as _jc # Reset module-level caches so prior tests don't leak data into this one. _jc._reset() @@ -201,7 +203,7 @@ class TestFlaskAnalysisPost: def test_nvda_post_returns_charts(self, client): """POST returns a skeleton; GET /render/statistical produces charts.""" - _seed_clean_prices("NVDA", 60) + _seed_clean_bars("NVDA", 60) resp = client.post( "/", @@ -230,7 +232,7 @@ def test_nvda_post_returns_charts(self, client): def test_nvda_post_futu_format_works(self, client): """POST with US.NVDA (futu format) should also work end-to-end.""" - _seed_clean_prices("NVDA", 60) + _seed_clean_bars("NVDA", 60) resp = client.post( "/", @@ -287,7 +289,7 @@ def test_failed_download_shows_error(self, client): def test_nan_only_db_shows_error(self, client): """DB with NaN-only rows should produce an error fragment from /render/statistical, not blank charts.""" - _seed_clean_prices("NVDA", 5, nan_only=True) + _seed_clean_bars("NVDA", 5, nan_only=True) resp = client.post( "/", data={ @@ -314,7 +316,7 @@ def test_nan_only_db_shows_error(self, client): def test_analysis_service_direct(self, _patch_downloads): """Direct AnalysisService call with good data produces charts.""" - _seed_clean_prices("NVDA", 60) + _seed_clean_bars("NVDA", 60) from services.market.analysis import AnalysisService form_data = { diff --git a/tests/test_pages_build.py b/tests/test_pages_build.py index f00bf2c..8e95846 100644 --- a/tests/test_pages_build.py +++ b/tests/test_pages_build.py @@ -24,7 +24,7 @@ def test_assemble_matches_flask_partials(tmp_path): # every tab body from templates/index.html must survive the static render for tab_id in ( - "tab-parameter", + "tab-portfolio", "tab-market-review", "tab-statistical-analysis", "tab-market-assessment", @@ -34,7 +34,6 @@ def test_assemble_matches_flask_partials(tmp_path): "tab-regime", "tab-simulation", "tab-option-pricing-matrix", - "tab-config", ): assert f'id="{tab_id}"' in html, tab_id @@ -46,7 +45,13 @@ def test_assemble_matches_flask_partials(tmp_path): # Pages delta: shim + demo banner + prefilled demo ticker assert "./pages-shim.js" in html assert "pages-demo-banner" in html - assert 'id="ticker" name="ticker" value="NVDA"' in html + # The demo snapshot pre-fills the Parameters bar's ticker input. Match the + # attributes independently: the assertion must not depend on their order + # (batch B6 inserted a class between `name` and `value`). + ticker_input = re.search(r']*id="ticker"[^>]*>', html) + assert ticker_input, "ticker input missing from the static build" + assert 'name="ticker"' in ticker_input.group(0) + assert 'value="NVDA"' in ticker_input.group(0) # snapshot content baked in (real analysis artefacts, not empty states) assert "data:image/png;base64," in html diff --git a/tests/test_portfolio.py b/tests/test_portfolio.py index ac1e4d3..1796178 100644 --- a/tests/test_portfolio.py +++ b/tests/test_portfolio.py @@ -59,7 +59,7 @@ def test_attribute_pnl_with_iv_drop_hurts_long_vega(): def test_create_and_list_position(): - from data_pipeline.db import init_db + from data_pipeline.store.db import init_db from services.portfolio.facade import create_position, list_positions init_db() @@ -88,7 +88,7 @@ def test_create_position_rejects_missing_ticker(): def test_portfolio_snapshot_uses_mocked_spots(monkeypatch): - from data_pipeline.db import init_db + from data_pipeline.store.db import init_db from services.portfolio import facade as ps init_db() diff --git a/tests/test_processing.py b/tests/test_processing.py index 923f0e8..7eedb34 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -1,12 +1,12 @@ -"""Tests for data_pipeline.processing — feature computation correctness.""" +"""Tests for data_pipeline.transform.processing — feature computation correctness.""" import datetime as dt import numpy as np import pandas as pd -from data_pipeline.db import init_db, upsert_many -from data_pipeline.processing import _agg_ohlcv, _features, process_frequencies +from data_pipeline.store.db import init_db, upsert_many +from data_pipeline.transform.processing import _agg_ohlcv, _features, process_frequencies # ── Helpers ─────────────────────────────────────────────────────── @@ -30,8 +30,8 @@ def _make_daily(n: int = 30, base_close: float = 100.0) -> pd.DataFrame: return df -def _seed_clean_prices(ticker: str, df: pd.DataFrame) -> None: - """Insert rows into clean_prices table for testing.""" +def _seed_clean_bars(ticker: str, df: pd.DataFrame) -> None: + """Insert rows into the clean_bars table for testing.""" init_db() rows = [] for d, r in df.iterrows(): @@ -53,7 +53,7 @@ def _seed_clean_prices(ticker: str, df: pd.DataFrame) -> None: ) ) upsert_many( - "clean_prices", + "clean_bars", [ "ticker", "date", @@ -176,7 +176,7 @@ class TestProcessFrequencies: def test_basic_pipeline(self): """Process 30 days of synthetic data through all frequencies.""" df = _make_daily(30) - _seed_clean_prices("TEST", df) + _seed_clean_bars("TEST", df) start = df.index[0].date() end = df.index[-1].date() result = process_frequencies("TEST", start, end) @@ -193,33 +193,33 @@ def test_empty_data_returns_zero_rows(self): def test_all_frequencies_present(self): """Check D, W, ME rows are produced.""" - from data_pipeline.db import fetch_df + from data_pipeline.store.db import fetch_df df = _make_daily(30) - _seed_clean_prices("FREQ", df) + _seed_clean_bars("FREQ", df) start = df.index[0].date() end = df.index[-1].date() process_frequencies("FREQ", start, end) for freq in ("D", "W", "ME"): out = fetch_df( - "SELECT * FROM processed_prices WHERE ticker=? AND frequency=?", + "SELECT * FROM feature_bars WHERE ticker=? AND frequency=?", ("FREQ", freq), ) assert not out.empty, f"No rows for frequency {freq}" def test_feature_columns_in_db(self): """Verify key feature columns are stored.""" - from data_pipeline.db import fetch_df + from data_pipeline.store.db import fetch_df df = _make_daily(30) - _seed_clean_prices("COLS", df) + _seed_clean_bars("COLS", df) start = df.index[0].date() end = df.index[-1].date() process_frequencies("COLS", start, end) out = fetch_df( - "SELECT * FROM processed_prices WHERE ticker=? AND frequency='D'", + "SELECT * FROM feature_bars WHERE ticker=? AND frequency='D'", ("COLS",), ) for col in ("log_return", "ma_5", "ma_20", "mom_10", "osc"): diff --git a/tests/test_provider_seam.py b/tests/test_provider_seam.py new file mode 100644 index 0000000..8d71801 --- /dev/null +++ b/tests/test_provider_seam.py @@ -0,0 +1,247 @@ +"""Contract tests for the data-provider seam (ADR 0011, batch B1). + +Domain: Tests — Provider Seam +Context: + - Batch B1 moved every yfinance call behind ``data_pipeline/providers/`` and + introduced the canonical schema + registry. These tests pin the parts of + that contract that are otherwise invisible: the single-import invariant, the + canonical mapping units, and the registry's resolution rules. +Contracts: + - Only production code under ``data_pipeline/providers/`` imports yfinance. + - ``to_canonical_bars`` / ``to_option_chain_snapshot`` apply the unit rules in + ``providers/base.py`` (decimal IV, nullable bid/ask, no ``inTheMoney``). + - ``get_provider`` defaults to yfinance and rejects unknown names loudly. + - ``data_pipeline.providers.yf_client`` still re-exports the legacy callables unchanged. +Dependencies UPWARD: + - (none — stdlib + pytest + the package under test) +""" + +from __future__ import annotations + +import ast +import subprocess +import sys +from pathlib import Path + +import pandas as pd +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +PROVIDER_DIR = REPO_ROOT / "data_pipeline" / "providers" + +# Production roots only: doc_guard's `single-yf-exit` rule deliberately exempts +# tests/ and scripts/ (test doubles patch `yfinance.download` on the module). +PRODUCTION_ROOTS = ("app.py", "routes", "core", "data_pipeline", "services", "utils") + + +def _production_python_files() -> list[Path]: + out: list[Path] = [] + for sub in PRODUCTION_ROOTS: + p = REPO_ROOT / sub + if p.is_file(): + out.append(p) + elif p.is_dir(): + out.extend(x for x in p.rglob("*.py") if "__pycache__" not in x.parts) + return sorted(out) + + +def _imports_yfinance(path: Path) -> bool: + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except SyntaxError: + return False + for node in ast.walk(tree): + if isinstance(node, ast.Import): + if any(a.name.split(".", 1)[0] == "yfinance" for a in node.names): + return True + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + if node.module.split(".", 1)[0] == "yfinance": + return True + return False + + +def test_only_provider_seam_imports_yfinance(): + """B1 exit criterion: production code imports yfinance only under providers/.""" + offenders = [str(p.relative_to(REPO_ROOT)) for p in _production_python_files() if _imports_yfinance(p)] + assert offenders, "expected at least one importer inside data_pipeline/providers/" + for rel in offenders: + assert Path(rel).is_relative_to(Path("data_pipeline") / "providers"), ( + f"{rel} imports yfinance outside data_pipeline/providers/ — see ADR 0011" + ) + + +def test_doc_guard_single_yf_exit_allows_provider_modules(): + """The rescoped guard must not flag the provider package itself.""" + result = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "doc_guard.py"), + "--json", + "--rule", + "single-yf-exit", + "--files", + str(PROVIDER_DIR / "yfinance_provider.py"), + str(PROVIDER_DIR / "yf_snapshot.py"), + ], + capture_output=True, + text=True, + ) + assert "single-yf-exit" not in (result.stdout or ""), result.stdout + assert result.returncode == 0, result.stdout + result.stderr + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- +def test_registry_defaults_to_yfinance(): + from data_pipeline.providers import available_providers, get_provider + from data_pipeline.providers.base import MarketDataProvider + + assert available_providers() == ("yfinance",) + provider = get_provider() + assert provider.name == "yfinance" + assert isinstance(provider, MarketDataProvider) + # Instances are cached per resolved name. + assert get_provider("yfinance") is provider + + +def test_registry_rejects_unknown_provider(): + from data_pipeline.providers import get_provider + + with pytest.raises(ValueError, match="unknown data provider"): + get_provider("does-not-exist") + + +def test_registry_honours_env_override(monkeypatch): + from data_pipeline.providers import get_provider + + monkeypatch.setenv("MARKET_DATA_PROVIDER", "nope") + with pytest.raises(ValueError): + get_provider() + + +# --------------------------------------------------------------------------- +# Canonical mapping +# --------------------------------------------------------------------------- +def _yf_bars_frame() -> pd.DataFrame: + idx = pd.DatetimeIndex(["2026-01-02", "2026-01-05"]) + return pd.DataFrame( + { + "Open": [100.0, 101.0], + "High": [102.0, 103.0], + "Low": [99.0, 100.0], + "Close": [101.0, 102.0], + "Adj Close": [100.5, 101.5], + "Volume": [1_000_000, 1_100_000], + }, + index=idx, + ) + + +def test_to_canonical_bars_renames_and_orders_columns(): + from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS + from data_pipeline.providers.yfinance_provider import to_canonical_bars + + out = to_canonical_bars(_yf_bars_frame()) + + assert tuple(out.columns) == CANONICAL_BAR_COLUMNS + assert out["adj_close"].tolist() == [100.5, 101.5] + assert out["close"].tolist() == [101.0, 102.0] + assert out.index.is_monotonic_increasing + + +def test_to_canonical_bars_handles_empty_and_missing_columns(): + from data_pipeline.providers.base import CANONICAL_BAR_COLUMNS + from data_pipeline.providers.yfinance_provider import to_canonical_bars + + assert tuple(to_canonical_bars(None).columns) == CANONICAL_BAR_COLUMNS + assert tuple(to_canonical_bars(pd.DataFrame()).columns) == CANONICAL_BAR_COLUMNS + + partial = pd.DataFrame({"Close": [1.0]}, index=pd.DatetimeIndex(["2026-01-02"])) + out = to_canonical_bars(partial) + assert tuple(out.columns) == CANONICAL_BAR_COLUMNS + assert out["adj_close"].isna().all() + + +def _legacy_chain_payload() -> dict: + calls = pd.DataFrame( + { + "strike": [100.0, 105.0], + "bid": [2.0, float("nan")], + "ask": [2.2, float("nan")], + "lastPrice": [2.1, 0.4], + "impliedVolatility": [0.25, 0.30], + "openInterest": [500.0, 120.0], + "volume": [10.0, 0.0], + "inTheMoney": [True, False], + } + ) + puts = pd.DataFrame( + { + "strike": [100.0], + "bid": [1.5], + "ask": [1.7], + "lastPrice": [1.6], + "impliedVolatility": [0.28], + "openInterest": [300.0], + "volume": [5.0], + "inTheMoney": [False], + } + ) + return { + "ticker": "AAPL", + "spot": 101.0, + "expiries": ["2026-01-16"], + "chain": {"2026-01-16": {"calls": calls, "puts": puts}}, + } + + +def test_to_option_chain_snapshot_applies_canonical_units(): + from data_pipeline.providers.yf_snapshot import to_option_chain_snapshot + + snap = to_option_chain_snapshot(_legacy_chain_payload()) + + assert snap.provider == "yfinance" + assert snap.symbol == "AAPL" + assert snap.spot == 101.0 + assert snap.expiries == ("2026-01-16",) + + calls = snap.legs("2026-01-16", "calls") + assert [leg.strike for leg in calls] == [100.0, 105.0] + # iv stays a decimal (0.25 == 25 %); futu's percent form is normalised at the + # provider boundary — see providers/base.py. + assert calls[0].iv == 0.25 + assert calls[1].iv == pytest.approx(0.30) + # NaN quotes become None rather than 0 — "absent" must stay expressible. + assert calls[1].bid is None + assert calls[1].ask is None + assert calls[0].open_interest == 500.0 + + # inTheMoney is deliberately NOT canonical (derivable, and futu has none). + assert not hasattr(calls[0], "in_the_money") + + assert len(snap.legs("2026-01-16", "puts")) == 1 + assert snap.legs("1999-01-01", "calls") == () + + +def test_to_option_chain_snapshot_tolerates_empty_payload(): + from data_pipeline.providers.base import OptionChainSnapshot + from data_pipeline.providers.yf_snapshot import to_option_chain_snapshot + + snap = to_option_chain_snapshot({"ticker": "MSFT", "spot": None, "expiries": [], "chain": {}}) + assert isinstance(snap, OptionChainSnapshot) + assert snap.expiries == () + assert snap.spot is None + + +# --------------------------------------------------------------------------- +# Compatibility shim +# --------------------------------------------------------------------------- +def test_yf_client_reexports_legacy_callables_unchanged(): + from data_pipeline.providers import yf_client, yf_snapshot, yfinance_provider + + assert yf_client.fetch_spot is yf_snapshot.fetch_spot + assert yf_client.fetch_spots_bulk is yf_snapshot.fetch_spots_bulk + assert yf_client.fetch_option_chain is yf_snapshot.fetch_option_chain + assert yf_client.fetch_close_panel is yfinance_provider.fetch_close_panel + assert yf_client.fetch_daily_ohlcv is yfinance_provider.fetch_daily_ohlcv diff --git a/tests/test_quality_log.py b/tests/test_quality_log.py index 1225811..4a27735 100644 --- a/tests/test_quality_log.py +++ b/tests/test_quality_log.py @@ -1,9 +1,9 @@ -"""Tests for data_pipeline/quality_log.py.""" +"""Tests for data_pipeline/store/quality_log.py.""" from __future__ import annotations -from data_pipeline.db import init_db -from data_pipeline.quality_log import failure_counts, log_failure, recent_failures +from data_pipeline.store.db import init_db +from data_pipeline.store.quality_log import failure_counts, log_failure, recent_failures def test_log_and_query_recent(): @@ -27,7 +27,7 @@ def test_failure_counts_aggregates_by_class(): def test_log_failure_swallows_db_errors(monkeypatch): """Logging path must never raise — caller is in an except block.""" - import data_pipeline.quality_log as ql + import data_pipeline.store.quality_log as ql def boom(*a, **kw): raise RuntimeError("db down") diff --git a/tests/test_readiness.py b/tests/test_readiness.py new file mode 100644 index 0000000..39d3a1e --- /dev/null +++ b/tests/test_readiness.py @@ -0,0 +1,265 @@ +"""Readiness planning + prefetch on submit (ADR 0012, batch B5). + +Domain: Tests — Data Readiness +Context: + - B5 turned the implicit, per-slice "discover missing coverage when the tab + loads" flow into an explicit plan computed at POST time. These tests pin the + plan union, the kick decision, the cold-start hold window, and the fact that a + kick really does populate the DB. +Contracts: + - ``plan_datasets`` is the union over the requested modules, one entry per + (ticker, dataset), with live-only modules contributing nothing. + - ``check_and_kick`` probes once per (ticker, range) and kicks exactly the + missing ones. + - ``hold_seconds_left`` holds only on a cold start and only for HOLD_SECONDS. + - ``create_job`` stores the plan; ``FormService.extract_modules`` defaults to + every known module and ignores unknown tokens. +Dependencies UPWARD: + - (none — stdlib + pytest + the packages under test) +""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from data_pipeline.orchestrate import backfill as _bf +from data_pipeline.orchestrate import readiness +from data_pipeline.orchestrate.job_cache import _reset as reset_jobs +from data_pipeline.orchestrate.job_cache import create_job, get_job +from data_pipeline.store.db import fetch_df, init_db + +TODAY = dt.date(2026, 9, 10) + + +# --------------------------------------------------------------------------- +# plan_datasets +# --------------------------------------------------------------------------- +def test_plan_datasets_is_the_union_over_modules(): + plan = readiness.plan_datasets(["AAPL"], ["market_review", "assessment"], today=TODAY) + pairs = {(r.ticker, r.dataset) for r in plan} + assert pairs == {("AAPL", "clean_bars"), ("AAPL", "feature_bars")} + assert {r.module for r in plan} == {"market_review", "assessment"} + + +def test_plan_datasets_dedupes_a_dataset_shared_by_two_modules(): + """market_review and options_chain both read clean_bars → one entry.""" + plan = readiness.plan_datasets(["AAPL"], ["market_review", "options_chain"], today=TODAY) + assert len(plan) == 1 + assert plan[0].dataset == "clean_bars" + assert plan[0].module == "market_review" # first module wins (documented) + + +def test_plan_datasets_covers_every_ticker(): + plan = readiness.plan_datasets(["AAPL", "MSFT"], ["statistical"], today=TODAY) + assert {r.ticker for r in plan} == {"AAPL", "MSFT"} + assert all(r.dataset == "feature_bars" for r in plan) + + +def test_plan_datasets_is_empty_for_live_only_modules(): + assert readiness.plan_datasets(["AAPL"], ["regime", "payoff_ratio", "simulation"], today=TODAY) == [] + + +def test_plan_datasets_horizon_defaults_and_overrides(): + default = readiness.plan_datasets(["AAPL"], ["statistical"], today=TODAY)[0] + assert default.end == TODAY + assert default.start == TODAY - dt.timedelta(days=readiness.DEFAULT_LOOKBACK_DAYS) + + explicit = readiness.plan_datasets( + ["AAPL"], ["statistical"], start=dt.date(2020, 1, 1), end=dt.date(2021, 1, 1), today=TODAY + )[0] + assert (explicit.start, explicit.end) == (dt.date(2020, 1, 1), dt.date(2021, 1, 1)) + + +# --------------------------------------------------------------------------- +# check_and_kick +# --------------------------------------------------------------------------- +def test_check_and_kick_kicks_missing_ranges_once_per_ticker(monkeypatch): + init_db() + monkeypatch.setattr(_bf, "needs_backfill", lambda ticker, start, end: True) + kicked: list[tuple] = [] + plan = readiness.plan_datasets(["AAPL", "MSFT"], ["market_review", "statistical"], today=TODAY) + + statuses = readiness.check_and_kick(plan, kick=lambda t, s, e: kicked.append((t, s, e))) + + expected_start = TODAY - dt.timedelta(days=readiness.DEFAULT_LOOKBACK_DAYS) + assert sorted(kicked) == [("AAPL", expected_start, TODAY), ("MSFT", expected_start, TODAY)] + assert len(statuses) == 4 + # One status per (ticker, dataset) — AAPL carries two datasets, MSFT two. + assert {s.ticker for s in statuses} == {"AAPL", "MSFT"} + assert all(s.state == "kicked" for s in statuses) + assert all(s.kicked_at > 0 for s in statuses) + + +def test_check_and_kick_skips_covered_ranges(monkeypatch): + init_db() + monkeypatch.setattr(_bf, "needs_backfill", lambda ticker, start, end: False) + kicked: list[tuple] = [] + + statuses = readiness.check_and_kick( + readiness.plan_datasets(["AAPL"], ["statistical"], today=TODAY), kick=lambda *a: kicked.append(a) + ) + + assert kicked == [] + assert [s.state for s in statuses] == ["covered"] + assert statuses[0].kicked_at == 0.0 + + +def test_check_and_kick_survives_a_failing_probe(monkeypatch): + """A probe must never break POST / — a failure degrades to 'not missing'.""" + init_db() + + def _boom(*_a, **_kw): + raise RuntimeError("db is unhappy") + + monkeypatch.setattr(_bf, "needs_backfill", _boom) + statuses = readiness.check_and_kick( + readiness.plan_datasets(["AAPL"], ["statistical"], today=TODAY), kick=lambda *a: None + ) + assert [s.state for s in statuses] == ["covered"] + + +# --------------------------------------------------------------------------- +# Cold-start hold window +# --------------------------------------------------------------------------- +def _status(state: str, *, has_data: bool, kicked_at: float = 100.0) -> readiness.ReadinessStatus: + return readiness.ReadinessStatus( + ticker="AAPL", dataset="feature_bars", module="statistical", state=state, kicked_at=kicked_at, has_data=has_data + ) + + +def test_hold_is_none_when_nothing_was_kicked(): + assert readiness.hold_seconds_left(None) is None + assert readiness.hold_seconds_left(_status("covered", has_data=False)) is None + + +def test_hold_is_none_when_the_ticker_already_has_usable_history(): + """A partial gap must paint now — only a cold start is worth holding.""" + assert readiness.hold_seconds_left(_status("kicked", has_data=True)) is None + + +def test_hold_counts_down_and_expires(): + cold = _status("kicked", has_data=False, kicked_at=100.0) + assert readiness.hold_seconds_left(cold, now=100.0) == pytest.approx(readiness.HOLD_SECONDS) + assert readiness.hold_seconds_left(cold, now=100.0 + readiness.HOLD_SECONDS - 1) == pytest.approx(1.0) + assert readiness.hold_seconds_left(cold, now=100.0 + readiness.HOLD_SECONDS + 1) is None + + +def test_should_hold_stops_as_soon_as_the_backfill_is_gone(): + """A dead (failed or finished) backfill must not keep the tab in 'preparing'. + + This is the difference between "still downloading" and "download failed": once + the thread exits, /render/* computes and the slice reports the real outcome. + """ + cold = _status("kicked", has_data=False, kicked_at=100.0) + assert readiness.should_hold(cold, now=100.0) is False # no live thread in this test + assert readiness.should_hold(None) is False + # Timer alone (hold_seconds_left) is what the count-down test above pins. + assert readiness.is_backfill_running(cold) is False + + +def test_status_for_maps_a_module_to_its_plan_entry(): + plan = [ + readiness.ReadinessStatus("AAPL", "clean_bars", "market_review", "covered"), + readiness.ReadinessStatus("AAPL", "feature_bars", "statistical", "kicked", kicked_at=5.0), + ] + assert readiness.status_for(plan, "AAPL", "statistical").dataset == "feature_bars" + assert readiness.status_for(plan, "AAPL", "regime") is None # live-only module + assert readiness.status_for(plan, "MSFT", "statistical") is None + assert readiness.status_for(None, "AAPL", "statistical") is None + + +# --------------------------------------------------------------------------- +# kick_backfill really populates the DB (offline TEST_ fixture) +# --------------------------------------------------------------------------- +def test_kick_backfill_populates_the_db(): + init_db() + ticker = "TEST_AAPL" + end = dt.date.today() + start = end - dt.timedelta(days=45) + + readiness.kick_backfill(ticker, start, end) + readiness.join_backfills(timeout=60) + + df = fetch_df("SELECT date FROM clean_bars WHERE ticker=?", (ticker,)) + assert len(df.index) > 0, "cold-start kick did not populate clean_bars" + + +def test_kick_backfill_dedupes_an_in_flight_range(): + """Two kicks for the same range must collapse into one thread.""" + init_db() + ticker = "TEST_AAPL" + end = dt.date.today() + start = end - dt.timedelta(days=30) + + readiness.kick_backfill(ticker, start, end) + with readiness._backfill_lock: + first = readiness._backfill_threads[(ticker, str(start), str(end))] + readiness.kick_backfill(ticker, start, end) + with readiness._backfill_lock: + second = readiness._backfill_threads.get((ticker, str(start), str(end))) + assert first is second + + readiness.join_backfills(timeout=60) + + +# --------------------------------------------------------------------------- +# job_cache carries the plan +# --------------------------------------------------------------------------- +def test_create_job_stores_the_plan(): + reset_jobs() + plan = [readiness.ReadinessStatus("AAPL", "clean_bars", "market_review", "covered")] + job_id = create_job({"ticker": "AAPL"}, ["AAPL"], plan) + assert get_job(job_id).plan == plan + + +def test_create_job_without_a_plan_defaults_to_empty(): + reset_jobs() + job_id = create_job({"ticker": "AAPL"}, ["AAPL"]) + assert get_job(job_id).plan == [] + + +# --------------------------------------------------------------------------- +# FormService.extract_modules +# --------------------------------------------------------------------------- +class _FakeRequest: + def __init__(self, values: list[str] | None): + self._values = values or [] + + class _Form(dict): + def __init__(self, values): + super().__init__() + self._values = values + + def getlist(self, key): + return self._values if key == "modules" else [] + + @property + def form(self): + return self._Form(self._values) + + +def test_extract_modules_defaults_to_every_known_module(): + from services.market.form import FormService + + assert FormService.extract_modules(_FakeRequest(None)) == list(readiness.ALL_MODULES) + + +def test_extract_modules_accepts_repeated_and_comma_separated_values(): + from services.market.form import FormService + + assert FormService.extract_modules(_FakeRequest(["market_review", "statistical"])) == [ + "market_review", + "statistical", + ] + assert FormService.extract_modules(_FakeRequest(["market_review,statistical"])) == [ + "market_review", + "statistical", + ] + + +def test_extract_modules_drops_unknown_tokens_and_dedupes(): + from services.market.form import FormService + + assert FormService.extract_modules(_FakeRequest(["statistical", "nope", "statistical"])) == ["statistical"] diff --git a/tests/test_regime.py b/tests/test_regime.py index 7d861bc..55a1805 100644 --- a/tests/test_regime.py +++ b/tests/test_regime.py @@ -197,7 +197,7 @@ def test_label_series_empty_inputs_returns_structured_df(): def test_fetch_df_aggregate_query_without_date_column(): """Regression: ``fetch_df`` must not try to index by 'date' on queries that don't select a date column (e.g. ``SELECT MAX(date)``).""" - from data_pipeline.db import fetch_df, init_db + from data_pipeline.store.db import fetch_df, init_db init_db() df = fetch_df("SELECT MAX(date) as max_date FROM raw_prices WHERE ticker=?", ("DOES_NOT_EXIST",)) diff --git a/tests/test_render_streaming.py b/tests/test_render_streaming.py index 9ffec1b..984faae 100644 --- a/tests/test_render_streaming.py +++ b/tests/test_render_streaming.py @@ -11,7 +11,7 @@ import pytest -from data_pipeline import job_cache as jc +from data_pipeline.orchestrate import job_cache as jc @pytest.fixture(autouse=True) diff --git a/tests/test_route_param_validation.py b/tests/test_route_param_validation.py index 2808d28..9aafbb1 100644 --- a/tests/test_route_param_validation.py +++ b/tests/test_route_param_validation.py @@ -96,14 +96,14 @@ def _boom(*args, **kwargs): # noqa: ARG001 class TestReposColumnWhitelist: def test_unknown_column_raises_before_sql(self): - from data_pipeline.repos import select_tracked_strategies + from data_pipeline.store.repos import select_tracked_strategies with pytest.raises(ValueError, match="unknown tracked_strategies columns"): select_tracked_strategies(["id", "notes; DROP TABLE tracked_strategies--"], None) def test_known_columns_accepted(self): - from data_pipeline.db import init_db - from data_pipeline.repos import select_tracked_strategies + from data_pipeline.store.db import init_db + from data_pipeline.store.repos import select_tracked_strategies init_db() rows = select_tracked_strategies(["id", "ticker", "status"], None) diff --git a/tests/test_scheduler_lock.py b/tests/test_scheduler_lock.py index 9c0e30c..3192c5a 100644 --- a/tests/test_scheduler_lock.py +++ b/tests/test_scheduler_lock.py @@ -3,7 +3,7 @@ import os import tempfile -from data_pipeline.scheduler import acquire_scheduler_lock +from data_pipeline.orchestrate.scheduler import acquire_scheduler_lock def test_first_acquire_succeeds_second_returns_none(monkeypatch): diff --git a/tests/test_scheduler_optional_dep.py b/tests/test_scheduler_optional_dep.py index 54b3eb7..71e764c 100644 --- a/tests/test_scheduler_optional_dep.py +++ b/tests/test_scheduler_optional_dep.py @@ -40,7 +40,7 @@ def test_importing_scheduler_module_does_not_import_apscheduler(): """Module import alone must not require the package (see constraints §6).""" result = _run( "import sys\n" - "import data_pipeline.scheduler\n" + "import data_pipeline.orchestrate.scheduler\n" "assert 'apscheduler' not in sys.modules, 'apscheduler imported at module scope'\n" ) @@ -52,7 +52,7 @@ def test_missing_apscheduler_raises_actionable_error(): result = _run( _BLOCK_APSCHEDULER + ( - "from data_pipeline.scheduler import UpdateScheduler\n" + "from data_pipeline.orchestrate.scheduler import UpdateScheduler\n" "try:\n" " UpdateScheduler()\n" "except ModuleNotFoundError as exc:\n" diff --git a/tests/test_strategy_builder.py b/tests/test_strategy_builder.py index b2dcbd8..c1f5f35 100644 --- a/tests/test_strategy_builder.py +++ b/tests/test_strategy_builder.py @@ -49,9 +49,9 @@ def _fake_chain(spot: float = 100.0, expiry: str = "2099-12-31"): def patched(monkeypatch): monkeypatch.setattr(sb, "fetch_option_chain", lambda t: _fake_chain()) # Skip DB lookup for vol context — return None - from data_pipeline import data_ops as _dops + from data_pipeline import read as _read_pkg - monkeypatch.setattr(_dops.DataService, "get_cleaned_daily", staticmethod(lambda *a, **kw: pd.DataFrame())) + monkeypatch.setattr(_read_pkg.DataService, "get_cleaned_daily", staticmethod(lambda *a, **kw: pd.DataFrame())) return monkeypatch diff --git a/tests/test_ticker_format_integration.py b/tests/test_ticker_format_integration.py index 4b31dfa..9725a97 100644 --- a/tests/test_ticker_format_integration.py +++ b/tests/test_ticker_format_integration.py @@ -216,45 +216,23 @@ class TestMarketAnalyzerFeaturesNotEmpty: """features_df must have rows when the horizon contains sufficient data.""" def _build_analyzer_with_mock_data(self, start_date, end_date=None, frequency="D", data_days=60): - """Create a MarketAnalyzer with mocked price data routed through the canonical DataContext path.""" + """Create a MarketAnalyzer from synthetic bars — no I/O (batch B4). + + WHY the pure builder: ``core/`` no longer fetches (ADR 0001), so the + context is assembled from the fixture frame and injected. + """ from core.market.analyzer import MarketAnalyzer - from core.market.data_context import DataContext - from core.market.models import Horizon + from core.market.data_context import Horizon, build_data_context fake_df = _make_daily_ohlcv(days=data_days, start="2025-12-01") - - with patch.object( - MarketAnalyzer, - "__init__", - lambda self, *a, **kw: None, - ): - analyzer = MarketAnalyzer.__new__(MarketAnalyzer) - - # Replicate _refrequency for D - resampled = fake_df.copy() - resampled["LastClose"] = resampled["Close"].shift(1) - resampled["LastAdjClose"] = resampled["Adj Close"].shift(1) - horizon = Horizon( start=start_date, end=end_date or dt.date.today(), user_provided_end=end_date is not None, frequency=frequency, ) - ctx = DataContext( - ticker="TEST", - frequency=frequency, - horizon=horizon, - bars=resampled, - daily_bars=fake_df, - ) - - analyzer._ctx = ctx - analyzer.ticker = "TEST" - analyzer.frequency = frequency - analyzer.end_date = end_date - analyzer.features_df = ctx.features_df - return analyzer + ctx = build_data_context(ticker="TEST", frequency=frequency, horizon=horizon, raw_data=fake_df) + return MarketAnalyzer(ctx) def test_features_not_empty_two_month_daily(self): """2-month daily horizon should produce a non-empty features_df.""" diff --git a/tests/test_yf_failure_injection.py b/tests/test_yf_failure_injection.py index ca1836f..72cdb96 100644 --- a/tests/test_yf_failure_injection.py +++ b/tests/test_yf_failure_injection.py @@ -30,15 +30,10 @@ import pytest from data_pipeline import PipelineResult -from data_pipeline.data_ops import ( - DataService, - _query_cache, - _query_cache_lock, - _update_lock_mutex, - _update_locks, -) -from data_pipeline.db import fetch_df, init_db, upsert_many -from data_pipeline.downloader import _download_yf, upsert_raw_prices +from data_pipeline._state import _query_cache, _query_cache_lock, _update_lock_mutex, _update_locks +from data_pipeline.ingest.ohlcv import download_bars, upsert_raw_prices +from data_pipeline.read import DataService +from data_pipeline.store.db import fetch_df, init_db, upsert_many # --------------------------------------------------------------------------- # Helpers @@ -99,8 +94,8 @@ class _FakeRateLimitError(Exception): class TestDownloadExceptions: """Network / yfinance exceptions must be caught and reported, not propagated.""" - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_rate_limit_429_returns_failed_result(self, mock_throttle, mock_dl): mock_dl.side_effect = _FakeRateLimitError("429 Too Many Requests") # Use a far-past start so the staleness check can't short-circuit. @@ -116,8 +111,8 @@ def test_rate_limit_429_returns_failed_result(self, mock_throttle, mock_dl): # Throttle must have been called once before the doomed download. assert mock_throttle.call_count == 1 - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_connection_timeout_returns_failed_result(self, mock_throttle, mock_dl): mock_dl.side_effect = TimeoutError("Connection timed out") end = dt.date(2024, 1, 10) @@ -130,8 +125,8 @@ def test_connection_timeout_returns_failed_result(self, mock_throttle, mock_dl): assert "timed out" in (result.error or "").lower() assert result.rows == 0 - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_generic_exception_does_not_crash(self, mock_throttle, mock_dl): mock_dl.side_effect = RuntimeError("yfinance internal boom") end = dt.date(2024, 1, 10) @@ -150,8 +145,8 @@ def test_generic_exception_does_not_crash(self, mock_throttle, mock_dl): class TestDownloadEmptyData: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_empty_dataframe_records_warning_no_crash(self, mock_throttle, mock_dl): mock_dl.return_value = pd.DataFrame() end = dt.date(2024, 1, 10) @@ -165,8 +160,8 @@ def test_empty_dataframe_records_warning_no_crash(self, mock_throttle, mock_dl): assert result.rows == 0 assert any("No new data" in w for w in result.warnings) - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_none_response_treated_as_empty(self, mock_throttle, mock_dl): mock_dl.return_value = None end = dt.date(2024, 1, 10) @@ -183,8 +178,8 @@ def test_none_response_treated_as_empty(self, mock_throttle, mock_dl): class TestStalenessSkip: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_fresh_db_skips_download(self, mock_throttle, mock_dl): """If every business day in [start, end] is already in raw_prices, no yfinance call is made.""" end = dt.date.today() @@ -203,8 +198,8 @@ def test_fresh_db_skips_download(self, mock_throttle, mock_dl): mock_dl.assert_not_called() mock_throttle.assert_not_called() - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_stale_db_triggers_download(self, mock_throttle, mock_dl): """If DB only has very old data, the download proceeds (and gets rate-limited in this test, but that's fine — we only assert that yf.download was attempted).""" @@ -225,8 +220,8 @@ def test_stale_db_triggers_download(self, mock_throttle, mock_dl): class TestDbSurvivesFailure: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_existing_rows_survive_429(self, mock_throttle, mock_dl): """A 429 during update must NOT delete or corrupt existing DB rows.""" end = dt.date.today() @@ -262,10 +257,10 @@ def test_throttle_called_before_download(self): parent.dl.return_value = _make_yf_frame(dt.date(2024, 1, 1), days=3) with ( - patch("data_pipeline.downloader.yf_throttle", parent.throttle), - patch("data_pipeline.downloader.yf.download", parent.dl), + patch("data_pipeline.providers.yfinance_provider.yf_throttle", parent.throttle), + patch("data_pipeline.providers.yfinance_provider.yf.download", parent.dl), ): - _download_yf("ORDER_TKR", dt.date(2024, 1, 1), dt.date(2024, 1, 5)) + download_bars("ORDER_TKR", dt.date(2024, 1, 1), dt.date(2024, 1, 5)) # First parent call must be throttle, then download. names = [c[0] for c in parent.mock_calls if c[0] in {"throttle", "dl"}] @@ -279,8 +274,8 @@ def test_throttle_called_before_download(self): class TestManualUpdateGracefulFailure: - @patch("data_pipeline.downloader.yf.download") - @patch("data_pipeline.downloader.yf_throttle") + @patch("data_pipeline.providers.yfinance_provider.yf.download") + @patch("data_pipeline.providers.yfinance_provider.yf_throttle") def test_manual_update_returns_false_on_429(self, mock_throttle, mock_dl): """`DataService.manual_update` must report False, not raise, on a 429.""" init_db() @@ -290,7 +285,7 @@ def test_manual_update_returns_false_on_429(self, mock_throttle, mock_dl): result = DataService.manual_update("E2E_TKR") assert result is False - @patch("data_pipeline.downloader.upsert_raw_prices") + @patch("data_pipeline.ingest.ohlcv.upsert_raw_prices") def test_manual_update_returns_false_on_pipeline_error_field(self, mock_upsert): """Even if downloader returns ok=False (rather than raising), we degrade gracefully.""" init_db() diff --git a/tests/unit/coverage.test.js b/tests/unit/coverage.test.js index 68c4e4b..24cf500 100644 --- a/tests/unit/coverage.test.js +++ b/tests/unit/coverage.test.js @@ -26,6 +26,12 @@ import '../../static/utils.js'; import '../../static/cache.js'; import '../../static/simulation.js'; import '../../static/theme.js'; +import '../../static/parametersBar.js'; +import '../../static/state/paramsStore.js'; +import '../../static/state/marketParamsState.js'; +import '../../static/state/assessmentParamsState.js'; +import '../../static/state/optionFilterState.js'; +import '../../static/moduleParams.js'; // Capture the post-import surface BEFORE the setup `beforeEach` runs and // wipes globals. We re-attach them in a local `beforeEach` so each `it` @@ -46,6 +52,11 @@ const _snapshot = { loadSimulationTab: window.loadSimulationTab, runSimulation: window.runSimulation, themeManager: window.themeManager, + parametersBar: window.parametersBar, + marketParams: window.appState.marketParams, + assessmentParams: window.appState.assessmentParams, + optionFilter: window.appState.optionFilter, + moduleParams: window.moduleParams, }; beforeEach(() => { @@ -83,6 +94,20 @@ describe('coverage smoke — every module publishes its surface', () => { expect(typeof window.parseTickers).toBe('function'); }); + it('parameters bar published its window surface', () => { + expect(window.parametersBar).toBeDefined(); + expect(typeof window.parametersBar.init).toBe('function'); + expect(window.parametersBar.STORAGE_KEY).toBe('parametersBarCollapsed'); + }); + + it('module parameter groups published their window surface', () => { + for (const key of ['marketParams', 'assessmentParams', 'optionFilter']) { + expect(typeof window.appState[key].get).toBe('function'); + expect(typeof window.appState[key].query).toBe('function'); + } + expect(typeof window.moduleParams.rerun).toBe('function'); + }); + it('simulation tab published its window surface', () => { expect(typeof window.loadSimulationTab).toBe('function'); expect(typeof window.runSimulation).toBe('function'); diff --git a/tests/unit/parametersBar.test.js b/tests/unit/parametersBar.test.js new file mode 100644 index 0000000..7fcd3ee --- /dev/null +++ b/tests/unit/parametersBar.test.js @@ -0,0 +1,124 @@ +/** + * Tests for static/parametersBar.js — the persistent Parameters bar (batch B6). + * + * Contract under test: + * - expands by default, collapses on the toggle, and remembers the choice; + * - the collapsed bar shows the current ticker as `▸ AAPL`; + * - a browser with storage denied must degrade to "not persisted", never throw. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { loadScript } from './_loadScript.js'; + +const BAR_HTML = ` +
+ + +
+ + +
+
`; + +function bar() { + return document.querySelector('.parameters-bar'); +} + +/** Mount the markup, then load the script — its auto-init runs on 'complete'. */ +function mount() { + document.body.innerHTML = BAR_HTML; + delete window.parametersBar; + loadScript('static/parametersBar.js'); +} + +beforeEach(() => { + window.localStorage.clear(); +}); + +describe('parametersBar — collapse persistence', () => { + it('expands by default when nothing is stored', () => { + mount(); + expect(bar().dataset.collapsed).toBe('false'); + expect(document.getElementById('parameters-bar-toggle').getAttribute('aria-expanded')).toBe('true'); + }); + + it('collapses on toggle and persists the choice', () => { + mount(); + document.getElementById('parameters-bar-toggle').click(); + + expect(bar().dataset.collapsed).toBe('true'); + expect(window.localStorage.getItem('parametersBarCollapsed')).toBe('true'); + expect(document.getElementById('parameters-bar-toggle').getAttribute('aria-expanded')).toBe('false'); + expect(document.getElementById('parameters-bar-toggle').title).toBe('Expand parameters'); + }); + + it('has a real element behind aria-controls (the collapsible fields)', () => { + mount(); + const toggle = document.getElementById('parameters-bar-toggle'); + const target = document.getElementById(toggle.getAttribute('aria-controls')); + expect(target).not.toBeNull(); + expect(target.contains(document.getElementById('ticker'))).toBe(true); + }); + + it('restores the collapsed state on the next page load', () => { + window.localStorage.setItem('parametersBarCollapsed', 'true'); + mount(); + expect(bar().dataset.collapsed).toBe('true'); + }); + + it('expands again when toggled twice', () => { + mount(); + const toggle = document.getElementById('parameters-bar-toggle'); + toggle.click(); + toggle.click(); + + expect(bar().dataset.collapsed).toBe('false'); + expect(window.localStorage.getItem('parametersBarCollapsed')).toBe('false'); + }); +}); + +describe('parametersBar — collapsed summary', () => { + it('mirrors the ticker input into the one-line summary', () => { + mount(); + const input = document.getElementById('ticker'); + input.value = '^SPX'; + input.dispatchEvent(new Event('input')); + + expect(document.getElementById('parameters-bar-summary').textContent).toBe('▸ ^SPX'); + }); + + it('is empty when the ticker is blank', () => { + mount(); + const input = document.getElementById('ticker'); + input.value = ' '; + input.dispatchEvent(new Event('input')); + + expect(document.getElementById('parameters-bar-summary').textContent).toBe(''); + }); +}); + +describe('parametersBar — degraded storage', () => { + it('does not throw when localStorage is denied', () => { + const getItem = vi.spyOn(window.localStorage, 'getItem').mockImplementation(() => { + throw new Error('storage denied'); + }); + const setItem = vi.spyOn(window.localStorage, 'setItem').mockImplementation(() => { + throw new Error('storage denied'); + }); + + expect(() => mount()).not.toThrow(); + // Toggling must still work — it only loses persistence. + expect(() => document.getElementById('parameters-bar-toggle').click()).not.toThrow(); + expect(bar().dataset.collapsed).toBe('true'); + + getItem.mockRestore(); + setItem.mockRestore(); + }); + + it('is inert when the bar is absent from the page', () => { + document.body.innerHTML = ''; + expect(() => mount()).not.toThrow(); + }); +}); diff --git a/tests/unit/paramsStore.test.js b/tests/unit/paramsStore.test.js new file mode 100644 index 0000000..947b835 --- /dev/null +++ b/tests/unit/paramsStore.test.js @@ -0,0 +1,213 @@ +/** + * Tests for static/state/paramsStore.js + the three module parameter groups + * (batch B7). + * + * Contract under test: + * - a group hydrates its toolbar inputs from its own localStorage key; + * - a change on any bound input commits the WHOLE group and emits + * `module:params-changed` with the modules that consume it; + * - the shared horizon is one field bound to three toolbars, so all three + * inputs stay in sync; + * - disabled storage degrades to defaults instead of throwing; + * - `init()` returns the store (callers chain `.init()` onto the factory). + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { loadScript } from './_loadScript.js'; + +const MARKET_TOOLBAR_HTML = ` + + + + + + + + + +`; + +const ASSESS_TOOLBAR_HTML = ` + + + + +`; + +const OPTION_TOOLBAR_HTML = ` + + + + +`; + +function mount(html) { + document.body.innerHTML = html; +} + +beforeEach(() => { + window.localStorage.clear(); + delete window.appState; + delete window.createParamsStore; + delete window.__paramsDebug; + loadScript('static/eventBus.js'); + loadScript('static/state/store.js'); + loadScript('static/state/paramsStore.js'); +}); + +/** Mount the markup, THEN load the store scripts — the real page order: the + * toolbars are parsed before the scripts hydrate and bind them. */ +function loadStores() { + loadScript('static/state/marketParamsState.js'); + loadScript('static/state/assessmentParamsState.js'); + loadScript('static/state/optionFilterState.js'); +} + +describe('marketParams — hydration at parse time', () => { + it('restores the group from its own localStorage key', () => { + window.localStorage.setItem( + 'marketParams', + JSON.stringify({ from: '2024-01', to: '2024-03', frequency: 'W' }), + ); + mount(MARKET_TOOLBAR_HTML); + loadStores(); + + expect(window.appState.marketParams.get()).toEqual({ + from: '2024-01', + to: '2024-03', + frequency: 'W', + }); + expect(document.getElementById('stat-frequency').value).toBe('W'); + expect(document.getElementById('assess-frequency').value).toBe('W'); + }); + + it('mirrors the horizon into the bar\u2019s submit-only inputs', () => { + window.localStorage.setItem( + 'marketParams', + JSON.stringify({ from: '2024-01', to: '2024-03' }), + ); + mount(MARKET_TOOLBAR_HTML); + loadStores(); + + expect(document.getElementById('start_time').value).toBe('2024-01'); + expect(document.getElementById('end_time').value).toBe('2024-03'); + }); +}); + +describe('marketParams — commit + emit', () => { + it('commits a toolbar change and emits only the modules that consume it', () => { + mount(MARKET_TOOLBAR_HTML); + loadStores(); + const seen = []; + window.bus.on('module:params-changed', (payload) => seen.push(payload)); + + const select = document.getElementById('stat-frequency'); + select.value = 'W'; + select.dispatchEvent(new Event('change', { bubbles: true })); + + expect(window.appState.marketParams.get().frequency).toBe('W'); + expect(window.localStorage.getItem('marketParams')).toContain('"frequency":"W"'); + expect(seen).toHaveLength(1); + expect(seen[0].modules).toEqual(['market_review', 'statistical', 'assessment']); + }); + + it('keeps the three toolbars in sync through the shared horizon', () => { + mount(MARKET_TOOLBAR_HTML); + loadStores(); + const input = document.getElementById('assess-from'); + input.value = '2024-05'; + input.dispatchEvent(new Event('change', { bubbles: true })); + + expect(document.getElementById('mr-from').value).toBe('2024-05'); + expect(document.getElementById('stat-from').value).toBe('2024-05'); + expect(document.getElementById('start_time').value).toBe('2024-05'); + }); + + it('exposes a query fragment for the module URL builder', () => { + window.localStorage.setItem( + 'marketParams', + JSON.stringify({ from: '2024-01', to: '2024-03', frequency: 'W' }), + ); + mount(MARKET_TOOLBAR_HTML); + loadStores(); + + expect(window.appState.marketParams.query(['from', 'to', 'frequency'])).toBe( + 'from=2024-01&to=2024-03&frequency=W', + ); + }); +}); + +describe('assessmentParams', () => { + it('commits its own knobs', () => { + mount(ASSESS_TOOLBAR_HTML); + loadStores(); + const input = document.getElementById('assess-risk-threshold'); + input.value = '85'; + input.dispatchEvent(new Event('change', { bubbles: true })); + + expect(window.appState.assessmentParams.get().risk_threshold).toBe('85'); + expect(window.localStorage.getItem('assessmentParams')).toContain('"risk_threshold":"85"'); + }); +}); + +describe('optionFilter', () => { + it('hydrates and commits the chain filters', () => { + window.localStorage.setItem( + 'optionFilter', + JSON.stringify({ max_dte: '30', moneyness_low: '0.80', moneyness_high: '1.20' }), + ); + mount(OPTION_TOOLBAR_HTML); + loadStores(); + + expect(document.getElementById('oc-max-dte').value).toBe('30'); + + const input = document.getElementById('oc-refresh-interval'); + input.value = '120'; + input.dispatchEvent(new Event('change', { bubbles: true })); + + expect(window.appState.optionFilter.get().refresh_interval).toBe('120'); + }); + + it('does not ask the module rerunner for a /render round trip', () => { + mount(OPTION_TOOLBAR_HTML); + loadStores(); + expect(window.appState.optionFilter.MODULES).toEqual([]); + }); +}); + +describe('paramsStore — degraded storage', () => { + it('falls back to defaults when localStorage is denied', () => { + mount(MARKET_TOOLBAR_HTML); + loadStores(); + vi.spyOn(window.localStorage, 'getItem').mockImplementation(() => { + throw new Error('denied'); + }); + vi.spyOn(window.localStorage, 'setItem').mockImplementation(() => { + throw new Error('denied'); + }); + + expect(() => loadScript('static/state/marketParamsState.js')).not.toThrow(); + expect(() => window.appState.marketParams.set('frequency', 'Q')).not.toThrow(); + expect(window.appState.marketParams.get().frequency).toBe('Q'); + + vi.restoreAllMocks(); + }); +}); + +describe('paramsStore — init returns the store', () => { + it('does not clobber the published global', () => { + mount(MARKET_TOOLBAR_HTML); + loadStores(); + const published = window.appState.marketParams; + expect(published).toBeTruthy(); + expect(typeof published.init).toBe('function'); + expect(published.init()).toBe(published); + }); +}); diff --git a/utils/ticker_utils.py b/utils/ticker_utils.py index fa051ce..26466da 100644 --- a/utils/ticker_utils.py +++ b/utils/ticker_utils.py @@ -32,7 +32,7 @@ # Syntactic whitelist for any ticker we are willing to forward to yfinance / DB. # WHY: Without this, validate_ticker() accepts arbitrary strings (including XSS -# payloads and SQL fragments) and persists rows for them in clean_prices — +# payloads and SQL fragments) and persists rows for them in clean_bars — # turning the DB into an attacker-writable surface and a yfinance request # amplifier. The pattern intentionally permits the formats that yfinance/our # code actually use: