From b0875b3da19c6669ffe5ece47a86a4cca327aa69 Mon Sep 17 00:00:00 2001 From: Co-Messi Date: Wed, 8 Jul 2026 11:51:37 +0800 Subject: [PATCH 1/2] Harden security, reliability, and CI from adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes every issue in .roast/REPORT-latest.md (6 High, 10 Medium, 5 Low). API security (High 1, 2): - Refuse non-loopback bind unless HYPERDATA_API_KEY is set (Bearer / X-API-Key auth on all non-health routes) or HYPERDATA_UNSAFE_PUBLIC_API=1 explicitly accepts the risk - CORS wildcard only on loopback; HYPERDATA_CORS_ORIGINS allowlist otherwise - Per-IP REST rate limiting (300 req/min sliding window) - WebSocket: bounded per-client send queues with one writer task per client (drop-and-count instead of unbounded task fan-out), inbound message size/rate limits, bad-message disconnect threshold, heartbeat loop errors logged instead of swallowed Reliability (High 3, 4, 5; Med 8, 9): - Hub startup contract: component failures recorded in status.failed_components, surfaced in /v1/health and an honest degraded log line; position_scanner/market_data report 'starting' until their first cycle succeeds instead of a blind 'ready' - Status-update loop body wrapped so one bad component can't kill the staleness watchdog; HLP z-score alerts debounced to once a minute - Explicit aiohttp timeouts on every external HTTP call (market data, position scanner, HL price poll, Telegram/Discord webhooks) - Exchange payload parse guards: malformed Binance/OKX/Bybit/HL records and orderbook levels are dropped individually and counted (parse_errors in liquidation stats) instead of crashing connection loops - Per-venue orderflow freshness (venue_freshness) so a dead venue can't hide behind the combined feed; hub warns when one venue goes silent Data integrity (Med 1, 3, 4, 5): - Liquidation dedup buckets on exchange event time and the cascade bypass is venue-scoped (also fixes bypass keys that never matched) - Corrupted SQLite files are quarantined to data/corrupted/ with a timestamp instead of deleted - schema_version table with versioned migrations; only duplicate-column errors are treated as already-applied - Wallet addresses validated (0x + 40 hex), normalized to lowercase, and capped at 50k with least-recently-seen expiry Strategy safety (Med 2, 6, 7): - Smart-money ranking requires 10+ trades and $50k+ volume, exposes a sample-size confidence score, and clears stale tiers on requalification - Paper trader: balance-checked position adds, weighted-average entry price, invalid-signal rejection — balance can never go negative - LLM agent: strict first-line-only action parsing (ambiguous output is rejected, never substring-matched), persistent worker thread, and an evals-per-hour budget (LLM_MAX_EVALS_PER_HOUR) Tests and CI (High 6, Low 1-5): - test_position_scanner.py rewritten against the SQLite address store and re-enabled in CI (all network mocked) - New tests/test_hardening.py: bind guard, auth/CORS/rate-limit middleware, WS abuse limits, dedup/cascade, malformed payloads, degraded startup, paper-trader invariants, LLM parsing, quarantine, schema version, alert redaction, per-venue freshness (43 tests) - CI: lint blocking (repo lint debt cleaned), actions pinned by SHA, requirements.lock used for installs, advisory pip-audit step - Dead copy-trading/smart-money handlers deleted from api_server - README/API docs foreground the exposure model; alert logs no longer persist full payloads (wallet intelligence redacted) - run_dashboard menu input no longer mutates executor internals --- .env.example | 13 + .github/workflows/ci.yml | 40 +- README.md | 12 + docs/DATA_INTEGRITY.md | 14 +- pyproject.toml | 5 + run_dashboard.py | 44 +- src/api_server.py | 526 ++++++++++------ src/dashboards/boot.py | 8 +- src/dashboards/combined_dashboard.py | 2 +- src/dashboards/cvd_dashboard.py | 6 +- src/dashboards/hub_panels.py | 4 +- src/dashboards/liquidation_heatmap.py | 5 +- src/dashboards/liquidation_stream.py | 5 +- src/dashboards/liquidation_watch.py | 8 +- src/dashboards/market_overview.py | 8 +- src/dashboards/whale_tracker.py | 6 +- src/data_layer/address_store.py | 65 +- src/data_layer/alerts.py | 30 +- src/data_layer/hlp_tracker.py | 5 +- src/data_layer/hub.py | 405 +++++++----- src/data_layer/liquidation_feed.py | 141 +++-- src/data_layer/market_data.py | 57 +- src/data_layer/orderbook.py | 31 +- src/data_layer/orderflow_engine.py | 27 + src/data_layer/persistence.py | 114 +++- src/data_layer/position_scanner.py | 27 +- src/data_layer/smart_money.py | 76 ++- src/strategies/__init__.py | 4 +- src/strategies/base.py | 3 +- src/strategies/examples/__init__.py | 2 + src/strategies/examples/cvd_momentum.py | 2 +- src/strategies/examples/funding_rate_arb.py | 2 +- .../examples/liquidation_cascade.py | 2 +- src/strategies/examples/whale_follow.py | 2 +- src/strategies/llm_agent.py | 90 +-- src/strategies/paper_trader.py | 53 +- tests/conftest.py | 2 +- tests/test_deribit.py | 4 +- tests/test_funding_rates.py | 4 +- tests/test_hardening.py | 595 ++++++++++++++++++ tests/test_long_short_ratio.py | 4 +- tests/test_market_data.py | 1 + tests/test_orderbook.py | 4 +- tests/test_orderflow.py | 4 +- tests/test_position_scanner.py | 258 ++++---- tests/test_spot_prices.py | 4 +- 46 files changed, 1977 insertions(+), 747 deletions(-) create mode 100644 tests/test_hardening.py diff --git a/.env.example b/.env.example index a323af7..a8038e6 100644 --- a/.env.example +++ b/.env.example @@ -20,3 +20,16 @@ TELEGRAM_CHAT_ID= # Discord: create a webhook in channel settings DISCORD_WEBHOOK_URL= + +# ── REST API exposure (see README "REST API" security note) ───── +# Loopback (127.0.0.1) needs none of these. A non-loopback bind is refused +# unless HYPERDATA_API_KEY is set (auth required on all non-health routes) +# or HYPERDATA_UNSAFE_PUBLIC_API=1 explicitly accepts unauthenticated exposure. +HYPERDATA_API_HOST=127.0.0.1 +HYPERDATA_API_KEY= +HYPERDATA_CORS_ORIGINS= +HYPERDATA_UNSAFE_PUBLIC_API= + +# ── LLM cost guardrail ──────────────────────────────────── +# Max LLM strategy evaluations per hour (default 60). +LLM_MAX_EVALS_PER_HOUR=60 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1da3a12..34161a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,35 +14,28 @@ jobs: python-version: ["3.12", "3.13"] steps: - - uses: actions/checkout@v4 + # Actions pinned by commit SHA (not mutable tags) so a compromised tag + # can't inject code into CI. + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - name: Install dependencies + # requirements.lock pins the runtime deps; -e . adds the package itself. run: | - pip install -e . - pip install pytest ruff + pip install -r requirements.lock + pip install -e . --no-deps + pip install pytest pytest-asyncio ruff pip-audit - name: Lint - # Non-blocking for now: the repo has pre-existing ruff debt (mostly - # E501/I001/F401) unrelated to current work. Tests below are the real - # gate. TODO: clean lint in a focused PR, then drop continue-on-error. - continue-on-error: true - run: ruff check src/ + run: ruff check src/ tests/ - - name: Syntax check all Python files - run: | - find src/ -name "*.py" -exec python -c " - import ast, sys - try: - ast.parse(open(sys.argv[1]).read()) - except SyntaxError as e: - print(f'FAIL: {sys.argv[1]}: {e}') - sys.exit(1) - " {} \; + - name: Dependency vulnerability audit + # Advisory: a newly-published CVE in a pinned dep should be visible + # without failing unrelated PRs. Review failures in the job log. + continue-on-error: true + run: pip-audit -r requirements.lock - name: Test imports run: | @@ -52,5 +45,6 @@ jobs: python -c "from src.dashboards.liquidation_heatmap import LiquidationHeatmapDashboard; print('heatmap: OK')" - name: Run tests - # test_position_scanner.py needs a live exchange connection — skip in CI. - run: python -m pytest tests/ -q --ignore=tests/test_position_scanner.py + # The full suite, position scanner included — its network calls are + # mocked; only tests marked `live` need real exchange connections. + run: python -m pytest tests/ -q diff --git a/README.md b/README.md index 0b5ea7f..bd125ab 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,18 @@ Start the API server alongside or instead of the terminal: python run_api.py --port 8420 ``` +> **⚠️ Security: local-only by default.** The API binds to `127.0.0.1` and is +> intended for loopback use. It serves wallet-derived positions, liquidation +> danger zones, and live order flow — trading intelligence you should not +> expose to a LAN or the internet. A non-loopback bind +> (`HYPERDATA_API_HOST=0.0.0.0`) is **refused** unless you either set +> `HYPERDATA_API_KEY=` (all non-health routes then require +> `Authorization: Bearer ` or `X-API-Key`) or explicitly accept the +> risk with `HYPERDATA_UNSAFE_PUBLIC_API=1`. Restrict browser access with +> `HYPERDATA_CORS_ORIGINS=https://your-app.example` (wildcard CORS applies to +> loopback binds only). Do not front this API with a public tunnel or reverse +> proxy without auth and rate limiting of your own. + ### Endpoints | Endpoint | Description | diff --git a/docs/DATA_INTEGRITY.md b/docs/DATA_INTEGRITY.md index cece95a..1d3ae4a 100644 --- a/docs/DATA_INTEGRITY.md +++ b/docs/DATA_INTEGRITY.md @@ -85,10 +85,16 @@ the WAL is checkpointed, so the DB stays bounded on long-running instances. ## API exposure -The REST/WebSocket API has no authentication and permissive CORS, so it binds to -**loopback (`127.0.0.1`) by default**. To expose it on the LAN, set -`HYPERDATA_API_HOST=0.0.0.0` — only do this behind a trusted network. Numeric -query params are validated (bad values return `400`, not `500`). +The REST/WebSocket API binds to **loopback (`127.0.0.1`) by default**. A +non-loopback bind (`HYPERDATA_API_HOST=0.0.0.0`) is refused at startup unless +either `HYPERDATA_API_KEY` is set — all non-health routes then require +`Authorization: Bearer ` or `X-API-Key: ` — or +`HYPERDATA_UNSAFE_PUBLIC_API=1` explicitly acknowledges the exposure. CORS is +wildcard only on loopback; non-loopback binds send CORS headers only for +origins allowlisted in `HYPERDATA_CORS_ORIGINS` (comma-separated). REST +requests are rate-limited per client IP, and WebSocket clients get bounded +per-client send queues plus inbound message size/rate limits. Numeric query +params are validated (bad values return `400`, not `500`). ## Timestamps diff --git a/pyproject.toml b/pyproject.toml index 3e1bfcb..eeba85c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,11 @@ target-version = "py312" [tool.ruff.lint] select = ["E", "F", "W", "I"] +[tool.ruff.lint.per-file-ignores] +# Presentation layer: ASCII-art boot screens and Rich table markup read +# better unwrapped; everything else must respect the line limit. +"src/dashboards/*" = ["E501"] + [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" diff --git a/run_dashboard.py b/run_dashboard.py index 2ecea5b..cae0ef8 100644 --- a/run_dashboard.py +++ b/run_dashboard.py @@ -76,6 +76,39 @@ async def run_dashboard(hub: HyperDataHub, key: str) -> None: await create_fn().run() +async def _ainput(prompt: str) -> str: + """Read one line of input without blocking the event loop. + + Uses a daemon thread per prompt (human-speed churn only) so a read that + is still pending at exit can never wedge interpreter shutdown — the + failure mode of parking input() inside a ThreadPoolExecutor, whose + non-daemon workers are joined at exit. + """ + import threading + + loop = asyncio.get_running_loop() + fut: asyncio.Future[str] = loop.create_future() + + def _set(value=None, exc=None): + if fut.done(): + return + if exc is not None: + fut.set_exception(exc) + else: + fut.set_result(value) + + def _worker(): + try: + line = input(prompt) + except BaseException as e: # EOFError / KeyboardInterrupt in the thread + loop.call_soon_threadsafe(_set, None, e) + else: + loop.call_soon_threadsafe(_set, line) + + threading.Thread(target=_worker, daemon=True, name="menu-input").start() + return await fut + + async def run_interactive(api_port: int | None = None) -> None: """Boot → menu → pick dashboard → run → back to menu on Ctrl+C.""" console = Console() @@ -89,19 +122,10 @@ async def run_interactive(api_port: int | None = None) -> None: while True: _build_menu(console) - # Read user choice — use a daemon thread so Ctrl+C exits cleanly - import concurrent.futures - _input_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) - _input_pool._threads = set() # ensure daemon threads try: - choice = await asyncio.get_event_loop().run_in_executor( - _input_pool, lambda: input(" Enter choice: ").strip().lower() - ) + choice = (await _ainput(" Enter choice: ")).strip().lower() except (EOFError, KeyboardInterrupt): - _input_pool.shutdown(wait=False) break - finally: - _input_pool.shutdown(wait=False) if choice in ("q", "quit", "exit"): break diff --git a/src/api_server.py b/src/api_server.py index 9f4e613..8dd3f57 100644 --- a/src/api_server.py +++ b/src/api_server.py @@ -12,18 +12,31 @@ # WS: ws://localhost:8420/v1/ws # Send: {"subscribe": ["trade", "liquidation"]} # Recv: {"type": "trade", "data": {...}, "ts": 1234567890.123} + +Security model: + - Binds to loopback by default; loopback needs no credentials. + - A non-loopback bind (HYPERDATA_API_HOST) is refused unless either + HYPERDATA_API_KEY is set (all non-health routes then require it) or + HYPERDATA_UNSAFE_PUBLIC_API=1 explicitly acknowledges the risk. + - CORS is wildcard only on loopback; non-loopback binds must allowlist + origins via HYPERDATA_CORS_ORIGINS (comma-separated), else no CORS. + - Per-IP REST rate limit + WebSocket per-client send queues. """ from __future__ import annotations import asyncio import dataclasses +import hmac +import ipaddress import json import logging import math +import os import time +from collections import deque from typing import Any -from aiohttp import web, WSMsgType +from aiohttp import WSMsgType, web logger = logging.getLogger(__name__) @@ -31,6 +44,29 @@ EVENT_TYPES = {"trade", "liquidation", "signal", "funding_update", "iv_update", "alert", "heartbeat"} MAX_WS_CONNECTIONS = 10 +# Per-client outbound queue depth. When a slow client's queue is full, new +# events are dropped for that client (counted) instead of spawning unbounded +# send tasks that compete with ingestion. +WS_SEND_QUEUE_SIZE = 200 +# Inbound WebSocket message limits: size cap, rate cap, and how many bad +# (non-JSON / oversized / too-fast) messages we tolerate before disconnecting. +WS_MAX_MSG_BYTES = 4096 +WS_MAX_MSGS_PER_10S = 20 +WS_BAD_MSG_LIMIT = 5 + +# Per-IP REST rate limit (sliding window). +RATE_LIMIT_REQUESTS = 300 +RATE_LIMIT_WINDOW_S = 60.0 + + +def _is_loopback_host(host: str) -> bool: + """True if the bind host is loopback-only ('localhost', 127.x, ::1).""" + if host in ("localhost",): + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False def _serialize(obj: Any) -> Any: @@ -49,30 +85,112 @@ def _serialize(obj: Any) -> Any: return obj -_CORS_HEADERS = { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", - "Access-Control-Allow-Headers": "Authorization, Content-Type", -} +def _make_cors_middleware(allowed_origins: set[str] | None): + """CORS middleware factory. + + allowed_origins=None means wildcard (loopback binds only); otherwise the + request Origin must be in the allowlist to receive CORS headers. + """ + + def _cors_headers(request: web.Request) -> dict[str, str]: + if allowed_origins is None: + origin = "*" + else: + req_origin = request.headers.get("Origin", "") + if req_origin not in allowed_origins: + return {} + origin = req_origin + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", + "Access-Control-Allow-Headers": "Authorization, Content-Type, X-API-Key", + **({"Vary": "Origin"} if origin != "*" else {}), + } + @web.middleware + async def cors_middleware(request: web.Request, handler): + headers = _cors_headers(request) + if request.method == "OPTIONS": + return web.Response(status=200, headers=headers) + try: + resp = await handler(request) + except web.HTTPNotFound: + return web.json_response( + {"error": "Endpoint not found", "path": request.path}, + status=404, + headers=headers, + ) + except web.HTTPException as exc: + exc.headers.update(headers) + raise + resp.headers.update(headers) + return resp + + return cors_middleware + + +# Paths reachable without an API key (liveness checks must not need secrets). +_UNAUTHENTICATED_PATHS = {"/v1/health", "/health"} + + +def _make_auth_middleware(api_key: str): + """Require the API key on every route except health checks. + + Accepts either ``Authorization: Bearer `` or ``X-API-Key: ``. + """ + + @web.middleware + async def auth_middleware(request: web.Request, handler): + if request.method == "OPTIONS" or request.path in _UNAUTHENTICATED_PATHS: + return await handler(request) + supplied = request.headers.get("X-API-Key", "") + auth = request.headers.get("Authorization", "") + if auth.startswith("Bearer "): + supplied = supplied or auth[len("Bearer "):] + if not supplied or not hmac.compare_digest(supplied, api_key): + return web.json_response({"error": "Unauthorized"}, status=401) + return await handler(request) + + return auth_middleware + + +class _RateLimiter: + """Sliding-window per-IP request limiter for the REST surface.""" + + def __init__(self, max_requests: int = RATE_LIMIT_REQUESTS, + window_s: float = RATE_LIMIT_WINDOW_S) -> None: + self.max_requests = max_requests + self.window_s = window_s + self._hits: dict[str, deque[float]] = {} + + def allow(self, key: str, now: float | None = None) -> bool: + now = time.time() if now is None else now + dq = self._hits.get(key) + if dq is None: + dq = self._hits.setdefault(key, deque()) + cutoff = now - self.window_s + while dq and dq[0] < cutoff: + dq.popleft() + if len(dq) >= self.max_requests: + return False + dq.append(now) + # Bound tracked IPs so a scan can't grow this dict forever. + if len(self._hits) > 10_000: + stale = [k for k, v in self._hits.items() if not v or v[-1] < cutoff] + for k in stale: + del self._hits[k] + return True -@web.middleware -async def cors_middleware(request: web.Request, handler): - if request.method == "OPTIONS": - return web.Response(status=200, headers=_CORS_HEADERS) - try: - resp = await handler(request) - except web.HTTPNotFound: - return web.json_response( - {"error": "Endpoint not found", "path": request.path}, - status=404, - headers=_CORS_HEADERS, - ) - except web.HTTPException as exc: - exc.headers.update(_CORS_HEADERS) - raise - resp.headers.update(_CORS_HEADERS) - return resp + +def _make_rate_limit_middleware(limiter: _RateLimiter): + @web.middleware + async def rate_limit_middleware(request: web.Request, handler): + remote = request.remote or "unknown" + if not limiter.allow(remote): + return web.json_response({"error": "Rate limit exceeded"}, status=429) + return await handler(request) + + return rate_limit_middleware def _int_param(request: web.Request, name: str, default: int, @@ -112,7 +230,8 @@ def _float_param(request: web.Request, name: str, default: float, # ── WebSocket client tracker ───────────────────────────────────────────── class _WSClient: - __slots__ = ("ws", "subscriptions", "ping_misses", "connected_at") + __slots__ = ("ws", "subscriptions", "ping_misses", "connected_at", + "queue", "writer_task", "dropped_msgs", "msg_times", "bad_msgs") def __init__(self, ws: web.WebSocketResponse, subscriptions: set[str] | None = None): self.ws = ws @@ -120,15 +239,22 @@ def __init__(self, ws: web.WebSocketResponse, subscriptions: set[str] | None = N self.subscriptions: set[str] = subscriptions if subscriptions is not None else set() self.ping_misses: int = 0 self.connected_at: float = time.time() + # Bounded outbound queue drained by a single writer task per client. + self.queue: asyncio.Queue[str] = asyncio.Queue(maxsize=WS_SEND_QUEUE_SIZE) + self.writer_task: asyncio.Task | None = None + self.dropped_msgs: int = 0 + # Inbound abuse tracking (message rate + malformed messages). + self.msg_times: deque[float] = deque(maxlen=WS_MAX_MSGS_PER_10S) + self.bad_msgs: int = 0 class HyperDataAPI: """REST API v1 + WebSocket streaming, backed by a live HyperDataHub.""" def __init__(self, hub, host: str = "127.0.0.1", port: int = 8420) -> None: - # Bind to loopback by default: the API has no auth and CORS is open, so - # it must not be reachable from the LAN unless the operator opts in - # (HYPERDATA_API_HOST=0.0.0.0). See docs/DATA_INTEGRITY.md / README. + # Bind to loopback by default. Non-loopback binds are refused in + # start() unless HYPERDATA_API_KEY is set (auth enforced) or + # HYPERDATA_UNSAFE_PUBLIC_API=1 explicitly acknowledges the risk. self.hub = hub self.host = host self.port = port @@ -136,11 +262,53 @@ def __init__(self, hub, host: str = "127.0.0.1", port: int = 8420) -> None: self._site: web.TCPSite | None = None self._ws_clients: list[_WSClient] = [] self._hooks_installed = False + self._rate_limiter = _RateLimiter() # ── Lifecycle ──────────────────────────────────────────────── + def _resolve_security(self) -> tuple[str, set[str] | None]: + """Validate bind/auth/CORS config. Returns (api_key, cors_allowlist). + + Raises RuntimeError for a non-loopback bind with neither an API key + nor an explicit unsafe acknowledgment. + """ + api_key = os.environ.get("HYPERDATA_API_KEY", "").strip() + unsafe_ack = os.environ.get("HYPERDATA_UNSAFE_PUBLIC_API", "") == "1" + origins_raw = os.environ.get("HYPERDATA_CORS_ORIGINS", "").strip() + origins: set[str] | None = ( + {o.strip() for o in origins_raw.split(",") if o.strip()} + if origins_raw else None + ) + + if _is_loopback_host(self.host): + # Loopback: wildcard CORS unless an allowlist was configured. + return api_key, origins + + if not api_key and not unsafe_ack: + raise RuntimeError( + f"Refusing to bind API to non-loopback host {self.host!r}: the " + "API would expose trading intelligence to the network. Set " + "HYPERDATA_API_KEY to require authentication, or set " + "HYPERDATA_UNSAFE_PUBLIC_API=1 to explicitly accept the risk." + ) + if not api_key: + logger.warning( + "SECURITY: API bound to %s WITHOUT authentication " + "(HYPERDATA_UNSAFE_PUBLIC_API=1). Anyone on the network can " + "read wallet/trading intelligence.", self.host, + ) + # Non-loopback: never wildcard CORS. No allowlist -> no CORS headers. + return api_key, (origins or set()) + async def start(self) -> None: - app = web.Application(middlewares=[cors_middleware]) + api_key, cors_origins = self._resolve_security() + self._api_key = api_key + + middlewares = [_make_rate_limit_middleware(self._rate_limiter)] + if api_key: + middlewares.append(_make_auth_middleware(api_key)) + middlewares.append(_make_cors_middleware(cors_origins)) + app = web.Application(middlewares=middlewares) # v1 routes v1 = [ @@ -200,6 +368,8 @@ async def stop(self) -> None: pass for client in list(self._ws_clients): + if client.writer_task: + client.writer_task.cancel() if not client.ws.closed: await client.ws.close() self._ws_clients.clear() @@ -257,16 +427,23 @@ def __init_dedup(self): self._liq_stats_ts = time.time() def _is_duplicate_liq(self, ev) -> bool: - """Check if this liquidation is a duplicate within the 3-second dedup window.""" + """Duplicate check within the dedup window, keyed per exchange. + + Buckets on the EXCHANGE event timestamp (not local receive time) so + two records of the same event dedup identically regardless of local + delivery jitter. The cascade bypass is also per-exchange: a Binance + cascade must not let Hyperliquid's heuristic events skip dedup. + """ self.__init_dedup() now = time.time() - bypass_key = f"{ev.symbol}_{ev.side}" + bypass_key = f"{ev.symbol}_{ev.side}_{ev.exchange}" if bypass_key in self._cascade_bypass and now < self._cascade_bypass[bypass_key]: return False + ev_ts = ev.timestamp if ev.timestamp > 0 else now size_rounded = round(ev.size_usd, -2) - h = f"{ev.symbol}_{ev.side}_{size_rounded}_{ev.exchange}_{int(now // self._DEDUP_WINDOW)}" + h = f"{ev.symbol}_{ev.side}_{size_rounded}_{ev.exchange}_{int(ev_ts // self._DEDUP_WINDOW)}" if len(self._liq_seen) > self._DEDUP_MAX: cutoff = now - self._DEDUP_WINDOW * 2 @@ -277,11 +454,16 @@ def _is_duplicate_liq(self, ev) -> bool: self._liq_seen[h] = now return False - def _check_cascade(self, symbol: str, side: str, exchange: str, size_usd: float) -> str | None: - """Track rapid successive liquidations. Returns cascade label if detected.""" + def _check_cascade(self, ev) -> str | None: + """Track rapid successive liquidations. Returns cascade label if detected. + + Keys on the RAW event fields (symbol/side/exchange) — the same domain + _is_duplicate_liq reads its bypass with — so a detected cascade + actually lifts dedup for the venue that is cascading. + """ self.__init_dedup() now = time.time() - key = f"{symbol}_{side}_{exchange}" + key = f"{ev.symbol}_{ev.side}_{ev.exchange}" if key not in self._cascade_tracker: self._cascade_tracker[key] = [] @@ -291,12 +473,13 @@ def _check_cascade(self, symbol: str, side: str, exchange: str, size_usd: float) if now - ts < self._CASCADE_WINDOW ] - self._cascade_tracker[key].append((now, size_usd)) + self._cascade_tracker[key].append((now, ev.size_usd)) entries = self._cascade_tracker[key] if len(entries) >= 3: - bypass_key = f"{symbol}_{side}" - self._cascade_bypass[bypass_key] = now + self._CASCADE_BYPASS_DURATION + # Bypass dedup only for this exchange's stream: cascades on one + # venue say nothing about duplicates on another. + self._cascade_bypass[key] = now + self._CASCADE_BYPASS_DURATION total = sum(sz for _, sz in entries) return f"cascade ${total:,.0f} ({len(entries)}x in {self._CASCADE_WINDOW}s)" @@ -360,7 +543,7 @@ def _on_liquidation(self, ev) -> None: ex_map = {"binance": "BIN", "bybit": "BYB", "okx": "OKX", "hyperliquid": "HYP"} ex_short = ex_map.get(ev.exchange, ev.exchange[:3].upper()) - cascade = self._check_cascade(symbol, ev.side.upper(), ex_short, ev.size_usd) + cascade = self._check_cascade(ev) self._broadcast("liquidation", { "exchange": ex_short, @@ -386,7 +569,12 @@ def _on_liquidation(self, ev) -> None: }) def _broadcast(self, event_type: str, data: dict) -> None: - """Send event to all subscribed WebSocket clients. Bulletproof.""" + """Enqueue event for all subscribed WebSocket clients. + + Each client has a bounded queue drained by its own writer task, so a + slow client drops ITS events (counted) instead of accumulating one + send task per client per event on the shared loop. + """ if not self._ws_clients: return msg = json.dumps({"type": event_type, "data": data, "ts": time.time()}) @@ -398,12 +586,42 @@ def _broadcast(self, event_type: str, data: dict) -> None: if event_type not in client.subscriptions: continue try: - asyncio.ensure_future(self._safe_send(client, msg)) - except Exception: - dead.append(client) + client.queue.put_nowait(msg) + except asyncio.QueueFull: + client.dropped_msgs += 1 + if client.dropped_msgs % 100 == 1: + logger.warning( + "[ws] Slow client: %d events dropped (queue full)", + client.dropped_msgs, + ) for d in dead: - if d in self._ws_clients: - self._ws_clients.remove(d) + self._remove_client(d) + + def _remove_client(self, client: _WSClient) -> None: + if client in self._ws_clients: + self._ws_clients.remove(client) + if client.writer_task and not client.writer_task.done(): + client.writer_task.cancel() + + async def _writer_loop(self, client: _WSClient) -> None: + """Single writer per client: drain the queue with a send timeout.""" + try: + while not client.ws.closed: + msg = await client.queue.get() + try: + await asyncio.wait_for(client.ws.send_str(msg), timeout=2.0) + except asyncio.CancelledError: + raise + except Exception: + # Send failed or timed out — this client is done. + self._remove_client(client) + try: + await client.ws.close() + except Exception: + pass + return + except asyncio.CancelledError: + pass async def _heartbeat_loop(self) -> None: """Push heartbeat every 10s. Evict clients that miss 3 consecutive pings.""" @@ -429,18 +647,20 @@ async def _heartbeat_loop(self) -> None: continue if "heartbeat" not in client.subscriptions: continue + # Enqueue through the same bounded queue as broadcasts. A + # full queue means the writer is stuck/slow — count it as + # a missed ping and evict after 3 in a row. try: - await asyncio.wait_for(client.ws.send_str(msg), timeout=2.0) + client.queue.put_nowait(msg) client.ping_misses = 0 - except Exception: + except asyncio.QueueFull: client.ping_misses += 1 if client.ping_misses >= 3: dead.append(client) - logger.debug("[ws] Evicting client after 3 missed pings") + logger.info("[ws] Evicting client after 3 missed heartbeats") for d in dead: - if d in self._ws_clients: - self._ws_clients.remove(d) + self._remove_client(d) try: await d.ws.close() except Exception: @@ -449,19 +669,9 @@ async def _heartbeat_loop(self) -> None: except asyncio.CancelledError: return except Exception: - pass - - async def _safe_send(self, client: _WSClient, msg: str) -> None: - """Send with timeout. Remove client on any failure.""" - try: - await asyncio.wait_for(client.ws.send_str(msg), timeout=2.0) - except Exception: - if client in self._ws_clients: - self._ws_clients.remove(client) - try: - await client.ws.close() - except Exception: - pass + # Never die silently: the heartbeat loop is also the WS + # liveness janitor, so log and keep going. + logger.exception("[ws] Heartbeat loop error") # ── Redirect helper ────────────────────────────────────────── @@ -476,35 +686,58 @@ async def redirect(request: web.Request) -> web.Response: async def handle_ws(self, request: web.Request) -> web.WebSocketResponse: if len(self._ws_clients) >= MAX_WS_CONNECTIONS: return web.json_response({"error": "Too many connections"}, status=429) - ws = web.WebSocketResponse(heartbeat=20) + ws = web.WebSocketResponse(heartbeat=20, max_msg_size=WS_MAX_MSG_BYTES) await ws.prepare(request) client = _WSClient(ws, subscriptions=set()) + client.writer_task = asyncio.create_task( + self._writer_loop(client), name="ws-writer" + ) self._ws_clients.append(client) logger.info("[ws] Client connected (%d total)", len(self._ws_clients)) try: async for msg in ws: if msg.type == WSMsgType.TEXT: - try: - data = json.loads(msg.data) - subs = data.get("subscribe") - if isinstance(subs, list): - client.subscriptions = {s for s in subs if s in EVENT_TYPES} - await ws.send_json({ - "type": "subscribed", - "channels": sorted(client.subscriptions), - }) - except json.JSONDecodeError: - pass + if self._ws_msg_violates_limits(client, msg.data): + break elif msg.type in (WSMsgType.CLOSED, WSMsgType.ERROR): break finally: - if client in self._ws_clients: - self._ws_clients.remove(client) + self._remove_client(client) logger.info("[ws] Client disconnected (%d remaining)", len(self._ws_clients)) return ws + def _ws_msg_violates_limits(self, client: _WSClient, raw: str) -> bool: + """Process one inbound message. Returns True if the client should be + disconnected (message flood or too many malformed messages).""" + now = time.time() + client.msg_times.append(now) + if (len(client.msg_times) == client.msg_times.maxlen + and now - client.msg_times[0] < 10.0): + logger.info("[ws] Disconnecting client: message rate limit exceeded") + return True + + if len(raw) > WS_MAX_MSG_BYTES: + client.bad_msgs += 1 + else: + try: + data = json.loads(raw) + subs = data.get("subscribe") if isinstance(data, dict) else None + if isinstance(subs, list): + client.subscriptions = {s for s in subs if s in EVENT_TYPES} + client.queue.put_nowait(json.dumps({ + "type": "subscribed", + "channels": sorted(client.subscriptions), + })) + except (json.JSONDecodeError, asyncio.QueueFull): + client.bad_msgs += 1 + + if client.bad_msgs >= WS_BAD_MSG_LIMIT: + logger.info("[ws] Disconnecting client after %d bad messages", client.bad_msgs) + return True + return False + # ── REST Handlers ──────────────────────────────────────────── async def handle_health(self, request: web.Request) -> web.Response: @@ -527,13 +760,26 @@ async def handle_health(self, request: web.Request) -> web.Response: "hlp": s.hlp_status, } + # Per-venue orderflow freshness: the combined status above follows the + # freshest venue, so a dead venue is only visible here. + orderflow_venues = None + try: + orderflow_venues = self.hub.orderflow.venue_freshness() + except Exception: + pass + # Top-level status reflects data health when available: 'ok' only when # nothing is stale/drifting. 'degraded' otherwise (server is still up). + # Components that failed to start also force 'degraded'. overall = data_health.get("overall") if data_health else None status = "ok" if overall in (None, "ok", "warn") else "degraded" + if s.failed_components: + status = "degraded" return web.json_response({ "status": status, + "failed_components": list(s.failed_components), + "orderflow_venues": orderflow_venues, "version": "1.0.0", "mode": s.mode, "uptime": f"{h}h {m}m", @@ -636,7 +882,10 @@ async def handle_funding_symbol(self, request: web.Request) -> web.Response: for ex_name, ex_rates in self.hub.funding.rates.items(): snap = ex_rates.get(sym) if snap: - rates[ex_name] = {"hourly": snap.funding_rate_hourly, "annualized_pct": snap.funding_rate_annualized * 100} + rates[ex_name] = { + "hourly": snap.funding_rate_hourly, + "annualized_pct": snap.funding_rate_annualized * 100, + } if not rates: return web.json_response({"error": f"No funding data for {sym}"}, status=404) return web.json_response({"symbol": sym, "rates": rates}) @@ -674,41 +923,6 @@ async def handle_deribit_iv(self, request: web.Request) -> web.Response: } return web.json_response(data) - async def handle_smart_money_rankings(self, request: web.Request) -> web.Response: - n = _int_param(request, "limit", 20, minimum=1, maximum=500) - smart = self.hub.get_smart_money(n) - dumb = self.hub.get_dumb_money(n) - stats = self.hub.smart_money.get_stats() - - def _fmt_wallet(w): - d = _serialize(w) - pnl = w.total_realized_pnl - if w.total_trades == 0: - d["pnl_display"] = "--" - elif abs(pnl) >= 1_000_000: - d["pnl_display"] = f"${pnl/1_000_000:+.1f}M" - elif abs(pnl) >= 1_000: - d["pnl_display"] = f"${pnl/1_000:+.1f}K" - elif abs(pnl) >= 1: - d["pnl_display"] = f"${pnl:+.0f}" - else: - d["pnl_display"] = "--" - return d - - return web.json_response({ - "stats": stats, - "smart": [_fmt_wallet(w) for w in smart], - "dumb": [_fmt_wallet(w) for w in dumb], - }) - - async def handle_smart_money_signals(self, request: web.Request) -> web.Response: - n = _int_param(request, "limit", 50, minimum=1, maximum=500) - signals = self.hub.get_smart_money_signals(n) - return web.json_response({ - "count": len(signals), - "signals": [_serialize(s) for s in signals], - }) - async def handle_orderbook(self, request: web.Request) -> web.Response: sym = request.match_info["symbol"].upper() snap = self.hub.get_orderbook(sym) @@ -741,93 +955,7 @@ async def handle_danger_zone(self, request: web.Request) -> web.Response: "positions": [_serialize(p) for p in positions], }) - # ── Copy-trading endpoints ──────────────────────────────────── - - _ct_signals_cache: dict | None = None - _ct_signals_cache_ts: float = 0 - - async def handle_copy_trading_signals(self, request: web.Request) -> web.Response: - """GET /v1/copy-trading/signals — recent copy/fade signals (10s cache).""" - try: - now = time.time() - if (self._ct_signals_cache is not None - and now - self._ct_signals_cache_ts < 10.0): - return web.json_response(self._ct_signals_cache) - - wc = getattr(self, '_wallet_cluster', None) - if wc is None: - return web.json_response({ - "signals": [], "active_count": 0, - "suppressed_count": 0, "last_updated": now, - }) - - signals = wc.get_signals(limit=20) - active = [s for s in signals if now - s.get("emitted_at", 0) < 300] - result = { - "signals": signals, - "active_count": len(active), - "suppressed_count": len(signals) - len(active), - "last_updated": now, - } - self._ct_signals_cache = result - self._ct_signals_cache_ts = now - return web.json_response(result) - except Exception as e: - return web.json_response({ - "error": str(e), "signals": [], "active_count": 0, - }) - - _ct_clusters_cache: dict | None = None - _ct_clusters_cache_ts: float = 0 - - async def handle_copy_trading_clusters(self, request: web.Request) -> web.Response: - """GET /v1/copy-trading/clusters — cluster breakdown (60s cache).""" - try: - now = time.time() - if (self._ct_clusters_cache is not None - and now - self._ct_clusters_cache_ts < 60.0): - return web.json_response(self._ct_clusters_cache) - - wc = getattr(self, '_wallet_cluster', None) - if wc is None: - return web.json_response({ - "clusters": [], "last_clustered": 0, - }) - - result = { - "clusters": wc.get_clusters(), - "last_clustered": wc._last_clustered, - } - self._ct_clusters_cache = result - self._ct_clusters_cache_ts = now - return web.json_response(result) - except Exception as e: - return web.json_response({"error": str(e), "clusters": []}) - - _ct_wallets_cache: dict | None = None - _ct_wallets_cache_ts: float = 0 - - async def handle_copy_trading_wallets(self, request: web.Request) -> web.Response: - """GET /v1/copy-trading/wallets — all tracked wallets (60s cache).""" - try: - now = time.time() - if (self._ct_wallets_cache is not None - and now - self._ct_wallets_cache_ts < 60.0): - return web.json_response(self._ct_wallets_cache) - - wc = getattr(self, '_wallet_cluster', None) - if wc is None: - return web.json_response({"wallets": [], "total": 0}) - - wallets = wc.get_wallets() - result = {"wallets": wallets, "total": len(wallets)} - self._ct_wallets_cache = result - self._ct_wallets_cache_ts = now - return web.json_response(result) - except Exception as e: - return web.json_response({"error": str(e), "wallets": []}) - - # ── Public metrics (live from backtest files + DB, 5-min cache) ── + # ── Public metrics ──────────────────────────────────────────── async def handle_public_metrics(self, request: web.Request) -> web.Response: """GET /v1/public/metrics — server status and data component health.""" diff --git a/src/dashboards/boot.py b/src/dashboards/boot.py index 4cc8129..e69d859 100644 --- a/src/dashboards/boot.py +++ b/src/dashboards/boot.py @@ -71,7 +71,7 @@ async def print_boot_sequence(console: Console, mode: str, dashboards: list[str] nw = 22 # name column width (same for both boxes) bw = 80 # box inner width - console.print(f" [bold bright_cyan]\u250c\u2500 INITIALIZING COMPONENTS " + "\u2500" * (bw - 26) + "\u2510[/]") + console.print(" [bold bright_cyan]\u250c\u2500 INITIALIZING COMPONENTS " + "\u2500" * (bw - 26) + "\u2510[/]") console.print(f" [bright_cyan]\u2502[/]{' ' * bw}[bright_cyan]\u2502[/]") for name, desc, color in components: @@ -85,10 +85,10 @@ async def print_boot_sequence(console: Console, mode: str, dashboards: list[str] console.print(f" [bright_cyan]\u2502[/]{rich}{' ' * max(pad_ck, 0)} [bold bright_green]\u2713[/][bright_cyan]\u2502[/]") console.print(f" [bright_cyan]\u2502[/]{' ' * bw}[bright_cyan]\u2502[/]") - console.print(f" [bold bright_cyan]\u2514" + "\u2500" * bw + "\u2518[/]") + console.print(" [bold bright_cyan]\u2514" + "\u2500" * bw + "\u2518[/]") console.print() - console.print(f" [bold bright_white]\u250c\u2500 ACTIVE DASHBOARDS " + "\u2500" * (bw - 21) + "\u2510[/]") + console.print(" [bold bright_white]\u250c\u2500 ACTIVE DASHBOARDS " + "\u2500" * (bw - 21) + "\u2510[/]") for d in dashboards: info = DASHBOARD_INFO.get(d, {"name": d, "desc": "", "color": "white"}) content = f" \u25b8 {info['name']:<{nw}}{info['desc']}" @@ -96,7 +96,7 @@ async def print_boot_sequence(console: Console, mode: str, dashboards: list[str] rich = f" [{info['color']}]\u25b8[/] [{info['color']}]{info['name']:<{nw}}[/][dim]{info['desc']}[/]" console.print(f" [bright_white]\u2502[/]{rich}{' ' * max(pad, 0)}[bright_white]\u2502[/]") await asyncio.sleep(0.15) - console.print(f" [bold bright_white]\u2514" + "\u2500" * bw + "\u2518[/]") + console.print(" [bold bright_white]\u2514" + "\u2500" * bw + "\u2518[/]") console.print() for i in range(bar_width + 1): diff --git a/src/dashboards/combined_dashboard.py b/src/dashboards/combined_dashboard.py index c101a99..519081a 100644 --- a/src/dashboards/combined_dashboard.py +++ b/src/dashboards/combined_dashboard.py @@ -11,7 +11,6 @@ from rich.panel import Panel from rich.text import Text -from src.data_layer.hub import HyperDataHub from src.dashboards.hub_panels import ( HubCVD, HubHLP, @@ -24,6 +23,7 @@ HubWhales, ) from src.dashboards.liquidation_heatmap import LiquidationHeatmapDashboard +from src.data_layer.hub import HyperDataHub class CombinedDashboard: diff --git a/src/dashboards/cvd_dashboard.py b/src/dashboards/cvd_dashboard.py index 2d3fb7f..76a8a96 100644 --- a/src/dashboards/cvd_dashboard.py +++ b/src/dashboards/cvd_dashboard.py @@ -23,12 +23,12 @@ from rich.table import Table from rich.text import Text +from src.data_layer.market_data import MarketData from src.data_layer.orderflow_engine import ( CVDSnapshot, OrderFlowEngine, Trade, ) -from src.data_layer.market_data import MarketData logger = logging.getLogger(__name__) @@ -206,7 +206,7 @@ def build_price_bar(self) -> Text: bar = Text() bar.append(" \u20bf ", style="bold bright_yellow") - bar.append(f"BITCOIN ", style="bold white") + bar.append("BITCOIN ", style="bold white") bar.append(f"${price:,.2f}", style="bold bright_white") bar.append(" ") @@ -391,7 +391,7 @@ def build_compact(self) -> Panel: now_str = datetime.now().strftime("%H:%M:%S") return Panel( table, - title=f"[bold bright_green]\U0001f4c8 BTC CVD[/]", + title="[bold bright_green]\U0001f4c8 BTC CVD[/]", subtitle=f"[dim]{now_str} #{self.cycle}[/]", border_style="bright_green", box=_box.ROUNDED, diff --git a/src/dashboards/hub_panels.py b/src/dashboards/hub_panels.py index 0d7f28b..4b0288b 100644 --- a/src/dashboards/hub_panels.py +++ b/src/dashboards/hub_panels.py @@ -14,7 +14,9 @@ from rich.text import Text from src.data_layer.hub import HyperDataHub -from src.utils.helpers import format_usd as fmt_usd, format_price as fmt_price, format_pct as fmt_pct +from src.utils.helpers import format_pct as fmt_pct +from src.utils.helpers import format_price as fmt_price +from src.utils.helpers import format_usd as fmt_usd class HubLiqWatch: diff --git a/src/dashboards/liquidation_heatmap.py b/src/dashboards/liquidation_heatmap.py index 82802f0..fdf1267 100644 --- a/src/dashboards/liquidation_heatmap.py +++ b/src/dashboards/liquidation_heatmap.py @@ -11,7 +11,6 @@ import asyncio import logging -import math from dataclasses import dataclass from rich import box @@ -203,9 +202,9 @@ def _build_asset_heatmap(self, symbol: str, current_price: float, n_buckets: int header = Text() header.append(f" {symbol} ", style="bold bright_white") header.append(f"${current_price:,.0f}", style=CURRENT_STYLE) - header.append(f" │ ", style="dim") + header.append(" │ ", style="dim") header.append(f"L: {_format_usd(total_long)}({long_count})", style=LONG_STYLE) - header.append(f" ", style="dim") + header.append(" ", style="dim") header.append(f"S: {_format_usd(total_short)}({short_count})", style=SHORT_STYLE) table = Table( diff --git a/src/dashboards/liquidation_stream.py b/src/dashboards/liquidation_stream.py index 121d1da..8b3d6e8 100644 --- a/src/dashboards/liquidation_stream.py +++ b/src/dashboards/liquidation_stream.py @@ -24,7 +24,6 @@ from rich.align import Align from rich.box import DOUBLE_EDGE, HEAVY, SIMPLE_HEAVY from rich.console import Console, Group -from rich.layout import Layout from rich.live import Live from rich.panel import Panel from rich.table import Table @@ -37,7 +36,7 @@ if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) -from src.data_layer.liquidation_feed import LiquidationEvent, LiquidationFeed +from src.data_layer.liquidation_feed import LiquidationEvent, LiquidationFeed # noqa: E402 logger = logging.getLogger(__name__) @@ -72,7 +71,7 @@ # --------------------------------------------------------------------------- # Formatting helpers (imported from central helpers) # --------------------------------------------------------------------------- -from src.utils.helpers import format_usd as fmt_usd +from src.utils.helpers import format_usd as fmt_usd # noqa: E402 def fmt_number(value: int) -> str: diff --git a/src/dashboards/liquidation_watch.py b/src/dashboards/liquidation_watch.py index 5cdb3a4..5b8a7f6 100644 --- a/src/dashboards/liquidation_watch.py +++ b/src/dashboards/liquidation_watch.py @@ -30,18 +30,18 @@ if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) -from src.data_layer.position_scanner import PositionScanner, TrackedPosition # noqa: E402 from config.settings import ( # noqa: E402 DASHBOARD_REFRESH_RATE, LIQ_ZONE_1_PCT, LIQ_ZONE_2_PCT, LIQ_ZONE_5_PCT, ) - +from src.data_layer.position_scanner import PositionScanner, TrackedPosition # noqa: E402 +from src.utils.helpers import format_pct_value as fmt_pct # noqa: E402 +from src.utils.helpers import format_price as fmt_price # noqa: E402 # ── Formatting helpers (imported from central helpers) ──────────────────── -from src.utils.helpers import format_usd as fmt_usd, format_price as fmt_price, format_pct_value as fmt_pct - +from src.utils.helpers import format_usd as fmt_usd # noqa: E402 # ── Zone summary with long/short breakdown ──────────────────────────────── diff --git a/src/dashboards/market_overview.py b/src/dashboards/market_overview.py index fc82864..802ccb6 100644 --- a/src/dashboards/market_overview.py +++ b/src/dashboards/market_overview.py @@ -32,11 +32,11 @@ sys.path.insert(0, str(_PROJECT_ROOT)) from src.data_layer.market_data import AssetInfo, MarketData # noqa: E402 -from config.settings import DASHBOARD_REFRESH_RATE, DEFAULT_SYMBOLS # noqa: E402 - +from src.utils.helpers import format_pct as fmt_pct # noqa: E402 +from src.utils.helpers import format_price as fmt_price # noqa: E402 # -- Formatting helpers (imported from central helpers) --------------------- -from src.utils.helpers import format_usd as fmt_usd, format_price as fmt_price, format_pct as fmt_pct +from src.utils.helpers import format_usd as fmt_usd # noqa: E402 def fmt_funding(value: float) -> str: @@ -258,10 +258,8 @@ def build_extreme_funding(self, assets: list[AssetInfo], limit: int = 10) -> Tab if asset.funding_rate >= 0: rate_style = "bold bright_green" - bar_char = "\U0001f7e9" # green square else: rate_style = "bold bright_red" - bar_char = "\U0001f7e5" # red square bar = make_bar(abs_ann, max_annualized, width=20) diff --git a/src/dashboards/whale_tracker.py b/src/dashboards/whale_tracker.py index ccfaaca..43eb67b 100644 --- a/src/dashboards/whale_tracker.py +++ b/src/dashboards/whale_tracker.py @@ -32,11 +32,11 @@ sys.path.insert(0, str(_PROJECT_ROOT)) from src.data_layer.position_scanner import PositionScanner, TrackedPosition # noqa: E402 -from config.settings import DASHBOARD_REFRESH_RATE # noqa: E402 - +from src.utils.helpers import format_pct_value as fmt_pct # noqa: E402 +from src.utils.helpers import format_price as fmt_price # noqa: E402 # -- Formatting helpers (imported from central helpers) --------------------- -from src.utils.helpers import format_usd as fmt_usd, format_price as fmt_price, format_pct_value as fmt_pct +from src.utils.helpers import format_usd as fmt_usd # noqa: E402 def shorten_addr(address: str) -> str: diff --git a/src/data_layer/address_store.py b/src/data_layer/address_store.py index 1db5657..5c3c0d1 100644 --- a/src/data_layer/address_store.py +++ b/src/data_layer/address_store.py @@ -7,6 +7,7 @@ """ import json import logging +import re import sqlite3 import threading import time @@ -18,6 +19,24 @@ DB_PATH = DATA_DIR / "hyperdata.db" LEGACY_JSON = DATA_DIR / "discovered_addresses.json" +# EVM wallet address: 0x + 40 hex chars. Anything else from an exchange +# payload is junk and must not be persisted (it would be re-scanned forever). +_ADDRESS_RE = re.compile(r"^0x[0-9a-fA-F]{40}$") + +# Retention cap: keep the most recently seen addresses; every tracked address +# costs a clearinghouseState call per scan cycle. +MAX_TRACKED_ADDRESSES = 50_000 + + +def is_valid_address(address: object) -> bool: + """True for a well-formed EVM wallet address string.""" + return isinstance(address, str) and bool(_ADDRESS_RE.match(address)) + + +def normalize_address(address: str) -> str: + """Canonical form: lowercase (EVM addresses are case-insensitive).""" + return address.lower() + _CREATE = """ CREATE TABLE IF NOT EXISTS discovered_addresses ( address TEXT PRIMARY KEY, @@ -72,31 +91,28 @@ def _init() -> None: def add_address(address: str, source: str = "unknown") -> None: - """Insert or update a single address. Idempotent.""" - _init() - now = time.time() - try: - with _lock: - conn = _get_conn() - conn.execute( - "INSERT INTO discovered_addresses (address, source, first_seen, last_seen) " - "VALUES (?, ?, ?, ?) " - "ON CONFLICT(address) DO UPDATE SET last_seen = excluded.last_seen", - (address, source, now, now), - ) - conn.commit() - conn.close() - except Exception: - logger.debug("[address_store] add failed", exc_info=True) + """Insert or update a single address. Idempotent. Invalid input is dropped.""" + add_addresses([address], source=source) def add_addresses(addresses: list[str] | set[str], source: str = "unknown") -> int: - """Batch insert. Returns count written.""" + """Batch insert (validated + normalized). Returns count written. + + Non-address strings from exchange payloads are dropped and counted here + so garbage identifiers never enter the store, and the table is capped at + MAX_TRACKED_ADDRESSES by expiring the least recently seen rows. + """ _init() if not addresses: return 0 now = time.time() - rows = [(a, source, now, now) for a in addresses] + valid = [normalize_address(a) for a in addresses if is_valid_address(a)] + dropped = len(list(addresses)) - len(valid) + if dropped: + logger.warning("[address_store] dropped %d invalid address strings", dropped) + if not valid: + return 0 + rows = [(a, source, now, now) for a in valid] try: with _lock: conn = _get_conn() @@ -106,6 +122,19 @@ def add_addresses(addresses: list[str] | set[str], source: str = "unknown") -> i "ON CONFLICT(address) DO UPDATE SET last_seen = excluded.last_seen", rows, ) + # Retention cap: expire the least-recently-seen overflow. + count = conn.execute( + "SELECT COUNT(*) FROM discovered_addresses" + ).fetchone()[0] + if count > MAX_TRACKED_ADDRESSES: + overflow = count - MAX_TRACKED_ADDRESSES + conn.execute( + "DELETE FROM discovered_addresses WHERE address IN (" + "SELECT address FROM discovered_addresses " + "ORDER BY last_seen ASC LIMIT ?)", + (overflow,), + ) + logger.info("[address_store] expired %d least-recently-seen addresses", overflow) conn.commit() conn.close() return len(rows) diff --git a/src/data_layer/alerts.py b/src/data_layer/alerts.py index 63c0044..a84d718 100644 --- a/src/data_layer/alerts.py +++ b/src/data_layer/alerts.py @@ -339,7 +339,6 @@ def _build_intel_report(self) -> str: delta = hlp_stats["net_delta"] zscore = hlp_stats["delta_zscore"] session_pnl = hlp_stats["session_pnl"] - absorptions = hlp_stats["liquidation_absorptions"] lines.append(f"AUM: {self._fmt_usd(aum)} | Delta: {self._fmt_usd_signed(delta)} | Z: {zscore:+.1f}") lines.append(f"Session PnL: {self._fmt_usd_signed(session_pnl)}") @@ -357,6 +356,10 @@ def _build_intel_report(self) -> str: return "\n".join(lines) + # Deadline for webhook posts: a stalled Telegram/Discord endpoint must + # not wedge whatever task is delivering the alert. + _SEND_TIMEOUT = aiohttp.ClientTimeout(total=10) + async def _send(self, message: str) -> None: """Send alert to all configured channels.""" if not self._session: @@ -368,27 +371,36 @@ async def _send(self, message: str) -> None: if self.telegram_token and self.telegram_chat_id: try: url = f"https://api.telegram.org/bot{self.telegram_token}/sendMessage" - await self._session.post(url, json={ + async with self._session.post(url, json={ "chat_id": self.telegram_chat_id, "text": message, "parse_mode": "HTML", - }) - logger.info("Telegram alert sent") + }, timeout=self._SEND_TIMEOUT) as resp: + if resp.status == 200: + logger.info("Telegram alert sent") + else: + logger.warning("Telegram send returned %d", resp.status) except Exception: logger.exception("Telegram send failed") # Discord if self.discord_webhook: try: - await self._session.post(self.discord_webhook, json={ + async with self._session.post(self.discord_webhook, json={ "content": message, - }) - logger.info("Discord alert sent") + }, timeout=self._SEND_TIMEOUT) as resp: + if resp.status in (200, 204): + logger.info("Discord alert sent") + else: + logger.warning("Discord send returned %d", resp.status) except Exception: logger.exception("Discord send failed") - # Always log to console - logger.warning("ALERT: %s", message.replace("\n", " | ")) + # Log that an alert fired, not its payload — alert bodies can contain + # wallet addresses and position intelligence that must not sit in + # rotating plaintext logs. + first_line = message.strip().splitlines()[0] if message.strip() else "" + logger.warning("ALERT sent (%d total): %.80s", self.alerts_sent, first_line) async def send_test(self) -> bool: """Send a test alert to verify configuration.""" diff --git a/src/data_layer/hlp_tracker.py b/src/data_layer/hlp_tracker.py index 3a7cf7b..b27bb10 100644 --- a/src/data_layer/hlp_tracker.py +++ b/src/data_layer/hlp_tracker.py @@ -13,7 +13,7 @@ import logging import time from collections import deque -from dataclasses import dataclass, field +from dataclasses import dataclass import aiohttp @@ -185,7 +185,8 @@ async def _take_snapshot(self, address: str) -> HLPSnapshot | None: # Current price from position data current_price = float(pos_info.get("positionValue", 0)) if abs(size_raw) > 0: - current_price = abs(float(pos_info.get("positionValue", 0)) / size_raw) if size_raw != 0 else entry_price + current_price = (abs(float(pos_info.get("positionValue", 0)) / size_raw) + if size_raw != 0 else entry_price) unrealized_pnl = float(pos_info.get("unrealizedPnl", 0)) leverage_info = pos_info.get("leverage", {}) diff --git a/src/data_layer/hub.py b/src/data_layer/hub.py index 8c02810..afab5a9 100644 --- a/src/data_layer/hub.py +++ b/src/data_layer/hub.py @@ -21,7 +21,6 @@ from __future__ import annotations import asyncio -import json import logging import os import time @@ -30,26 +29,27 @@ from typing import Any, Callable from config.settings import DEFAULT_SYMBOLS +from src.api_server import HyperDataAPI from src.data_layer.alerts import AlertManager -from src.data_layer.liquidation_feed import LiquidationFeed, LiquidationEvent -from src.data_layer.position_scanner import PositionScanner, TrackedPosition +from src.data_layer.deribit import DeribitFeed, DeribitIVSnapshot +from src.data_layer.funding_rates import FundingRateCollector, FundingRateSnapshot +from src.data_layer.health_monitor import DataHealthMonitor +from src.data_layer.hlp_tracker import HLPPosition, HLPSnapshot, HLPTracker, HLPTrade +from src.data_layer.liquidation_feed import LiquidationEvent, LiquidationFeed +from src.data_layer.long_short_ratio import LongShortCollector, LongShortSnapshot +from src.data_layer.market_data import AssetInfo, MarketData +from src.data_layer.orderbook import OrderBookEngine, OrderBookSnapshot +from src.data_layer.orderflow_engine import ( + STALE_AFTER_SECONDS as ORDERFLOW_STALE_AFTER, +) from src.data_layer.orderflow_engine import ( OrderFlowEngine, Trade, - CVDSnapshot, - STALE_AFTER_SECONDS as ORDERFLOW_STALE_AFTER, ) -from src.data_layer.market_data import MarketData, AssetInfo from src.data_layer.persistence import DataStore +from src.data_layer.position_scanner import PositionScanner, TrackedPosition from src.data_layer.smart_money import SmartMoneyEngine, SmartMoneySignal, WalletProfile -from src.data_layer.hlp_tracker import HLPTracker, HLPSnapshot, HLPPosition, HLPTrade -from src.data_layer.funding_rates import FundingRateCollector, FundingRateSnapshot -from src.data_layer.long_short_ratio import LongShortCollector, LongShortSnapshot -from src.data_layer.orderbook import OrderBookEngine, OrderBookSnapshot from src.data_layer.spot_prices import SpotPriceCollector, SpotPriceSnapshot -from src.data_layer.deribit import DeribitFeed, DeribitIVSnapshot -from src.data_layer.health_monitor import DataHealthMonitor -from src.api_server import HyperDataAPI logger = logging.getLogger(__name__) @@ -72,6 +72,9 @@ class HubStatus: orderbook_feed: str = "offline" market_data: str = "offline" + # Components that raised during start(); non-empty means degraded mode. + failed_components: list = field(default_factory=list) + # Counters total_liquidations: int = 0 total_trades_processed: int = 0 @@ -174,6 +177,8 @@ def __init__( # ── Background tasks ───────────────────────────────────── self._tasks: list[asyncio.Task] = [] self._running = False + # Debounce for per-venue orderflow staleness warnings. + self._venue_stale_warned_at: dict[str, float] = {} # Wire up internal callbacks self.liquidations.on_liquidation(self._handle_liquidation) @@ -254,12 +259,15 @@ async def start(self) -> None: # Start REST API server if port is configured (both modes) if self._api_port: try: - # Loopback by default; set HYPERDATA_API_HOST=0.0.0.0 to expose - # on the LAN (no auth — only do this behind a trusted network). + # Loopback by default; a non-loopback HYPERDATA_API_HOST is + # refused unless HYPERDATA_API_KEY or the explicit unsafe + # acknowledgment is set (see HyperDataAPI._resolve_security). api_host = os.environ.get("HYPERDATA_API_HOST", "127.0.0.1") self._api_server = HyperDataAPI(self, host=api_host, port=self._api_port) await self._api_server.start() except Exception: + self._api_server = None + self.status.failed_components.append("api_server") logger.exception("Failed to start API server") # Background loops that run in both modes @@ -283,81 +291,88 @@ async def start(self) -> None: self.store.attach(self) # ── Alerts ───────────────────────────────────────────── - await self.alerts.start() - self.alerts.attach(self) - - logger.info("HyperDataHub started — all components online") - - async def _start_live(self) -> None: - """Connect to real exchange APIs.""" - # Start liquidation WebSocket feeds - try: - await self.liquidations.start() - self.status.liquidation_feed = "connected" - logger.info("Liquidation feed: connected") - except Exception: - self.status.liquidation_feed = "error" - logger.exception("Failed to start liquidation feed") - - # Start order flow WebSocket try: - await self.orderflow.start() - self.status.orderflow_engine = "connected" - logger.info("Order flow engine: connected") + await self.alerts.start() + self.alerts.attach(self) except Exception: - self.status.orderflow_engine = "error" - logger.exception("Failed to start order flow engine") - - # Start smart money engine - try: - await self.smart_money.start() - logger.info("Smart money engine: started") - except Exception: - logger.exception("Failed to start smart money engine") - - # Start HLP tracker - try: - await self.hlp.start() - self.status.hlp_status = "connected" - logger.info("HLP tracker: started") - except Exception: - self.status.hlp_status = "error" - logger.exception("Failed to start HLP tracker") - - try: - await self.funding.start() - logger.info("Funding rate collector: started") - except Exception: - logger.exception("Failed to start funding rate collector") + self.status.failed_components.append("alerts") + logger.exception("Failed to start alert manager") + + if self.status.failed_components: + logger.error( + "HyperDataHub started DEGRADED — failed components: %s. " + "Data from these sources will be missing or stale.", + ", ".join(self.status.failed_components), + ) + else: + logger.info("HyperDataHub started — all components online") - try: - await self.lsr.start() - logger.info("Long/short ratio collector: started") - except Exception: - logger.exception("Failed to start long/short ratio collector") + async def _start_component(self, name: str, coro, required: bool = False, + on_ok=None, on_fail=None) -> bool: + """Start one component, recording failures instead of hiding them. + A failed *required* component raises and aborts startup; a failed + optional component is appended to status.failed_components so health + surfaces (log line, /v1/health) report degraded mode honestly. + """ try: - await self.orderbook.start() - self.status.orderbook_feed = "connected" - logger.info("OrderBook engine: started") + await coro + if on_ok: + on_ok() + logger.info("%s: started", name) + return True except Exception: - self.status.orderbook_feed = "error" - logger.exception("Failed to start orderbook engine") + if on_fail: + on_fail() + if required: + logger.exception("Required component %s failed to start", name) + raise + self.status.failed_components.append(name) + logger.exception("Failed to start %s (continuing degraded)", name) + return False - try: - await self.spot.start(perp_price_fn=lambda sym: self.market.assets.get(sym)) - logger.info("Spot price collector: started") - except Exception: - logger.exception("Failed to start spot price collector") + async def _start_live(self) -> None: + """Connect to real exchange APIs. - try: - await self.deribit.start() - logger.info("Deribit IV feed: started") - except Exception: - logger.exception("Failed to start Deribit IV feed") + Every component is optional-but-reported: a failure puts it in + status.failed_components (surfaced via /v1/health and the startup + log) instead of being silently swallowed. + """ + s = self.status + await self._start_component( + "liquidation_feed", self.liquidations.start(), + on_ok=lambda: setattr(s, "liquidation_feed", "connected"), + on_fail=lambda: setattr(s, "liquidation_feed", "error"), + ) + await self._start_component( + "orderflow_engine", self.orderflow.start(), + on_ok=lambda: setattr(s, "orderflow_engine", "connected"), + on_fail=lambda: setattr(s, "orderflow_engine", "error"), + ) + await self._start_component("smart_money", self.smart_money.start()) + await self._start_component( + "hlp_tracker", self.hlp.start(), + on_ok=lambda: setattr(s, "hlp_status", "connected"), + on_fail=lambda: setattr(s, "hlp_status", "error"), + ) + await self._start_component("funding_rates", self.funding.start()) + await self._start_component("long_short_ratio", self.lsr.start()) + await self._start_component( + "orderbook", self.orderbook.start(), + on_ok=lambda: setattr(s, "orderbook_feed", "connected"), + on_fail=lambda: setattr(s, "orderbook_feed", "error"), + ) + await self._start_component( + "spot_prices", + self.spot.start(perp_price_fn=lambda sym: self.market.assets.get(sym)), + ) + await self._start_component("deribit_iv", self.deribit.start()) - self.status.position_scanner = "ready" - self.status.market_data = "ready" + # Loop-driven components: 'starting' until their first cycle actually + # succeeds (the loops flip these to 'connected'/'error'). Never claim + # 'ready' for something that has not fetched anything yet. + self.status.position_scanner = "starting" + self.status.market_data = "starting" async def _start_demo(self) -> None: """Start mock data generators.""" @@ -468,6 +483,8 @@ async def _position_scan_loop(self) -> None: all_positions = await self.positions.scan() self.status.tracked_positions = len(all_positions) self.status.discovered_addresses = len(self.positions.discovered_addresses) + # 'connected' only after a scan actually succeeded. + self.status.position_scanner = "connected" self.status.last_position_scan = time.time() self.status.scan_cycle += 1 @@ -482,6 +499,8 @@ async def _position_scan_loop(self) -> None: except asyncio.CancelledError: break except Exception: + if not self.demo: + self.status.position_scanner = "error" logger.exception("Position scan error") await asyncio.sleep(self.scan_interval) @@ -494,6 +513,8 @@ async def _market_refresh_loop(self) -> None: await self._demo_market_refresh() else: await self.market.refresh() + # 'connected' only after a refresh actually succeeded. + self.status.market_data = "connected" self.status.last_market_refresh = time.time() self.status.tracked_assets = len(self.market.assets) @@ -501,104 +522,129 @@ async def _market_refresh_loop(self) -> None: except asyncio.CancelledError: break except Exception: + if not self.demo: + self.status.market_data = "error" logger.exception("Market refresh error") await asyncio.sleep(self.market_refresh_interval) async def _status_update_loop(self) -> None: - """Update uptime counter and periodically refresh DB stats.""" + """Update uptime counter and periodically refresh DB stats. + + The body is wrapped so one component's bad stats shape can't kill the + loop — this loop is also the staleness watchdog and the source of + /v1/health data, so it must outlive individual component errors. + """ _db_tick = 0 + _last_hlp_alert_check = 0.0 while self._running: - self.status.uptime_seconds = time.time() - self.status.started_at - - # Update smart money stats every tick - sm_stats = self.smart_money.get_stats() - self.status.tracked_wallets = sm_stats["total_wallets"] - self.status.ranked_wallets = sm_stats["ranked_wallets"] - self.status.smart_money_signals = sm_stats["total_signals"] - - # Update HLP stats every tick - hlp_stats = self.hlp.get_stats() - self.status.hlp_account_value = hlp_stats["account_value"] - self.status.hlp_net_delta = hlp_stats["net_delta"] - self.status.hlp_delta_zscore = hlp_stats["delta_zscore"] - self.status.hlp_positions = hlp_stats["num_positions"] - self.status.hlp_trades = hlp_stats["total_trades"] - self.status.hlp_liquidation_absorptions = hlp_stats["liquidation_absorptions"] - self.status.hlp_session_pnl = hlp_stats["session_pnl"] - - # Check HLP Z-score for alert - if abs(hlp_stats.get("delta_zscore", 0)) > 2.0: - asyncio.create_task(self.alerts._check_hlp_zscore(hlp_stats)) - - # Persist HLP snapshots periodically - self.store.maybe_save_hlp_snapshot() - - _db_tick += 1 - # Prune old rows + checkpoint the WAL roughly hourly so the DB and - # the COUNT(*) below stay bounded on long-running instances. - if _db_tick % 3600 == 0: - try: - self.store.prune() - except Exception: - logger.exception("Error pruning DB") - # Update persistence stats every 30 seconds - if _db_tick % 30 == 0: - try: - db_stats = self.store.get_db_stats() - self.status.db_size_mb = db_stats["db_size_mb"] - self.status.events_persisted = ( - db_stats["liquidations_stored"] + db_stats["trades_stored"] + try: + _db_tick = await self._status_update_tick(_db_tick) + + # Check HLP Z-score for alert — at most once a minute while + # extreme, not one new task per tick. + hlp_zscore = self.status.hlp_delta_zscore + now = time.time() + if abs(hlp_zscore) > 2.0 and now - _last_hlp_alert_check > 60.0: + _last_hlp_alert_check = now + asyncio.create_task( + self.alerts._check_hlp_zscore(self.hlp.get_stats()) ) - except Exception: - logger.exception("Error fetching DB stats") - try: - for ex_rates in self.funding.rates.values(): - for snap in ex_rates.values(): - self.store.save_funding_rate(snap) - except Exception: - logger.exception("Error saving funding rate snapshots") - try: - for snap in self.lsr.ratios.values(): - self.store.save_long_short_ratio(snap) - except Exception: - logger.exception("Error saving LSR snapshots") - try: - for snap in self.deribit.snapshots.values(): - self.store.save_options_snapshot(snap) - except Exception: - logger.exception("Error saving Deribit IV snapshots") - - # Update new component status fields every tick - self.status.funding_rate_symbols_binance = len(self.funding.rates.get("binance", {})) - self.status.funding_rate_symbols_bybit = len(self.funding.rates.get("bybit", {})) - - btc_lsr = self.lsr.get_latest("BTC") - eth_lsr = self.lsr.get_latest("ETH") - self.status.lsr_btc_ratio = btc_lsr.long_short_ratio if btc_lsr else 0.0 - self.status.lsr_eth_ratio = eth_lsr.long_short_ratio if eth_lsr else 0.0 - - self.status.orderbook_symbols = len(self.orderbook.snapshots) - - btc_spot = self.spot.get_latest("BTC") - eth_spot = self.spot.get_latest("ETH") - self.status.spot_btc_basis_pct = btc_spot.basis_pct if btc_spot else 0.0 - self.status.spot_eth_basis_pct = eth_spot.basis_pct if eth_spot else 0.0 - - btc_iv = self.deribit.get_latest("BTC") - eth_iv = self.deribit.get_latest("ETH") - self.status.deribit_btc_iv = btc_iv.mark_iv if btc_iv else 0.0 - self.status.deribit_eth_iv = eth_iv.mark_iv if eth_iv else 0.0 - # ── Staleness watchdog ────────────────────────────────── - # Flag WS feeds that have stopped delivering data as 'stale' so the - # UI/API never present frozen numbers as live, and force a reconnect - # on a socket that's alive-but-silent (heartbeat only catches - # half-open connections, not a venue that quietly stops sending). - if not self.demo: - await self._update_feed_staleness() + except asyncio.CancelledError: + break + except Exception: + logger.exception("Status update loop error") await asyncio.sleep(1) + async def _status_update_tick(self, _db_tick: int) -> int: + """One status-loop iteration. Returns the incremented DB tick.""" + self.status.uptime_seconds = time.time() - self.status.started_at + + # Update smart money stats every tick + sm_stats = self.smart_money.get_stats() + self.status.tracked_wallets = sm_stats["total_wallets"] + self.status.ranked_wallets = sm_stats["ranked_wallets"] + self.status.smart_money_signals = sm_stats["total_signals"] + + # Update HLP stats every tick + hlp_stats = self.hlp.get_stats() + self.status.hlp_account_value = hlp_stats["account_value"] + self.status.hlp_net_delta = hlp_stats["net_delta"] + self.status.hlp_delta_zscore = hlp_stats["delta_zscore"] + self.status.hlp_positions = hlp_stats["num_positions"] + self.status.hlp_trades = hlp_stats["total_trades"] + self.status.hlp_liquidation_absorptions = hlp_stats["liquidation_absorptions"] + self.status.hlp_session_pnl = hlp_stats["session_pnl"] + + # Persist HLP snapshots periodically + self.store.maybe_save_hlp_snapshot() + + _db_tick += 1 + # Prune old rows + checkpoint the WAL roughly hourly so the DB and + # the COUNT(*) below stay bounded on long-running instances. + if _db_tick % 3600 == 0: + try: + self.store.prune() + except Exception: + logger.exception("Error pruning DB") + # Update persistence stats every 30 seconds + if _db_tick % 30 == 0: + try: + db_stats = self.store.get_db_stats() + self.status.db_size_mb = db_stats["db_size_mb"] + self.status.events_persisted = ( + db_stats["liquidations_stored"] + db_stats["trades_stored"] + ) + except Exception: + logger.exception("Error fetching DB stats") + try: + for ex_rates in self.funding.rates.values(): + for snap in ex_rates.values(): + self.store.save_funding_rate(snap) + except Exception: + logger.exception("Error saving funding rate snapshots") + try: + for snap in self.lsr.ratios.values(): + self.store.save_long_short_ratio(snap) + except Exception: + logger.exception("Error saving LSR snapshots") + try: + for snap in self.deribit.snapshots.values(): + self.store.save_options_snapshot(snap) + except Exception: + logger.exception("Error saving Deribit IV snapshots") + + # Update new component status fields every tick + self.status.funding_rate_symbols_binance = len(self.funding.rates.get("binance", {})) + self.status.funding_rate_symbols_bybit = len(self.funding.rates.get("bybit", {})) + + btc_lsr = self.lsr.get_latest("BTC") + eth_lsr = self.lsr.get_latest("ETH") + self.status.lsr_btc_ratio = btc_lsr.long_short_ratio if btc_lsr else 0.0 + self.status.lsr_eth_ratio = eth_lsr.long_short_ratio if eth_lsr else 0.0 + + self.status.orderbook_symbols = len(self.orderbook.snapshots) + + btc_spot = self.spot.get_latest("BTC") + eth_spot = self.spot.get_latest("ETH") + self.status.spot_btc_basis_pct = btc_spot.basis_pct if btc_spot else 0.0 + self.status.spot_eth_basis_pct = eth_spot.basis_pct if eth_spot else 0.0 + + btc_iv = self.deribit.get_latest("BTC") + eth_iv = self.deribit.get_latest("ETH") + self.status.deribit_btc_iv = btc_iv.mark_iv if btc_iv else 0.0 + self.status.deribit_eth_iv = eth_iv.mark_iv if eth_iv else 0.0 + # ── Staleness watchdog ────────────────────────────────── + # Flag WS feeds that have stopped delivering data as 'stale' so the + # UI/API never present frozen numbers as live, and force a reconnect + # on a socket that's alive-but-silent (heartbeat only catches + # half-open connections, not a venue that quietly stops sending). + if not self.demo: + await self._update_feed_staleness() + + return _db_tick + async def _health_monitor_loop(self) -> None: """Run data-integrity checks against external sources on an interval. @@ -628,6 +674,22 @@ async def _update_feed_staleness(self) -> None: self.status.orderflow_engine = ( "stale" if self.orderflow.is_stale() else "connected" ) + # Combined freshness follows the freshest venue, so one dead venue + # can hide behind the other. Warn (debounced) when that happens so + # "orderflow connected" is never silently half-true. + if not self.orderflow.is_stale(): + now_w = time.time() + for venue in ("hyperliquid", "binance"): + if (self.orderflow.venue_is_stale(venue) + and self.orderflow.venue_data_age(venue) != float("inf") + and now_w - self._venue_stale_warned_at.get(venue, 0.0) > 300): + self._venue_stale_warned_at[venue] = now_w + logger.warning( + "[hub] order flow venue %s silent %.0fs while the " + "combined feed is still fresh — venue-specific " + "data (per-venue CVD) is stale", + venue, self.orderflow.venue_data_age(venue), + ) # Both venues silent for well past the threshold → kick the HL # socket so its backoff loop rebuilds it. The Binance loop self-heals # via its own heartbeat, and if Binance were still feeding, the @@ -802,7 +864,8 @@ async def _demo_smart_money(self) -> None: total_realized_pnl=total_pnl, total_volume_usd=volume, largest_win=abs(total_pnl) * random.uniform(0.05, 0.3) if total_pnl > 0 else random.uniform(100, 50000), - largest_loss=-abs(total_pnl) * random.uniform(0.02, 0.15) if total_pnl < 0 else -random.uniform(100, 30000), + largest_loss=(-abs(total_pnl) * random.uniform(0.02, 0.15) + if total_pnl < 0 else -random.uniform(100, 30000)), avg_hold_time_seconds=random.uniform(60, 86400), win_rate=win_rate, sharpe_ratio=sharpe, @@ -992,6 +1055,7 @@ async def _demo_hlp(self) -> None: async def _demo_market_refresh(self) -> None: """Generate mock market data for demo mode.""" import random + from src.data_layer.market_data import AssetInfo mock_assets = [ @@ -1032,6 +1096,7 @@ async def _demo_market_refresh(self) -> None: async def _demo_deribit(self) -> None: """Generate synthetic Deribit DVOL data for demo mode.""" import random + from src.data_layer.deribit import DeribitIVSnapshot btc_iv = 55.0 @@ -1063,6 +1128,7 @@ async def _demo_deribit(self) -> None: async def _demo_basis(self) -> None: """Generate synthetic spot/perp basis data for demo mode.""" import random + from src.data_layer.spot_prices import SpotPriceSnapshot bases = {"BTC": 0.05, "ETH": 0.03, "SOL": 0.08} @@ -1094,6 +1160,7 @@ async def _demo_basis(self) -> None: async def _demo_lsr(self) -> None: """Generate synthetic long/short ratio data for demo mode.""" import random + from src.data_layer.long_short_ratio import LongShortSnapshot ratios = {"BTC": 1.1, "ETH": 0.95, "SOL": 1.2} diff --git a/src/data_layer/liquidation_feed.py b/src/data_layer/liquidation_feed.py index c9f3734..ed78602 100644 --- a/src/data_layer/liquidation_feed.py +++ b/src/data_layer/liquidation_feed.py @@ -46,6 +46,10 @@ def normalize_symbol(raw: str, exchange: str) -> str: # trades at least this large (USD). These are estimates, not confirmed events. HL_LIQUIDATION_MIN_USD = 10_000 +# Deadline for REST polls (price context); a hung endpoint must not wedge the +# poll loop. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10) + def exchange_coverage() -> dict[str, dict[str, str]]: """Per-exchange description of HOW liquidations are collected, so consumers @@ -159,19 +163,25 @@ def __init__(self, feed: LiquidationFeed): async def _on_message(self, data: Any) -> None: if isinstance(data, dict) and data.get("e") == "forceOrder": - o = data["o"] - price = float(o["p"]) - qty = float(o["q"]) - side_raw = o["S"].upper() - event = LiquidationEvent( - timestamp=o["T"] / 1000.0, - exchange="binance", - symbol=normalize_symbol(o["s"], "binance"), - side="long" if side_raw == "SELL" else "short", - size_usd=price * qty, - price=price, - quantity=qty, - ) + # Exchange payloads are untrusted: one malformed record must not + # raise out of the connection loop and trigger a reconnect. + try: + o = data["o"] + price = float(o["p"]) + qty = float(o["q"]) + side_raw = str(o["S"]).upper() + event = LiquidationEvent( + timestamp=float(o["T"]) / 1000.0, + exchange="binance", + symbol=normalize_symbol(str(o["s"]), "binance"), + side="long" if side_raw == "SELL" else "short", + size_usd=price * qty, + price=price, + quantity=qty, + ) + except (KeyError, TypeError, ValueError): + self.feed.record_parse_error("binance", data) + return await self.feed.emit(event) @@ -217,8 +227,8 @@ async def _on_message(self, data: Any) -> None: confirmed=True, ) await self.feed.emit(event) - except Exception: - logger.debug("[bybit] failed to parse liquidation: %s", d) + except (KeyError, TypeError, ValueError): + self.feed.record_parse_error("bybit", d) class OKXConnection(ExchangeConnection): @@ -239,24 +249,40 @@ async def _on_connected(self, ws: aiohttp.ClientWebSocketResponse) -> None: async def _on_message(self, data: Any) -> None: if not isinstance(data, dict) or "data" not in data: return + if not isinstance(data["data"], list): + self.feed.record_parse_error("okx", data) + return for d in data["data"]: - details = d.get("details", []) - inst_id = d.get("instId", "") + # Drop malformed records individually — one bad detail must not + # kill the whole message or the connection loop. + try: + details = d.get("details", []) + inst_id = d.get("instId", "") + if not isinstance(details, list): + self.feed.record_parse_error("okx", d) + continue + except AttributeError: + self.feed.record_parse_error("okx", d) + continue for det in details: - price = float(det.get("bkPx", 0)) - qty = float(det.get("sz", 0)) - side_raw = det.get("side", "").lower() - ts_raw = det.get("ts", "0") - event = LiquidationEvent( - timestamp=int(ts_raw) / 1000.0, - exchange="okx", - symbol=normalize_symbol(inst_id, "okx"), - side="long" if side_raw == "sell" else "short", - size_usd=price * qty, - price=price, - quantity=qty, - ) + try: + price = float(det.get("bkPx", 0) or 0) + qty = float(det.get("sz", 0) or 0) + side_raw = str(det.get("side", "")).lower() + ts_raw = det.get("ts", "0") or "0" + event = LiquidationEvent( + timestamp=int(ts_raw) / 1000.0, + exchange="okx", + symbol=normalize_symbol(inst_id, "okx"), + side="long" if side_raw == "sell" else "short", + size_usd=price * qty, + price=price, + quantity=qty, + ) + except (KeyError, TypeError, ValueError, AttributeError): + self.feed.record_parse_error("okx", det) + continue await self.feed.emit(event) @@ -302,17 +328,25 @@ async def stop(self) -> None: async def _price_poll(self) -> None: """Poll mid prices to have context for liquidation detection.""" + consecutive_failures = 0 while self._running: try: async with self._session.post( - self.API_URL, json={"type": "allMids"} + self.API_URL, json={"type": "allMids"}, timeout=HTTP_TIMEOUT ) as resp: if resp.status == 200: self._mid_prices = {k: float(v) for k, v in (await resp.json()).items()} + consecutive_failures = 0 except asyncio.CancelledError: return except Exception: - pass + consecutive_failures += 1 + # Silent-pass hid outages for hours; warn once it looks real. + if consecutive_failures in (3, 10) or consecutive_failures % 100 == 0: + logger.warning( + "[hyperliquid] price poll failing (%d consecutive)", + consecutive_failures, exc_info=True, + ) await asyncio.sleep(self.POLL_INTERVAL) async def _ws_loop(self) -> None: @@ -351,18 +385,23 @@ async def _process_trades(self, trades: list[dict]) -> None: Heuristic: Large trades that move price aggressively are likely liquidations. """ for t in trades: - tid = t.get("tid", 0) - if tid in self._seen_tids: + try: + tid = t.get("tid", 0) + if tid in self._seen_tids: + continue + self._seen_tids[tid] = None + while len(self._seen_tids) > 100_000: + self._seen_tids.popitem(last=False) # Remove oldest + + coin = t.get("coin", "") + price = float(t.get("px", 0) or 0) + qty = float(t.get("sz", 0) or 0) + ts_ms = int(t.get("time", 0) or 0) + side = t.get("side", "") # "B" = buyer taker, "A" = seller taker + except (TypeError, ValueError, AttributeError): + self.feed.record_parse_error("hyperliquid", t) continue - self._seen_tids[tid] = None - while len(self._seen_tids) > 100_000: - self._seen_tids.popitem(last=False) # Remove oldest - - coin = t.get("coin", "") - price = float(t.get("px", 0)) - qty = float(t.get("sz", 0)) size_usd = price * qty - side = t.get("side", "") # "B" = buyer taker, "A" = seller taker # Only flag large trades as potential liquidations if size_usd < self.LARGE_TRADE_USD: @@ -371,7 +410,7 @@ async def _process_trades(self, trades: list[dict]) -> None: # Side logic: "A" (ask/sell taker) = someone is aggressively selling = long liquidation # "B" (bid/buy taker) = someone aggressively buying = short liquidation event = LiquidationEvent( - timestamp=int(t.get("time", 0)) / 1000.0, + timestamp=ts_ms / 1000.0, exchange="hyperliquid", symbol=coin, side="long" if side == "A" else "short", @@ -400,6 +439,21 @@ def __init__(self, max_events: int = 10_000): self._connections: list[ExchangeConnection | HyperliquidConnection] = [] self._lock = asyncio.Lock() self._running = False + # Per-exchange count of records dropped at the parse boundary. + # Surfaced via get_stats() so schema drift is visible, not silent. + self.parse_errors: dict[str, int] = {} + + def record_parse_error(self, exchange: str, payload: Any = None) -> None: + """Count a malformed record dropped at the parse boundary.""" + count = self.parse_errors.get(exchange, 0) + 1 + self.parse_errors[exchange] = count + # Log the first few and then sample, so a schema change is visible + # without a malformed-message flood drowning the logs. + if count <= 3 or count % 1000 == 0: + logger.warning( + "[%s] dropped malformed record (%d total): %.300s", + exchange, count, payload, + ) async def start(self) -> None: logger.info("starting liquidation feed") @@ -494,6 +548,7 @@ def get_stats(self, window_minutes: int = 60) -> dict[str, Any]: "heuristic_count": heuristic_count, "confirmed_volume_usd": confirmed_volume_usd, "heuristic_volume_usd": heuristic_volume_usd, + "parse_errors": dict(self.parse_errors), "coverage": _coverage, "by_exchange": { k: { diff --git a/src/data_layer/market_data.py b/src/data_layer/market_data.py index 11b6dc9..04b6968 100644 --- a/src/data_layer/market_data.py +++ b/src/data_layer/market_data.py @@ -11,6 +11,10 @@ logger = logging.getLogger(__name__) +# Every outbound request gets an explicit deadline: a hung exchange endpoint +# must fail the refresh cycle, not stall the hub's market-refresh loop forever. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10) + @dataclass class AssetInfo: @@ -132,9 +136,21 @@ async def get_orderbook(self, symbol: str, depth: int = 20) -> dict: return {"bids": [], "asks": []} levels = data.get("levels", [[], []]) - bids = [{"price": float(b["px"]), "size": float(b["sz"])} for b in levels[0][:depth]] - asks = [{"price": float(a["px"]), "size": float(a["sz"])} for a in levels[1][:depth]] - return {"bids": bids, "asks": asks} + if not isinstance(levels, list) or len(levels) < 2: + return {"bids": [], "asks": []} + + def _parse_side(raw_levels) -> list[dict]: + parsed: list[dict] = [] + if not isinstance(raw_levels, list): + return parsed + for lvl in raw_levels[:depth]: + try: + parsed.append({"price": float(lvl["px"]), "size": float(lvl["sz"])}) + except (KeyError, TypeError, ValueError): + logger.debug("Dropped malformed orderbook level: %.100s", lvl) + return parsed + + return {"bids": _parse_side(levels[0]), "asks": _parse_side(levels[1])} async def get_candles( self, @@ -164,14 +180,17 @@ async def get_candles( candles: list[dict] = [] for c in data: - candles.append({ - "timestamp": c.get("t"), - "open": float(c.get("o", 0)), - "high": float(c.get("h", 0)), - "low": float(c.get("l", 0)), - "close": float(c.get("c", 0)), - "volume": float(c.get("v", 0)), - }) + try: + candles.append({ + "timestamp": c.get("t"), + "open": float(c.get("o", 0)), + "high": float(c.get("h", 0)), + "low": float(c.get("l", 0)), + "close": float(c.get("c", 0)), + "volume": float(c.get("v", 0)), + }) + except (TypeError, ValueError, AttributeError): + logger.debug("Dropped malformed candle: %.100s", c) return candles async def get_recent_trades(self, symbol: str, limit: int = 100) -> list[dict]: @@ -187,12 +206,15 @@ async def get_recent_trades(self, symbol: str, limit: int = 100) -> list[dict]: trades: list[dict] = [] for t in data[:limit]: - trades.append({ - "time": t.get("time"), - "price": float(t.get("px", 0)), - "size": float(t.get("sz", 0)), - "side": t.get("side", ""), - }) + try: + trades.append({ + "time": t.get("time"), + "price": float(t.get("px", 0)), + "size": float(t.get("sz", 0)), + "side": t.get("side", ""), + }) + except (TypeError, ValueError, AttributeError): + logger.debug("Dropped malformed trade: %.100s", t) return trades # ── Internals ───────────────────────────────────────────────── @@ -211,6 +233,7 @@ async def _post(self, session: aiohttp.ClientSession, payload: dict) -> dict | l HYPERLIQUID_INFO_URL, json=payload, headers={"Content-Type": "application/json"}, + timeout=HTTP_TIMEOUT, ) as resp: resp.raise_for_status() return await resp.json() diff --git a/src/data_layer/orderbook.py b/src/data_layer/orderbook.py index 8fc2b09..e462b6a 100644 --- a/src/data_layer/orderbook.py +++ b/src/data_layer/orderbook.py @@ -137,21 +137,34 @@ def is_stale(self, now: float | None = None) -> bool: # ── Book update (public for testability) ───────────────────── def _update_book(self, symbol: str, data: dict) -> None: - """Parse l2Book levels data and update the in-memory book.""" + """Parse l2Book levels data and update the in-memory book. + + Malformed levels are dropped individually so one bad entry can't + raise out of the WS read loop and force a reconnect (or poison the + whole book). + """ if symbol not in self.books: return levels = data.get("levels", [[], []]) + if not isinstance(levels, list): + logger.warning("[orderbook] malformed levels for %s: %.200s", symbol, levels) + return bids_raw = levels[0] if len(levels) > 0 else [] asks_raw = levels[1] if len(levels) > 1 else [] - bids = [ - OrderBookLevel(price=float(b["px"]), size=float(b["sz"])) - for b in bids_raw[:self.depth] - ] - asks = [ - OrderBookLevel(price=float(a["px"]), size=float(a["sz"])) - for a in asks_raw[:self.depth] - ] + def _parse_side(raw_levels) -> list[OrderBookLevel]: + parsed: list[OrderBookLevel] = [] + if not isinstance(raw_levels, list): + return parsed + for lvl in raw_levels[:self.depth]: + try: + parsed.append(OrderBookLevel(price=float(lvl["px"]), size=float(lvl["sz"]))) + except (KeyError, TypeError, ValueError): + logger.debug("[orderbook] dropped malformed level for %s: %.100s", symbol, lvl) + return parsed + + bids = _parse_side(bids_raw) + asks = _parse_side(asks_raw) self.books[symbol]["bids"] = bids self.books[symbol]["asks"] = asks diff --git a/src/data_layer/orderflow_engine.py b/src/data_layer/orderflow_engine.py index 3399b78..d963aac 100644 --- a/src/data_layer/orderflow_engine.py +++ b/src/data_layer/orderflow_engine.py @@ -231,6 +231,33 @@ def data_age(self, now: float | None = None) -> float: def is_stale(self, now: float | None = None) -> bool: return self.data_age(now) > STALE_AFTER_SECONDS + def venue_data_age(self, venue: str, now: float | None = None) -> float: + """Seconds since the last trade from ONE venue (inf if none yet).""" + last = (self.last_hl_message_at if venue == "hyperliquid" + else self.last_binance_message_at) + if last <= 0: + return float("inf") + return (now if now is not None else time.time()) - last + + def venue_is_stale(self, venue: str, now: float | None = None) -> bool: + return self.venue_data_age(venue, now) > STALE_AFTER_SECONDS + + def venue_freshness(self, now: float | None = None) -> dict[str, dict]: + """Per-venue freshness so a dead venue can't hide behind a live one. + + The combined is_stale() uses the freshest venue (intentional: the + blended CVD is still moving), but consumers of venue-specific data + need to know when THEIR venue went quiet. + """ + out: dict[str, dict] = {} + for venue in ("hyperliquid", "binance"): + age = self.venue_data_age(venue, now) + out[venue] = { + "data_age_seconds": None if age == float("inf") else round(age, 1), + "stale": age > STALE_AFTER_SECONDS, + } + return out + async def start(self) -> None: """Open WebSocket(s), subscribe, and begin processing in background.""" if self._running: diff --git a/src/data_layer/persistence.py b/src/data_layer/persistence.py index bc97ceb..a8afe45 100644 --- a/src/data_layer/persistence.py +++ b/src/data_layer/persistence.py @@ -13,12 +13,11 @@ """ import atexit +import logging import sqlite3 -import time import threading -import logging +import time from pathlib import Path -from dataclasses import asdict logger = logging.getLogger(__name__) @@ -58,14 +57,28 @@ def __init__(self, db_path: str | Path = DB_PATH): self._conn = conn self._init_tables() except sqlite3.DatabaseError: - logger.warning("Database corrupted at %s — recreating", self.db_path) if conn is not None: conn.close() - # Remove corrupted db and WAL/SHM files + # Quarantine, never delete: move the corrupted DB (and WAL/SHM) + # aside with a timestamp so history survives for postmortem and + # possible `.recover`, then start fresh. + quarantine_dir = self.db_path.parent / "corrupted" + quarantine_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S") for suffix in ("", "-wal", "-shm"): p = Path(str(self.db_path) + suffix) if p.exists(): - p.unlink() + dest = quarantine_dir / f"{p.name}.{stamp}" + try: + p.rename(dest) + except OSError: + logger.exception("Failed to quarantine %s", p) + p.unlink() # last resort so we can still start + logger.error( + "Database corrupted at %s — quarantined to %s and recreated. " + "Historical data is preserved there for recovery.", + self.db_path, quarantine_dir, + ) self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False, timeout=10) self._conn.execute("PRAGMA journal_mode=WAL") self._conn.execute("PRAGMA synchronous=NORMAL") @@ -263,18 +276,54 @@ def _init_tables(self): CREATE INDEX IF NOT EXISTS idx_options_ts ON options_data(timestamp); CREATE INDEX IF NOT EXISTS idx_options_underlying ON options_data(underlying); """) - # Migration: add new columns to existing tables if not present - for table, col, col_type in [ - ("snapshots", "premium_pct", "REAL DEFAULT 0.0"), - ("snapshots", "basis_pct", "REAL DEFAULT 0.0"), - ("paper_trades", "funding_collected", "REAL DEFAULT 0.0"), - ]: - try: - self._conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} {col_type}") - except Exception: - pass # Column already exists + self._run_migrations() self._conn.commit() + # Bump when adding a migration below. The schema_version table lets a + # future release tell an old DB from a new one instead of guessing from + # ALTER TABLE failures. + SCHEMA_VERSION = 2 + + def _run_migrations(self) -> None: + """Versioned, idempotent migrations. Caller holds the lock. + + Only "duplicate column" is treated as already-applied; any other + migration failure is a real error and is raised so the app doesn't + keep running against a half-migrated schema. + """ + self._conn.execute( + "CREATE TABLE IF NOT EXISTS schema_version " + "(version INTEGER NOT NULL, applied_at REAL NOT NULL)" + ) + row = self._conn.execute("SELECT MAX(version) FROM schema_version").fetchone() + current = row[0] or 0 + + # v1: original schema (implicit for pre-versioning DBs). + # v2: extra columns on snapshots / paper_trades. + for table, col, col_type in [ + ("snapshots", "premium_pct", "REAL DEFAULT 0.0"), + ("snapshots", "basis_pct", "REAL DEFAULT 0.0"), + ("paper_trades", "funding_collected", "REAL DEFAULT 0.0"), + ]: + try: + self._conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} {col_type}") + except sqlite3.OperationalError as exc: + if "duplicate column" not in str(exc).lower(): + logger.error("Migration failed for %s.%s: %s", table, col, exc) + raise + + if current < self.SCHEMA_VERSION: + self._conn.execute( + "INSERT INTO schema_version (version, applied_at) VALUES (?, ?)", + (self.SCHEMA_VERSION, time.time()), + ) + + def get_schema_version(self) -> int: + """Highest applied schema version (0 for a brand-new/legacy DB).""" + with self._lock: + row = self._conn.execute("SELECT MAX(version) FROM schema_version").fetchone() + return row[0] or 0 + def attach(self, hub) -> None: """Attach to a HyperDataHub — automatically persists all events.""" hub.on_liquidation(self._save_liquidation) @@ -292,7 +341,8 @@ def _save_liquidation(self, event) -> None: """Callback: save a liquidation event.""" with self._lock: self._conn.execute( - "INSERT INTO liquidations (timestamp, exchange, symbol, side, size_usd, price, quantity, confirmed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO liquidations (timestamp, exchange, symbol, side, size_usd, " + "price, quantity, confirmed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (event.timestamp, event.exchange, event.symbol, event.side, event.size_usd, event.price, event.quantity, 1 if getattr(event, 'confirmed', True) else 0, @@ -317,7 +367,8 @@ def _save_trade(self, trade) -> None: if self._trade_count % self.TRADE_SAMPLE_RATE != 0: return self._conn.execute( - "INSERT INTO trades (timestamp, symbol, side, price, size, size_usd, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO trades (timestamp, symbol, side, price, size, size_usd, " + "created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", (trade.timestamp, trade.symbol, trade.side, trade.price, trade.size, trade.size_usd, time.time()) ) @@ -372,7 +423,8 @@ def get_liquidations(self, since_hours: float = 24, exchange: str | None = None, symbol: str | None = None, limit: int = 1000) -> list[dict]: """Get historical liquidation events.""" cutoff = time.time() - (since_hours * 3600) - query = "SELECT timestamp, exchange, symbol, side, size_usd, price, quantity, confirmed FROM liquidations WHERE timestamp > ?" + query = ("SELECT timestamp, exchange, symbol, side, size_usd, price, quantity, " + "confirmed FROM liquidations WHERE timestamp > ?") params: list = [cutoff] if exchange: query += " AND exchange = ?" @@ -455,7 +507,8 @@ def _save_smart_money_signal(self, signal) -> None: """Callback: save a smart money signal.""" with self._lock: self._conn.execute( - "INSERT INTO smart_money_signals (timestamp, address, tier, action, symbol, size_usd, wallet_rank, signal_type, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO smart_money_signals (timestamp, address, tier, action, symbol, " + "size_usd, wallet_rank, signal_type, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (signal.timestamp, signal.address, signal.tier, signal.action, signal.symbol, signal.size_usd, signal.wallet_rank, signal.signal_type, time.time()), @@ -672,7 +725,8 @@ def save_funding_rate(self, snap) -> None: """Save a funding rate snapshot.""" with self._lock: self._conn.execute( - "INSERT INTO funding_rates (timestamp, exchange, symbol, funding_rate_hourly, funding_rate_annualized, created_at) VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO funding_rates (timestamp, exchange, symbol, funding_rate_hourly, " + "funding_rate_annualized, created_at) VALUES (?, ?, ?, ?, ?, ?)", (snap.timestamp, snap.exchange, snap.symbol, snap.funding_rate_hourly, snap.funding_rate_annualized, time.time()), ) @@ -683,7 +737,8 @@ def get_funding_rates(self, exchange: str | None = None, symbol: str | None = No hours: float = 24, limit: int = 500) -> list[dict]: """Get historical funding rate snapshots.""" cutoff = time.time() - (hours * 3600) - query = "SELECT timestamp, exchange, symbol, funding_rate_hourly, funding_rate_annualized FROM funding_rates WHERE timestamp > ?" + query = ("SELECT timestamp, exchange, symbol, funding_rate_hourly, " + "funding_rate_annualized FROM funding_rates WHERE timestamp > ?") params: list = [cutoff] if exchange: query += " AND exchange = ?" @@ -704,7 +759,8 @@ def get_funding_rates(self, exchange: str | None = None, symbol: str | None = No def save_long_short_ratio(self, snap) -> None: with self._lock: self._conn.execute( - "INSERT INTO long_short_ratios (timestamp, symbol, long_ratio, short_ratio, long_short_ratio, created_at) VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO long_short_ratios (timestamp, symbol, long_ratio, short_ratio, " + "long_short_ratio, created_at) VALUES (?, ?, ?, ?, ?, ?)", (snap.timestamp, snap.symbol, snap.long_ratio, snap.short_ratio, snap.long_short_ratio, time.time()), ) self._event_count += 1 @@ -712,7 +768,8 @@ def save_long_short_ratio(self, snap) -> None: def get_long_short_ratios(self, symbol: str | None = None, hours: float = 24, limit: int = 200) -> list[dict]: cutoff = time.time() - (hours * 3600) - query = "SELECT timestamp, symbol, long_ratio, short_ratio, long_short_ratio FROM long_short_ratios WHERE timestamp > ?" + query = ("SELECT timestamp, symbol, long_ratio, short_ratio, long_short_ratio " + "FROM long_short_ratios WHERE timestamp > ?") params: list = [cutoff] if symbol: query += " AND symbol = ?" @@ -731,8 +788,10 @@ def save_options_snapshot(self, snap) -> None: """Save a Deribit IV snapshot.""" with self._lock: self._conn.execute( - "INSERT INTO options_data (timestamp, underlying, mark_iv, bid_iv, ask_iv, oi_usd, index_price, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - (snap.timestamp, snap.underlying, snap.mark_iv, snap.bid_iv, snap.ask_iv, snap.oi_usd, snap.index_price, time.time()), + "INSERT INTO options_data (timestamp, underlying, mark_iv, bid_iv, ask_iv, " + "oi_usd, index_price, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (snap.timestamp, snap.underlying, snap.mark_iv, snap.bid_iv, snap.ask_iv, + snap.oi_usd, snap.index_price, time.time()), ) self._event_count += 1 self._maybe_commit() @@ -740,7 +799,8 @@ def save_options_snapshot(self, snap) -> None: def get_options_data(self, underlying: str | None = None, hours: float = 24, limit: int = 200) -> list[dict]: """Get historical Deribit IV snapshots.""" cutoff = time.time() - (hours * 3600) - query = "SELECT timestamp, underlying, mark_iv, bid_iv, ask_iv, oi_usd, index_price FROM options_data WHERE timestamp > ?" + query = ("SELECT timestamp, underlying, mark_iv, bid_iv, ask_iv, oi_usd, " + "index_price FROM options_data WHERE timestamp > ?") params: list = [cutoff] if underlying: query += " AND underlying = ?" diff --git a/src/data_layer/position_scanner.py b/src/data_layer/position_scanner.py index 296ba40..6e3c8f5 100644 --- a/src/data_layer/position_scanner.py +++ b/src/data_layer/position_scanner.py @@ -15,6 +15,10 @@ RATE_LIMIT_PER_SEC = 10 META_CACHE_TTL = 300 # 5 minutes +# Explicit deadline on every request so a hung endpoint fails the scan cycle +# instead of blocking the hub's position-scan loop indefinitely. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10) + @dataclass class TrackedPosition: @@ -102,12 +106,18 @@ async def discover_addresses(self, limit: int = 100) -> set[str]: }) if isinstance(data, list): for trade in data: + # Validate at the boundary: exchange payloads are + # untrusted, and a junk identifier persisted here gets + # re-scanned (one API call per cycle) forever. + candidates: list[object] = [] for side_key in ("buyer", "seller", "users"): if side_key in trade and isinstance(trade[side_key], str): - new_addresses.add(trade[side_key]) + candidates.append(trade[side_key]) if "users" in trade and isinstance(trade["users"], list): - for addr in trade["users"]: - new_addresses.add(addr) + candidates.extend(trade["users"]) + for addr in candidates: + if address_store.is_valid_address(addr): + new_addresses.add(address_store.normalize_address(addr)) if len(new_addresses) >= limit: break except Exception: @@ -279,6 +289,7 @@ async def _post(self, payload: dict) -> dict | list | None: API_URL, json=payload, headers={"Content-Type": "application/json"}, + timeout=HTTP_TIMEOUT, ) as resp: resp.raise_for_status() return await resp.json() @@ -294,6 +305,10 @@ def _save_discovered_addresses(self): address_store.add_addresses(self.discovered_addresses, source="position_scanner") def add_addresses(self, addresses: list[str]): - """Manually add addresses to track.""" - self.discovered_addresses.update(addresses) - address_store.add_addresses(addresses, source="position_scanner_manual") + """Manually add addresses to track (validated + normalized).""" + valid = [ + address_store.normalize_address(a) + for a in addresses if address_store.is_valid_address(a) + ] + self.discovered_addresses.update(valid) + address_store.add_addresses(valid, source="position_scanner_manual") diff --git a/src/data_layer/smart_money.py b/src/data_layer/smart_money.py index a45859f..fde584d 100644 --- a/src/data_layer/smart_money.py +++ b/src/data_layer/smart_money.py @@ -62,6 +62,7 @@ class WalletProfile: pnl_score: float = 0.0 # log-scaled PnL sharpe_ratio: float = 0.0 # risk-adjusted returns composite_score: float = 0.0 # final weighted score + confidence: float = 0.0 # 0-1 sample-size confidence (see _compute_confidence) # Classification rank: int = 0 # 1 = best performer @@ -101,8 +102,13 @@ class SmartMoneyEngine: BETA = 0.40 # PnL weight (log-scaled) GAMMA = 0.25 # Sharpe weight - # Thresholds - MIN_TRADES_FOR_RANKING = 3 # Low for early data collection; tighten later + # Thresholds. Three closed trades says nothing about skill — a coin flip + # "wins" three in a row 12.5% of the time — so ranking requires a + # minimally meaningful sample plus real volume, and every profile carries + # a sample-size confidence that consumers must surface alongside tiers. + MIN_TRADES_FOR_RANKING = 10 + MIN_VOLUME_FOR_RANKING = 50_000 # Total traded volume (USD) + FULL_CONFIDENCE_TRADES = 50 # Trades at which confidence saturates SMART_MONEY_TOP_N = 100 # Top 100 = smart money DUMB_MONEY_BOTTOM_N = 100 # Bottom 100 = dumb money ANALYSIS_INTERVAL = 300 # Analyze wallets every 5 minutes @@ -200,8 +206,8 @@ async def fetch_leaderboard_wallets(self, top_n: int = 50) -> list[str]: or row.get("user") or "" ) - if isinstance(addr, str) and addr.startswith("0x"): - addresses.append(addr.lower()) + if address_store.is_valid_address(addr): + addresses.append(address_store.normalize_address(addr)) if addresses: logger.info( "[smart_money] Fetched %d leaderboard wallets from %s", @@ -229,8 +235,8 @@ async def seed_from_leaderboard(self) -> int: raw = json.loads(ANCHOR_PATH.read_text()) for entry in raw.get("anchors", []): addr = entry.get("address", "") - if isinstance(addr, str) and addr.startswith("0x"): - seed_addrs.append(addr.lower()) + if address_store.is_valid_address(addr): + seed_addrs.append(address_store.normalize_address(addr)) anchor_count += 1 except Exception: logger.debug("[smart_money] Failed to load anchor_wallets.json") @@ -337,14 +343,19 @@ async def _discovery_loop(self) -> None: if data.get("channel") == "trades": for trade in data.get("data", []): for addr in trade.get("users", []): - if addr and addr not in self.wallets: + # Untrusted payload: only well-formed + # wallet addresses become profiles. + if not address_store.is_valid_address(addr): + continue + addr = address_store.normalize_address(addr) + if addr not in self.wallets: self.wallets[addr] = WalletProfile( address=addr, discovered_at=time.time(), last_seen=time.time(), last_analyzed=0, ) - elif addr and addr in self.wallets: + else: self.wallets[addr].last_seen = time.time() except asyncio.CancelledError: return @@ -376,7 +387,10 @@ async def discover_from_trades(self, symbols: list[str] | None = None, duration: if data.get("channel") == "trades": for trade in data.get("data", []): for addr in trade.get("users", []): - if addr and addr not in self.wallets: + if not address_store.is_valid_address(addr): + continue + addr = address_store.normalize_address(addr) + if addr not in self.wallets: self.wallets[addr] = WalletProfile( address=addr, discovered_at=time.time(), @@ -425,7 +439,7 @@ async def _analysis_loop(self) -> None: # Re-rank after each batch self.rank_all() - ranked_count = sum(1 for w in self.wallets.values() if w.total_trades >= self.MIN_TRADES_FOR_RANKING) + ranked_count = sum(1 for w in self.wallets.values() if self._qualifies_for_ranking(w)) logger.info( "[smart_money] Analyzed %d wallets, %d total tracked, %d ranked", len(batch), len(self.wallets), ranked_count, @@ -519,6 +533,7 @@ async def analyze_wallet(self, address: str) -> WalletProfile: wallet.sharpe_ratio = self._compute_sharpe(close_pnls) wallet.pnl_score = self._compute_pnl_score(total_pnl) wallet.composite_score = self._compute_composite(wallet) + wallet.confidence = self._compute_confidence(wallet) # 5. Fetch clearinghouse state for account value + open positions ch = await self._fetch_clearinghouse(address) @@ -578,16 +593,36 @@ def _compute_composite(self, w: WalletProfile) -> float: return self.ALPHA * wr + self.BETA * pnl + self.GAMMA * sharpe + def _compute_confidence(self, w: WalletProfile) -> float: + """Sample-size confidence in [0, 1]. + + Linear in closed-trade count up to FULL_CONFIDENCE_TRADES. This is a + coverage heuristic (recent fills only, no confidence interval on win + rate) — consumers must show it next to any smart/dumb label rather + than presenting tiers as certainty. + """ + return min(1.0, w.total_trades / float(self.FULL_CONFIDENCE_TRADES)) + + def _qualifies_for_ranking(self, w: WalletProfile) -> bool: + return (w.total_trades >= self.MIN_TRADES_FOR_RANKING + and w.total_volume_usd >= self.MIN_VOLUME_FOR_RANKING) + # ── Ranking ─────────────────────────────────────────────────────── def rank_all(self) -> None: """Re-rank all wallets by composite_score.""" - qualified = [ - w for w in self.wallets.values() - if w.total_trades >= self.MIN_TRADES_FOR_RANKING - ] + qualified = [w for w in self.wallets.values() if self._qualifies_for_ranking(w)] qualified.sort(key=lambda w: w.composite_score, reverse=True) + qualified_set = {w.address for w in qualified} + # Wallets that no longer qualify must lose their old rank/tier, or a + # stale "smart" label survives after the sample stops qualifying. + for w in self.wallets.values(): + if w.address not in qualified_set: + w.rank = 0 + if w.tier in ("smart", "average", "dumb"): + w.tier = "unknown" + for i, w in enumerate(qualified, 1): w.rank = i if i <= self.SMART_MONEY_TOP_N: @@ -707,7 +742,7 @@ def get_wallet(self, address: str) -> WalletProfile | None: def get_stats(self) -> dict: """Summary stats: total wallets, ranked wallets, signals generated.""" - ranked = sum(1 for w in self.wallets.values() if w.total_trades >= self.MIN_TRADES_FOR_RANKING) + ranked = sum(1 for w in self.wallets.values() if self._qualifies_for_ranking(w)) smart = sum(1 for w in self.wallets.values() if w.tier == "smart") dumb = sum(1 for w in self.wallets.values() if w.tier == "dumb") return { @@ -716,4 +751,15 @@ def get_stats(self) -> dict: "smart_wallets": smart, "dumb_wallets": dumb, "total_signals": len(self.signals), + # Coverage caveats: rankings come from recent fills only and small + # samples — consumers should show these limits, not just tiers. + "ranking_criteria": { + "min_trades": self.MIN_TRADES_FOR_RANKING, + "min_volume_usd": self.MIN_VOLUME_FOR_RANKING, + "note": ( + "Performance is computed from recent fills only; tiers are " + "heuristic. Check each wallet's `confidence` (0-1 sample-" + "size score) before acting on smart/dumb labels." + ), + }, } diff --git a/src/strategies/__init__.py b/src/strategies/__init__.py index 0f1644e..6cc6a66 100644 --- a/src/strategies/__init__.py +++ b/src/strategies/__init__.py @@ -1,2 +1,4 @@ -from .base import Strategy, Signal +from .base import Signal, Strategy from .paper_trader import PaperTrader + +__all__ = ["Signal", "Strategy", "PaperTrader"] diff --git a/src/strategies/base.py b/src/strategies/base.py index 78b305c..7a08212 100644 --- a/src/strategies/base.py +++ b/src/strategies/base.py @@ -8,8 +8,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from dataclasses import dataclass, field - +from dataclasses import dataclass # --------------------------------------------------------------------------- # Signal — the output of every strategy evaluation diff --git a/src/strategies/examples/__init__.py b/src/strategies/examples/__init__.py index a0ac70a..33099ee 100644 --- a/src/strategies/examples/__init__.py +++ b/src/strategies/examples/__init__.py @@ -2,3 +2,5 @@ from .funding_rate_arb import FundingRateArb from .liquidation_cascade import LiquidationCascade from .whale_follow import WhaleFollow + +__all__ = ["CVDMomentum", "FundingRateArb", "LiquidationCascade", "WhaleFollow"] diff --git a/src/strategies/examples/cvd_momentum.py b/src/strategies/examples/cvd_momentum.py index e3fc78c..6700b9c 100644 --- a/src/strategies/examples/cvd_momentum.py +++ b/src/strategies/examples/cvd_momentum.py @@ -11,7 +11,7 @@ from __future__ import annotations -from src.strategies.base import Strategy, Signal +from src.strategies.base import Signal, Strategy class CVDMomentum(Strategy): diff --git a/src/strategies/examples/funding_rate_arb.py b/src/strategies/examples/funding_rate_arb.py index b684716..23d6274 100644 --- a/src/strategies/examples/funding_rate_arb.py +++ b/src/strategies/examples/funding_rate_arb.py @@ -12,7 +12,7 @@ from __future__ import annotations -from src.strategies.base import Strategy, Signal +from src.strategies.base import Signal, Strategy class FundingRateArb(Strategy): diff --git a/src/strategies/examples/liquidation_cascade.py b/src/strategies/examples/liquidation_cascade.py index c33c368..b8387ca 100644 --- a/src/strategies/examples/liquidation_cascade.py +++ b/src/strategies/examples/liquidation_cascade.py @@ -12,7 +12,7 @@ import time -from src.strategies.base import Strategy, Signal +from src.strategies.base import Signal, Strategy class LiquidationCascade(Strategy): diff --git a/src/strategies/examples/whale_follow.py b/src/strategies/examples/whale_follow.py index 00390be..fe08e19 100644 --- a/src/strategies/examples/whale_follow.py +++ b/src/strategies/examples/whale_follow.py @@ -10,7 +10,7 @@ """ from __future__ import annotations -from src.strategies.base import Strategy, Signal +from src.strategies.base import Signal, Strategy class WhaleFollow(Strategy): diff --git a/src/strategies/llm_agent.py b/src/strategies/llm_agent.py index 357350d..27c7921 100644 --- a/src/strategies/llm_agent.py +++ b/src/strategies/llm_agent.py @@ -14,9 +14,12 @@ from __future__ import annotations import asyncio +import concurrent.futures import json import logging import os +import time +from collections import deque import aiohttp @@ -42,6 +45,10 @@ class LLMAgent(Strategy): """Strategy that delegates trading decisions to a language model.""" + # Budget guardrail: an LLM call per check interval adds up. Configurable + # via LLM_MAX_EVALS_PER_HOUR; evaluations beyond the budget are skipped. + DEFAULT_MAX_EVALS_PER_HOUR = 60 + def __init__(self, symbol: str = "BTC") -> None: self.symbol = symbol @@ -49,11 +56,32 @@ def __init__(self, symbol: str = "BTC") -> None: self.base_url = os.environ.get("LLM_BASE_URL", "http://localhost:11434/v1") self.model = os.environ.get("LLM_MODEL", "llama3") self.api_key = os.environ.get("LLM_API_KEY", "") + try: + self.max_evals_per_hour = int( + os.environ.get("LLM_MAX_EVALS_PER_HOUR", self.DEFAULT_MAX_EVALS_PER_HOUR) + ) + except ValueError: + self.max_evals_per_hour = self.DEFAULT_MAX_EVALS_PER_HOUR + self._eval_times: deque[float] = deque(maxlen=max(self.max_evals_per_hour, 1)) + # One long-lived worker thread — not a new executor per evaluation. + self._pool = concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix="llm-agent" + ) @property def name(self) -> str: return "llm_agent" + def _within_budget(self, now: float | None = None) -> bool: + """Sliding-window cap on LLM calls per hour.""" + now = time.time() if now is None else now + while self._eval_times and now - self._eval_times[0] > 3600: + self._eval_times.popleft() + if len(self._eval_times) >= self.max_evals_per_hour: + return False + self._eval_times.append(now) + return True + def evaluate(self, hub) -> Signal | None: """Build a market summary and ask the LLM for a decision. @@ -69,12 +97,18 @@ def evaluate(self, hub) -> Signal | None: ) return None + if not self._within_budget(): + logger.warning( + "LLM eval budget exhausted (%d/hour) — skipping evaluation", + self.max_evals_per_hour, + ) + return None + try: - # Run blocking LLM call in a thread so it doesn't stall the async loop - import concurrent.futures - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - future = pool.submit(self._sync_evaluate, hub) - return future.result(timeout=20) + # Run blocking LLM call in the persistent worker thread so it + # doesn't stall the async loop. + future = self._pool.submit(self._sync_evaluate, hub) + return future.result(timeout=20) except Exception: logger.exception("LLM agent error") return None @@ -161,50 +195,26 @@ async def _async_evaluate(self, hub) -> Signal | None: # ---- Parse response ---- try: text = data["choices"][0]["message"]["content"].strip() - except (KeyError, IndexError): + except (KeyError, IndexError, TypeError, AttributeError): logger.warning("Unexpected LLM response format: %s", json.dumps(data)[:200]) return None - lines = text.split("\n", 1) - action_word = lines[0].strip().upper() - reason = lines[1].strip() if len(lines) > 1 else "" - - # Validate action - if action_word not in ("BUY", "SELL", "HOLD"): - # Try to find the action word somewhere in the first line - for word in ("BUY", "SELL", "HOLD"): - if word in action_word: - action_word = word - break - else: - logger.warning("LLM returned unparseable action: %s", lines[0]) - return None - - if action_word == "HOLD": - return None - - return Signal( - symbol=self.symbol, - action=action_word, - size_usd=100.0, - confidence=0.6, - reason=f"[LLM] {reason}", - ) + return self._parse_response(text) def _parse_response(self, text: str) -> Signal | None: - """Parse LLM response text into a Signal.""" + """Parse LLM response text into a Signal — deterministic, reject-on-ambiguous. + + The first line must be exactly BUY, SELL, or HOLD (case-insensitive, + surrounding punctuation tolerated). Substring matching is deliberately + NOT done: "I would not BUY here" must never resolve to a BUY. + """ lines = text.split("\n", 1) - action_word = lines[0].strip().upper() + action_word = lines[0].strip().upper().strip(".!:*# ") reason = lines[1].strip() if len(lines) > 1 else "" if action_word not in ("BUY", "SELL", "HOLD"): - for word in ("BUY", "SELL", "HOLD"): - if word in action_word: - action_word = word - break - else: - logger.warning("LLM returned unparseable action: %s", lines[0]) - return None + logger.warning("LLM returned ambiguous action, rejecting: %r", lines[0][:100]) + return None if action_word == "HOLD": return None diff --git a/src/strategies/paper_trader.py b/src/strategies/paper_trader.py index aa9ef38..5477d0d 100644 --- a/src/strategies/paper_trader.py +++ b/src/strategies/paper_trader.py @@ -20,6 +20,7 @@ import asyncio import logging +import math import sqlite3 import time from pathlib import Path @@ -27,7 +28,7 @@ from rich.console import Console -from .base import Strategy, Signal +from .base import Signal, Strategy logger = logging.getLogger(__name__) console = Console() @@ -153,18 +154,47 @@ async def _loop(self) -> None: # Trade execution # ------------------------------------------------------------------ + @staticmethod + def _signal_is_valid(signal: Signal) -> bool: + """Reject malformed signals before they can corrupt the books.""" + try: + size = float(signal.size_usd) + except (TypeError, ValueError): + return False + return ( + signal.action in ("BUY", "SELL") + and isinstance(signal.symbol, str) and bool(signal.symbol) + and size > 0 + and math.isfinite(size) + ) + def _execute_trade(self, strategy_name: str, signal: Signal) -> None: - """Execute a paper trade: update positions, log to SQLite, print.""" + """Execute a paper trade: update positions, log to SQLite, print. + + Accounting invariants: + - balance never goes negative (adds to a position are balance-checked + exactly like opens); + - adding to a position updates the size-weighted average entry price; + - an opposite-side signal closes the whole position (explicit + close-all semantics; partial reduction is not modeled). + """ + if not self._signal_is_valid(signal): + logger.warning( + "Rejected invalid signal from %s: action=%r symbol=%r size_usd=%r", + strategy_name, signal.action, signal.symbol, signal.size_usd, + ) + return + # Get current market price for the symbol asset = self.hub.market.assets.get(signal.symbol) - if asset is None: + if asset is None or not asset.price or asset.price <= 0: logger.warning( "Cannot execute trade for %s — no market data", signal.symbol ) return price = asset.price - # Calculate PnL if closing/reducing an existing position + # Calculate PnL if closing an existing position pnl = 0.0 if signal.symbol in self.positions: pos = self.positions[signal.symbol] @@ -178,8 +208,19 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: self.balance += pos["size_usd"] + pnl del self.positions[signal.symbol] else: - # Adding to position in same direction — just increase size - pos["size_usd"] += signal.size_usd + # Adding in the same direction: balance-checked like an open, + # entry price becomes the size-weighted average. + if signal.size_usd > self.balance: + logger.warning( + "Insufficient balance to add to %s (need $%.2f, have $%.2f)", + signal.symbol, signal.size_usd, self.balance, + ) + return + new_size = pos["size_usd"] + signal.size_usd + pos["entry_price"] = ( + pos["entry_price"] * pos["size_usd"] + price * signal.size_usd + ) / new_size + pos["size_usd"] = new_size self.balance -= signal.size_usd else: # Open a new position diff --git a/tests/conftest.py b/tests/conftest.py index 5842a73..4677a82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,8 @@ """Pytest configuration and shared fixtures for HyperData tests.""" from __future__ import annotations -import sys import os +import sys # Ensure project root and src/ are importable _PROJECT_ROOT = os.path.join(os.path.dirname(__file__), "..") diff --git a/tests/test_deribit.py b/tests/test_deribit.py index 693ef7b..309c9d3 100644 --- a/tests/test_deribit.py +++ b/tests/test_deribit.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.deribit import DeribitIVSnapshot, DeribitFeed + +from data_layer.deribit import DeribitFeed, DeribitIVSnapshot def test_snapshot_fields(): diff --git a/tests/test_funding_rates.py b/tests/test_funding_rates.py index 0707153..d27d3f9 100644 --- a/tests/test_funding_rates.py +++ b/tests/test_funding_rates.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.funding_rates import FundingRateSnapshot, FundingRateCollector, normalise_fr_symbol + +from data_layer.funding_rates import FundingRateCollector, FundingRateSnapshot, normalise_fr_symbol def test_normalise_fr_symbol(): diff --git a/tests/test_hardening.py b/tests/test_hardening.py new file mode 100644 index 0000000..3ee8aef --- /dev/null +++ b/tests/test_hardening.py @@ -0,0 +1,595 @@ +"""Tests for the adversarial-review hardening pass. + +Covers: API bind guard / auth / CORS / rate limiting, WebSocket abuse limits, +liquidation dedup + cascade bypass, malformed exchange payloads, degraded hub +startup, paper-trader accounting invariants, LLM response parsing, persistence +corruption quarantine + schema versioning, and alert log redaction. +""" +from __future__ import annotations + +import json +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer + +from data_layer.liquidation_feed import ( + BinanceConnection, + LiquidationEvent, + LiquidationFeed, + OKXConnection, +) +from data_layer.orderbook import OrderBookEngine +from data_layer.persistence import DataStore +from src.api_server import ( + WS_BAD_MSG_LIMIT, + HyperDataAPI, + _is_loopback_host, + _make_auth_middleware, + _make_cors_middleware, + _make_rate_limit_middleware, + _RateLimiter, + _WSClient, +) +from src.strategies.base import Signal +from src.strategies.llm_agent import LLMAgent +from src.strategies.paper_trader import PaperTrader + +# ── Helpers ────────────────────────────────────────────────────── + +def _liq_event(**overrides) -> LiquidationEvent: + defaults = dict( + timestamp=time.time(), + exchange="binance", + symbol="BTC", + side="long", + size_usd=25_000.0, + price=70_000.0, + quantity=0.357, + ) + defaults.update(overrides) + return LiquidationEvent(**defaults) + + +def _api(monkeypatch=None, host="127.0.0.1") -> HyperDataAPI: + return HyperDataAPI(hub=MagicMock(), host=host) + + +async def _client_for(middlewares) -> TestClient: + app = web.Application(middlewares=middlewares) + + async def ok(request): + return web.json_response({"ok": True}) + + app.router.add_get("/v1/health", ok) + app.router.add_get("/v1/whales", ok) + client = TestClient(TestServer(app)) + await client.start_server() + return client + + +# ── Bind guard / security resolution ───────────────────────────── + +class TestBindGuard: + def test_loopback_hosts(self): + assert _is_loopback_host("127.0.0.1") + assert _is_loopback_host("localhost") + assert _is_loopback_host("::1") + assert not _is_loopback_host("0.0.0.0") + assert not _is_loopback_host("192.168.1.10") + assert not _is_loopback_host("") + + def test_loopback_needs_no_key(self, monkeypatch): + monkeypatch.delenv("HYPERDATA_API_KEY", raising=False) + monkeypatch.delenv("HYPERDATA_UNSAFE_PUBLIC_API", raising=False) + monkeypatch.delenv("HYPERDATA_CORS_ORIGINS", raising=False) + key, origins = _api(host="127.0.0.1")._resolve_security() + assert key == "" + assert origins is None # wildcard CORS allowed on loopback + + def test_public_bind_refused_without_key_or_ack(self, monkeypatch): + monkeypatch.delenv("HYPERDATA_API_KEY", raising=False) + monkeypatch.delenv("HYPERDATA_UNSAFE_PUBLIC_API", raising=False) + with pytest.raises(RuntimeError, match="Refusing to bind"): + _api(host="0.0.0.0")._resolve_security() + + def test_public_bind_allowed_with_key(self, monkeypatch): + monkeypatch.setenv("HYPERDATA_API_KEY", "sekrit") + monkeypatch.delenv("HYPERDATA_CORS_ORIGINS", raising=False) + key, origins = _api(host="0.0.0.0")._resolve_security() + assert key == "sekrit" + assert origins == set() # never wildcard CORS off loopback + + def test_public_bind_allowed_with_explicit_ack(self, monkeypatch): + monkeypatch.delenv("HYPERDATA_API_KEY", raising=False) + monkeypatch.setenv("HYPERDATA_UNSAFE_PUBLIC_API", "1") + key, origins = _api(host="0.0.0.0")._resolve_security() + assert key == "" + assert origins == set() + + def test_cors_allowlist_parsed(self, monkeypatch): + monkeypatch.setenv("HYPERDATA_API_KEY", "k") + monkeypatch.setenv("HYPERDATA_CORS_ORIGINS", "https://a.example, https://b.example") + _, origins = _api(host="0.0.0.0")._resolve_security() + assert origins == {"https://a.example", "https://b.example"} + + +# ── Middlewares over a live test server ────────────────────────── + +class TestAuthMiddleware: + @pytest.mark.asyncio + async def test_key_required_except_health(self): + client = await _client_for([ + _make_auth_middleware("sekrit"), + _make_cors_middleware(None), + ]) + try: + assert (await client.get("/v1/health")).status == 200 + assert (await client.get("/v1/whales")).status == 401 + ok_bearer = await client.get( + "/v1/whales", headers={"Authorization": "Bearer sekrit"}) + assert ok_bearer.status == 200 + ok_header = await client.get( + "/v1/whales", headers={"X-API-Key": "sekrit"}) + assert ok_header.status == 200 + bad = await client.get( + "/v1/whales", headers={"Authorization": "Bearer wrong"}) + assert bad.status == 401 + finally: + await client.close() + + @pytest.mark.asyncio + async def test_cors_allowlist_echoes_only_allowed_origin(self): + client = await _client_for([ + _make_cors_middleware({"https://ok.example"}), + ]) + try: + allowed = await client.get( + "/v1/health", headers={"Origin": "https://ok.example"}) + assert allowed.headers.get("Access-Control-Allow-Origin") == "https://ok.example" + denied = await client.get( + "/v1/health", headers={"Origin": "https://evil.example"}) + assert "Access-Control-Allow-Origin" not in denied.headers + finally: + await client.close() + + @pytest.mark.asyncio + async def test_rate_limit_returns_429(self): + limiter = _RateLimiter(max_requests=3, window_s=60) + client = await _client_for([_make_rate_limit_middleware(limiter)]) + try: + for _ in range(3): + assert (await client.get("/v1/health")).status == 200 + assert (await client.get("/v1/health")).status == 429 + finally: + await client.close() + + +class TestRateLimiterUnit: + def test_sliding_window(self): + limiter = _RateLimiter(max_requests=2, window_s=10) + assert limiter.allow("ip", now=100.0) + assert limiter.allow("ip", now=101.0) + assert not limiter.allow("ip", now=102.0) + # Window slides: the first hit expires. + assert limiter.allow("ip", now=110.5) + + def test_per_key_isolation(self): + limiter = _RateLimiter(max_requests=1, window_s=10) + assert limiter.allow("a", now=1.0) + assert limiter.allow("b", now=1.0) + assert not limiter.allow("a", now=2.0) + + +# ── WebSocket abuse limits ─────────────────────────────────────── + +class TestWSLimits: + def _client(self) -> _WSClient: + ws = MagicMock() + ws.closed = False + return _WSClient(ws) + + @pytest.mark.asyncio + async def test_bad_messages_disconnect(self): + api = _api() + client = self._client() + for i in range(WS_BAD_MSG_LIMIT - 1): + assert api._ws_msg_violates_limits(client, "{not json") is False + assert api._ws_msg_violates_limits(client, "{not json") is True + + @pytest.mark.asyncio + async def test_subscribe_still_works(self): + api = _api() + client = self._client() + raw = json.dumps({"subscribe": ["trade", "bogus_channel"]}) + assert api._ws_msg_violates_limits(client, raw) is False + assert client.subscriptions == {"trade"} + # Confirmation got queued for the writer task. + assert client.queue.qsize() == 1 + + @pytest.mark.asyncio + async def test_message_flood_disconnects(self): + api = _api() + client = self._client() + raw = json.dumps({"subscribe": ["trade"]}) + violated = False + for _ in range(50): + if api._ws_msg_violates_limits(client, raw): + violated = True + break + assert violated + + @pytest.mark.asyncio + async def test_broadcast_drops_when_queue_full(self): + api = _api() + client = self._client() + client.subscriptions = {"trade"} + api._ws_clients.append(client) + # No writer task draining -> queue fills to maxsize then drops. + for _ in range(client.queue.maxsize + 10): + api._broadcast("trade", {"x": 1}) + assert client.queue.qsize() == client.queue.maxsize + assert client.dropped_msgs == 10 + + +# ── Liquidation dedup + cascade bypass ─────────────────────────── + +class TestLiquidationDedup: + def test_same_event_deduped(self): + api = _api() + ev = _liq_event() + assert api._is_duplicate_liq(ev) is False + assert api._is_duplicate_liq(ev) is True + + def test_different_exchanges_not_deduped(self): + api = _api() + ts = time.time() + assert api._is_duplicate_liq(_liq_event(exchange="binance", timestamp=ts)) is False + assert api._is_duplicate_liq(_liq_event(exchange="okx", timestamp=ts)) is False + + def test_dedup_uses_exchange_timestamp_not_local_clock(self): + api = _api() + # Two records of the same event in different dedup buckets by + # exchange time are distinct regardless of local arrival time. + assert api._is_duplicate_liq(_liq_event(timestamp=1000.0)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=1009.0)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=1000.5)) is True + + def test_cascade_bypass_lifts_dedup_for_own_venue_only(self): + api = _api() + ts = time.time() + binance = _liq_event(exchange="binance", timestamp=ts) + # Three rapid events trigger the cascade bypass for binance/BTC/long. + for _ in range(3): + api._check_cascade(binance) + # A duplicate binance record now passes (cascade mode)... + assert api._is_duplicate_liq(binance) is False + assert api._is_duplicate_liq(binance) is False + # ...but hyperliquid's heuristic stream still dedups normally. + hl = _liq_event(exchange="hyperliquid", timestamp=ts) + assert api._is_duplicate_liq(hl) is False + assert api._is_duplicate_liq(hl) is True + + +# ── Malformed exchange payloads ────────────────────────────────── + +class TestMalformedPayloads: + @pytest.mark.asyncio + async def test_binance_malformed_dropped_not_raised(self): + feed = LiquidationFeed() + conn = BinanceConnection(feed) + received = [] + feed.on_liquidation(received.append) + + await conn._on_message({"e": "forceOrder", "o": {"p": "", "q": "1", "S": "SELL", + "s": "BTCUSDT", "T": 1}}) + await conn._on_message({"e": "forceOrder", "o": {}}) + await conn._on_message({"e": "forceOrder"}) + assert received == [] + assert feed.parse_errors["binance"] == 3 + + # A valid message still parses after the bad ones. + await conn._on_message({"e": "forceOrder", "o": { + "p": "70000", "q": "0.5", "S": "SELL", "s": "BTCUSDT", "T": 1700000000000, + }}) + assert len(received) == 1 + assert received[0].symbol == "BTC" + assert received[0].side == "long" + + @pytest.mark.asyncio + async def test_okx_malformed_detail_dropped_individually(self): + feed = LiquidationFeed() + conn = OKXConnection(feed) + received = [] + feed.on_liquidation(received.append) + + await conn._on_message({"data": [{ + "instId": "BTC-USDT-SWAP", + "details": [ + {"bkPx": "", "sz": "bogus", "side": "sell", "ts": "x"}, # bad + {"bkPx": "70000", "sz": "1", "side": "sell", "ts": "1700000000000"}, + ], + }]}) + assert len(received) == 1 + assert received[0].symbol == "BTC" + assert feed.parse_errors["okx"] == 1 + + @pytest.mark.asyncio + async def test_okx_non_list_shapes(self): + feed = LiquidationFeed() + conn = OKXConnection(feed) + await conn._on_message({"data": "not-a-list"}) + await conn._on_message({"data": [{"instId": "X", "details": "not-a-list"}]}) + assert feed.parse_errors["okx"] == 2 + + def test_orderbook_malformed_levels_dropped(self): + engine = OrderBookEngine(symbols=["BTC"]) + engine._update_book("BTC", {"levels": [ + [{"px": "70000", "sz": "1"}, {"px": "", "sz": "zzz"}, "garbage"], + [{"px": "70010", "sz": "2"}], + ]}) + book = engine.books["BTC"] + assert len(book["bids"]) == 1 + assert book["bids"][0].price == 70000.0 + assert len(book["asks"]) == 1 + + def test_orderbook_non_list_levels_ignored(self): + engine = OrderBookEngine(symbols=["BTC"]) + engine._update_book("BTC", {"levels": {"bad": "shape"}}) + assert engine.books["BTC"]["bids"] == [] + + +# ── Degraded hub startup ───────────────────────────────────────── + +class TestHubDegradedStartup: + @pytest.mark.asyncio + async def test_failed_component_recorded_not_swallowed(self, tmp_path, monkeypatch): + from src.data_layer import address_store, persistence + monkeypatch.setattr(persistence, "DB_PATH", tmp_path / "hub.db") + monkeypatch.setattr(address_store, "DATA_DIR", tmp_path) + monkeypatch.setattr(address_store, "DB_PATH", tmp_path / "hub.db") + monkeypatch.setattr(address_store, "LEGACY_JSON", tmp_path / "legacy.json") + monkeypatch.setattr(address_store, "_initialized", False) + + from src.data_layer.hub import HyperDataHub + hub = HyperDataHub() + # Every component start is stubbed: one fails, the rest succeed. + hub.liquidations.start = AsyncMock(side_effect=ConnectionError("down")) + for comp in (hub.orderflow, hub.smart_money, hub.hlp, hub.funding, + hub.lsr, hub.orderbook, hub.deribit): + comp.start = AsyncMock() + hub.spot.start = AsyncMock() + + await hub._start_live() + + assert hub.status.failed_components == ["liquidation_feed"] + assert hub.status.liquidation_feed == "error" + assert hub.status.orderflow_engine == "connected" + # Loop-driven components are 'starting', never a blind 'ready'. + assert hub.status.position_scanner == "starting" + assert hub.status.market_data == "starting" + + hub.store.close() + + +# ── Paper trader accounting invariants ─────────────────────────── + +class TestPaperTraderAccounting: + def _trader(self, price=100.0, balance=10_000.0) -> PaperTrader: + hub = MagicMock() + hub.market.assets = {"BTC": SimpleNamespace(price=price)} + return PaperTrader(hub, [], starting_balance=balance) + + def test_add_to_position_is_balance_checked(self): + trader = self._trader(balance=10_000.0) + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=6_000.0)) + assert trader.balance == 4_000.0 + # Second same-direction BUY exceeds the remaining balance -> rejected. + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=6_000.0)) + assert trader.balance == 4_000.0 + assert trader.positions["BTC"]["size_usd"] == 6_000.0 + assert trader.balance >= 0 + + def test_repeated_buys_never_go_negative(self): + trader = self._trader(balance=1_000.0) + for _ in range(50): + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=400.0)) + assert trader.balance >= 0 + assert trader.positions["BTC"]["size_usd"] == 800.0 + + def test_weighted_average_entry_price(self): + trader = self._trader(price=100.0) + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=1_000.0)) + trader.hub.market.assets["BTC"].price = 200.0 + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=1_000.0)) + pos = trader.positions["BTC"] + # (100*1000 + 200*1000) / 2000 = 150 + assert abs(pos["entry_price"] - 150.0) < 1e-9 + assert pos["size_usd"] == 2_000.0 + + def test_close_realizes_pnl(self): + trader = self._trader(price=100.0, balance=1_000.0) + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=500.0)) + trader.hub.market.assets["BTC"].price = 110.0 + trader._execute_trade("t", Signal("BTC", "SELL", size_usd=500.0)) + # +10% on 500 = +50 + assert abs(trader.balance - 1_050.0) < 1e-9 + assert "BTC" not in trader.positions + + def test_invalid_signals_rejected(self): + trader = self._trader() + for bad in [ + Signal("BTC", "BUY", size_usd=-5.0), + Signal("BTC", "BUY", size_usd=0.0), + Signal("BTC", "BUY", size_usd=float("nan")), + Signal("BTC", "BUY", size_usd=float("inf")), + Signal("BTC", "HOLD", size_usd=100.0), + Signal("", "BUY", size_usd=100.0), + ]: + trader._execute_trade("t", bad) + assert trader.positions == {} + assert trader.balance == 10_000.0 + + +# ── LLM response parsing ───────────────────────────────────────── + +class TestLLMParsing: + def _agent(self) -> LLMAgent: + return LLMAgent(symbol="BTC") + + def test_exact_actions(self): + agent = self._agent() + assert agent._parse_response("BUY\nmomentum").action == "BUY" + assert agent._parse_response("SELL\nfunding").action == "SELL" + assert agent._parse_response("HOLD\nchop") is None + + def test_case_and_punctuation_tolerated(self): + agent = self._agent() + assert agent._parse_response("buy.").action == "BUY" + assert agent._parse_response("**SELL**\nx").action == "SELL" + + def test_ambiguous_rejected_not_substring_matched(self): + agent = self._agent() + assert agent._parse_response("I would not BUY here") is None + assert agent._parse_response("BUY or SELL depending on funding") is None + assert agent._parse_response("Definitely bullish") is None + assert agent._parse_response("") is None + + def test_eval_budget(self): + agent = self._agent() + agent.max_evals_per_hour = 2 + agent._eval_times.clear() + assert agent._within_budget(now=100.0) + assert agent._within_budget(now=101.0) + assert not agent._within_budget(now=102.0) + # Window slides after an hour. + assert agent._within_budget(now=100.0 + 3601) + + +# ── Persistence: quarantine + schema version ───────────────────── + +class TestPersistence: + def test_corrupted_db_quarantined_not_deleted(self, tmp_path): + db_path = tmp_path / "hyperdata.db" + db_path.write_bytes(b"this is not a sqlite database " * 100) + + store = DataStore(db_path) + try: + quarantine = tmp_path / "corrupted" + quarantined = list(quarantine.glob("hyperdata.db.*")) + assert len(quarantined) == 1 + # Original bytes preserved for postmortem. + assert quarantined[0].read_bytes().startswith(b"this is not") + # Fresh DB works. + assert store.get_db_stats()["liquidations_stored"] == 0 + finally: + store.close() + + def test_schema_version_recorded(self, tmp_path): + store = DataStore(tmp_path / "fresh.db") + try: + assert store.get_schema_version() == DataStore.SCHEMA_VERSION + finally: + store.close() + + def test_reopen_keeps_schema_version(self, tmp_path): + path = tmp_path / "reopen.db" + DataStore(path).close() + store = DataStore(path) + try: + assert store.get_schema_version() == DataStore.SCHEMA_VERSION + finally: + store.close() + + +# ── Alert redaction ────────────────────────────────────────────── + +class TestAlertRedaction: + @pytest.mark.asyncio + async def test_alert_payload_not_logged(self, caplog): + from data_layer.alerts import AlertManager + mgr = AlertManager() + mgr.telegram_token = "" # no channels configured + mgr.discord_webhook = "" + secret_wallet = "0x" + "ab" * 20 + message = f"whale alert\nwallet {secret_wallet} is near liquidation" + + with caplog.at_level("WARNING", logger="data_layer.alerts"): + await mgr._send(message) + log_text = caplog.text + assert secret_wallet not in log_text + assert "ALERT sent" in log_text + await mgr.stop() + + +# ── Smart money ranking thresholds ─────────────────────────────── + +class TestSmartMoneyThresholds: + def _wallet(self, engine, addr, trades, volume, score): + from data_layer.smart_money import WalletProfile + w = WalletProfile( + address=addr, discovered_at=0, last_seen=0, last_analyzed=0, + total_trades=trades, total_volume_usd=volume, composite_score=score, + ) + engine.wallets[addr] = w + return w + + def test_small_samples_not_ranked(self): + from data_layer.smart_money import SmartMoneyEngine + engine = SmartMoneyEngine() + tiny = self._wallet(engine, "0x" + "1" * 40, trades=3, volume=1e6, score=0.9) + thin = self._wallet(engine, "0x" + "2" * 40, trades=50, volume=100.0, score=0.9) + solid = self._wallet(engine, "0x" + "3" * 40, + trades=SmartMoneyEngine.MIN_TRADES_FOR_RANKING, + volume=SmartMoneyEngine.MIN_VOLUME_FOR_RANKING, score=0.5) + engine.rank_all() + assert tiny.rank == 0 and tiny.tier == "unknown" + assert thin.rank == 0 and thin.tier == "unknown" + assert solid.rank == 1 and solid.tier == "smart" + + def test_disqualified_wallet_loses_stale_tier(self): + from data_layer.smart_money import SmartMoneyEngine + engine = SmartMoneyEngine() + w = self._wallet(engine, "0x" + "4" * 40, trades=20, volume=1e6, score=0.8) + engine.rank_all() + assert w.tier == "smart" + w.total_trades = 2 # sample no longer qualifies + engine.rank_all() + assert w.rank == 0 + assert w.tier == "unknown" + + def test_confidence_scales_with_sample(self): + from data_layer.smart_money import SmartMoneyEngine, WalletProfile + engine = SmartMoneyEngine() + w = WalletProfile(address="0x" + "5" * 40, discovered_at=0, + last_seen=0, last_analyzed=0, total_trades=25) + assert engine._compute_confidence(w) == 0.5 + w.total_trades = 500 + assert engine._compute_confidence(w) == 1.0 + + +# ── Per-venue orderflow freshness ──────────────────────────────── + +class TestPerVenueFreshness: + def test_dead_venue_visible_while_combined_fresh(self): + from data_layer.orderflow_engine import OrderFlowEngine + e = OrderFlowEngine(symbols=["BTC"]) + now = time.time() + e.last_hl_message_at = now - 1000 # HL dead + e.last_binance_message_at = now - 1 # Binance fresh + assert e.is_stale() is False # combined follows freshest + assert e.venue_is_stale("hyperliquid") is True + assert e.venue_is_stale("binance") is False + fresh = e.venue_freshness() + assert fresh["hyperliquid"]["stale"] is True + assert fresh["binance"]["stale"] is False + + def test_no_data_reports_none_age(self): + from data_layer.orderflow_engine import OrderFlowEngine + e = OrderFlowEngine(symbols=["BTC"]) + fresh = e.venue_freshness() + assert fresh["hyperliquid"]["data_age_seconds"] is None + assert fresh["hyperliquid"]["stale"] is True diff --git a/tests/test_long_short_ratio.py b/tests/test_long_short_ratio.py index 279e422..bdab713 100644 --- a/tests/test_long_short_ratio.py +++ b/tests/test_long_short_ratio.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.long_short_ratio import LongShortSnapshot, LongShortCollector + +from data_layer.long_short_ratio import LongShortCollector, LongShortSnapshot def test_snapshot_fields(): diff --git a/tests/test_market_data.py b/tests/test_market_data.py index 9625e9a..5988706 100644 --- a/tests/test_market_data.py +++ b/tests/test_market_data.py @@ -2,6 +2,7 @@ from __future__ import annotations import pytest + from data_layer.market_data import AssetInfo, _timeframe_to_ms diff --git a/tests/test_orderbook.py b/tests/test_orderbook.py index 6a46f82..35274fd 100644 --- a/tests/test_orderbook.py +++ b/tests/test_orderbook.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.orderbook import OrderBookLevel, OrderBookSnapshot, OrderBookEngine, compute_imbalance + +from data_layer.orderbook import OrderBookEngine, OrderBookLevel, compute_imbalance def test_order_book_level(): diff --git a/tests/test_orderflow.py b/tests/test_orderflow.py index f78c687..b1f853a 100644 --- a/tests/test_orderflow.py +++ b/tests/test_orderflow.py @@ -6,9 +6,9 @@ import pytest from data_layer.orderflow_engine import ( - Trade, - TimeframeBucket, OrderFlowEngine, + TimeframeBucket, + Trade, classify_signal, ) diff --git a/tests/test_position_scanner.py b/tests/test_position_scanner.py index ed0205f..01e4eb8 100644 --- a/tests/test_position_scanner.py +++ b/tests/test_position_scanner.py @@ -1,23 +1,38 @@ +"""PositionScanner unit tests. + +All network calls are mocked; the SQLite-backed address store is redirected to +a per-test temp directory so tests never touch the repo's data/ files. +""" from __future__ import annotations -import asyncio -import json -from unittest.mock import AsyncMock, MagicMock, patch +import sqlite3 +from unittest.mock import AsyncMock, MagicMock import pytest -from src.data_layer.position_scanner import ( - DISCOVERED_ADDRESSES_PATH, - PositionScanner, - TrackedPosition, -) - +from data_layer.position_scanner import PositionScanner, TrackedPosition +from src.data_layer import address_store # ── Fixtures ───────────────────────────────────────────────────── +@pytest.fixture(autouse=True) +def isolated_address_store(tmp_path, monkeypatch): + """Point the address store at a temp SQLite DB for every test.""" + monkeypatch.setattr(address_store, "DATA_DIR", tmp_path) + monkeypatch.setattr(address_store, "DB_PATH", tmp_path / "hyperdata.db") + monkeypatch.setattr(address_store, "LEGACY_JSON", tmp_path / "discovered_addresses.json") + monkeypatch.setattr(address_store, "_initialized", False) + yield tmp_path + + +def _addr(seed: str) -> str: + """Build a valid 0x + 40-hex wallet address from a short seed.""" + return "0x" + (seed * 40)[:40] + + def _make_position(**overrides) -> TrackedPosition: defaults = dict( - address="0xabc", + address=_addr("abc"), symbol="BTC", side="long", size_usd=50_000.0, @@ -33,6 +48,17 @@ def _make_position(**overrides) -> TrackedPosition: return TrackedPosition(**defaults) +def _mock_session(json_payload) -> MagicMock: + session = MagicMock() + response = AsyncMock() + response.json = AsyncMock(return_value=json_payload) + response.raise_for_status = MagicMock() + response.__aenter__ = AsyncMock(return_value=response) + response.__aexit__ = AsyncMock(return_value=False) + session.post = MagicMock(return_value=response) + return session + + MOCK_ALL_MIDS = {"BTC": "71000.0", "ETH": "3500.0", "SOL": "150.0"} MOCK_META = { @@ -77,7 +103,7 @@ def _make_position(**overrides) -> TrackedPosition: class TestTrackedPosition: def test_creation(self): p = _make_position() - assert p.address == "0xabc" + assert p.address == _addr("abc") assert p.side == "long" assert p.leverage == 10.0 @@ -137,9 +163,9 @@ def test_unknown_coin_defaults(self): class TestFilterMethods: - def setup_method(self): - self.scanner = PositionScanner() - self.scanner.positions = [ + def setup_positions(self): + scanner = PositionScanner() + scanner.positions = [ _make_position(side="long", distance_pct=0.5, size_usd=100_000), _make_position(side="long", distance_pct=1.5, size_usd=50_000), _make_position(side="short", distance_pct=0.8, size_usd=200_000), @@ -147,79 +173,111 @@ def setup_method(self): _make_position(side="long", distance_pct=4.5, size_usd=30_000), _make_position(side="short", distance_pct=10.0, size_usd=10_000), ] + return scanner def test_get_danger_zone_default(self): - danger = self.scanner.get_danger_zone() + danger = self.setup_positions().get_danger_zone() assert len(danger) == 3 assert all(p.distance_pct <= 2.0 for p in danger) def test_get_danger_zone_custom_threshold(self): - danger = self.scanner.get_danger_zone(threshold_pct=1.0) + danger = self.setup_positions().get_danger_zone(threshold_pct=1.0) assert len(danger) == 2 def test_get_closest_longs(self): - longs = self.scanner.get_closest_longs(n=2) + longs = self.setup_positions().get_closest_longs(n=2) assert len(longs) == 2 assert all(p.side == "long" for p in longs) assert longs[0].distance_pct < longs[1].distance_pct def test_get_closest_shorts(self): - shorts = self.scanner.get_closest_shorts(n=2) + shorts = self.setup_positions().get_closest_shorts(n=2) assert len(shorts) == 2 assert all(p.side == "short" for p in shorts) assert shorts[0].distance_pct < shorts[1].distance_pct def test_get_closest_longs_more_than_available(self): - longs = self.scanner.get_closest_longs(n=100) + longs = self.setup_positions().get_closest_longs(n=100) assert len(longs) == 3 def test_get_zone_summary(self): - summary = self.scanner.get_zone_summary() + summary = self.setup_positions().get_zone_summary() assert summary["within_1pct"]["count"] == 2 assert summary["within_1pct"]["total_value"] == 300_000 assert summary["within_2pct"]["count"] == 3 assert summary["within_5pct"]["count"] == 5 def test_get_zone_summary_empty(self): - self.scanner.positions = [] - summary = self.scanner.get_zone_summary() + scanner = self.setup_positions() + scanner.positions = [] + summary = scanner.get_zone_summary() assert summary["within_1pct"]["count"] == 0 assert summary["within_5pct"]["total_value"] == 0.0 class TestAddressPersistence: - def test_add_addresses(self, tmp_path, monkeypatch): - addr_file = tmp_path / "discovered_addresses.json" - monkeypatch.setattr( - "src.data_layer.position_scanner.DISCOVERED_ADDRESSES_PATH", addr_file - ) + """Address persistence is SQLite-backed (data_layer.address_store).""" + + def test_add_addresses_persists_to_sqlite(self, isolated_address_store): scanner = PositionScanner() - scanner.add_addresses(["0xaaa", "0xbbb"]) - assert "0xaaa" in scanner.discovered_addresses - assert addr_file.exists() - - loaded = json.loads(addr_file.read_text()) - assert "0xaaa" in loaded - assert "0xbbb" in loaded - - def test_load_existing_addresses(self, tmp_path, monkeypatch): - addr_file = tmp_path / "discovered_addresses.json" - addr_file.write_text(json.dumps(["0x111", "0x222"])) - monkeypatch.setattr( - "src.data_layer.position_scanner.DISCOVERED_ADDRESSES_PATH", addr_file - ) + a1, a2 = _addr("aaa"), _addr("bbb") + scanner.add_addresses([a1, a2]) + + assert a1 in scanner.discovered_addresses + assert a2 in scanner.discovered_addresses + + # Rows actually landed in the discovered_addresses table. + conn = sqlite3.connect(str(isolated_address_store / "hyperdata.db")) + rows = { + r[0] for r in + conn.execute("SELECT address FROM discovered_addresses").fetchall() + } + conn.close() + assert {a1, a2} <= rows + + def test_load_existing_addresses(self): + a1, a2 = _addr("111"), _addr("222") + address_store.add_addresses([a1, a2], source="test") + + scanner = PositionScanner() + assert a1 in scanner.discovered_addresses + assert a2 in scanner.discovered_addresses + + def test_invalid_addresses_rejected(self, isolated_address_store): scanner = PositionScanner() - assert "0x111" in scanner.discovered_addresses - assert "0x222" in scanner.discovered_addresses - - def test_corrupted_file_handled(self, tmp_path, monkeypatch): - addr_file = tmp_path / "discovered_addresses.json" - addr_file.write_text("NOT VALID JSON {{{") - monkeypatch.setattr( - "src.data_layer.position_scanner.DISCOVERED_ADDRESSES_PATH", addr_file - ) + scanner.add_addresses([ + "not-an-address", + "0xTOOSHORT", + "0x" + "g" * 40, # non-hex + _addr("c0ffee"), # the only valid one + ]) + assert scanner.discovered_addresses == {_addr("c0ffee")} + + conn = sqlite3.connect(str(isolated_address_store / "hyperdata.db")) + count = conn.execute("SELECT COUNT(*) FROM discovered_addresses").fetchone()[0] + conn.close() + assert count == 1 + + def test_addresses_normalized_to_lowercase(self): + mixed = "0x" + "AbCdEf0123456789aBcDeF0123456789ABCDEF01" scanner = PositionScanner() - assert len(scanner.discovered_addresses) == 0 + scanner.add_addresses([mixed]) + assert mixed.lower() in scanner.discovered_addresses + assert mixed not in scanner.discovered_addresses + + def test_retention_cap_expires_oldest(self, monkeypatch): + monkeypatch.setattr(address_store, "MAX_TRACKED_ADDRESSES", 3) + for i in range(5): + address_store.add_addresses([_addr(f"{i}{i}{i}")], source="test") + remaining = address_store.get_all_addresses() + assert len(remaining) == 3 + + def test_store_validator(self): + assert address_store.is_valid_address(_addr("abc")) + assert not address_store.is_valid_address("0xabc") + assert not address_store.is_valid_address(None) + assert not address_store.is_valid_address(42) + assert not address_store.is_valid_address("0x" + "z" * 40) class TestGetPositionsForAddress: @@ -228,18 +286,9 @@ async def test_parses_clearinghouse_state(self): scanner = PositionScanner() scanner.market_prices = {"BTC": 71_000.0, "ETH": 3_500.0} scanner.market_meta = MOCK_META + scanner._session = _mock_session(MOCK_CLEARINGHOUSE_STATE) - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=MOCK_CLEARINGHOUSE_STATE) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) - - scanner._session = mock_session - - positions = await scanner.get_positions_for_address("0xtest") + positions = await scanner.get_positions_for_address(_addr("fed")) assert len(positions) == 2 @@ -259,18 +308,9 @@ async def test_empty_positions(self): scanner = PositionScanner() scanner.market_prices = {} scanner.market_meta = MOCK_META + scanner._session = _mock_session({"assetPositions": [], "marginSummary": {}}) - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value={"assetPositions": [], "marginSummary": {}}) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) - - scanner._session = mock_session - - positions = await scanner.get_positions_for_address("0xempty") + positions = await scanner.get_positions_for_address(_addr("e")) assert positions == [] @pytest.mark.asyncio @@ -295,36 +335,38 @@ async def test_fallback_liq_price_when_missing(self): scanner = PositionScanner() scanner.market_prices = {"BTC": 70_000.0} scanner.market_meta = MOCK_META + scanner._session = _mock_session(state) - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=state) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) - - scanner._session = mock_session - - positions = await scanner.get_positions_for_address("0xfallback") + positions = await scanner.get_positions_for_address(_addr("f")) assert len(positions) == 1 assert abs(positions[0].liq_price - 63_210.0) < 0.01 -class TestUpdatePrices: +class TestDiscoverAddresses: @pytest.mark.asyncio - async def test_update_prices(self): + async def test_discovery_validates_payload_addresses(self): + """Junk strings in exchange trade payloads must never be persisted.""" + good = _addr("dead") + trades = [ + {"buyer": good, "seller": "junk-string"}, + {"users": [good, "0xshort", 12345, None]}, + {"users": "not-a-list"}, + ] scanner = PositionScanner() + scanner._session = _mock_session(trades) - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=MOCK_ALL_MIDS) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) + discovered = await scanner.discover_addresses(limit=10) + assert good in discovered + assert "junk-string" not in discovered + assert "0xshort" not in discovered + assert address_store.get_all_addresses() == {good} - scanner._session = mock_session + +class TestUpdatePrices: + @pytest.mark.asyncio + async def test_update_prices(self): + scanner = PositionScanner() + scanner._session = _mock_session(MOCK_ALL_MIDS) await scanner.update_prices() assert scanner.market_prices["BTC"] == 71_000.0 @@ -336,16 +378,7 @@ class TestUpdateMeta: @pytest.mark.asyncio async def test_update_meta(self): scanner = PositionScanner() - - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=MOCK_META) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) - - scanner._session = mock_session + scanner._session = _mock_session(MOCK_META) await scanner.update_meta() assert scanner.market_meta == MOCK_META @@ -367,10 +400,6 @@ async def test_meta_caching(self): class TestDistanceCalculation: def test_distance_for_long(self): - scanner = PositionScanner() - scanner.market_prices = {"BTC": 70_000.0} - scanner.market_meta = MOCK_META - # liq at 63500, current at 70000 # distance = |70000 - 63500| / 70000 * 100 = 9.2857% current = 70_000.0 @@ -399,3 +428,16 @@ async def test_rate_limit_tracks_requests(self): for _ in range(5): await scanner._rate_limit() assert len(scanner._request_times) == 5 + + +class TestPostTimeout: + @pytest.mark.asyncio + async def test_post_sends_explicit_timeout(self): + """Every outbound request must carry an explicit deadline (High 4).""" + scanner = PositionScanner() + session = _mock_session({}) + scanner._session = session + await scanner._post({"type": "allMids"}) + _, kwargs = session.post.call_args + assert kwargs.get("timeout") is not None + assert kwargs["timeout"].total == 10 diff --git a/tests/test_spot_prices.py b/tests/test_spot_prices.py index d1cad11..63c3a4a 100644 --- a/tests/test_spot_prices.py +++ b/tests/test_spot_prices.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.spot_prices import SpotPriceSnapshot, SpotPriceCollector + +from data_layer.spot_prices import SpotPriceCollector, SpotPriceSnapshot def test_snapshot_fields(): From 0961b0dc68d43a852331c5a8e9dc6d547e3750df Mon Sep 17 00:00:00 2001 From: Co-Messi Date: Wed, 8 Jul 2026 13:05:24 +0800 Subject: [PATCH 2/2] Address PR #2 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI: - Restore 'python-version: ${{ matrix.python-version }}' on setup-python (the matrix was display-only after the rewrite) - Pin test tooling versions in the workflow; make pip-audit blocking - Bump aiohttp 3.13.3 -> 3.14.1 in requirements.lock (4 published CVEs) - Add .github/dependabot.yml (pip + github-actions, weekly) API server: - WebSocket upgrades with a browser Origin header are rejected unless the origin is allowlisted (browser WS is not gated by the same-origin policy, so any webpage could read the loopback stream) - Split /v1/live (minimal, unauthenticated) from /v1/health (detailed, requires the API key when one is set — the payload is operational recon) - Health/liveness paths exempt from per-IP rate limiting (LB/monitor probes share a NAT IP) - Dedup: events without a plausible epoch-seconds timestamp are never deduped (no local-clock fallback, no ms-scale hashing); the hash uses the exact size instead of round(-2) so distinct ~$100k events cannot collapse; cascade bypass has an absolute 2x-duration cap so replayed duplicates cannot extend it forever; cascade tracker bounded per key - /v1/health reports degraded when position scanner or market data loops are in 'error' (previously only startup failures counted) Feeds: - Binance forceOrder handler accepts array frames (@arr batching) instead of silently dropping them - Bybit handler accepts v5 list payloads and short keys (p/v/S/s/T), catches AttributeError, and drops malformed records individually - Exchange WS reconnect escalates to a single ERROR after 20 consecutive failures and quiets the log spam; still retries at max backoff (a hard stop would permanently kill the feed across a venue outage) - Split connect/sock_read timeouts on all external HTTP deadlines so a slow TLS handshake can't consume the whole budget - Position scanner price/meta updates use return_exceptions so one dead endpoint doesn't discard the other's result Strategies: - Paper trader: close credit floored at zero balance (a >100% adverse move on a short could previously drive balance negative); trades persist to SQLite BEFORE mutating the books, so a DB error can no longer diverge the portfolio from the audit trail - LLM agent: evaluate() is now async and awaited by the paper trader (a slow LLM no longer blocks the event loop); transport-level failures refund their budget slot (a down provider can't exhaust the hourly budget; parse failures still consume theirs since tokens were billed); leading blank lines in responses are tolerated - Cascade example strategy actually fires: getattr on the stats dict always returned 0 (and the key name was wrong) — now reads stats['long_volume_usd'] Persistence & alerts: - 'database is locked/busy' raises instead of quarantining a healthy DB held by another process; quarantine uses shutil.move and falls back to an in-memory store if the corrupted file can't be moved or removed - Alert send failures log the exception type only (aiohttp error strings can embed the bot-token URL); wallet addresses masked on the logged line - Removed dead HLP z-score alert scheduling (the alert body is disabled) Declined with rationale (see PR discussion): config/ stays in packaging (settings.py is a runtime import, contains no secrets); smart-money tier persistence is not wired (save_wallet/load_wallets have no callers), so the restart-resurrection scenario cannot occur. 18 new tests cover the above; 166 passing total. --- .github/dependabot.yml | 14 + .github/workflows/ci.yml | 12 +- README.md | 3 +- requirements.lock | 2 +- src/api_server.py | 88 +++++- src/data_layer/alerts.py | 20 +- src/data_layer/hub.py | 15 +- src/data_layer/liquidation_feed.py | 92 ++++-- src/data_layer/market_data.py | 4 +- src/data_layer/persistence.py | 38 ++- src/data_layer/position_scanner.py | 19 +- .../examples/liquidation_cascade.py | 7 +- src/strategies/llm_agent.py | 51 +++- src/strategies/paper_trader.py | 64 ++-- tests/test_hardening.py | 285 +++++++++++++++++- 15 files changed, 594 insertions(+), 120 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..98b43bb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: + # Keep requirements.lock / pyproject deps patched (security + routine). + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + + # Keep SHA-pinned GitHub Actions fresh. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34161a0..89c7106 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,21 +20,23 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ matrix.python-version }} - name: Install dependencies - # requirements.lock pins the runtime deps; -e . adds the package itself. + # requirements.lock pins the runtime deps; -e . adds the package + # itself; test tooling is pinned so CI is deterministic. run: | pip install -r requirements.lock pip install -e . --no-deps - pip install pytest pytest-asyncio ruff pip-audit + pip install pytest==9.1.1 pytest-asyncio==1.4.0 ruff==0.15.20 pip-audit==2.10.1 - name: Lint run: ruff check src/ tests/ - name: Dependency vulnerability audit - # Advisory: a newly-published CVE in a pinned dep should be visible - # without failing unrelated PRs. Review failures in the job log. - continue-on-error: true + # Blocking: a known CVE in a pinned runtime dep fails CI. Dependabot + # (.github/dependabot.yml) keeps the lockfile and action SHAs fresh. run: pip-audit -r requirements.lock - name: Test imports diff --git a/README.md b/README.md index bd125ab..c093d5a 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,8 @@ python run_api.py --port 8420 | Endpoint | Description | |---|---| -| `GET /v1/health` | Server status and uptime | +| `GET /v1/live` | Minimal liveness probe (always unauthenticated) | +| `GET /v1/health` | Server status and uptime (requires the API key when one is set) | | `GET /v1/market` | All assets — prices, OI, funding | | `GET /v1/market/{symbol}` | Single asset detail | | `GET /v1/liquidations` | Recent liquidation events | diff --git a/requirements.lock b/requirements.lock index 79ef770..ea1debc 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,7 +1,7 @@ # Generated with pip freeze — do not edit manually # Pinned versions for reproducible installs # To install: pip install -r requirements.lock -aiohttp==3.13.3 +aiohttp==3.14.1 numpy==2.4.3 pandas==3.0.1 pyfiglet==1.0.4 diff --git a/src/api_server.py b/src/api_server.py index 8dd3f57..bc7c39c 100644 --- a/src/api_server.py +++ b/src/api_server.py @@ -129,8 +129,14 @@ async def cors_middleware(request: web.Request, handler): return cors_middleware -# Paths reachable without an API key (liveness checks must not need secrets). -_UNAUTHENTICATED_PATHS = {"/v1/health", "/health"} +# Paths reachable without an API key. Only the minimal liveness probe is +# exempt — the detailed /v1/health payload (mode, feed states, counters) is +# operational recon and requires the key on authenticated deployments. +_UNAUTHENTICATED_PATHS = {"/v1/live"} + +# Paths exempt from per-IP rate limiting: load balancers, uptime monitors, +# and liveness probes often share one NAT egress IP and poll continuously. +_RATE_LIMIT_EXEMPT_PATHS = {"/v1/live", "/v1/health", "/health"} def _make_auth_middleware(api_key: str): @@ -185,6 +191,8 @@ def allow(self, key: str, now: float | None = None) -> bool: def _make_rate_limit_middleware(limiter: _RateLimiter): @web.middleware async def rate_limit_middleware(request: web.Request, handler): + if request.path in _RATE_LIMIT_EXEMPT_PATHS: + return await handler(request) remote = request.remote or "unknown" if not limiter.allow(remote): return web.json_response({"error": "Rate limit exceeded"}, status=429) @@ -303,6 +311,7 @@ def _resolve_security(self) -> tuple[str, set[str] | None]: async def start(self) -> None: api_key, cors_origins = self._resolve_security() self._api_key = api_key + self._cors_origins = cors_origins middlewares = [_make_rate_limit_middleware(self._rate_limiter)] if api_key: @@ -312,6 +321,7 @@ async def start(self) -> None: # v1 routes v1 = [ + ("GET", "/v1/live", self.handle_live), ("GET", "/v1/health", self.handle_health), ("GET", "/v1/market", self.handle_market), ("GET", "/v1/market/{symbol}", self.handle_market_symbol), @@ -415,6 +425,7 @@ def _on_trade(self, trade) -> None: _DEDUP_MAX = 500 _CASCADE_WINDOW = 30 _CASCADE_BYPASS_DURATION = 30 + _CASCADE_TRACKER_MAX = 200 # entries kept per symbol/side/exchange key def __init_dedup(self): if not hasattr(self, '_liq_seen'): @@ -423,27 +434,45 @@ def __init_dedup(self): self._heartbeat_task: asyncio.Task | None = None self._cascade_tracker: dict[str, list] = {} self._cascade_bypass: dict[str, float] = {} + self._cascade_bypass_started: dict[str, float] = {} self._liq_stats = {"received": 0, "broadcast": 0, "deduped": 0, "filtered": 0} self._liq_stats_ts = time.time() + # Plausible epoch-seconds range for exchange event times (2001..5138). + # A timestamp outside this range means a connector skipped ms→s + # normalization (or sent 0) — such events cannot be safely hashed. + _TS_SANE_MIN = 1e9 + _TS_SANE_MAX = 1e11 + def _is_duplicate_liq(self, ev) -> bool: """Duplicate check within the dedup window, keyed per exchange. Buckets on the EXCHANGE event timestamp (not local receive time) so two records of the same event dedup identically regardless of local - delivery jitter. The cascade bypass is also per-exchange: a Binance - cascade must not let Hyperliquid's heuristic events skip dedup. + delivery jitter. Events without a plausible exchange timestamp are + never deduped — substituting the local clock would collide distinct + events that merely arrived together. The hash uses the exact size: + replayed duplicates carry identical payloads, while distinct events + of similar size must not collapse into one. The cascade bypass is + also per-exchange: a Binance cascade must not let Hyperliquid's + heuristic events skip dedup. """ self.__init_dedup() now = time.time() + ev_ts = ev.timestamp + if not (self._TS_SANE_MIN < ev_ts < self._TS_SANE_MAX): + logger.warning( + "[liq] %s event has implausible timestamp %r — skipping dedup", + ev.exchange, ev_ts, + ) + return False + bypass_key = f"{ev.symbol}_{ev.side}_{ev.exchange}" if bypass_key in self._cascade_bypass and now < self._cascade_bypass[bypass_key]: return False - ev_ts = ev.timestamp if ev.timestamp > 0 else now - size_rounded = round(ev.size_usd, -2) - h = f"{ev.symbol}_{ev.side}_{size_rounded}_{ev.exchange}_{int(ev_ts // self._DEDUP_WINDOW)}" + h = f"{ev.symbol}_{ev.side}_{ev.size_usd:.2f}_{ev.exchange}_{int(ev_ts // self._DEDUP_WINDOW)}" if len(self._liq_seen) > self._DEDUP_MAX: cutoff = now - self._DEDUP_WINDOW * 2 @@ -474,15 +503,27 @@ def _check_cascade(self, ev) -> str | None: ] self._cascade_tracker[key].append((now, ev.size_usd)) + # Bound per-key memory: only the most recent window entries matter. + if len(self._cascade_tracker[key]) > self._CASCADE_TRACKER_MAX: + self._cascade_tracker[key] = self._cascade_tracker[key][-self._CASCADE_TRACKER_MAX:] entries = self._cascade_tracker[key] if len(entries) >= 3: # Bypass dedup only for this exchange's stream: cascades on one - # venue say nothing about duplicates on another. - self._cascade_bypass[key] = now + self._CASCADE_BYPASS_DURATION + # venue say nothing about duplicates on another. The bypass has + # an ABSOLUTE cap: without it, events passing dedup during the + # bypass re-trigger cascade detection and extend it forever + # (replayed duplicates would keep the floodgate open). + first = self._cascade_bypass_started.setdefault(key, now) + cap = first + 2 * self._CASCADE_BYPASS_DURATION + self._cascade_bypass[key] = min(now + self._CASCADE_BYPASS_DURATION, cap) total = sum(sz for _, sz in entries) return f"cascade ${total:,.0f} ({len(entries)}x in {self._CASCADE_WINDOW}s)" + # Quiet again: allow a future cascade to start a fresh bypass window. + if key in self._cascade_bypass_started and now > self._cascade_bypass.get(key, 0): + del self._cascade_bypass_started[key] + return None def _log_liq_stats(self) -> None: @@ -683,7 +724,19 @@ async def redirect(request: web.Request) -> web.Response: # ── WebSocket handler ──────────────────────────────────────── - async def handle_ws(self, request: web.Request) -> web.WebSocketResponse: + async def handle_ws(self, request: web.Request) -> web.WebSocketResponse | web.Response: + # Browser WebSockets are NOT gated by the same-origin policy: any + # webpage can open ws://127.0.0.1 and read the stream. An Origin + # header means a browser context — reject it unless the origin was + # explicitly allowlisted (HYPERDATA_CORS_ORIGINS). Non-browser + # clients (curl, bots, SDKs) send no Origin and are unaffected. + origin = request.headers.get("Origin") + if origin is not None: + allowed = getattr(self, "_cors_origins", None) or set() + if origin not in allowed: + logger.warning("[ws] Rejected cross-origin upgrade from %s", origin) + return web.json_response({"error": "Origin not allowed"}, status=403) + if len(self._ws_clients) >= MAX_WS_CONNECTIONS: return web.json_response({"error": "Too many connections"}, status=429) ws = web.WebSocketResponse(heartbeat=20, max_msg_size=WS_MAX_MSG_BYTES) @@ -740,6 +793,15 @@ def _ws_msg_violates_limits(self, client: _WSClient, raw: str) -> bool: # ── REST Handlers ──────────────────────────────────────────── + async def handle_live(self, request: web.Request) -> web.Response: + """Minimal liveness probe — safe to expose unauthenticated. + + Deliberately says nothing about mode, feeds, or counters: the + detailed /v1/health payload is operational recon and requires the + API key on authenticated deployments. + """ + return web.json_response({"status": "ok"}) + async def handle_health(self, request: web.Request) -> web.Response: s = self.hub.status uptime = int(s.uptime_seconds) @@ -770,10 +832,12 @@ async def handle_health(self, request: web.Request) -> web.Response: # Top-level status reflects data health when available: 'ok' only when # nothing is stale/drifting. 'degraded' otherwise (server is still up). - # Components that failed to start also force 'degraded'. + # Components that failed to start OR whose loops are currently erroring + # (position scanner / market data flip to 'error' at runtime without + # touching failed_components) also force 'degraded'. overall = data_health.get("overall") if data_health else None status = "ok" if overall in (None, "ok", "warn") else "degraded" - if s.failed_components: + if s.failed_components or any(v == "error" for v in feeds.values()): status = "degraded" return web.json_response({ diff --git a/src/data_layer/alerts.py b/src/data_layer/alerts.py index a84d718..09c7520 100644 --- a/src/data_layer/alerts.py +++ b/src/data_layer/alerts.py @@ -15,6 +15,7 @@ import asyncio import logging import os +import re import time from dataclasses import dataclass @@ -358,7 +359,7 @@ def _build_intel_report(self) -> str: # Deadline for webhook posts: a stalled Telegram/Discord endpoint must # not wedge whatever task is delivering the alert. - _SEND_TIMEOUT = aiohttp.ClientTimeout(total=10) + _SEND_TIMEOUT = aiohttp.ClientTimeout(total=10, connect=3, sock_connect=3, sock_read=5) async def _send(self, message: str) -> None: """Send alert to all configured channels.""" @@ -380,8 +381,10 @@ async def _send(self, message: str) -> None: logger.info("Telegram alert sent") else: logger.warning("Telegram send returned %d", resp.status) - except Exception: - logger.exception("Telegram send failed") + except Exception as exc: + # Exception type only — aiohttp error messages can embed the + # request URL, which contains the bot token. + logger.warning("Telegram send failed: %s", type(exc).__name__) # Discord if self.discord_webhook: @@ -393,14 +396,17 @@ async def _send(self, message: str) -> None: logger.info("Discord alert sent") else: logger.warning("Discord send returned %d", resp.status) - except Exception: - logger.exception("Discord send failed") + except Exception as exc: + # Exception type only — error messages can embed the webhook URL. + logger.warning("Discord send failed: %s", type(exc).__name__) # Log that an alert fired, not its payload — alert bodies can contain # wallet addresses and position intelligence that must not sit in - # rotating plaintext logs. + # rotating plaintext logs. Wallet addresses are masked even on the + # first line in case a future alert format leads with one. first_line = message.strip().splitlines()[0] if message.strip() else "" - logger.warning("ALERT sent (%d total): %.80s", self.alerts_sent, first_line) + redacted = re.sub(r"0x[0-9a-fA-F]{40}", "0x…[redacted]", first_line) + logger.warning("ALERT sent (%d total): %.80s", self.alerts_sent, redacted) async def send_test(self) -> bool: """Send a test alert to verify configuration.""" diff --git a/src/data_layer/hub.py b/src/data_layer/hub.py index afab5a9..6fb5d81 100644 --- a/src/data_layer/hub.py +++ b/src/data_layer/hub.py @@ -536,20 +536,13 @@ async def _status_update_loop(self) -> None: /v1/health data, so it must outlive individual component errors. """ _db_tick = 0 - _last_hlp_alert_check = 0.0 while self._running: try: _db_tick = await self._status_update_tick(_db_tick) - - # Check HLP Z-score for alert — at most once a minute while - # extreme, not one new task per tick. - hlp_zscore = self.status.hlp_delta_zscore - now = time.time() - if abs(hlp_zscore) > 2.0 and now - _last_hlp_alert_check > 60.0: - _last_hlp_alert_check = now - asyncio.create_task( - self.alerts._check_hlp_zscore(self.hlp.get_stats()) - ) + # NOTE: HLP z-score alert dispatch was removed here — the + # AlertManager z-score/cascade sends are deliberately disabled + # (too noisy), so scheduling tasks for them was dead work. + # Re-add scheduling here if those alerts are re-enabled. except asyncio.CancelledError: break except Exception: diff --git a/src/data_layer/liquidation_feed.py b/src/data_layer/liquidation_feed.py index ed78602..e2e9501 100644 --- a/src/data_layer/liquidation_feed.py +++ b/src/data_layer/liquidation_feed.py @@ -47,8 +47,8 @@ def normalize_symbol(raw: str, exchange: str) -> str: HL_LIQUIDATION_MIN_USD = 10_000 # Deadline for REST polls (price context); a hung endpoint must not wedge the -# poll loop. -HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10) +# poll loop. Split connect/read so a slow handshake can't eat the budget. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10, connect=3, sock_connect=3, sock_read=5) def exchange_coverage() -> dict[str, dict[str, str]]: @@ -92,6 +92,12 @@ def exchange_coverage() -> dict[str, dict[str, str]]: class ExchangeConnection: MAX_BACKOFF = 60.0 + # After this many consecutive failed connections, escalate once to ERROR + # and demote further reconnect chatter to debug. We deliberately keep + # retrying (at MAX_BACKOFF) rather than stopping: a market-data feed that + # permanently kills itself during a long venue outage never recovers, + # and one handshake per minute is negligible load. + FAILURE_ESCALATION_THRESHOLD = 20 def __init__(self, name: str, ws_url: str, feed: LiquidationFeed): self.name = name @@ -102,6 +108,7 @@ def __init__(self, name: str, ws_url: str, feed: LiquidationFeed): self._ws: aiohttp.ClientWebSocketResponse | None = None self._running = False self._backoff = 1.0 + self.consecutive_failures = 0 async def start(self) -> None: self._running = True @@ -128,6 +135,7 @@ async def _run_loop(self) -> None: async with self._session.ws_connect(self.ws_url, heartbeat=20) as ws: self._ws = ws self._backoff = 1.0 + self.consecutive_failures = 0 logger.info("[%s] connected", self.name) await self._on_connected(ws) async for msg in ws: @@ -139,10 +147,22 @@ async def _run_loop(self) -> None: except asyncio.CancelledError: return except Exception: - logger.exception("[%s] connection error", self.name) + self.consecutive_failures += 1 + if self.consecutive_failures == self.FAILURE_ESCALATION_THRESHOLD: + logger.error( + "[%s] %d consecutive connection failures — endpoint looks " + "dead/deprecated; will keep retrying every %.0fs quietly", + self.name, self.consecutive_failures, self.MAX_BACKOFF, + ) + elif self.consecutive_failures < self.FAILURE_ESCALATION_THRESHOLD: + logger.exception("[%s] connection error", self.name) + else: + logger.debug("[%s] connection error (%d consecutive)", + self.name, self.consecutive_failures) if self._running: - logger.info("[%s] reconnecting in %.1fs", self.name, self._backoff) + if self.consecutive_failures < self.FAILURE_ESCALATION_THRESHOLD: + logger.info("[%s] reconnecting in %.1fs", self.name, self._backoff) await asyncio.sleep(self._backoff) self._backoff = min(self._backoff * 2, self.MAX_BACKOFF) @@ -162,11 +182,17 @@ def __init__(self, feed: LiquidationFeed): ) async def _on_message(self, data: Any) -> None: - if isinstance(data, dict) and data.get("e") == "forceOrder": + # `!forceOrder@arr` frames normally carry one object, but the `@arr` + # family can batch events into a JSON array — handle both shapes so + # an array frame is parsed instead of silently failing a dict check. + records = data if isinstance(data, list) else [data] + for rec in records: + if not isinstance(rec, dict) or rec.get("e") != "forceOrder": + continue # Exchange payloads are untrusted: one malformed record must not # raise out of the connection loop and trigger a reconnect. try: - o = data["o"] + o = rec["o"] price = float(o["p"]) qty = float(o["q"]) side_raw = str(o["S"]).upper() @@ -179,9 +205,9 @@ async def _on_message(self, data: Any) -> None: price=price, quantity=qty, ) - except (KeyError, TypeError, ValueError): - self.feed.record_parse_error("binance", data) - return + except (KeyError, TypeError, ValueError, AttributeError): + self.feed.record_parse_error("binance", rec) + continue await self.feed.emit(event) @@ -207,28 +233,36 @@ async def _on_connected(self, ws: aiohttp.ClientWebSocketResponse) -> None: async def _on_message(self, data: Any) -> None: if not isinstance(data, dict) or "data" not in data: return - if not data.get("topic", "").startswith("allLiquidation."): + if not str(data.get("topic", "")).startswith("allLiquidation."): return - d = data["data"] - try: - price = float(d.get("price", 0)) - qty = float(d.get("qty", 0) or d.get("size", 0)) - side_raw = d.get("side", "") # "Sell" = long liquidated; "Buy" = short liquidated - symbol_raw = d.get("symbol", "") - ts_ms = int(d.get("updatedTime", 0)) - event = LiquidationEvent( - timestamp=ts_ms / 1000.0, - exchange="bybit", - symbol=normalize_symbol(symbol_raw, "bybit"), - side="long" if side_raw == "Sell" else "short", - size_usd=price * qty, - price=price, - quantity=qty, - confirmed=True, - ) + # Bybit v5 sends `data` as a list of records (older topics used a + # single dict) — accept both, and drop malformed records individually + # so schema drift can never raise out of the WS loop and reconnect. + payload = data["data"] + records = payload if isinstance(payload, list) else [payload] + for d in records: + try: + # v5 allLiquidation uses short keys (p/v/S/s/T); the long + # names cover the legacy `liquidation` topic shape. + price = float(d.get("price") or d.get("p") or 0) + qty = float(d.get("qty") or d.get("size") or d.get("v") or 0) + side_raw = d.get("side") or d.get("S") or "" # "Sell" = long liquidated + symbol_raw = d.get("symbol") or d.get("s") or "" + ts_ms = int(d.get("updatedTime") or d.get("T") or 0) + event = LiquidationEvent( + timestamp=ts_ms / 1000.0, + exchange="bybit", + symbol=normalize_symbol(str(symbol_raw), "bybit"), + side="long" if side_raw == "Sell" else "short", + size_usd=price * qty, + price=price, + quantity=qty, + confirmed=True, + ) + except (KeyError, TypeError, ValueError, AttributeError): + self.feed.record_parse_error("bybit", d) + continue await self.feed.emit(event) - except (KeyError, TypeError, ValueError): - self.feed.record_parse_error("bybit", d) class OKXConnection(ExchangeConnection): diff --git a/src/data_layer/market_data.py b/src/data_layer/market_data.py index 04b6968..f14c628 100644 --- a/src/data_layer/market_data.py +++ b/src/data_layer/market_data.py @@ -13,7 +13,9 @@ # Every outbound request gets an explicit deadline: a hung exchange endpoint # must fail the refresh cycle, not stall the hub's market-refresh loop forever. -HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10) +# connect/sock_read are split so a slow TLS handshake (venue throttling) can't +# consume the whole budget and masquerade as a partial-read error. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10, connect=3, sock_connect=3, sock_read=5) @dataclass diff --git a/src/data_layer/persistence.py b/src/data_layer/persistence.py index a8afe45..e996bfe 100644 --- a/src/data_layer/persistence.py +++ b/src/data_layer/persistence.py @@ -14,6 +14,7 @@ import atexit import logging +import shutil import sqlite3 import threading import time @@ -56,9 +57,18 @@ def __init__(self, db_path: str | Path = DB_PATH): conn.execute("PRAGMA integrity_check") self._conn = conn self._init_tables() - except sqlite3.DatabaseError: + except sqlite3.DatabaseError as exc: if conn is not None: conn.close() + # Lock/busy contention is NOT corruption: another process (a + # dashboard, a verification run) holding the DB must not get the + # healthy database quarantined out from under it. + if "lock" in str(exc).lower() or "busy" in str(exc).lower(): + logger.error( + "Database at %s is locked/busy — failing startup rather " + "than quarantining a healthy DB: %s", self.db_path, exc, + ) + raise # Quarantine, never delete: move the corrupted DB (and WAL/SHM) # aside with a timestamp so history survives for postmortem and # possible `.recover`, then start fresh. @@ -70,19 +80,33 @@ def __init__(self, db_path: str | Path = DB_PATH): if p.exists(): dest = quarantine_dir / f"{p.name}.{stamp}" try: - p.rename(dest) + shutil.move(str(p), str(dest)) # handles cross-device except OSError: logger.exception("Failed to quarantine %s", p) - p.unlink() # last resort so we can still start + try: + p.unlink() # last resort so we can still start + except OSError: + logger.exception("Could not remove %s either", p) logger.error( "Database corrupted at %s — quarantined to %s and recreated. " "Historical data is preserved there for recovery.", self.db_path, quarantine_dir, ) - self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False, timeout=10) - self._conn.execute("PRAGMA journal_mode=WAL") - self._conn.execute("PRAGMA synchronous=NORMAL") - self._init_tables() + try: + self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False, timeout=10) + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA synchronous=NORMAL") + self._init_tables() + except sqlite3.DatabaseError: + # The corrupted file could not be moved OR removed (held + # handle, read-only mount). Run on an in-memory DB so the + # terminal stays alive; persistence is lost for this session. + logger.critical( + "Could not recreate database at %s — falling back to an " + "in-memory store (NO persistence this session)", self.db_path, + ) + self._conn = sqlite3.connect(":memory:", check_same_thread=False) + self._init_tables() # Safety net for graceful exits (normal return, unhandled exception, # Ctrl-C → KeyboardInterrupt unwinds to interpreter exit). The diff --git a/src/data_layer/position_scanner.py b/src/data_layer/position_scanner.py index 6e3c8f5..0d202d4 100644 --- a/src/data_layer/position_scanner.py +++ b/src/data_layer/position_scanner.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging import time from dataclasses import dataclass, field from pathlib import Path @@ -9,6 +10,8 @@ from src.data_layer import address_store +logger = logging.getLogger(__name__) + API_URL = "https://api.hyperliquid.xyz/info" DATA_DIR = Path(__file__).resolve().parents[2] / "data" @@ -16,8 +19,9 @@ META_CACHE_TTL = 300 # 5 minutes # Explicit deadline on every request so a hung endpoint fails the scan cycle -# instead of blocking the hub's position-scan loop indefinitely. -HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10) +# instead of blocking the hub's position-scan loop indefinitely. Split +# connect/read so a slow handshake can't consume the entire budget. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10, connect=3, sock_connect=3, sock_read=5) @dataclass @@ -56,7 +60,16 @@ async def scan(self) -> list[TrackedPosition]: async with aiohttp.ClientSession() as session: self._session = session try: - await asyncio.gather(self.update_prices(), self.update_meta()) + # Independent updates: one endpoint failing must not discard + # the other's result (meta is a 5-min cache — losing a refresh + # means stale maintenance margins for the whole window). + results = await asyncio.gather( + self.update_prices(), self.update_meta(), + return_exceptions=True, + ) + for name, res in zip(("update_prices", "update_meta"), results): + if isinstance(res, BaseException): + logger.warning("[scanner] %s failed: %r", name, res) # Discover new addresses: always on first run, then every 30 minutes import time as _time diff --git a/src/strategies/examples/liquidation_cascade.py b/src/strategies/examples/liquidation_cascade.py index b8387ca..9485b9e 100644 --- a/src/strategies/examples/liquidation_cascade.py +++ b/src/strategies/examples/liquidation_cascade.py @@ -42,8 +42,11 @@ def evaluate(self, hub) -> Signal | None: if not stats: return None - # Look for a spike in long liquidations (longs getting wiped = price dropping) - long_liq_usd = getattr(stats, "total_long_usd", 0) or 0 + # Look for a spike in long liquidations (longs getting wiped = price + # dropping). get_stats() returns a dict; the long-side dollar volume + # key is long_volume_usd (getattr on a dict always returned 0 and + # silently disabled this strategy). + long_liq_usd = stats.get("long_volume_usd", 0) or 0 if long_liq_usd >= self.cascade_threshold_usd: self._last_signal_time = now diff --git a/src/strategies/llm_agent.py b/src/strategies/llm_agent.py index 27c7921..7859e4d 100644 --- a/src/strategies/llm_agent.py +++ b/src/strategies/llm_agent.py @@ -82,12 +82,23 @@ def _within_budget(self, now: float | None = None) -> bool: self._eval_times.append(now) return True - def evaluate(self, hub) -> Signal | None: + def _refund_eval_slot(self) -> None: + """Return the most recent budget slot. + + Called when the call failed at the TRANSPORT level (timeout, refused + connection, HTTP error) — no tokens were consumed, so a flaky + provider must not exhaust the hourly budget. Parse failures keep + their slot: the provider did the work and billed for it. + """ + if self._eval_times: + self._eval_times.pop() + + async def evaluate(self, hub) -> Signal | None: """Build a market summary and ask the LLM for a decision. - NOTE: This calls an async HTTP endpoint. Since evaluate() is - synchronous, we use asyncio to run the coroutine. If you're - already in an async context, see _async_evaluate() directly. + Async: the blocking HTTP call runs in the persistent worker thread + and is awaited, so a slow LLM response cannot stall the paper + trader's event loop (other strategies keep evaluating). """ # If no API key and not using a local model, warn and skip if not self.api_key and "localhost" not in self.base_url: @@ -104,11 +115,16 @@ def evaluate(self, hub) -> Signal | None: ) return None + loop = asyncio.get_running_loop() try: - # Run blocking LLM call in the persistent worker thread so it - # doesn't stall the async loop. - future = self._pool.submit(self._sync_evaluate, hub) - return future.result(timeout=20) + return await asyncio.wait_for( + loop.run_in_executor(self._pool, self._sync_evaluate, hub), + timeout=20, + ) + except asyncio.TimeoutError: + self._refund_eval_slot() + logger.warning("LLM evaluation timed out after 20s") + return None except Exception: logger.exception("LLM agent error") return None @@ -147,6 +163,9 @@ def _sync_evaluate(self, hub) -> Signal | None: data = _json.loads(resp.read()) text = data["choices"][0]["message"]["content"].strip() except Exception as e: + # Transport-level failure: no tokens consumed — give the budget + # slot back so a down provider can't burn the hourly allowance. + self._refund_eval_slot() logger.warning("LLM API call failed: %s", e) return None @@ -204,13 +223,17 @@ async def _async_evaluate(self, hub) -> Signal | None: def _parse_response(self, text: str) -> Signal | None: """Parse LLM response text into a Signal — deterministic, reject-on-ambiguous. - The first line must be exactly BUY, SELL, or HOLD (case-insensitive, - surrounding punctuation tolerated). Substring matching is deliberately - NOT done: "I would not BUY here" must never resolve to a BUY. + The first NON-EMPTY line must be exactly BUY, SELL, or HOLD + (case-insensitive, surrounding punctuation tolerated; leading blank + lines are ignored). Substring matching is deliberately NOT done: + "I would not BUY here" must never resolve to a BUY. """ - lines = text.split("\n", 1) - action_word = lines[0].strip().upper().strip(".!:*# ") - reason = lines[1].strip() if len(lines) > 1 else "" + lines = [ln.strip() for ln in text.splitlines() if ln.strip()] + if not lines: + logger.warning("LLM returned empty response") + return None + action_word = lines[0].upper().strip(".!:*# ") + reason = " ".join(lines[1:]) if len(lines) > 1 else "" if action_word not in ("BUY", "SELL", "HOLD"): logger.warning("LLM returned ambiguous action, rejecting: %r", lines[0][:100]) diff --git a/src/strategies/paper_trader.py b/src/strategies/paper_trader.py index 5477d0d..9d75f8a 100644 --- a/src/strategies/paper_trader.py +++ b/src/strategies/paper_trader.py @@ -19,6 +19,7 @@ from __future__ import annotations import asyncio +import inspect import logging import math import sqlite3 @@ -131,12 +132,18 @@ async def stop(self) -> None: # ------------------------------------------------------------------ async def _loop(self) -> None: - """Evaluate all strategies every check_interval seconds.""" + """Evaluate all strategies every check_interval seconds. + + Strategies may implement evaluate() as sync or async; async ones + (e.g. the LLM agent) are awaited so a slow evaluation never blocks + the event loop for the other strategies. + """ while self._running: try: for strategy in self.strategies: try: - signal = strategy.evaluate(self.hub) + result = strategy.evaluate(self.hub) + signal = await result if inspect.isawaitable(result) else result if signal is None: continue if signal.action in ("BUY", "SELL"): @@ -194,8 +201,12 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: return price = asset.price - # Calculate PnL if closing an existing position + # Plan the state mutation WITHOUT applying it yet: the trade is + # persisted to SQLite first, and only a logged trade mutates the + # books. Otherwise a DB error silently diverges get_portfolio() + # from the audit trail. pnl = 0.0 + apply_mutation: Any if signal.symbol in self.positions: pos = self.positions[signal.symbol] # Closing a long (SELL) or closing a short (BUY) @@ -205,8 +216,14 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: if pos["side"] == "short": price_change_pct = -price_change_pct pnl = pos["size_usd"] * price_change_pct - self.balance += pos["size_usd"] + pnl - del self.positions[signal.symbol] + credit = pos["size_usd"] + pnl + + def apply_mutation() -> None: + # A loss beyond the margin posted would take the account + # negative; a real venue liquidates first. Floor at zero + # (position is wiped, balance cannot go below broke). + self.balance = max(0.0, self.balance + credit) + del self.positions[signal.symbol] else: # Adding in the same direction: balance-checked like an open, # entry price becomes the size-weighted average. @@ -217,11 +234,14 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: ) return new_size = pos["size_usd"] + signal.size_usd - pos["entry_price"] = ( + new_entry = ( pos["entry_price"] * pos["size_usd"] + price * signal.size_usd ) / new_size - pos["size_usd"] = new_size - self.balance -= signal.size_usd + + def apply_mutation() -> None: + pos["entry_price"] = new_entry + pos["size_usd"] = new_size + self.balance -= signal.size_usd else: # Open a new position if signal.size_usd > self.balance: @@ -231,13 +251,15 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: ) return side = "long" if signal.action == "BUY" else "short" - self.positions[signal.symbol] = { - "side": side, - "entry_price": price, - "size_usd": signal.size_usd, - "opened_at": time.time(), - } - self.balance -= signal.size_usd + + def apply_mutation() -> None: + self.positions[signal.symbol] = { + "side": side, + "entry_price": price, + "size_usd": signal.size_usd, + "opened_at": time.time(), + } + self.balance -= signal.size_usd # Build trade record trade = { @@ -251,9 +273,8 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: "reason": signal.reason, "pnl": pnl, } - self.trades.append(trade) - # Persist to SQLite + # Persist FIRST; a trade that cannot be logged is not executed. if self._db: try: self._db.execute( @@ -268,7 +289,14 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: ) self._db.commit() except sqlite3.Error: - logger.exception("Failed to persist trade to SQLite") + logger.exception( + "Failed to persist trade to SQLite — trade NOT executed " + "(books stay consistent with the audit log)" + ) + return + + apply_mutation() + self.trades.append(trade) # Print to console with Rich color = "green" if signal.action == "BUY" else "red" diff --git a/tests/test_hardening.py b/tests/test_hardening.py index 3ee8aef..518f5b9 100644 --- a/tests/test_hardening.py +++ b/tests/test_hardening.py @@ -7,7 +7,9 @@ """ from __future__ import annotations +import inspect import json +import sqlite3 import time from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -18,6 +20,7 @@ from data_layer.liquidation_feed import ( BinanceConnection, + BybitConnection, LiquidationEvent, LiquidationFeed, OKXConnection, @@ -64,6 +67,7 @@ async def _client_for(middlewares) -> TestClient: async def ok(request): return web.json_response({"ok": True}) + app.router.add_get("/v1/live", ok) app.router.add_get("/v1/health", ok) app.router.add_get("/v1/whales", ok) client = TestClient(TestServer(app)) @@ -121,20 +125,23 @@ def test_cors_allowlist_parsed(self, monkeypatch): class TestAuthMiddleware: @pytest.mark.asyncio - async def test_key_required_except_health(self): + async def test_key_required_except_liveness(self): client = await _client_for([ _make_auth_middleware("sekrit"), _make_cors_middleware(None), ]) try: - assert (await client.get("/v1/health")).status == 200 + # Only the minimal liveness probe is exempt; the detailed health + # payload is operational recon and requires the key. + assert (await client.get("/v1/live")).status == 200 + assert (await client.get("/v1/health")).status == 401 assert (await client.get("/v1/whales")).status == 401 ok_bearer = await client.get( "/v1/whales", headers={"Authorization": "Bearer sekrit"}) assert ok_bearer.status == 200 - ok_header = await client.get( - "/v1/whales", headers={"X-API-Key": "sekrit"}) - assert ok_header.status == 200 + ok_health = await client.get( + "/v1/health", headers={"X-API-Key": "sekrit"}) + assert ok_health.status == 200 bad = await client.get( "/v1/whales", headers={"Authorization": "Bearer wrong"}) assert bad.status == 401 @@ -162,8 +169,25 @@ async def test_rate_limit_returns_429(self): client = await _client_for([_make_rate_limit_middleware(limiter)]) try: for _ in range(3): + assert (await client.get("/v1/whales")).status == 200 + assert (await client.get("/v1/whales")).status == 429 + finally: + await client.close() + + @pytest.mark.asyncio + async def test_health_probes_exempt_from_rate_limit(self): + """LBs/monitors behind one NAT IP poll health continuously — a 429 + there makes the balancer mark the backend down.""" + limiter = _RateLimiter(max_requests=2, window_s=60) + client = await _client_for([_make_rate_limit_middleware(limiter)]) + try: + for _ in range(10): assert (await client.get("/v1/health")).status == 200 - assert (await client.get("/v1/health")).status == 429 + assert (await client.get("/v1/live")).status == 200 + # Non-exempt routes still consume the budget normally. + assert (await client.get("/v1/whales")).status == 200 + assert (await client.get("/v1/whales")).status == 200 + assert (await client.get("/v1/whales")).status == 429 finally: await client.close() @@ -252,11 +276,46 @@ def test_different_exchanges_not_deduped(self): def test_dedup_uses_exchange_timestamp_not_local_clock(self): api = _api() + base = 1_700_000_001.0 # Two records of the same event in different dedup buckets by # exchange time are distinct regardless of local arrival time. - assert api._is_duplicate_liq(_liq_event(timestamp=1000.0)) is False - assert api._is_duplicate_liq(_liq_event(timestamp=1009.0)) is False - assert api._is_duplicate_liq(_liq_event(timestamp=1000.5)) is True + assert api._is_duplicate_liq(_liq_event(timestamp=base)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=base + 9.0)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=base + 0.5)) is True + + def test_implausible_timestamp_skips_dedup_never_local_clock(self): + """ts=0 (parse fallback) or ms-scale ts must not be hashed — falling + back to the local clock would collide distinct events that merely + arrived together.""" + api = _api() + # Missing/zero timestamp: identical-looking events both broadcast. + assert api._is_duplicate_liq(_liq_event(timestamp=0.0)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=0.0)) is False + # Millisecond-scale (connector forgot /1000): not safely dedupable. + assert api._is_duplicate_liq(_liq_event(timestamp=1.7e12)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=1.7e12)) is False + + def test_similar_but_distinct_sizes_not_deduped(self): + """$99,950 and $100,050 both round to $100k at round(-2) — the hash + must use the exact size so distinct events never collapse.""" + api = _api() + ts = 1_700_000_001.0 + assert api._is_duplicate_liq(_liq_event(timestamp=ts, size_usd=99_950.0)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=ts, size_usd=100_050.0)) is False + # An exact replay (identical payload) still dedups. + assert api._is_duplicate_liq(_liq_event(timestamp=ts, size_usd=99_950.0)) is True + + def test_cascade_bypass_has_absolute_cap(self): + """Continuous cascade re-triggers must not extend the dedup bypass + forever (replayed duplicates would keep the floodgate open).""" + api = _api() + ev = _liq_event() + key = f"{ev.symbol}_{ev.side}_{ev.exchange}" + for _ in range(30): + api._check_cascade(ev) + started = api._cascade_bypass_started[key] + cap = started + 2 * api._CASCADE_BYPASS_DURATION + assert api._cascade_bypass[key] <= cap + 1e-6 def test_cascade_bypass_lifts_dedup_for_own_venue_only(self): api = _api() @@ -593,3 +652,211 @@ def test_no_data_reports_none_age(self): fresh = e.venue_freshness() assert fresh["hyperliquid"]["data_age_seconds"] is None assert fresh["hyperliquid"]["stale"] is True + + +# ── Review round 2: WS origin, balance floor, persist-first, feeds ── + +class TestWSOriginCheck: + @pytest.mark.asyncio + async def test_browser_origin_rejected_by_default(self): + """Browser WS is not gated by SOP: any webpage can open + ws://127.0.0.1 — an unlisted Origin must be refused.""" + api = _api() + api._cors_origins = None # loopback wildcard REST CORS + request = MagicMock() + request.headers = {"Origin": "https://evil.example"} + resp = await api.handle_ws(request) + assert resp.status == 403 + + @pytest.mark.asyncio + async def test_allowlisted_origin_passes_the_gate(self): + api = _api() + api._cors_origins = {"https://ok.example"} + request = MagicMock() + request.headers = {"Origin": "https://bad.example"} + assert (await api.handle_ws(request)).status == 403 + # An allowlisted origin proceeds past the origin gate (the next + # check is the connection cap, exercised here by filling it). + api._ws_clients = [MagicMock()] * 100 + request.headers = {"Origin": "https://ok.example"} + assert (await api.handle_ws(request)).status == 429 + + +class TestPaperTraderRound2: + def _trader(self, price=100.0, balance=10_000.0) -> PaperTrader: + hub = MagicMock() + hub.market.assets = {"BTC": SimpleNamespace(price=price)} + return PaperTrader(hub, [], starting_balance=balance) + + def test_catastrophic_close_floors_at_zero(self): + """A short losing far more than the posted margin must not drive + the account balance negative.""" + trader = self._trader(price=100.0, balance=1_000.0) + trader._execute_trade("t", Signal("BTC", "SELL", size_usd=1_000.0)) # short + trader.hub.market.assets["BTC"].price = 10_000.0 # +9900% against us + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=1_000.0)) # close + assert trader.balance == 0.0 + assert "BTC" not in trader.positions + + def test_db_error_means_trade_not_executed(self): + """Persist-first: a trade that cannot be logged must not mutate the + books, or the portfolio silently diverges from the audit trail.""" + trader = self._trader(balance=5_000.0) + db = MagicMock() + db.execute.side_effect = sqlite3.OperationalError("disk full") + trader._db = db + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=1_000.0)) + assert trader.balance == 5_000.0 + assert trader.positions == {} + assert trader.trades == [] + + +class TestLLMRound2: + def test_blank_lines_before_action_tolerated(self): + agent = LLMAgent(symbol="BTC") + assert agent._parse_response("\n\nBUY\nmomentum").action == "BUY" + assert agent._parse_response(" \nSELL") .action == "SELL" + assert agent._parse_response("\n\n") is None + + def test_transport_failure_refunds_budget_slot(self, monkeypatch): + """A down provider must not exhaust the hourly budget: transport + failures consumed no tokens, so their slots are returned.""" + agent = LLMAgent(symbol="BTC") + agent.api_key = "k" + agent.base_url = "https://llm.example/v1" + assert agent._within_budget(now=100.0) + assert len(agent._eval_times) == 1 + + def boom(*a, **kw): + raise OSError("connection refused") + + monkeypatch.setattr("urllib.request.urlopen", boom) + hub = MagicMock() + hub.market.assets = {"BTC": SimpleNamespace(price=100.0, funding_rate=0.0)} + assert agent._sync_evaluate(hub) is None + assert len(agent._eval_times) == 0 # slot refunded + + @pytest.mark.asyncio + async def test_evaluate_is_async_and_skips_without_key(self): + """evaluate() is awaited by the paper trader so a slow LLM cannot + block the event loop for other strategies.""" + agent = LLMAgent(symbol="BTC") + agent.api_key = "" + agent.base_url = "https://api.example.com/v1" # non-local, no key + result = agent.evaluate(MagicMock()) + assert inspect.isawaitable(result) + assert await result is None + + +class TestFeedsRound2: + @pytest.mark.asyncio + async def test_binance_array_frame_parsed(self): + """@arr frames may batch events into a JSON array — an array frame + must parse instead of silently failing an isinstance-dict check.""" + feed = LiquidationFeed() + conn = BinanceConnection(feed) + received = [] + feed.on_liquidation(received.append) + frame = [ + {"e": "forceOrder", "o": {"p": "70000", "q": "0.5", "S": "SELL", + "s": "BTCUSDT", "T": 1700000000000}}, + {"e": "forceOrder", "o": {"p": "3500", "q": "2", "S": "BUY", + "s": "ETHUSDT", "T": 1700000000001}}, + ] + await conn._on_message(frame) + assert [ev.symbol for ev in received] == ["BTC", "ETH"] + + @pytest.mark.asyncio + async def test_bybit_v5_list_payload_with_short_keys(self): + """Bybit v5 allLiquidation sends data as a LIST of records with + short keys (p/v/S/s/T) — both shapes must parse without raising.""" + feed = LiquidationFeed() + conn = BybitConnection(feed) + received = [] + feed.on_liquidation(received.append) + await conn._on_message({ + "topic": "allLiquidation.BTCUSDT", + "data": [ + {"p": "70000", "v": "0.5", "S": "Sell", "s": "BTCUSDT", + "T": 1700000000000}, + {"p": "", "v": None, "S": None, "s": None, "T": "x"}, # malformed + ], + }) + assert len(received) == 1 + assert received[0].symbol == "BTC" + assert received[0].side == "long" + assert feed.parse_errors["bybit"] == 1 + + @pytest.mark.asyncio + async def test_bybit_non_dict_data_does_not_raise(self): + feed = LiquidationFeed() + conn = BybitConnection(feed) + await conn._on_message({"topic": "allLiquidation.BTCUSDT", "data": "junk"}) + await conn._on_message({"topic": "allLiquidation.BTCUSDT", "data": [None, 42]}) + assert feed.parse_errors["bybit"] >= 2 + + +class TestPersistenceRound2: + def test_locked_db_raises_instead_of_quarantining(self, tmp_path, monkeypatch): + """'database is locked' is contention, not corruption — a healthy DB + held by another process must never be quarantined.""" + import data_layer.persistence as persistence_mod + + real_connect = persistence_mod.sqlite3.connect + + def locked_connect(*args, **kwargs): + raise sqlite3.OperationalError("database is locked") + + monkeypatch.setattr(persistence_mod.sqlite3, "connect", locked_connect) + with pytest.raises(sqlite3.OperationalError, match="locked"): + DataStore(tmp_path / "locked.db") + monkeypatch.setattr(persistence_mod.sqlite3, "connect", real_connect) + assert not (tmp_path / "corrupted").exists() + + +class TestAlertsRound2: + @pytest.mark.asyncio + async def test_send_failure_does_not_leak_token(self, caplog): + """aiohttp error messages can embed the request URL — which contains + the bot token — so failure logs carry the exception type only.""" + from data_layer.alerts import AlertManager + mgr = AlertManager() + mgr.telegram_token = "123456:SECRET-TOKEN-VALUE" + mgr.telegram_chat_id = "42" + mgr.discord_webhook = "" + session = MagicMock() + session.post = MagicMock(side_effect=RuntimeError( + f"cannot connect to https://api.telegram.org/bot{mgr.telegram_token}/sendMessage" + )) + mgr._session = session + + with caplog.at_level("DEBUG", logger="data_layer.alerts"): + await mgr._send("test message") + assert "SECRET-TOKEN-VALUE" not in caplog.text + assert "Telegram send failed" in caplog.text + + @pytest.mark.asyncio + async def test_wallet_on_first_line_still_redacted(self, caplog): + from data_layer.alerts import AlertManager + mgr = AlertManager() + mgr.telegram_token = "" + mgr.discord_webhook = "" + wallet = "0x" + "cd" * 20 + with caplog.at_level("WARNING", logger="data_layer.alerts"): + await mgr._send(f"whale {wallet} near liquidation") + assert wallet not in caplog.text + assert "ALERT sent" in caplog.text + await mgr.stop() + + +class TestCascadeExampleStrategy: + def test_cascade_strategy_actually_fires(self): + """getattr on the stats dict always returned 0 and silently disabled + this strategy — dict access must read the real key.""" + from src.strategies.examples import LiquidationCascade + hub = MagicMock() + hub.liquidations.get_stats.return_value = {"long_volume_usd": 2_000_000.0} + strat = LiquidationCascade(symbol="BTC", cascade_threshold_usd=1_000_000) + signal = strat.evaluate(hub) + assert signal is not None + assert signal.action == "BUY"