diff --git a/.env.example b/.env.example index a323af7..a8038e6 100644 --- a/.env.example +++ b/.env.example @@ -20,3 +20,16 @@ TELEGRAM_CHAT_ID= # Discord: create a webhook in channel settings DISCORD_WEBHOOK_URL= + +# ── REST API exposure (see README "REST API" security note) ───── +# Loopback (127.0.0.1) needs none of these. A non-loopback bind is refused +# unless HYPERDATA_API_KEY is set (auth required on all non-health routes) +# or HYPERDATA_UNSAFE_PUBLIC_API=1 explicitly accepts unauthenticated exposure. +HYPERDATA_API_HOST=127.0.0.1 +HYPERDATA_API_KEY= +HYPERDATA_CORS_ORIGINS= +HYPERDATA_UNSAFE_PUBLIC_API= + +# ── LLM cost guardrail ──────────────────────────────────── +# Max LLM strategy evaluations per hour (default 60). +LLM_MAX_EVALS_PER_HOUR=60 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..98b43bb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: + # Keep requirements.lock / pyproject deps patched (security + routine). + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + + # Keep SHA-pinned GitHub Actions fresh. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1da3a12..89c7106 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,35 +14,30 @@ jobs: python-version: ["3.12", "3.13"] steps: - - uses: actions/checkout@v4 + # Actions pinned by commit SHA (not mutable tags) so a compromised tag + # can't inject code into CI. + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies + # requirements.lock pins the runtime deps; -e . adds the package + # itself; test tooling is pinned so CI is deterministic. run: | - pip install -e . - pip install pytest ruff + pip install -r requirements.lock + pip install -e . --no-deps + pip install pytest==9.1.1 pytest-asyncio==1.4.0 ruff==0.15.20 pip-audit==2.10.1 - name: Lint - # Non-blocking for now: the repo has pre-existing ruff debt (mostly - # E501/I001/F401) unrelated to current work. Tests below are the real - # gate. TODO: clean lint in a focused PR, then drop continue-on-error. - continue-on-error: true - run: ruff check src/ + run: ruff check src/ tests/ - - name: Syntax check all Python files - run: | - find src/ -name "*.py" -exec python -c " - import ast, sys - try: - ast.parse(open(sys.argv[1]).read()) - except SyntaxError as e: - print(f'FAIL: {sys.argv[1]}: {e}') - sys.exit(1) - " {} \; + - name: Dependency vulnerability audit + # Blocking: a known CVE in a pinned runtime dep fails CI. Dependabot + # (.github/dependabot.yml) keeps the lockfile and action SHAs fresh. + run: pip-audit -r requirements.lock - name: Test imports run: | @@ -52,5 +47,6 @@ jobs: python -c "from src.dashboards.liquidation_heatmap import LiquidationHeatmapDashboard; print('heatmap: OK')" - name: Run tests - # test_position_scanner.py needs a live exchange connection — skip in CI. - run: python -m pytest tests/ -q --ignore=tests/test_position_scanner.py + # The full suite, position scanner included — its network calls are + # mocked; only tests marked `live` need real exchange connections. + run: python -m pytest tests/ -q diff --git a/README.md b/README.md index 0b5ea7f..c093d5a 100644 --- a/README.md +++ b/README.md @@ -105,11 +105,24 @@ Start the API server alongside or instead of the terminal: python run_api.py --port 8420 ``` +> **⚠️ Security: local-only by default.** The API binds to `127.0.0.1` and is +> intended for loopback use. It serves wallet-derived positions, liquidation +> danger zones, and live order flow — trading intelligence you should not +> expose to a LAN or the internet. A non-loopback bind +> (`HYPERDATA_API_HOST=0.0.0.0`) is **refused** unless you either set +> `HYPERDATA_API_KEY=` (all non-health routes then require +> `Authorization: Bearer ` or `X-API-Key`) or explicitly accept the +> risk with `HYPERDATA_UNSAFE_PUBLIC_API=1`. Restrict browser access with +> `HYPERDATA_CORS_ORIGINS=https://your-app.example` (wildcard CORS applies to +> loopback binds only). Do not front this API with a public tunnel or reverse +> proxy without auth and rate limiting of your own. + ### Endpoints | Endpoint | Description | |---|---| -| `GET /v1/health` | Server status and uptime | +| `GET /v1/live` | Minimal liveness probe (always unauthenticated) | +| `GET /v1/health` | Server status and uptime (requires the API key when one is set) | | `GET /v1/market` | All assets — prices, OI, funding | | `GET /v1/market/{symbol}` | Single asset detail | | `GET /v1/liquidations` | Recent liquidation events | diff --git a/docs/DATA_INTEGRITY.md b/docs/DATA_INTEGRITY.md index cece95a..1d3ae4a 100644 --- a/docs/DATA_INTEGRITY.md +++ b/docs/DATA_INTEGRITY.md @@ -85,10 +85,16 @@ the WAL is checkpointed, so the DB stays bounded on long-running instances. ## API exposure -The REST/WebSocket API has no authentication and permissive CORS, so it binds to -**loopback (`127.0.0.1`) by default**. To expose it on the LAN, set -`HYPERDATA_API_HOST=0.0.0.0` — only do this behind a trusted network. Numeric -query params are validated (bad values return `400`, not `500`). +The REST/WebSocket API binds to **loopback (`127.0.0.1`) by default**. A +non-loopback bind (`HYPERDATA_API_HOST=0.0.0.0`) is refused at startup unless +either `HYPERDATA_API_KEY` is set — all non-health routes then require +`Authorization: Bearer ` or `X-API-Key: ` — or +`HYPERDATA_UNSAFE_PUBLIC_API=1` explicitly acknowledges the exposure. CORS is +wildcard only on loopback; non-loopback binds send CORS headers only for +origins allowlisted in `HYPERDATA_CORS_ORIGINS` (comma-separated). REST +requests are rate-limited per client IP, and WebSocket clients get bounded +per-client send queues plus inbound message size/rate limits. Numeric query +params are validated (bad values return `400`, not `500`). ## Timestamps diff --git a/pyproject.toml b/pyproject.toml index 3e1bfcb..eeba85c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,11 @@ target-version = "py312" [tool.ruff.lint] select = ["E", "F", "W", "I"] +[tool.ruff.lint.per-file-ignores] +# Presentation layer: ASCII-art boot screens and Rich table markup read +# better unwrapped; everything else must respect the line limit. +"src/dashboards/*" = ["E501"] + [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" diff --git a/requirements.lock b/requirements.lock index 79ef770..ea1debc 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,7 +1,7 @@ # Generated with pip freeze — do not edit manually # Pinned versions for reproducible installs # To install: pip install -r requirements.lock -aiohttp==3.13.3 +aiohttp==3.14.1 numpy==2.4.3 pandas==3.0.1 pyfiglet==1.0.4 diff --git a/run_dashboard.py b/run_dashboard.py index 2ecea5b..cae0ef8 100644 --- a/run_dashboard.py +++ b/run_dashboard.py @@ -76,6 +76,39 @@ async def run_dashboard(hub: HyperDataHub, key: str) -> None: await create_fn().run() +async def _ainput(prompt: str) -> str: + """Read one line of input without blocking the event loop. + + Uses a daemon thread per prompt (human-speed churn only) so a read that + is still pending at exit can never wedge interpreter shutdown — the + failure mode of parking input() inside a ThreadPoolExecutor, whose + non-daemon workers are joined at exit. + """ + import threading + + loop = asyncio.get_running_loop() + fut: asyncio.Future[str] = loop.create_future() + + def _set(value=None, exc=None): + if fut.done(): + return + if exc is not None: + fut.set_exception(exc) + else: + fut.set_result(value) + + def _worker(): + try: + line = input(prompt) + except BaseException as e: # EOFError / KeyboardInterrupt in the thread + loop.call_soon_threadsafe(_set, None, e) + else: + loop.call_soon_threadsafe(_set, line) + + threading.Thread(target=_worker, daemon=True, name="menu-input").start() + return await fut + + async def run_interactive(api_port: int | None = None) -> None: """Boot → menu → pick dashboard → run → back to menu on Ctrl+C.""" console = Console() @@ -89,19 +122,10 @@ async def run_interactive(api_port: int | None = None) -> None: while True: _build_menu(console) - # Read user choice — use a daemon thread so Ctrl+C exits cleanly - import concurrent.futures - _input_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) - _input_pool._threads = set() # ensure daemon threads try: - choice = await asyncio.get_event_loop().run_in_executor( - _input_pool, lambda: input(" Enter choice: ").strip().lower() - ) + choice = (await _ainput(" Enter choice: ")).strip().lower() except (EOFError, KeyboardInterrupt): - _input_pool.shutdown(wait=False) break - finally: - _input_pool.shutdown(wait=False) if choice in ("q", "quit", "exit"): break diff --git a/src/api_server.py b/src/api_server.py index 9f4e613..bc7c39c 100644 --- a/src/api_server.py +++ b/src/api_server.py @@ -12,18 +12,31 @@ # WS: ws://localhost:8420/v1/ws # Send: {"subscribe": ["trade", "liquidation"]} # Recv: {"type": "trade", "data": {...}, "ts": 1234567890.123} + +Security model: + - Binds to loopback by default; loopback needs no credentials. + - A non-loopback bind (HYPERDATA_API_HOST) is refused unless either + HYPERDATA_API_KEY is set (all non-health routes then require it) or + HYPERDATA_UNSAFE_PUBLIC_API=1 explicitly acknowledges the risk. + - CORS is wildcard only on loopback; non-loopback binds must allowlist + origins via HYPERDATA_CORS_ORIGINS (comma-separated), else no CORS. + - Per-IP REST rate limit + WebSocket per-client send queues. """ from __future__ import annotations import asyncio import dataclasses +import hmac +import ipaddress import json import logging import math +import os import time +from collections import deque from typing import Any -from aiohttp import web, WSMsgType +from aiohttp import WSMsgType, web logger = logging.getLogger(__name__) @@ -31,6 +44,29 @@ EVENT_TYPES = {"trade", "liquidation", "signal", "funding_update", "iv_update", "alert", "heartbeat"} MAX_WS_CONNECTIONS = 10 +# Per-client outbound queue depth. When a slow client's queue is full, new +# events are dropped for that client (counted) instead of spawning unbounded +# send tasks that compete with ingestion. +WS_SEND_QUEUE_SIZE = 200 +# Inbound WebSocket message limits: size cap, rate cap, and how many bad +# (non-JSON / oversized / too-fast) messages we tolerate before disconnecting. +WS_MAX_MSG_BYTES = 4096 +WS_MAX_MSGS_PER_10S = 20 +WS_BAD_MSG_LIMIT = 5 + +# Per-IP REST rate limit (sliding window). +RATE_LIMIT_REQUESTS = 300 +RATE_LIMIT_WINDOW_S = 60.0 + + +def _is_loopback_host(host: str) -> bool: + """True if the bind host is loopback-only ('localhost', 127.x, ::1).""" + if host in ("localhost",): + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False def _serialize(obj: Any) -> Any: @@ -49,30 +85,120 @@ def _serialize(obj: Any) -> Any: return obj -_CORS_HEADERS = { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", - "Access-Control-Allow-Headers": "Authorization, Content-Type", -} +def _make_cors_middleware(allowed_origins: set[str] | None): + """CORS middleware factory. + + allowed_origins=None means wildcard (loopback binds only); otherwise the + request Origin must be in the allowlist to receive CORS headers. + """ + + def _cors_headers(request: web.Request) -> dict[str, str]: + if allowed_origins is None: + origin = "*" + else: + req_origin = request.headers.get("Origin", "") + if req_origin not in allowed_origins: + return {} + origin = req_origin + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", + "Access-Control-Allow-Headers": "Authorization, Content-Type, X-API-Key", + **({"Vary": "Origin"} if origin != "*" else {}), + } + @web.middleware + async def cors_middleware(request: web.Request, handler): + headers = _cors_headers(request) + if request.method == "OPTIONS": + return web.Response(status=200, headers=headers) + try: + resp = await handler(request) + except web.HTTPNotFound: + return web.json_response( + {"error": "Endpoint not found", "path": request.path}, + status=404, + headers=headers, + ) + except web.HTTPException as exc: + exc.headers.update(headers) + raise + resp.headers.update(headers) + return resp + + return cors_middleware + + +# Paths reachable without an API key. Only the minimal liveness probe is +# exempt — the detailed /v1/health payload (mode, feed states, counters) is +# operational recon and requires the key on authenticated deployments. +_UNAUTHENTICATED_PATHS = {"/v1/live"} + +# Paths exempt from per-IP rate limiting: load balancers, uptime monitors, +# and liveness probes often share one NAT egress IP and poll continuously. +_RATE_LIMIT_EXEMPT_PATHS = {"/v1/live", "/v1/health", "/health"} + + +def _make_auth_middleware(api_key: str): + """Require the API key on every route except health checks. + + Accepts either ``Authorization: Bearer `` or ``X-API-Key: ``. + """ + + @web.middleware + async def auth_middleware(request: web.Request, handler): + if request.method == "OPTIONS" or request.path in _UNAUTHENTICATED_PATHS: + return await handler(request) + supplied = request.headers.get("X-API-Key", "") + auth = request.headers.get("Authorization", "") + if auth.startswith("Bearer "): + supplied = supplied or auth[len("Bearer "):] + if not supplied or not hmac.compare_digest(supplied, api_key): + return web.json_response({"error": "Unauthorized"}, status=401) + return await handler(request) + + return auth_middleware + + +class _RateLimiter: + """Sliding-window per-IP request limiter for the REST surface.""" + + def __init__(self, max_requests: int = RATE_LIMIT_REQUESTS, + window_s: float = RATE_LIMIT_WINDOW_S) -> None: + self.max_requests = max_requests + self.window_s = window_s + self._hits: dict[str, deque[float]] = {} + + def allow(self, key: str, now: float | None = None) -> bool: + now = time.time() if now is None else now + dq = self._hits.get(key) + if dq is None: + dq = self._hits.setdefault(key, deque()) + cutoff = now - self.window_s + while dq and dq[0] < cutoff: + dq.popleft() + if len(dq) >= self.max_requests: + return False + dq.append(now) + # Bound tracked IPs so a scan can't grow this dict forever. + if len(self._hits) > 10_000: + stale = [k for k, v in self._hits.items() if not v or v[-1] < cutoff] + for k in stale: + del self._hits[k] + return True -@web.middleware -async def cors_middleware(request: web.Request, handler): - if request.method == "OPTIONS": - return web.Response(status=200, headers=_CORS_HEADERS) - try: - resp = await handler(request) - except web.HTTPNotFound: - return web.json_response( - {"error": "Endpoint not found", "path": request.path}, - status=404, - headers=_CORS_HEADERS, - ) - except web.HTTPException as exc: - exc.headers.update(_CORS_HEADERS) - raise - resp.headers.update(_CORS_HEADERS) - return resp + +def _make_rate_limit_middleware(limiter: _RateLimiter): + @web.middleware + async def rate_limit_middleware(request: web.Request, handler): + if request.path in _RATE_LIMIT_EXEMPT_PATHS: + return await handler(request) + remote = request.remote or "unknown" + if not limiter.allow(remote): + return web.json_response({"error": "Rate limit exceeded"}, status=429) + return await handler(request) + + return rate_limit_middleware def _int_param(request: web.Request, name: str, default: int, @@ -112,7 +238,8 @@ def _float_param(request: web.Request, name: str, default: float, # ── WebSocket client tracker ───────────────────────────────────────────── class _WSClient: - __slots__ = ("ws", "subscriptions", "ping_misses", "connected_at") + __slots__ = ("ws", "subscriptions", "ping_misses", "connected_at", + "queue", "writer_task", "dropped_msgs", "msg_times", "bad_msgs") def __init__(self, ws: web.WebSocketResponse, subscriptions: set[str] | None = None): self.ws = ws @@ -120,15 +247,22 @@ def __init__(self, ws: web.WebSocketResponse, subscriptions: set[str] | None = N self.subscriptions: set[str] = subscriptions if subscriptions is not None else set() self.ping_misses: int = 0 self.connected_at: float = time.time() + # Bounded outbound queue drained by a single writer task per client. + self.queue: asyncio.Queue[str] = asyncio.Queue(maxsize=WS_SEND_QUEUE_SIZE) + self.writer_task: asyncio.Task | None = None + self.dropped_msgs: int = 0 + # Inbound abuse tracking (message rate + malformed messages). + self.msg_times: deque[float] = deque(maxlen=WS_MAX_MSGS_PER_10S) + self.bad_msgs: int = 0 class HyperDataAPI: """REST API v1 + WebSocket streaming, backed by a live HyperDataHub.""" def __init__(self, hub, host: str = "127.0.0.1", port: int = 8420) -> None: - # Bind to loopback by default: the API has no auth and CORS is open, so - # it must not be reachable from the LAN unless the operator opts in - # (HYPERDATA_API_HOST=0.0.0.0). See docs/DATA_INTEGRITY.md / README. + # Bind to loopback by default. Non-loopback binds are refused in + # start() unless HYPERDATA_API_KEY is set (auth enforced) or + # HYPERDATA_UNSAFE_PUBLIC_API=1 explicitly acknowledges the risk. self.hub = hub self.host = host self.port = port @@ -136,14 +270,58 @@ def __init__(self, hub, host: str = "127.0.0.1", port: int = 8420) -> None: self._site: web.TCPSite | None = None self._ws_clients: list[_WSClient] = [] self._hooks_installed = False + self._rate_limiter = _RateLimiter() # ── Lifecycle ──────────────────────────────────────────────── + def _resolve_security(self) -> tuple[str, set[str] | None]: + """Validate bind/auth/CORS config. Returns (api_key, cors_allowlist). + + Raises RuntimeError for a non-loopback bind with neither an API key + nor an explicit unsafe acknowledgment. + """ + api_key = os.environ.get("HYPERDATA_API_KEY", "").strip() + unsafe_ack = os.environ.get("HYPERDATA_UNSAFE_PUBLIC_API", "") == "1" + origins_raw = os.environ.get("HYPERDATA_CORS_ORIGINS", "").strip() + origins: set[str] | None = ( + {o.strip() for o in origins_raw.split(",") if o.strip()} + if origins_raw else None + ) + + if _is_loopback_host(self.host): + # Loopback: wildcard CORS unless an allowlist was configured. + return api_key, origins + + if not api_key and not unsafe_ack: + raise RuntimeError( + f"Refusing to bind API to non-loopback host {self.host!r}: the " + "API would expose trading intelligence to the network. Set " + "HYPERDATA_API_KEY to require authentication, or set " + "HYPERDATA_UNSAFE_PUBLIC_API=1 to explicitly accept the risk." + ) + if not api_key: + logger.warning( + "SECURITY: API bound to %s WITHOUT authentication " + "(HYPERDATA_UNSAFE_PUBLIC_API=1). Anyone on the network can " + "read wallet/trading intelligence.", self.host, + ) + # Non-loopback: never wildcard CORS. No allowlist -> no CORS headers. + return api_key, (origins or set()) + async def start(self) -> None: - app = web.Application(middlewares=[cors_middleware]) + api_key, cors_origins = self._resolve_security() + self._api_key = api_key + self._cors_origins = cors_origins + + middlewares = [_make_rate_limit_middleware(self._rate_limiter)] + if api_key: + middlewares.append(_make_auth_middleware(api_key)) + middlewares.append(_make_cors_middleware(cors_origins)) + app = web.Application(middlewares=middlewares) # v1 routes v1 = [ + ("GET", "/v1/live", self.handle_live), ("GET", "/v1/health", self.handle_health), ("GET", "/v1/market", self.handle_market), ("GET", "/v1/market/{symbol}", self.handle_market_symbol), @@ -200,6 +378,8 @@ async def stop(self) -> None: pass for client in list(self._ws_clients): + if client.writer_task: + client.writer_task.cancel() if not client.ws.closed: await client.ws.close() self._ws_clients.clear() @@ -245,6 +425,7 @@ def _on_trade(self, trade) -> None: _DEDUP_MAX = 500 _CASCADE_WINDOW = 30 _CASCADE_BYPASS_DURATION = 30 + _CASCADE_TRACKER_MAX = 200 # entries kept per symbol/side/exchange key def __init_dedup(self): if not hasattr(self, '_liq_seen'): @@ -253,20 +434,45 @@ def __init_dedup(self): self._heartbeat_task: asyncio.Task | None = None self._cascade_tracker: dict[str, list] = {} self._cascade_bypass: dict[str, float] = {} + self._cascade_bypass_started: dict[str, float] = {} self._liq_stats = {"received": 0, "broadcast": 0, "deduped": 0, "filtered": 0} self._liq_stats_ts = time.time() + # Plausible epoch-seconds range for exchange event times (2001..5138). + # A timestamp outside this range means a connector skipped ms→s + # normalization (or sent 0) — such events cannot be safely hashed. + _TS_SANE_MIN = 1e9 + _TS_SANE_MAX = 1e11 + def _is_duplicate_liq(self, ev) -> bool: - """Check if this liquidation is a duplicate within the 3-second dedup window.""" + """Duplicate check within the dedup window, keyed per exchange. + + Buckets on the EXCHANGE event timestamp (not local receive time) so + two records of the same event dedup identically regardless of local + delivery jitter. Events without a plausible exchange timestamp are + never deduped — substituting the local clock would collide distinct + events that merely arrived together. The hash uses the exact size: + replayed duplicates carry identical payloads, while distinct events + of similar size must not collapse into one. The cascade bypass is + also per-exchange: a Binance cascade must not let Hyperliquid's + heuristic events skip dedup. + """ self.__init_dedup() now = time.time() - bypass_key = f"{ev.symbol}_{ev.side}" + ev_ts = ev.timestamp + if not (self._TS_SANE_MIN < ev_ts < self._TS_SANE_MAX): + logger.warning( + "[liq] %s event has implausible timestamp %r — skipping dedup", + ev.exchange, ev_ts, + ) + return False + + bypass_key = f"{ev.symbol}_{ev.side}_{ev.exchange}" if bypass_key in self._cascade_bypass and now < self._cascade_bypass[bypass_key]: return False - size_rounded = round(ev.size_usd, -2) - h = f"{ev.symbol}_{ev.side}_{size_rounded}_{ev.exchange}_{int(now // self._DEDUP_WINDOW)}" + h = f"{ev.symbol}_{ev.side}_{ev.size_usd:.2f}_{ev.exchange}_{int(ev_ts // self._DEDUP_WINDOW)}" if len(self._liq_seen) > self._DEDUP_MAX: cutoff = now - self._DEDUP_WINDOW * 2 @@ -277,11 +483,16 @@ def _is_duplicate_liq(self, ev) -> bool: self._liq_seen[h] = now return False - def _check_cascade(self, symbol: str, side: str, exchange: str, size_usd: float) -> str | None: - """Track rapid successive liquidations. Returns cascade label if detected.""" + def _check_cascade(self, ev) -> str | None: + """Track rapid successive liquidations. Returns cascade label if detected. + + Keys on the RAW event fields (symbol/side/exchange) — the same domain + _is_duplicate_liq reads its bypass with — so a detected cascade + actually lifts dedup for the venue that is cascading. + """ self.__init_dedup() now = time.time() - key = f"{symbol}_{side}_{exchange}" + key = f"{ev.symbol}_{ev.side}_{ev.exchange}" if key not in self._cascade_tracker: self._cascade_tracker[key] = [] @@ -291,15 +502,28 @@ def _check_cascade(self, symbol: str, side: str, exchange: str, size_usd: float) if now - ts < self._CASCADE_WINDOW ] - self._cascade_tracker[key].append((now, size_usd)) + self._cascade_tracker[key].append((now, ev.size_usd)) + # Bound per-key memory: only the most recent window entries matter. + if len(self._cascade_tracker[key]) > self._CASCADE_TRACKER_MAX: + self._cascade_tracker[key] = self._cascade_tracker[key][-self._CASCADE_TRACKER_MAX:] entries = self._cascade_tracker[key] if len(entries) >= 3: - bypass_key = f"{symbol}_{side}" - self._cascade_bypass[bypass_key] = now + self._CASCADE_BYPASS_DURATION + # Bypass dedup only for this exchange's stream: cascades on one + # venue say nothing about duplicates on another. The bypass has + # an ABSOLUTE cap: without it, events passing dedup during the + # bypass re-trigger cascade detection and extend it forever + # (replayed duplicates would keep the floodgate open). + first = self._cascade_bypass_started.setdefault(key, now) + cap = first + 2 * self._CASCADE_BYPASS_DURATION + self._cascade_bypass[key] = min(now + self._CASCADE_BYPASS_DURATION, cap) total = sum(sz for _, sz in entries) return f"cascade ${total:,.0f} ({len(entries)}x in {self._CASCADE_WINDOW}s)" + # Quiet again: allow a future cascade to start a fresh bypass window. + if key in self._cascade_bypass_started and now > self._cascade_bypass.get(key, 0): + del self._cascade_bypass_started[key] + return None def _log_liq_stats(self) -> None: @@ -360,7 +584,7 @@ def _on_liquidation(self, ev) -> None: ex_map = {"binance": "BIN", "bybit": "BYB", "okx": "OKX", "hyperliquid": "HYP"} ex_short = ex_map.get(ev.exchange, ev.exchange[:3].upper()) - cascade = self._check_cascade(symbol, ev.side.upper(), ex_short, ev.size_usd) + cascade = self._check_cascade(ev) self._broadcast("liquidation", { "exchange": ex_short, @@ -386,7 +610,12 @@ def _on_liquidation(self, ev) -> None: }) def _broadcast(self, event_type: str, data: dict) -> None: - """Send event to all subscribed WebSocket clients. Bulletproof.""" + """Enqueue event for all subscribed WebSocket clients. + + Each client has a bounded queue drained by its own writer task, so a + slow client drops ITS events (counted) instead of accumulating one + send task per client per event on the shared loop. + """ if not self._ws_clients: return msg = json.dumps({"type": event_type, "data": data, "ts": time.time()}) @@ -398,12 +627,42 @@ def _broadcast(self, event_type: str, data: dict) -> None: if event_type not in client.subscriptions: continue try: - asyncio.ensure_future(self._safe_send(client, msg)) - except Exception: - dead.append(client) + client.queue.put_nowait(msg) + except asyncio.QueueFull: + client.dropped_msgs += 1 + if client.dropped_msgs % 100 == 1: + logger.warning( + "[ws] Slow client: %d events dropped (queue full)", + client.dropped_msgs, + ) for d in dead: - if d in self._ws_clients: - self._ws_clients.remove(d) + self._remove_client(d) + + def _remove_client(self, client: _WSClient) -> None: + if client in self._ws_clients: + self._ws_clients.remove(client) + if client.writer_task and not client.writer_task.done(): + client.writer_task.cancel() + + async def _writer_loop(self, client: _WSClient) -> None: + """Single writer per client: drain the queue with a send timeout.""" + try: + while not client.ws.closed: + msg = await client.queue.get() + try: + await asyncio.wait_for(client.ws.send_str(msg), timeout=2.0) + except asyncio.CancelledError: + raise + except Exception: + # Send failed or timed out — this client is done. + self._remove_client(client) + try: + await client.ws.close() + except Exception: + pass + return + except asyncio.CancelledError: + pass async def _heartbeat_loop(self) -> None: """Push heartbeat every 10s. Evict clients that miss 3 consecutive pings.""" @@ -429,18 +688,20 @@ async def _heartbeat_loop(self) -> None: continue if "heartbeat" not in client.subscriptions: continue + # Enqueue through the same bounded queue as broadcasts. A + # full queue means the writer is stuck/slow — count it as + # a missed ping and evict after 3 in a row. try: - await asyncio.wait_for(client.ws.send_str(msg), timeout=2.0) + client.queue.put_nowait(msg) client.ping_misses = 0 - except Exception: + except asyncio.QueueFull: client.ping_misses += 1 if client.ping_misses >= 3: dead.append(client) - logger.debug("[ws] Evicting client after 3 missed pings") + logger.info("[ws] Evicting client after 3 missed heartbeats") for d in dead: - if d in self._ws_clients: - self._ws_clients.remove(d) + self._remove_client(d) try: await d.ws.close() except Exception: @@ -449,19 +710,9 @@ async def _heartbeat_loop(self) -> None: except asyncio.CancelledError: return except Exception: - pass - - async def _safe_send(self, client: _WSClient, msg: str) -> None: - """Send with timeout. Remove client on any failure.""" - try: - await asyncio.wait_for(client.ws.send_str(msg), timeout=2.0) - except Exception: - if client in self._ws_clients: - self._ws_clients.remove(client) - try: - await client.ws.close() - except Exception: - pass + # Never die silently: the heartbeat loop is also the WS + # liveness janitor, so log and keep going. + logger.exception("[ws] Heartbeat loop error") # ── Redirect helper ────────────────────────────────────────── @@ -473,40 +724,84 @@ async def redirect(request: web.Request) -> web.Response: # ── WebSocket handler ──────────────────────────────────────── - async def handle_ws(self, request: web.Request) -> web.WebSocketResponse: + async def handle_ws(self, request: web.Request) -> web.WebSocketResponse | web.Response: + # Browser WebSockets are NOT gated by the same-origin policy: any + # webpage can open ws://127.0.0.1 and read the stream. An Origin + # header means a browser context — reject it unless the origin was + # explicitly allowlisted (HYPERDATA_CORS_ORIGINS). Non-browser + # clients (curl, bots, SDKs) send no Origin and are unaffected. + origin = request.headers.get("Origin") + if origin is not None: + allowed = getattr(self, "_cors_origins", None) or set() + if origin not in allowed: + logger.warning("[ws] Rejected cross-origin upgrade from %s", origin) + return web.json_response({"error": "Origin not allowed"}, status=403) + if len(self._ws_clients) >= MAX_WS_CONNECTIONS: return web.json_response({"error": "Too many connections"}, status=429) - ws = web.WebSocketResponse(heartbeat=20) + ws = web.WebSocketResponse(heartbeat=20, max_msg_size=WS_MAX_MSG_BYTES) await ws.prepare(request) client = _WSClient(ws, subscriptions=set()) + client.writer_task = asyncio.create_task( + self._writer_loop(client), name="ws-writer" + ) self._ws_clients.append(client) logger.info("[ws] Client connected (%d total)", len(self._ws_clients)) try: async for msg in ws: if msg.type == WSMsgType.TEXT: - try: - data = json.loads(msg.data) - subs = data.get("subscribe") - if isinstance(subs, list): - client.subscriptions = {s for s in subs if s in EVENT_TYPES} - await ws.send_json({ - "type": "subscribed", - "channels": sorted(client.subscriptions), - }) - except json.JSONDecodeError: - pass + if self._ws_msg_violates_limits(client, msg.data): + break elif msg.type in (WSMsgType.CLOSED, WSMsgType.ERROR): break finally: - if client in self._ws_clients: - self._ws_clients.remove(client) + self._remove_client(client) logger.info("[ws] Client disconnected (%d remaining)", len(self._ws_clients)) return ws + def _ws_msg_violates_limits(self, client: _WSClient, raw: str) -> bool: + """Process one inbound message. Returns True if the client should be + disconnected (message flood or too many malformed messages).""" + now = time.time() + client.msg_times.append(now) + if (len(client.msg_times) == client.msg_times.maxlen + and now - client.msg_times[0] < 10.0): + logger.info("[ws] Disconnecting client: message rate limit exceeded") + return True + + if len(raw) > WS_MAX_MSG_BYTES: + client.bad_msgs += 1 + else: + try: + data = json.loads(raw) + subs = data.get("subscribe") if isinstance(data, dict) else None + if isinstance(subs, list): + client.subscriptions = {s for s in subs if s in EVENT_TYPES} + client.queue.put_nowait(json.dumps({ + "type": "subscribed", + "channels": sorted(client.subscriptions), + })) + except (json.JSONDecodeError, asyncio.QueueFull): + client.bad_msgs += 1 + + if client.bad_msgs >= WS_BAD_MSG_LIMIT: + logger.info("[ws] Disconnecting client after %d bad messages", client.bad_msgs) + return True + return False + # ── REST Handlers ──────────────────────────────────────────── + async def handle_live(self, request: web.Request) -> web.Response: + """Minimal liveness probe — safe to expose unauthenticated. + + Deliberately says nothing about mode, feeds, or counters: the + detailed /v1/health payload is operational recon and requires the + API key on authenticated deployments. + """ + return web.json_response({"status": "ok"}) + async def handle_health(self, request: web.Request) -> web.Response: s = self.hub.status uptime = int(s.uptime_seconds) @@ -527,13 +822,28 @@ async def handle_health(self, request: web.Request) -> web.Response: "hlp": s.hlp_status, } + # Per-venue orderflow freshness: the combined status above follows the + # freshest venue, so a dead venue is only visible here. + orderflow_venues = None + try: + orderflow_venues = self.hub.orderflow.venue_freshness() + except Exception: + pass + # Top-level status reflects data health when available: 'ok' only when # nothing is stale/drifting. 'degraded' otherwise (server is still up). + # Components that failed to start OR whose loops are currently erroring + # (position scanner / market data flip to 'error' at runtime without + # touching failed_components) also force 'degraded'. overall = data_health.get("overall") if data_health else None status = "ok" if overall in (None, "ok", "warn") else "degraded" + if s.failed_components or any(v == "error" for v in feeds.values()): + status = "degraded" return web.json_response({ "status": status, + "failed_components": list(s.failed_components), + "orderflow_venues": orderflow_venues, "version": "1.0.0", "mode": s.mode, "uptime": f"{h}h {m}m", @@ -636,7 +946,10 @@ async def handle_funding_symbol(self, request: web.Request) -> web.Response: for ex_name, ex_rates in self.hub.funding.rates.items(): snap = ex_rates.get(sym) if snap: - rates[ex_name] = {"hourly": snap.funding_rate_hourly, "annualized_pct": snap.funding_rate_annualized * 100} + rates[ex_name] = { + "hourly": snap.funding_rate_hourly, + "annualized_pct": snap.funding_rate_annualized * 100, + } if not rates: return web.json_response({"error": f"No funding data for {sym}"}, status=404) return web.json_response({"symbol": sym, "rates": rates}) @@ -674,41 +987,6 @@ async def handle_deribit_iv(self, request: web.Request) -> web.Response: } return web.json_response(data) - async def handle_smart_money_rankings(self, request: web.Request) -> web.Response: - n = _int_param(request, "limit", 20, minimum=1, maximum=500) - smart = self.hub.get_smart_money(n) - dumb = self.hub.get_dumb_money(n) - stats = self.hub.smart_money.get_stats() - - def _fmt_wallet(w): - d = _serialize(w) - pnl = w.total_realized_pnl - if w.total_trades == 0: - d["pnl_display"] = "--" - elif abs(pnl) >= 1_000_000: - d["pnl_display"] = f"${pnl/1_000_000:+.1f}M" - elif abs(pnl) >= 1_000: - d["pnl_display"] = f"${pnl/1_000:+.1f}K" - elif abs(pnl) >= 1: - d["pnl_display"] = f"${pnl:+.0f}" - else: - d["pnl_display"] = "--" - return d - - return web.json_response({ - "stats": stats, - "smart": [_fmt_wallet(w) for w in smart], - "dumb": [_fmt_wallet(w) for w in dumb], - }) - - async def handle_smart_money_signals(self, request: web.Request) -> web.Response: - n = _int_param(request, "limit", 50, minimum=1, maximum=500) - signals = self.hub.get_smart_money_signals(n) - return web.json_response({ - "count": len(signals), - "signals": [_serialize(s) for s in signals], - }) - async def handle_orderbook(self, request: web.Request) -> web.Response: sym = request.match_info["symbol"].upper() snap = self.hub.get_orderbook(sym) @@ -741,93 +1019,7 @@ async def handle_danger_zone(self, request: web.Request) -> web.Response: "positions": [_serialize(p) for p in positions], }) - # ── Copy-trading endpoints ──────────────────────────────────── - - _ct_signals_cache: dict | None = None - _ct_signals_cache_ts: float = 0 - - async def handle_copy_trading_signals(self, request: web.Request) -> web.Response: - """GET /v1/copy-trading/signals — recent copy/fade signals (10s cache).""" - try: - now = time.time() - if (self._ct_signals_cache is not None - and now - self._ct_signals_cache_ts < 10.0): - return web.json_response(self._ct_signals_cache) - - wc = getattr(self, '_wallet_cluster', None) - if wc is None: - return web.json_response({ - "signals": [], "active_count": 0, - "suppressed_count": 0, "last_updated": now, - }) - - signals = wc.get_signals(limit=20) - active = [s for s in signals if now - s.get("emitted_at", 0) < 300] - result = { - "signals": signals, - "active_count": len(active), - "suppressed_count": len(signals) - len(active), - "last_updated": now, - } - self._ct_signals_cache = result - self._ct_signals_cache_ts = now - return web.json_response(result) - except Exception as e: - return web.json_response({ - "error": str(e), "signals": [], "active_count": 0, - }) - - _ct_clusters_cache: dict | None = None - _ct_clusters_cache_ts: float = 0 - - async def handle_copy_trading_clusters(self, request: web.Request) -> web.Response: - """GET /v1/copy-trading/clusters — cluster breakdown (60s cache).""" - try: - now = time.time() - if (self._ct_clusters_cache is not None - and now - self._ct_clusters_cache_ts < 60.0): - return web.json_response(self._ct_clusters_cache) - - wc = getattr(self, '_wallet_cluster', None) - if wc is None: - return web.json_response({ - "clusters": [], "last_clustered": 0, - }) - - result = { - "clusters": wc.get_clusters(), - "last_clustered": wc._last_clustered, - } - self._ct_clusters_cache = result - self._ct_clusters_cache_ts = now - return web.json_response(result) - except Exception as e: - return web.json_response({"error": str(e), "clusters": []}) - - _ct_wallets_cache: dict | None = None - _ct_wallets_cache_ts: float = 0 - - async def handle_copy_trading_wallets(self, request: web.Request) -> web.Response: - """GET /v1/copy-trading/wallets — all tracked wallets (60s cache).""" - try: - now = time.time() - if (self._ct_wallets_cache is not None - and now - self._ct_wallets_cache_ts < 60.0): - return web.json_response(self._ct_wallets_cache) - - wc = getattr(self, '_wallet_cluster', None) - if wc is None: - return web.json_response({"wallets": [], "total": 0}) - - wallets = wc.get_wallets() - result = {"wallets": wallets, "total": len(wallets)} - self._ct_wallets_cache = result - self._ct_wallets_cache_ts = now - return web.json_response(result) - except Exception as e: - return web.json_response({"error": str(e), "wallets": []}) - - # ── Public metrics (live from backtest files + DB, 5-min cache) ── + # ── Public metrics ──────────────────────────────────────────── async def handle_public_metrics(self, request: web.Request) -> web.Response: """GET /v1/public/metrics — server status and data component health.""" diff --git a/src/dashboards/boot.py b/src/dashboards/boot.py index 4cc8129..e69d859 100644 --- a/src/dashboards/boot.py +++ b/src/dashboards/boot.py @@ -71,7 +71,7 @@ async def print_boot_sequence(console: Console, mode: str, dashboards: list[str] nw = 22 # name column width (same for both boxes) bw = 80 # box inner width - console.print(f" [bold bright_cyan]\u250c\u2500 INITIALIZING COMPONENTS " + "\u2500" * (bw - 26) + "\u2510[/]") + console.print(" [bold bright_cyan]\u250c\u2500 INITIALIZING COMPONENTS " + "\u2500" * (bw - 26) + "\u2510[/]") console.print(f" [bright_cyan]\u2502[/]{' ' * bw}[bright_cyan]\u2502[/]") for name, desc, color in components: @@ -85,10 +85,10 @@ async def print_boot_sequence(console: Console, mode: str, dashboards: list[str] console.print(f" [bright_cyan]\u2502[/]{rich}{' ' * max(pad_ck, 0)} [bold bright_green]\u2713[/][bright_cyan]\u2502[/]") console.print(f" [bright_cyan]\u2502[/]{' ' * bw}[bright_cyan]\u2502[/]") - console.print(f" [bold bright_cyan]\u2514" + "\u2500" * bw + "\u2518[/]") + console.print(" [bold bright_cyan]\u2514" + "\u2500" * bw + "\u2518[/]") console.print() - console.print(f" [bold bright_white]\u250c\u2500 ACTIVE DASHBOARDS " + "\u2500" * (bw - 21) + "\u2510[/]") + console.print(" [bold bright_white]\u250c\u2500 ACTIVE DASHBOARDS " + "\u2500" * (bw - 21) + "\u2510[/]") for d in dashboards: info = DASHBOARD_INFO.get(d, {"name": d, "desc": "", "color": "white"}) content = f" \u25b8 {info['name']:<{nw}}{info['desc']}" @@ -96,7 +96,7 @@ async def print_boot_sequence(console: Console, mode: str, dashboards: list[str] rich = f" [{info['color']}]\u25b8[/] [{info['color']}]{info['name']:<{nw}}[/][dim]{info['desc']}[/]" console.print(f" [bright_white]\u2502[/]{rich}{' ' * max(pad, 0)}[bright_white]\u2502[/]") await asyncio.sleep(0.15) - console.print(f" [bold bright_white]\u2514" + "\u2500" * bw + "\u2518[/]") + console.print(" [bold bright_white]\u2514" + "\u2500" * bw + "\u2518[/]") console.print() for i in range(bar_width + 1): diff --git a/src/dashboards/combined_dashboard.py b/src/dashboards/combined_dashboard.py index c101a99..519081a 100644 --- a/src/dashboards/combined_dashboard.py +++ b/src/dashboards/combined_dashboard.py @@ -11,7 +11,6 @@ from rich.panel import Panel from rich.text import Text -from src.data_layer.hub import HyperDataHub from src.dashboards.hub_panels import ( HubCVD, HubHLP, @@ -24,6 +23,7 @@ HubWhales, ) from src.dashboards.liquidation_heatmap import LiquidationHeatmapDashboard +from src.data_layer.hub import HyperDataHub class CombinedDashboard: diff --git a/src/dashboards/cvd_dashboard.py b/src/dashboards/cvd_dashboard.py index 2d3fb7f..76a8a96 100644 --- a/src/dashboards/cvd_dashboard.py +++ b/src/dashboards/cvd_dashboard.py @@ -23,12 +23,12 @@ from rich.table import Table from rich.text import Text +from src.data_layer.market_data import MarketData from src.data_layer.orderflow_engine import ( CVDSnapshot, OrderFlowEngine, Trade, ) -from src.data_layer.market_data import MarketData logger = logging.getLogger(__name__) @@ -206,7 +206,7 @@ def build_price_bar(self) -> Text: bar = Text() bar.append(" \u20bf ", style="bold bright_yellow") - bar.append(f"BITCOIN ", style="bold white") + bar.append("BITCOIN ", style="bold white") bar.append(f"${price:,.2f}", style="bold bright_white") bar.append(" ") @@ -391,7 +391,7 @@ def build_compact(self) -> Panel: now_str = datetime.now().strftime("%H:%M:%S") return Panel( table, - title=f"[bold bright_green]\U0001f4c8 BTC CVD[/]", + title="[bold bright_green]\U0001f4c8 BTC CVD[/]", subtitle=f"[dim]{now_str} #{self.cycle}[/]", border_style="bright_green", box=_box.ROUNDED, diff --git a/src/dashboards/hub_panels.py b/src/dashboards/hub_panels.py index 0d7f28b..4b0288b 100644 --- a/src/dashboards/hub_panels.py +++ b/src/dashboards/hub_panels.py @@ -14,7 +14,9 @@ from rich.text import Text from src.data_layer.hub import HyperDataHub -from src.utils.helpers import format_usd as fmt_usd, format_price as fmt_price, format_pct as fmt_pct +from src.utils.helpers import format_pct as fmt_pct +from src.utils.helpers import format_price as fmt_price +from src.utils.helpers import format_usd as fmt_usd class HubLiqWatch: diff --git a/src/dashboards/liquidation_heatmap.py b/src/dashboards/liquidation_heatmap.py index 82802f0..fdf1267 100644 --- a/src/dashboards/liquidation_heatmap.py +++ b/src/dashboards/liquidation_heatmap.py @@ -11,7 +11,6 @@ import asyncio import logging -import math from dataclasses import dataclass from rich import box @@ -203,9 +202,9 @@ def _build_asset_heatmap(self, symbol: str, current_price: float, n_buckets: int header = Text() header.append(f" {symbol} ", style="bold bright_white") header.append(f"${current_price:,.0f}", style=CURRENT_STYLE) - header.append(f" │ ", style="dim") + header.append(" │ ", style="dim") header.append(f"L: {_format_usd(total_long)}({long_count})", style=LONG_STYLE) - header.append(f" ", style="dim") + header.append(" ", style="dim") header.append(f"S: {_format_usd(total_short)}({short_count})", style=SHORT_STYLE) table = Table( diff --git a/src/dashboards/liquidation_stream.py b/src/dashboards/liquidation_stream.py index 121d1da..8b3d6e8 100644 --- a/src/dashboards/liquidation_stream.py +++ b/src/dashboards/liquidation_stream.py @@ -24,7 +24,6 @@ from rich.align import Align from rich.box import DOUBLE_EDGE, HEAVY, SIMPLE_HEAVY from rich.console import Console, Group -from rich.layout import Layout from rich.live import Live from rich.panel import Panel from rich.table import Table @@ -37,7 +36,7 @@ if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) -from src.data_layer.liquidation_feed import LiquidationEvent, LiquidationFeed +from src.data_layer.liquidation_feed import LiquidationEvent, LiquidationFeed # noqa: E402 logger = logging.getLogger(__name__) @@ -72,7 +71,7 @@ # --------------------------------------------------------------------------- # Formatting helpers (imported from central helpers) # --------------------------------------------------------------------------- -from src.utils.helpers import format_usd as fmt_usd +from src.utils.helpers import format_usd as fmt_usd # noqa: E402 def fmt_number(value: int) -> str: diff --git a/src/dashboards/liquidation_watch.py b/src/dashboards/liquidation_watch.py index 5cdb3a4..5b8a7f6 100644 --- a/src/dashboards/liquidation_watch.py +++ b/src/dashboards/liquidation_watch.py @@ -30,18 +30,18 @@ if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) -from src.data_layer.position_scanner import PositionScanner, TrackedPosition # noqa: E402 from config.settings import ( # noqa: E402 DASHBOARD_REFRESH_RATE, LIQ_ZONE_1_PCT, LIQ_ZONE_2_PCT, LIQ_ZONE_5_PCT, ) - +from src.data_layer.position_scanner import PositionScanner, TrackedPosition # noqa: E402 +from src.utils.helpers import format_pct_value as fmt_pct # noqa: E402 +from src.utils.helpers import format_price as fmt_price # noqa: E402 # ── Formatting helpers (imported from central helpers) ──────────────────── -from src.utils.helpers import format_usd as fmt_usd, format_price as fmt_price, format_pct_value as fmt_pct - +from src.utils.helpers import format_usd as fmt_usd # noqa: E402 # ── Zone summary with long/short breakdown ──────────────────────────────── diff --git a/src/dashboards/market_overview.py b/src/dashboards/market_overview.py index fc82864..802ccb6 100644 --- a/src/dashboards/market_overview.py +++ b/src/dashboards/market_overview.py @@ -32,11 +32,11 @@ sys.path.insert(0, str(_PROJECT_ROOT)) from src.data_layer.market_data import AssetInfo, MarketData # noqa: E402 -from config.settings import DASHBOARD_REFRESH_RATE, DEFAULT_SYMBOLS # noqa: E402 - +from src.utils.helpers import format_pct as fmt_pct # noqa: E402 +from src.utils.helpers import format_price as fmt_price # noqa: E402 # -- Formatting helpers (imported from central helpers) --------------------- -from src.utils.helpers import format_usd as fmt_usd, format_price as fmt_price, format_pct as fmt_pct +from src.utils.helpers import format_usd as fmt_usd # noqa: E402 def fmt_funding(value: float) -> str: @@ -258,10 +258,8 @@ def build_extreme_funding(self, assets: list[AssetInfo], limit: int = 10) -> Tab if asset.funding_rate >= 0: rate_style = "bold bright_green" - bar_char = "\U0001f7e9" # green square else: rate_style = "bold bright_red" - bar_char = "\U0001f7e5" # red square bar = make_bar(abs_ann, max_annualized, width=20) diff --git a/src/dashboards/whale_tracker.py b/src/dashboards/whale_tracker.py index ccfaaca..43eb67b 100644 --- a/src/dashboards/whale_tracker.py +++ b/src/dashboards/whale_tracker.py @@ -32,11 +32,11 @@ sys.path.insert(0, str(_PROJECT_ROOT)) from src.data_layer.position_scanner import PositionScanner, TrackedPosition # noqa: E402 -from config.settings import DASHBOARD_REFRESH_RATE # noqa: E402 - +from src.utils.helpers import format_pct_value as fmt_pct # noqa: E402 +from src.utils.helpers import format_price as fmt_price # noqa: E402 # -- Formatting helpers (imported from central helpers) --------------------- -from src.utils.helpers import format_usd as fmt_usd, format_price as fmt_price, format_pct_value as fmt_pct +from src.utils.helpers import format_usd as fmt_usd # noqa: E402 def shorten_addr(address: str) -> str: diff --git a/src/data_layer/address_store.py b/src/data_layer/address_store.py index 1db5657..5c3c0d1 100644 --- a/src/data_layer/address_store.py +++ b/src/data_layer/address_store.py @@ -7,6 +7,7 @@ """ import json import logging +import re import sqlite3 import threading import time @@ -18,6 +19,24 @@ DB_PATH = DATA_DIR / "hyperdata.db" LEGACY_JSON = DATA_DIR / "discovered_addresses.json" +# EVM wallet address: 0x + 40 hex chars. Anything else from an exchange +# payload is junk and must not be persisted (it would be re-scanned forever). +_ADDRESS_RE = re.compile(r"^0x[0-9a-fA-F]{40}$") + +# Retention cap: keep the most recently seen addresses; every tracked address +# costs a clearinghouseState call per scan cycle. +MAX_TRACKED_ADDRESSES = 50_000 + + +def is_valid_address(address: object) -> bool: + """True for a well-formed EVM wallet address string.""" + return isinstance(address, str) and bool(_ADDRESS_RE.match(address)) + + +def normalize_address(address: str) -> str: + """Canonical form: lowercase (EVM addresses are case-insensitive).""" + return address.lower() + _CREATE = """ CREATE TABLE IF NOT EXISTS discovered_addresses ( address TEXT PRIMARY KEY, @@ -72,31 +91,28 @@ def _init() -> None: def add_address(address: str, source: str = "unknown") -> None: - """Insert or update a single address. Idempotent.""" - _init() - now = time.time() - try: - with _lock: - conn = _get_conn() - conn.execute( - "INSERT INTO discovered_addresses (address, source, first_seen, last_seen) " - "VALUES (?, ?, ?, ?) " - "ON CONFLICT(address) DO UPDATE SET last_seen = excluded.last_seen", - (address, source, now, now), - ) - conn.commit() - conn.close() - except Exception: - logger.debug("[address_store] add failed", exc_info=True) + """Insert or update a single address. Idempotent. Invalid input is dropped.""" + add_addresses([address], source=source) def add_addresses(addresses: list[str] | set[str], source: str = "unknown") -> int: - """Batch insert. Returns count written.""" + """Batch insert (validated + normalized). Returns count written. + + Non-address strings from exchange payloads are dropped and counted here + so garbage identifiers never enter the store, and the table is capped at + MAX_TRACKED_ADDRESSES by expiring the least recently seen rows. + """ _init() if not addresses: return 0 now = time.time() - rows = [(a, source, now, now) for a in addresses] + valid = [normalize_address(a) for a in addresses if is_valid_address(a)] + dropped = len(list(addresses)) - len(valid) + if dropped: + logger.warning("[address_store] dropped %d invalid address strings", dropped) + if not valid: + return 0 + rows = [(a, source, now, now) for a in valid] try: with _lock: conn = _get_conn() @@ -106,6 +122,19 @@ def add_addresses(addresses: list[str] | set[str], source: str = "unknown") -> i "ON CONFLICT(address) DO UPDATE SET last_seen = excluded.last_seen", rows, ) + # Retention cap: expire the least-recently-seen overflow. + count = conn.execute( + "SELECT COUNT(*) FROM discovered_addresses" + ).fetchone()[0] + if count > MAX_TRACKED_ADDRESSES: + overflow = count - MAX_TRACKED_ADDRESSES + conn.execute( + "DELETE FROM discovered_addresses WHERE address IN (" + "SELECT address FROM discovered_addresses " + "ORDER BY last_seen ASC LIMIT ?)", + (overflow,), + ) + logger.info("[address_store] expired %d least-recently-seen addresses", overflow) conn.commit() conn.close() return len(rows) diff --git a/src/data_layer/alerts.py b/src/data_layer/alerts.py index 63c0044..09c7520 100644 --- a/src/data_layer/alerts.py +++ b/src/data_layer/alerts.py @@ -15,6 +15,7 @@ import asyncio import logging import os +import re import time from dataclasses import dataclass @@ -339,7 +340,6 @@ def _build_intel_report(self) -> str: delta = hlp_stats["net_delta"] zscore = hlp_stats["delta_zscore"] session_pnl = hlp_stats["session_pnl"] - absorptions = hlp_stats["liquidation_absorptions"] lines.append(f"AUM: {self._fmt_usd(aum)} | Delta: {self._fmt_usd_signed(delta)} | Z: {zscore:+.1f}") lines.append(f"Session PnL: {self._fmt_usd_signed(session_pnl)}") @@ -357,6 +357,10 @@ def _build_intel_report(self) -> str: return "\n".join(lines) + # Deadline for webhook posts: a stalled Telegram/Discord endpoint must + # not wedge whatever task is delivering the alert. + _SEND_TIMEOUT = aiohttp.ClientTimeout(total=10, connect=3, sock_connect=3, sock_read=5) + async def _send(self, message: str) -> None: """Send alert to all configured channels.""" if not self._session: @@ -368,27 +372,41 @@ async def _send(self, message: str) -> None: if self.telegram_token and self.telegram_chat_id: try: url = f"https://api.telegram.org/bot{self.telegram_token}/sendMessage" - await self._session.post(url, json={ + async with self._session.post(url, json={ "chat_id": self.telegram_chat_id, "text": message, "parse_mode": "HTML", - }) - logger.info("Telegram alert sent") - except Exception: - logger.exception("Telegram send failed") + }, timeout=self._SEND_TIMEOUT) as resp: + if resp.status == 200: + logger.info("Telegram alert sent") + else: + logger.warning("Telegram send returned %d", resp.status) + except Exception as exc: + # Exception type only — aiohttp error messages can embed the + # request URL, which contains the bot token. + logger.warning("Telegram send failed: %s", type(exc).__name__) # Discord if self.discord_webhook: try: - await self._session.post(self.discord_webhook, json={ + async with self._session.post(self.discord_webhook, json={ "content": message, - }) - logger.info("Discord alert sent") - except Exception: - logger.exception("Discord send failed") - - # Always log to console - logger.warning("ALERT: %s", message.replace("\n", " | ")) + }, timeout=self._SEND_TIMEOUT) as resp: + if resp.status in (200, 204): + logger.info("Discord alert sent") + else: + logger.warning("Discord send returned %d", resp.status) + except Exception as exc: + # Exception type only — error messages can embed the webhook URL. + logger.warning("Discord send failed: %s", type(exc).__name__) + + # Log that an alert fired, not its payload — alert bodies can contain + # wallet addresses and position intelligence that must not sit in + # rotating plaintext logs. Wallet addresses are masked even on the + # first line in case a future alert format leads with one. + first_line = message.strip().splitlines()[0] if message.strip() else "" + redacted = re.sub(r"0x[0-9a-fA-F]{40}", "0x…[redacted]", first_line) + logger.warning("ALERT sent (%d total): %.80s", self.alerts_sent, redacted) async def send_test(self) -> bool: """Send a test alert to verify configuration.""" diff --git a/src/data_layer/hlp_tracker.py b/src/data_layer/hlp_tracker.py index 3a7cf7b..b27bb10 100644 --- a/src/data_layer/hlp_tracker.py +++ b/src/data_layer/hlp_tracker.py @@ -13,7 +13,7 @@ import logging import time from collections import deque -from dataclasses import dataclass, field +from dataclasses import dataclass import aiohttp @@ -185,7 +185,8 @@ async def _take_snapshot(self, address: str) -> HLPSnapshot | None: # Current price from position data current_price = float(pos_info.get("positionValue", 0)) if abs(size_raw) > 0: - current_price = abs(float(pos_info.get("positionValue", 0)) / size_raw) if size_raw != 0 else entry_price + current_price = (abs(float(pos_info.get("positionValue", 0)) / size_raw) + if size_raw != 0 else entry_price) unrealized_pnl = float(pos_info.get("unrealizedPnl", 0)) leverage_info = pos_info.get("leverage", {}) diff --git a/src/data_layer/hub.py b/src/data_layer/hub.py index 8c02810..6fb5d81 100644 --- a/src/data_layer/hub.py +++ b/src/data_layer/hub.py @@ -21,7 +21,6 @@ from __future__ import annotations import asyncio -import json import logging import os import time @@ -30,26 +29,27 @@ from typing import Any, Callable from config.settings import DEFAULT_SYMBOLS +from src.api_server import HyperDataAPI from src.data_layer.alerts import AlertManager -from src.data_layer.liquidation_feed import LiquidationFeed, LiquidationEvent -from src.data_layer.position_scanner import PositionScanner, TrackedPosition +from src.data_layer.deribit import DeribitFeed, DeribitIVSnapshot +from src.data_layer.funding_rates import FundingRateCollector, FundingRateSnapshot +from src.data_layer.health_monitor import DataHealthMonitor +from src.data_layer.hlp_tracker import HLPPosition, HLPSnapshot, HLPTracker, HLPTrade +from src.data_layer.liquidation_feed import LiquidationEvent, LiquidationFeed +from src.data_layer.long_short_ratio import LongShortCollector, LongShortSnapshot +from src.data_layer.market_data import AssetInfo, MarketData +from src.data_layer.orderbook import OrderBookEngine, OrderBookSnapshot +from src.data_layer.orderflow_engine import ( + STALE_AFTER_SECONDS as ORDERFLOW_STALE_AFTER, +) from src.data_layer.orderflow_engine import ( OrderFlowEngine, Trade, - CVDSnapshot, - STALE_AFTER_SECONDS as ORDERFLOW_STALE_AFTER, ) -from src.data_layer.market_data import MarketData, AssetInfo from src.data_layer.persistence import DataStore +from src.data_layer.position_scanner import PositionScanner, TrackedPosition from src.data_layer.smart_money import SmartMoneyEngine, SmartMoneySignal, WalletProfile -from src.data_layer.hlp_tracker import HLPTracker, HLPSnapshot, HLPPosition, HLPTrade -from src.data_layer.funding_rates import FundingRateCollector, FundingRateSnapshot -from src.data_layer.long_short_ratio import LongShortCollector, LongShortSnapshot -from src.data_layer.orderbook import OrderBookEngine, OrderBookSnapshot from src.data_layer.spot_prices import SpotPriceCollector, SpotPriceSnapshot -from src.data_layer.deribit import DeribitFeed, DeribitIVSnapshot -from src.data_layer.health_monitor import DataHealthMonitor -from src.api_server import HyperDataAPI logger = logging.getLogger(__name__) @@ -72,6 +72,9 @@ class HubStatus: orderbook_feed: str = "offline" market_data: str = "offline" + # Components that raised during start(); non-empty means degraded mode. + failed_components: list = field(default_factory=list) + # Counters total_liquidations: int = 0 total_trades_processed: int = 0 @@ -174,6 +177,8 @@ def __init__( # ── Background tasks ───────────────────────────────────── self._tasks: list[asyncio.Task] = [] self._running = False + # Debounce for per-venue orderflow staleness warnings. + self._venue_stale_warned_at: dict[str, float] = {} # Wire up internal callbacks self.liquidations.on_liquidation(self._handle_liquidation) @@ -254,12 +259,15 @@ async def start(self) -> None: # Start REST API server if port is configured (both modes) if self._api_port: try: - # Loopback by default; set HYPERDATA_API_HOST=0.0.0.0 to expose - # on the LAN (no auth — only do this behind a trusted network). + # Loopback by default; a non-loopback HYPERDATA_API_HOST is + # refused unless HYPERDATA_API_KEY or the explicit unsafe + # acknowledgment is set (see HyperDataAPI._resolve_security). api_host = os.environ.get("HYPERDATA_API_HOST", "127.0.0.1") self._api_server = HyperDataAPI(self, host=api_host, port=self._api_port) await self._api_server.start() except Exception: + self._api_server = None + self.status.failed_components.append("api_server") logger.exception("Failed to start API server") # Background loops that run in both modes @@ -283,81 +291,88 @@ async def start(self) -> None: self.store.attach(self) # ── Alerts ───────────────────────────────────────────── - await self.alerts.start() - self.alerts.attach(self) - - logger.info("HyperDataHub started — all components online") - - async def _start_live(self) -> None: - """Connect to real exchange APIs.""" - # Start liquidation WebSocket feeds - try: - await self.liquidations.start() - self.status.liquidation_feed = "connected" - logger.info("Liquidation feed: connected") - except Exception: - self.status.liquidation_feed = "error" - logger.exception("Failed to start liquidation feed") - - # Start order flow WebSocket try: - await self.orderflow.start() - self.status.orderflow_engine = "connected" - logger.info("Order flow engine: connected") + await self.alerts.start() + self.alerts.attach(self) except Exception: - self.status.orderflow_engine = "error" - logger.exception("Failed to start order flow engine") - - # Start smart money engine - try: - await self.smart_money.start() - logger.info("Smart money engine: started") - except Exception: - logger.exception("Failed to start smart money engine") - - # Start HLP tracker - try: - await self.hlp.start() - self.status.hlp_status = "connected" - logger.info("HLP tracker: started") - except Exception: - self.status.hlp_status = "error" - logger.exception("Failed to start HLP tracker") - - try: - await self.funding.start() - logger.info("Funding rate collector: started") - except Exception: - logger.exception("Failed to start funding rate collector") + self.status.failed_components.append("alerts") + logger.exception("Failed to start alert manager") + + if self.status.failed_components: + logger.error( + "HyperDataHub started DEGRADED — failed components: %s. " + "Data from these sources will be missing or stale.", + ", ".join(self.status.failed_components), + ) + else: + logger.info("HyperDataHub started — all components online") - try: - await self.lsr.start() - logger.info("Long/short ratio collector: started") - except Exception: - logger.exception("Failed to start long/short ratio collector") + async def _start_component(self, name: str, coro, required: bool = False, + on_ok=None, on_fail=None) -> bool: + """Start one component, recording failures instead of hiding them. + A failed *required* component raises and aborts startup; a failed + optional component is appended to status.failed_components so health + surfaces (log line, /v1/health) report degraded mode honestly. + """ try: - await self.orderbook.start() - self.status.orderbook_feed = "connected" - logger.info("OrderBook engine: started") + await coro + if on_ok: + on_ok() + logger.info("%s: started", name) + return True except Exception: - self.status.orderbook_feed = "error" - logger.exception("Failed to start orderbook engine") + if on_fail: + on_fail() + if required: + logger.exception("Required component %s failed to start", name) + raise + self.status.failed_components.append(name) + logger.exception("Failed to start %s (continuing degraded)", name) + return False - try: - await self.spot.start(perp_price_fn=lambda sym: self.market.assets.get(sym)) - logger.info("Spot price collector: started") - except Exception: - logger.exception("Failed to start spot price collector") + async def _start_live(self) -> None: + """Connect to real exchange APIs. - try: - await self.deribit.start() - logger.info("Deribit IV feed: started") - except Exception: - logger.exception("Failed to start Deribit IV feed") + Every component is optional-but-reported: a failure puts it in + status.failed_components (surfaced via /v1/health and the startup + log) instead of being silently swallowed. + """ + s = self.status + await self._start_component( + "liquidation_feed", self.liquidations.start(), + on_ok=lambda: setattr(s, "liquidation_feed", "connected"), + on_fail=lambda: setattr(s, "liquidation_feed", "error"), + ) + await self._start_component( + "orderflow_engine", self.orderflow.start(), + on_ok=lambda: setattr(s, "orderflow_engine", "connected"), + on_fail=lambda: setattr(s, "orderflow_engine", "error"), + ) + await self._start_component("smart_money", self.smart_money.start()) + await self._start_component( + "hlp_tracker", self.hlp.start(), + on_ok=lambda: setattr(s, "hlp_status", "connected"), + on_fail=lambda: setattr(s, "hlp_status", "error"), + ) + await self._start_component("funding_rates", self.funding.start()) + await self._start_component("long_short_ratio", self.lsr.start()) + await self._start_component( + "orderbook", self.orderbook.start(), + on_ok=lambda: setattr(s, "orderbook_feed", "connected"), + on_fail=lambda: setattr(s, "orderbook_feed", "error"), + ) + await self._start_component( + "spot_prices", + self.spot.start(perp_price_fn=lambda sym: self.market.assets.get(sym)), + ) + await self._start_component("deribit_iv", self.deribit.start()) - self.status.position_scanner = "ready" - self.status.market_data = "ready" + # Loop-driven components: 'starting' until their first cycle actually + # succeeds (the loops flip these to 'connected'/'error'). Never claim + # 'ready' for something that has not fetched anything yet. + self.status.position_scanner = "starting" + self.status.market_data = "starting" async def _start_demo(self) -> None: """Start mock data generators.""" @@ -468,6 +483,8 @@ async def _position_scan_loop(self) -> None: all_positions = await self.positions.scan() self.status.tracked_positions = len(all_positions) self.status.discovered_addresses = len(self.positions.discovered_addresses) + # 'connected' only after a scan actually succeeded. + self.status.position_scanner = "connected" self.status.last_position_scan = time.time() self.status.scan_cycle += 1 @@ -482,6 +499,8 @@ async def _position_scan_loop(self) -> None: except asyncio.CancelledError: break except Exception: + if not self.demo: + self.status.position_scanner = "error" logger.exception("Position scan error") await asyncio.sleep(self.scan_interval) @@ -494,6 +513,8 @@ async def _market_refresh_loop(self) -> None: await self._demo_market_refresh() else: await self.market.refresh() + # 'connected' only after a refresh actually succeeded. + self.status.market_data = "connected" self.status.last_market_refresh = time.time() self.status.tracked_assets = len(self.market.assets) @@ -501,104 +522,122 @@ async def _market_refresh_loop(self) -> None: except asyncio.CancelledError: break except Exception: + if not self.demo: + self.status.market_data = "error" logger.exception("Market refresh error") await asyncio.sleep(self.market_refresh_interval) async def _status_update_loop(self) -> None: - """Update uptime counter and periodically refresh DB stats.""" + """Update uptime counter and periodically refresh DB stats. + + The body is wrapped so one component's bad stats shape can't kill the + loop — this loop is also the staleness watchdog and the source of + /v1/health data, so it must outlive individual component errors. + """ _db_tick = 0 while self._running: - self.status.uptime_seconds = time.time() - self.status.started_at - - # Update smart money stats every tick - sm_stats = self.smart_money.get_stats() - self.status.tracked_wallets = sm_stats["total_wallets"] - self.status.ranked_wallets = sm_stats["ranked_wallets"] - self.status.smart_money_signals = sm_stats["total_signals"] - - # Update HLP stats every tick - hlp_stats = self.hlp.get_stats() - self.status.hlp_account_value = hlp_stats["account_value"] - self.status.hlp_net_delta = hlp_stats["net_delta"] - self.status.hlp_delta_zscore = hlp_stats["delta_zscore"] - self.status.hlp_positions = hlp_stats["num_positions"] - self.status.hlp_trades = hlp_stats["total_trades"] - self.status.hlp_liquidation_absorptions = hlp_stats["liquidation_absorptions"] - self.status.hlp_session_pnl = hlp_stats["session_pnl"] - - # Check HLP Z-score for alert - if abs(hlp_stats.get("delta_zscore", 0)) > 2.0: - asyncio.create_task(self.alerts._check_hlp_zscore(hlp_stats)) - - # Persist HLP snapshots periodically - self.store.maybe_save_hlp_snapshot() - - _db_tick += 1 - # Prune old rows + checkpoint the WAL roughly hourly so the DB and - # the COUNT(*) below stay bounded on long-running instances. - if _db_tick % 3600 == 0: - try: - self.store.prune() - except Exception: - logger.exception("Error pruning DB") - # Update persistence stats every 30 seconds - if _db_tick % 30 == 0: - try: - db_stats = self.store.get_db_stats() - self.status.db_size_mb = db_stats["db_size_mb"] - self.status.events_persisted = ( - db_stats["liquidations_stored"] + db_stats["trades_stored"] - ) - except Exception: - logger.exception("Error fetching DB stats") - try: - for ex_rates in self.funding.rates.values(): - for snap in ex_rates.values(): - self.store.save_funding_rate(snap) - except Exception: - logger.exception("Error saving funding rate snapshots") - try: - for snap in self.lsr.ratios.values(): - self.store.save_long_short_ratio(snap) - except Exception: - logger.exception("Error saving LSR snapshots") - try: - for snap in self.deribit.snapshots.values(): - self.store.save_options_snapshot(snap) - except Exception: - logger.exception("Error saving Deribit IV snapshots") - - # Update new component status fields every tick - self.status.funding_rate_symbols_binance = len(self.funding.rates.get("binance", {})) - self.status.funding_rate_symbols_bybit = len(self.funding.rates.get("bybit", {})) - - btc_lsr = self.lsr.get_latest("BTC") - eth_lsr = self.lsr.get_latest("ETH") - self.status.lsr_btc_ratio = btc_lsr.long_short_ratio if btc_lsr else 0.0 - self.status.lsr_eth_ratio = eth_lsr.long_short_ratio if eth_lsr else 0.0 - - self.status.orderbook_symbols = len(self.orderbook.snapshots) - - btc_spot = self.spot.get_latest("BTC") - eth_spot = self.spot.get_latest("ETH") - self.status.spot_btc_basis_pct = btc_spot.basis_pct if btc_spot else 0.0 - self.status.spot_eth_basis_pct = eth_spot.basis_pct if eth_spot else 0.0 - - btc_iv = self.deribit.get_latest("BTC") - eth_iv = self.deribit.get_latest("ETH") - self.status.deribit_btc_iv = btc_iv.mark_iv if btc_iv else 0.0 - self.status.deribit_eth_iv = eth_iv.mark_iv if eth_iv else 0.0 - # ── Staleness watchdog ────────────────────────────────── - # Flag WS feeds that have stopped delivering data as 'stale' so the - # UI/API never present frozen numbers as live, and force a reconnect - # on a socket that's alive-but-silent (heartbeat only catches - # half-open connections, not a venue that quietly stops sending). - if not self.demo: - await self._update_feed_staleness() + try: + _db_tick = await self._status_update_tick(_db_tick) + # NOTE: HLP z-score alert dispatch was removed here — the + # AlertManager z-score/cascade sends are deliberately disabled + # (too noisy), so scheduling tasks for them was dead work. + # Re-add scheduling here if those alerts are re-enabled. + except asyncio.CancelledError: + break + except Exception: + logger.exception("Status update loop error") await asyncio.sleep(1) + async def _status_update_tick(self, _db_tick: int) -> int: + """One status-loop iteration. Returns the incremented DB tick.""" + self.status.uptime_seconds = time.time() - self.status.started_at + + # Update smart money stats every tick + sm_stats = self.smart_money.get_stats() + self.status.tracked_wallets = sm_stats["total_wallets"] + self.status.ranked_wallets = sm_stats["ranked_wallets"] + self.status.smart_money_signals = sm_stats["total_signals"] + + # Update HLP stats every tick + hlp_stats = self.hlp.get_stats() + self.status.hlp_account_value = hlp_stats["account_value"] + self.status.hlp_net_delta = hlp_stats["net_delta"] + self.status.hlp_delta_zscore = hlp_stats["delta_zscore"] + self.status.hlp_positions = hlp_stats["num_positions"] + self.status.hlp_trades = hlp_stats["total_trades"] + self.status.hlp_liquidation_absorptions = hlp_stats["liquidation_absorptions"] + self.status.hlp_session_pnl = hlp_stats["session_pnl"] + + # Persist HLP snapshots periodically + self.store.maybe_save_hlp_snapshot() + + _db_tick += 1 + # Prune old rows + checkpoint the WAL roughly hourly so the DB and + # the COUNT(*) below stay bounded on long-running instances. + if _db_tick % 3600 == 0: + try: + self.store.prune() + except Exception: + logger.exception("Error pruning DB") + # Update persistence stats every 30 seconds + if _db_tick % 30 == 0: + try: + db_stats = self.store.get_db_stats() + self.status.db_size_mb = db_stats["db_size_mb"] + self.status.events_persisted = ( + db_stats["liquidations_stored"] + db_stats["trades_stored"] + ) + except Exception: + logger.exception("Error fetching DB stats") + try: + for ex_rates in self.funding.rates.values(): + for snap in ex_rates.values(): + self.store.save_funding_rate(snap) + except Exception: + logger.exception("Error saving funding rate snapshots") + try: + for snap in self.lsr.ratios.values(): + self.store.save_long_short_ratio(snap) + except Exception: + logger.exception("Error saving LSR snapshots") + try: + for snap in self.deribit.snapshots.values(): + self.store.save_options_snapshot(snap) + except Exception: + logger.exception("Error saving Deribit IV snapshots") + + # Update new component status fields every tick + self.status.funding_rate_symbols_binance = len(self.funding.rates.get("binance", {})) + self.status.funding_rate_symbols_bybit = len(self.funding.rates.get("bybit", {})) + + btc_lsr = self.lsr.get_latest("BTC") + eth_lsr = self.lsr.get_latest("ETH") + self.status.lsr_btc_ratio = btc_lsr.long_short_ratio if btc_lsr else 0.0 + self.status.lsr_eth_ratio = eth_lsr.long_short_ratio if eth_lsr else 0.0 + + self.status.orderbook_symbols = len(self.orderbook.snapshots) + + btc_spot = self.spot.get_latest("BTC") + eth_spot = self.spot.get_latest("ETH") + self.status.spot_btc_basis_pct = btc_spot.basis_pct if btc_spot else 0.0 + self.status.spot_eth_basis_pct = eth_spot.basis_pct if eth_spot else 0.0 + + btc_iv = self.deribit.get_latest("BTC") + eth_iv = self.deribit.get_latest("ETH") + self.status.deribit_btc_iv = btc_iv.mark_iv if btc_iv else 0.0 + self.status.deribit_eth_iv = eth_iv.mark_iv if eth_iv else 0.0 + # ── Staleness watchdog ────────────────────────────────── + # Flag WS feeds that have stopped delivering data as 'stale' so the + # UI/API never present frozen numbers as live, and force a reconnect + # on a socket that's alive-but-silent (heartbeat only catches + # half-open connections, not a venue that quietly stops sending). + if not self.demo: + await self._update_feed_staleness() + + return _db_tick + async def _health_monitor_loop(self) -> None: """Run data-integrity checks against external sources on an interval. @@ -628,6 +667,22 @@ async def _update_feed_staleness(self) -> None: self.status.orderflow_engine = ( "stale" if self.orderflow.is_stale() else "connected" ) + # Combined freshness follows the freshest venue, so one dead venue + # can hide behind the other. Warn (debounced) when that happens so + # "orderflow connected" is never silently half-true. + if not self.orderflow.is_stale(): + now_w = time.time() + for venue in ("hyperliquid", "binance"): + if (self.orderflow.venue_is_stale(venue) + and self.orderflow.venue_data_age(venue) != float("inf") + and now_w - self._venue_stale_warned_at.get(venue, 0.0) > 300): + self._venue_stale_warned_at[venue] = now_w + logger.warning( + "[hub] order flow venue %s silent %.0fs while the " + "combined feed is still fresh — venue-specific " + "data (per-venue CVD) is stale", + venue, self.orderflow.venue_data_age(venue), + ) # Both venues silent for well past the threshold → kick the HL # socket so its backoff loop rebuilds it. The Binance loop self-heals # via its own heartbeat, and if Binance were still feeding, the @@ -802,7 +857,8 @@ async def _demo_smart_money(self) -> None: total_realized_pnl=total_pnl, total_volume_usd=volume, largest_win=abs(total_pnl) * random.uniform(0.05, 0.3) if total_pnl > 0 else random.uniform(100, 50000), - largest_loss=-abs(total_pnl) * random.uniform(0.02, 0.15) if total_pnl < 0 else -random.uniform(100, 30000), + largest_loss=(-abs(total_pnl) * random.uniform(0.02, 0.15) + if total_pnl < 0 else -random.uniform(100, 30000)), avg_hold_time_seconds=random.uniform(60, 86400), win_rate=win_rate, sharpe_ratio=sharpe, @@ -992,6 +1048,7 @@ async def _demo_hlp(self) -> None: async def _demo_market_refresh(self) -> None: """Generate mock market data for demo mode.""" import random + from src.data_layer.market_data import AssetInfo mock_assets = [ @@ -1032,6 +1089,7 @@ async def _demo_market_refresh(self) -> None: async def _demo_deribit(self) -> None: """Generate synthetic Deribit DVOL data for demo mode.""" import random + from src.data_layer.deribit import DeribitIVSnapshot btc_iv = 55.0 @@ -1063,6 +1121,7 @@ async def _demo_deribit(self) -> None: async def _demo_basis(self) -> None: """Generate synthetic spot/perp basis data for demo mode.""" import random + from src.data_layer.spot_prices import SpotPriceSnapshot bases = {"BTC": 0.05, "ETH": 0.03, "SOL": 0.08} @@ -1094,6 +1153,7 @@ async def _demo_basis(self) -> None: async def _demo_lsr(self) -> None: """Generate synthetic long/short ratio data for demo mode.""" import random + from src.data_layer.long_short_ratio import LongShortSnapshot ratios = {"BTC": 1.1, "ETH": 0.95, "SOL": 1.2} diff --git a/src/data_layer/liquidation_feed.py b/src/data_layer/liquidation_feed.py index c9f3734..e2e9501 100644 --- a/src/data_layer/liquidation_feed.py +++ b/src/data_layer/liquidation_feed.py @@ -46,6 +46,10 @@ def normalize_symbol(raw: str, exchange: str) -> str: # trades at least this large (USD). These are estimates, not confirmed events. HL_LIQUIDATION_MIN_USD = 10_000 +# Deadline for REST polls (price context); a hung endpoint must not wedge the +# poll loop. Split connect/read so a slow handshake can't eat the budget. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10, connect=3, sock_connect=3, sock_read=5) + def exchange_coverage() -> dict[str, dict[str, str]]: """Per-exchange description of HOW liquidations are collected, so consumers @@ -88,6 +92,12 @@ def exchange_coverage() -> dict[str, dict[str, str]]: class ExchangeConnection: MAX_BACKOFF = 60.0 + # After this many consecutive failed connections, escalate once to ERROR + # and demote further reconnect chatter to debug. We deliberately keep + # retrying (at MAX_BACKOFF) rather than stopping: a market-data feed that + # permanently kills itself during a long venue outage never recovers, + # and one handshake per minute is negligible load. + FAILURE_ESCALATION_THRESHOLD = 20 def __init__(self, name: str, ws_url: str, feed: LiquidationFeed): self.name = name @@ -98,6 +108,7 @@ def __init__(self, name: str, ws_url: str, feed: LiquidationFeed): self._ws: aiohttp.ClientWebSocketResponse | None = None self._running = False self._backoff = 1.0 + self.consecutive_failures = 0 async def start(self) -> None: self._running = True @@ -124,6 +135,7 @@ async def _run_loop(self) -> None: async with self._session.ws_connect(self.ws_url, heartbeat=20) as ws: self._ws = ws self._backoff = 1.0 + self.consecutive_failures = 0 logger.info("[%s] connected", self.name) await self._on_connected(ws) async for msg in ws: @@ -135,10 +147,22 @@ async def _run_loop(self) -> None: except asyncio.CancelledError: return except Exception: - logger.exception("[%s] connection error", self.name) + self.consecutive_failures += 1 + if self.consecutive_failures == self.FAILURE_ESCALATION_THRESHOLD: + logger.error( + "[%s] %d consecutive connection failures — endpoint looks " + "dead/deprecated; will keep retrying every %.0fs quietly", + self.name, self.consecutive_failures, self.MAX_BACKOFF, + ) + elif self.consecutive_failures < self.FAILURE_ESCALATION_THRESHOLD: + logger.exception("[%s] connection error", self.name) + else: + logger.debug("[%s] connection error (%d consecutive)", + self.name, self.consecutive_failures) if self._running: - logger.info("[%s] reconnecting in %.1fs", self.name, self._backoff) + if self.consecutive_failures < self.FAILURE_ESCALATION_THRESHOLD: + logger.info("[%s] reconnecting in %.1fs", self.name, self._backoff) await asyncio.sleep(self._backoff) self._backoff = min(self._backoff * 2, self.MAX_BACKOFF) @@ -158,20 +182,32 @@ def __init__(self, feed: LiquidationFeed): ) async def _on_message(self, data: Any) -> None: - if isinstance(data, dict) and data.get("e") == "forceOrder": - o = data["o"] - price = float(o["p"]) - qty = float(o["q"]) - side_raw = o["S"].upper() - event = LiquidationEvent( - timestamp=o["T"] / 1000.0, - exchange="binance", - symbol=normalize_symbol(o["s"], "binance"), - side="long" if side_raw == "SELL" else "short", - size_usd=price * qty, - price=price, - quantity=qty, - ) + # `!forceOrder@arr` frames normally carry one object, but the `@arr` + # family can batch events into a JSON array — handle both shapes so + # an array frame is parsed instead of silently failing a dict check. + records = data if isinstance(data, list) else [data] + for rec in records: + if not isinstance(rec, dict) or rec.get("e") != "forceOrder": + continue + # Exchange payloads are untrusted: one malformed record must not + # raise out of the connection loop and trigger a reconnect. + try: + o = rec["o"] + price = float(o["p"]) + qty = float(o["q"]) + side_raw = str(o["S"]).upper() + event = LiquidationEvent( + timestamp=float(o["T"]) / 1000.0, + exchange="binance", + symbol=normalize_symbol(str(o["s"]), "binance"), + side="long" if side_raw == "SELL" else "short", + size_usd=price * qty, + price=price, + quantity=qty, + ) + except (KeyError, TypeError, ValueError, AttributeError): + self.feed.record_parse_error("binance", rec) + continue await self.feed.emit(event) @@ -197,28 +233,36 @@ async def _on_connected(self, ws: aiohttp.ClientWebSocketResponse) -> None: async def _on_message(self, data: Any) -> None: if not isinstance(data, dict) or "data" not in data: return - if not data.get("topic", "").startswith("allLiquidation."): + if not str(data.get("topic", "")).startswith("allLiquidation."): return - d = data["data"] - try: - price = float(d.get("price", 0)) - qty = float(d.get("qty", 0) or d.get("size", 0)) - side_raw = d.get("side", "") # "Sell" = long liquidated; "Buy" = short liquidated - symbol_raw = d.get("symbol", "") - ts_ms = int(d.get("updatedTime", 0)) - event = LiquidationEvent( - timestamp=ts_ms / 1000.0, - exchange="bybit", - symbol=normalize_symbol(symbol_raw, "bybit"), - side="long" if side_raw == "Sell" else "short", - size_usd=price * qty, - price=price, - quantity=qty, - confirmed=True, - ) + # Bybit v5 sends `data` as a list of records (older topics used a + # single dict) — accept both, and drop malformed records individually + # so schema drift can never raise out of the WS loop and reconnect. + payload = data["data"] + records = payload if isinstance(payload, list) else [payload] + for d in records: + try: + # v5 allLiquidation uses short keys (p/v/S/s/T); the long + # names cover the legacy `liquidation` topic shape. + price = float(d.get("price") or d.get("p") or 0) + qty = float(d.get("qty") or d.get("size") or d.get("v") or 0) + side_raw = d.get("side") or d.get("S") or "" # "Sell" = long liquidated + symbol_raw = d.get("symbol") or d.get("s") or "" + ts_ms = int(d.get("updatedTime") or d.get("T") or 0) + event = LiquidationEvent( + timestamp=ts_ms / 1000.0, + exchange="bybit", + symbol=normalize_symbol(str(symbol_raw), "bybit"), + side="long" if side_raw == "Sell" else "short", + size_usd=price * qty, + price=price, + quantity=qty, + confirmed=True, + ) + except (KeyError, TypeError, ValueError, AttributeError): + self.feed.record_parse_error("bybit", d) + continue await self.feed.emit(event) - except Exception: - logger.debug("[bybit] failed to parse liquidation: %s", d) class OKXConnection(ExchangeConnection): @@ -239,24 +283,40 @@ async def _on_connected(self, ws: aiohttp.ClientWebSocketResponse) -> None: async def _on_message(self, data: Any) -> None: if not isinstance(data, dict) or "data" not in data: return + if not isinstance(data["data"], list): + self.feed.record_parse_error("okx", data) + return for d in data["data"]: - details = d.get("details", []) - inst_id = d.get("instId", "") + # Drop malformed records individually — one bad detail must not + # kill the whole message or the connection loop. + try: + details = d.get("details", []) + inst_id = d.get("instId", "") + if not isinstance(details, list): + self.feed.record_parse_error("okx", d) + continue + except AttributeError: + self.feed.record_parse_error("okx", d) + continue for det in details: - price = float(det.get("bkPx", 0)) - qty = float(det.get("sz", 0)) - side_raw = det.get("side", "").lower() - ts_raw = det.get("ts", "0") - event = LiquidationEvent( - timestamp=int(ts_raw) / 1000.0, - exchange="okx", - symbol=normalize_symbol(inst_id, "okx"), - side="long" if side_raw == "sell" else "short", - size_usd=price * qty, - price=price, - quantity=qty, - ) + try: + price = float(det.get("bkPx", 0) or 0) + qty = float(det.get("sz", 0) or 0) + side_raw = str(det.get("side", "")).lower() + ts_raw = det.get("ts", "0") or "0" + event = LiquidationEvent( + timestamp=int(ts_raw) / 1000.0, + exchange="okx", + symbol=normalize_symbol(inst_id, "okx"), + side="long" if side_raw == "sell" else "short", + size_usd=price * qty, + price=price, + quantity=qty, + ) + except (KeyError, TypeError, ValueError, AttributeError): + self.feed.record_parse_error("okx", det) + continue await self.feed.emit(event) @@ -302,17 +362,25 @@ async def stop(self) -> None: async def _price_poll(self) -> None: """Poll mid prices to have context for liquidation detection.""" + consecutive_failures = 0 while self._running: try: async with self._session.post( - self.API_URL, json={"type": "allMids"} + self.API_URL, json={"type": "allMids"}, timeout=HTTP_TIMEOUT ) as resp: if resp.status == 200: self._mid_prices = {k: float(v) for k, v in (await resp.json()).items()} + consecutive_failures = 0 except asyncio.CancelledError: return except Exception: - pass + consecutive_failures += 1 + # Silent-pass hid outages for hours; warn once it looks real. + if consecutive_failures in (3, 10) or consecutive_failures % 100 == 0: + logger.warning( + "[hyperliquid] price poll failing (%d consecutive)", + consecutive_failures, exc_info=True, + ) await asyncio.sleep(self.POLL_INTERVAL) async def _ws_loop(self) -> None: @@ -351,18 +419,23 @@ async def _process_trades(self, trades: list[dict]) -> None: Heuristic: Large trades that move price aggressively are likely liquidations. """ for t in trades: - tid = t.get("tid", 0) - if tid in self._seen_tids: + try: + tid = t.get("tid", 0) + if tid in self._seen_tids: + continue + self._seen_tids[tid] = None + while len(self._seen_tids) > 100_000: + self._seen_tids.popitem(last=False) # Remove oldest + + coin = t.get("coin", "") + price = float(t.get("px", 0) or 0) + qty = float(t.get("sz", 0) or 0) + ts_ms = int(t.get("time", 0) or 0) + side = t.get("side", "") # "B" = buyer taker, "A" = seller taker + except (TypeError, ValueError, AttributeError): + self.feed.record_parse_error("hyperliquid", t) continue - self._seen_tids[tid] = None - while len(self._seen_tids) > 100_000: - self._seen_tids.popitem(last=False) # Remove oldest - - coin = t.get("coin", "") - price = float(t.get("px", 0)) - qty = float(t.get("sz", 0)) size_usd = price * qty - side = t.get("side", "") # "B" = buyer taker, "A" = seller taker # Only flag large trades as potential liquidations if size_usd < self.LARGE_TRADE_USD: @@ -371,7 +444,7 @@ async def _process_trades(self, trades: list[dict]) -> None: # Side logic: "A" (ask/sell taker) = someone is aggressively selling = long liquidation # "B" (bid/buy taker) = someone aggressively buying = short liquidation event = LiquidationEvent( - timestamp=int(t.get("time", 0)) / 1000.0, + timestamp=ts_ms / 1000.0, exchange="hyperliquid", symbol=coin, side="long" if side == "A" else "short", @@ -400,6 +473,21 @@ def __init__(self, max_events: int = 10_000): self._connections: list[ExchangeConnection | HyperliquidConnection] = [] self._lock = asyncio.Lock() self._running = False + # Per-exchange count of records dropped at the parse boundary. + # Surfaced via get_stats() so schema drift is visible, not silent. + self.parse_errors: dict[str, int] = {} + + def record_parse_error(self, exchange: str, payload: Any = None) -> None: + """Count a malformed record dropped at the parse boundary.""" + count = self.parse_errors.get(exchange, 0) + 1 + self.parse_errors[exchange] = count + # Log the first few and then sample, so a schema change is visible + # without a malformed-message flood drowning the logs. + if count <= 3 or count % 1000 == 0: + logger.warning( + "[%s] dropped malformed record (%d total): %.300s", + exchange, count, payload, + ) async def start(self) -> None: logger.info("starting liquidation feed") @@ -494,6 +582,7 @@ def get_stats(self, window_minutes: int = 60) -> dict[str, Any]: "heuristic_count": heuristic_count, "confirmed_volume_usd": confirmed_volume_usd, "heuristic_volume_usd": heuristic_volume_usd, + "parse_errors": dict(self.parse_errors), "coverage": _coverage, "by_exchange": { k: { diff --git a/src/data_layer/market_data.py b/src/data_layer/market_data.py index 11b6dc9..f14c628 100644 --- a/src/data_layer/market_data.py +++ b/src/data_layer/market_data.py @@ -11,6 +11,12 @@ logger = logging.getLogger(__name__) +# Every outbound request gets an explicit deadline: a hung exchange endpoint +# must fail the refresh cycle, not stall the hub's market-refresh loop forever. +# connect/sock_read are split so a slow TLS handshake (venue throttling) can't +# consume the whole budget and masquerade as a partial-read error. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10, connect=3, sock_connect=3, sock_read=5) + @dataclass class AssetInfo: @@ -132,9 +138,21 @@ async def get_orderbook(self, symbol: str, depth: int = 20) -> dict: return {"bids": [], "asks": []} levels = data.get("levels", [[], []]) - bids = [{"price": float(b["px"]), "size": float(b["sz"])} for b in levels[0][:depth]] - asks = [{"price": float(a["px"]), "size": float(a["sz"])} for a in levels[1][:depth]] - return {"bids": bids, "asks": asks} + if not isinstance(levels, list) or len(levels) < 2: + return {"bids": [], "asks": []} + + def _parse_side(raw_levels) -> list[dict]: + parsed: list[dict] = [] + if not isinstance(raw_levels, list): + return parsed + for lvl in raw_levels[:depth]: + try: + parsed.append({"price": float(lvl["px"]), "size": float(lvl["sz"])}) + except (KeyError, TypeError, ValueError): + logger.debug("Dropped malformed orderbook level: %.100s", lvl) + return parsed + + return {"bids": _parse_side(levels[0]), "asks": _parse_side(levels[1])} async def get_candles( self, @@ -164,14 +182,17 @@ async def get_candles( candles: list[dict] = [] for c in data: - candles.append({ - "timestamp": c.get("t"), - "open": float(c.get("o", 0)), - "high": float(c.get("h", 0)), - "low": float(c.get("l", 0)), - "close": float(c.get("c", 0)), - "volume": float(c.get("v", 0)), - }) + try: + candles.append({ + "timestamp": c.get("t"), + "open": float(c.get("o", 0)), + "high": float(c.get("h", 0)), + "low": float(c.get("l", 0)), + "close": float(c.get("c", 0)), + "volume": float(c.get("v", 0)), + }) + except (TypeError, ValueError, AttributeError): + logger.debug("Dropped malformed candle: %.100s", c) return candles async def get_recent_trades(self, symbol: str, limit: int = 100) -> list[dict]: @@ -187,12 +208,15 @@ async def get_recent_trades(self, symbol: str, limit: int = 100) -> list[dict]: trades: list[dict] = [] for t in data[:limit]: - trades.append({ - "time": t.get("time"), - "price": float(t.get("px", 0)), - "size": float(t.get("sz", 0)), - "side": t.get("side", ""), - }) + try: + trades.append({ + "time": t.get("time"), + "price": float(t.get("px", 0)), + "size": float(t.get("sz", 0)), + "side": t.get("side", ""), + }) + except (TypeError, ValueError, AttributeError): + logger.debug("Dropped malformed trade: %.100s", t) return trades # ── Internals ───────────────────────────────────────────────── @@ -211,6 +235,7 @@ async def _post(self, session: aiohttp.ClientSession, payload: dict) -> dict | l HYPERLIQUID_INFO_URL, json=payload, headers={"Content-Type": "application/json"}, + timeout=HTTP_TIMEOUT, ) as resp: resp.raise_for_status() return await resp.json() diff --git a/src/data_layer/orderbook.py b/src/data_layer/orderbook.py index 8fc2b09..e462b6a 100644 --- a/src/data_layer/orderbook.py +++ b/src/data_layer/orderbook.py @@ -137,21 +137,34 @@ def is_stale(self, now: float | None = None) -> bool: # ── Book update (public for testability) ───────────────────── def _update_book(self, symbol: str, data: dict) -> None: - """Parse l2Book levels data and update the in-memory book.""" + """Parse l2Book levels data and update the in-memory book. + + Malformed levels are dropped individually so one bad entry can't + raise out of the WS read loop and force a reconnect (or poison the + whole book). + """ if symbol not in self.books: return levels = data.get("levels", [[], []]) + if not isinstance(levels, list): + logger.warning("[orderbook] malformed levels for %s: %.200s", symbol, levels) + return bids_raw = levels[0] if len(levels) > 0 else [] asks_raw = levels[1] if len(levels) > 1 else [] - bids = [ - OrderBookLevel(price=float(b["px"]), size=float(b["sz"])) - for b in bids_raw[:self.depth] - ] - asks = [ - OrderBookLevel(price=float(a["px"]), size=float(a["sz"])) - for a in asks_raw[:self.depth] - ] + def _parse_side(raw_levels) -> list[OrderBookLevel]: + parsed: list[OrderBookLevel] = [] + if not isinstance(raw_levels, list): + return parsed + for lvl in raw_levels[:self.depth]: + try: + parsed.append(OrderBookLevel(price=float(lvl["px"]), size=float(lvl["sz"]))) + except (KeyError, TypeError, ValueError): + logger.debug("[orderbook] dropped malformed level for %s: %.100s", symbol, lvl) + return parsed + + bids = _parse_side(bids_raw) + asks = _parse_side(asks_raw) self.books[symbol]["bids"] = bids self.books[symbol]["asks"] = asks diff --git a/src/data_layer/orderflow_engine.py b/src/data_layer/orderflow_engine.py index 3399b78..d963aac 100644 --- a/src/data_layer/orderflow_engine.py +++ b/src/data_layer/orderflow_engine.py @@ -231,6 +231,33 @@ def data_age(self, now: float | None = None) -> float: def is_stale(self, now: float | None = None) -> bool: return self.data_age(now) > STALE_AFTER_SECONDS + def venue_data_age(self, venue: str, now: float | None = None) -> float: + """Seconds since the last trade from ONE venue (inf if none yet).""" + last = (self.last_hl_message_at if venue == "hyperliquid" + else self.last_binance_message_at) + if last <= 0: + return float("inf") + return (now if now is not None else time.time()) - last + + def venue_is_stale(self, venue: str, now: float | None = None) -> bool: + return self.venue_data_age(venue, now) > STALE_AFTER_SECONDS + + def venue_freshness(self, now: float | None = None) -> dict[str, dict]: + """Per-venue freshness so a dead venue can't hide behind a live one. + + The combined is_stale() uses the freshest venue (intentional: the + blended CVD is still moving), but consumers of venue-specific data + need to know when THEIR venue went quiet. + """ + out: dict[str, dict] = {} + for venue in ("hyperliquid", "binance"): + age = self.venue_data_age(venue, now) + out[venue] = { + "data_age_seconds": None if age == float("inf") else round(age, 1), + "stale": age > STALE_AFTER_SECONDS, + } + return out + async def start(self) -> None: """Open WebSocket(s), subscribe, and begin processing in background.""" if self._running: diff --git a/src/data_layer/persistence.py b/src/data_layer/persistence.py index bc97ceb..e996bfe 100644 --- a/src/data_layer/persistence.py +++ b/src/data_layer/persistence.py @@ -13,12 +13,12 @@ """ import atexit +import logging +import shutil import sqlite3 -import time import threading -import logging +import time from pathlib import Path -from dataclasses import asdict logger = logging.getLogger(__name__) @@ -57,19 +57,56 @@ def __init__(self, db_path: str | Path = DB_PATH): conn.execute("PRAGMA integrity_check") self._conn = conn self._init_tables() - except sqlite3.DatabaseError: - logger.warning("Database corrupted at %s — recreating", self.db_path) + except sqlite3.DatabaseError as exc: if conn is not None: conn.close() - # Remove corrupted db and WAL/SHM files + # Lock/busy contention is NOT corruption: another process (a + # dashboard, a verification run) holding the DB must not get the + # healthy database quarantined out from under it. + if "lock" in str(exc).lower() or "busy" in str(exc).lower(): + logger.error( + "Database at %s is locked/busy — failing startup rather " + "than quarantining a healthy DB: %s", self.db_path, exc, + ) + raise + # Quarantine, never delete: move the corrupted DB (and WAL/SHM) + # aside with a timestamp so history survives for postmortem and + # possible `.recover`, then start fresh. + quarantine_dir = self.db_path.parent / "corrupted" + quarantine_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S") for suffix in ("", "-wal", "-shm"): p = Path(str(self.db_path) + suffix) if p.exists(): - p.unlink() - self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False, timeout=10) - self._conn.execute("PRAGMA journal_mode=WAL") - self._conn.execute("PRAGMA synchronous=NORMAL") - self._init_tables() + dest = quarantine_dir / f"{p.name}.{stamp}" + try: + shutil.move(str(p), str(dest)) # handles cross-device + except OSError: + logger.exception("Failed to quarantine %s", p) + try: + p.unlink() # last resort so we can still start + except OSError: + logger.exception("Could not remove %s either", p) + logger.error( + "Database corrupted at %s — quarantined to %s and recreated. " + "Historical data is preserved there for recovery.", + self.db_path, quarantine_dir, + ) + try: + self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False, timeout=10) + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA synchronous=NORMAL") + self._init_tables() + except sqlite3.DatabaseError: + # The corrupted file could not be moved OR removed (held + # handle, read-only mount). Run on an in-memory DB so the + # terminal stays alive; persistence is lost for this session. + logger.critical( + "Could not recreate database at %s — falling back to an " + "in-memory store (NO persistence this session)", self.db_path, + ) + self._conn = sqlite3.connect(":memory:", check_same_thread=False) + self._init_tables() # Safety net for graceful exits (normal return, unhandled exception, # Ctrl-C → KeyboardInterrupt unwinds to interpreter exit). The @@ -263,18 +300,54 @@ def _init_tables(self): CREATE INDEX IF NOT EXISTS idx_options_ts ON options_data(timestamp); CREATE INDEX IF NOT EXISTS idx_options_underlying ON options_data(underlying); """) - # Migration: add new columns to existing tables if not present - for table, col, col_type in [ - ("snapshots", "premium_pct", "REAL DEFAULT 0.0"), - ("snapshots", "basis_pct", "REAL DEFAULT 0.0"), - ("paper_trades", "funding_collected", "REAL DEFAULT 0.0"), - ]: - try: - self._conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} {col_type}") - except Exception: - pass # Column already exists + self._run_migrations() self._conn.commit() + # Bump when adding a migration below. The schema_version table lets a + # future release tell an old DB from a new one instead of guessing from + # ALTER TABLE failures. + SCHEMA_VERSION = 2 + + def _run_migrations(self) -> None: + """Versioned, idempotent migrations. Caller holds the lock. + + Only "duplicate column" is treated as already-applied; any other + migration failure is a real error and is raised so the app doesn't + keep running against a half-migrated schema. + """ + self._conn.execute( + "CREATE TABLE IF NOT EXISTS schema_version " + "(version INTEGER NOT NULL, applied_at REAL NOT NULL)" + ) + row = self._conn.execute("SELECT MAX(version) FROM schema_version").fetchone() + current = row[0] or 0 + + # v1: original schema (implicit for pre-versioning DBs). + # v2: extra columns on snapshots / paper_trades. + for table, col, col_type in [ + ("snapshots", "premium_pct", "REAL DEFAULT 0.0"), + ("snapshots", "basis_pct", "REAL DEFAULT 0.0"), + ("paper_trades", "funding_collected", "REAL DEFAULT 0.0"), + ]: + try: + self._conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} {col_type}") + except sqlite3.OperationalError as exc: + if "duplicate column" not in str(exc).lower(): + logger.error("Migration failed for %s.%s: %s", table, col, exc) + raise + + if current < self.SCHEMA_VERSION: + self._conn.execute( + "INSERT INTO schema_version (version, applied_at) VALUES (?, ?)", + (self.SCHEMA_VERSION, time.time()), + ) + + def get_schema_version(self) -> int: + """Highest applied schema version (0 for a brand-new/legacy DB).""" + with self._lock: + row = self._conn.execute("SELECT MAX(version) FROM schema_version").fetchone() + return row[0] or 0 + def attach(self, hub) -> None: """Attach to a HyperDataHub — automatically persists all events.""" hub.on_liquidation(self._save_liquidation) @@ -292,7 +365,8 @@ def _save_liquidation(self, event) -> None: """Callback: save a liquidation event.""" with self._lock: self._conn.execute( - "INSERT INTO liquidations (timestamp, exchange, symbol, side, size_usd, price, quantity, confirmed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO liquidations (timestamp, exchange, symbol, side, size_usd, " + "price, quantity, confirmed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (event.timestamp, event.exchange, event.symbol, event.side, event.size_usd, event.price, event.quantity, 1 if getattr(event, 'confirmed', True) else 0, @@ -317,7 +391,8 @@ def _save_trade(self, trade) -> None: if self._trade_count % self.TRADE_SAMPLE_RATE != 0: return self._conn.execute( - "INSERT INTO trades (timestamp, symbol, side, price, size, size_usd, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO trades (timestamp, symbol, side, price, size, size_usd, " + "created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", (trade.timestamp, trade.symbol, trade.side, trade.price, trade.size, trade.size_usd, time.time()) ) @@ -372,7 +447,8 @@ def get_liquidations(self, since_hours: float = 24, exchange: str | None = None, symbol: str | None = None, limit: int = 1000) -> list[dict]: """Get historical liquidation events.""" cutoff = time.time() - (since_hours * 3600) - query = "SELECT timestamp, exchange, symbol, side, size_usd, price, quantity, confirmed FROM liquidations WHERE timestamp > ?" + query = ("SELECT timestamp, exchange, symbol, side, size_usd, price, quantity, " + "confirmed FROM liquidations WHERE timestamp > ?") params: list = [cutoff] if exchange: query += " AND exchange = ?" @@ -455,7 +531,8 @@ def _save_smart_money_signal(self, signal) -> None: """Callback: save a smart money signal.""" with self._lock: self._conn.execute( - "INSERT INTO smart_money_signals (timestamp, address, tier, action, symbol, size_usd, wallet_rank, signal_type, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO smart_money_signals (timestamp, address, tier, action, symbol, " + "size_usd, wallet_rank, signal_type, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (signal.timestamp, signal.address, signal.tier, signal.action, signal.symbol, signal.size_usd, signal.wallet_rank, signal.signal_type, time.time()), @@ -672,7 +749,8 @@ def save_funding_rate(self, snap) -> None: """Save a funding rate snapshot.""" with self._lock: self._conn.execute( - "INSERT INTO funding_rates (timestamp, exchange, symbol, funding_rate_hourly, funding_rate_annualized, created_at) VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO funding_rates (timestamp, exchange, symbol, funding_rate_hourly, " + "funding_rate_annualized, created_at) VALUES (?, ?, ?, ?, ?, ?)", (snap.timestamp, snap.exchange, snap.symbol, snap.funding_rate_hourly, snap.funding_rate_annualized, time.time()), ) @@ -683,7 +761,8 @@ def get_funding_rates(self, exchange: str | None = None, symbol: str | None = No hours: float = 24, limit: int = 500) -> list[dict]: """Get historical funding rate snapshots.""" cutoff = time.time() - (hours * 3600) - query = "SELECT timestamp, exchange, symbol, funding_rate_hourly, funding_rate_annualized FROM funding_rates WHERE timestamp > ?" + query = ("SELECT timestamp, exchange, symbol, funding_rate_hourly, " + "funding_rate_annualized FROM funding_rates WHERE timestamp > ?") params: list = [cutoff] if exchange: query += " AND exchange = ?" @@ -704,7 +783,8 @@ def get_funding_rates(self, exchange: str | None = None, symbol: str | None = No def save_long_short_ratio(self, snap) -> None: with self._lock: self._conn.execute( - "INSERT INTO long_short_ratios (timestamp, symbol, long_ratio, short_ratio, long_short_ratio, created_at) VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO long_short_ratios (timestamp, symbol, long_ratio, short_ratio, " + "long_short_ratio, created_at) VALUES (?, ?, ?, ?, ?, ?)", (snap.timestamp, snap.symbol, snap.long_ratio, snap.short_ratio, snap.long_short_ratio, time.time()), ) self._event_count += 1 @@ -712,7 +792,8 @@ def save_long_short_ratio(self, snap) -> None: def get_long_short_ratios(self, symbol: str | None = None, hours: float = 24, limit: int = 200) -> list[dict]: cutoff = time.time() - (hours * 3600) - query = "SELECT timestamp, symbol, long_ratio, short_ratio, long_short_ratio FROM long_short_ratios WHERE timestamp > ?" + query = ("SELECT timestamp, symbol, long_ratio, short_ratio, long_short_ratio " + "FROM long_short_ratios WHERE timestamp > ?") params: list = [cutoff] if symbol: query += " AND symbol = ?" @@ -731,8 +812,10 @@ def save_options_snapshot(self, snap) -> None: """Save a Deribit IV snapshot.""" with self._lock: self._conn.execute( - "INSERT INTO options_data (timestamp, underlying, mark_iv, bid_iv, ask_iv, oi_usd, index_price, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - (snap.timestamp, snap.underlying, snap.mark_iv, snap.bid_iv, snap.ask_iv, snap.oi_usd, snap.index_price, time.time()), + "INSERT INTO options_data (timestamp, underlying, mark_iv, bid_iv, ask_iv, " + "oi_usd, index_price, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (snap.timestamp, snap.underlying, snap.mark_iv, snap.bid_iv, snap.ask_iv, + snap.oi_usd, snap.index_price, time.time()), ) self._event_count += 1 self._maybe_commit() @@ -740,7 +823,8 @@ def save_options_snapshot(self, snap) -> None: def get_options_data(self, underlying: str | None = None, hours: float = 24, limit: int = 200) -> list[dict]: """Get historical Deribit IV snapshots.""" cutoff = time.time() - (hours * 3600) - query = "SELECT timestamp, underlying, mark_iv, bid_iv, ask_iv, oi_usd, index_price FROM options_data WHERE timestamp > ?" + query = ("SELECT timestamp, underlying, mark_iv, bid_iv, ask_iv, oi_usd, " + "index_price FROM options_data WHERE timestamp > ?") params: list = [cutoff] if underlying: query += " AND underlying = ?" diff --git a/src/data_layer/position_scanner.py b/src/data_layer/position_scanner.py index 296ba40..0d202d4 100644 --- a/src/data_layer/position_scanner.py +++ b/src/data_layer/position_scanner.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging import time from dataclasses import dataclass, field from pathlib import Path @@ -9,12 +10,19 @@ from src.data_layer import address_store +logger = logging.getLogger(__name__) + API_URL = "https://api.hyperliquid.xyz/info" DATA_DIR = Path(__file__).resolve().parents[2] / "data" RATE_LIMIT_PER_SEC = 10 META_CACHE_TTL = 300 # 5 minutes +# Explicit deadline on every request so a hung endpoint fails the scan cycle +# instead of blocking the hub's position-scan loop indefinitely. Split +# connect/read so a slow handshake can't consume the entire budget. +HTTP_TIMEOUT = aiohttp.ClientTimeout(total=10, connect=3, sock_connect=3, sock_read=5) + @dataclass class TrackedPosition: @@ -52,7 +60,16 @@ async def scan(self) -> list[TrackedPosition]: async with aiohttp.ClientSession() as session: self._session = session try: - await asyncio.gather(self.update_prices(), self.update_meta()) + # Independent updates: one endpoint failing must not discard + # the other's result (meta is a 5-min cache — losing a refresh + # means stale maintenance margins for the whole window). + results = await asyncio.gather( + self.update_prices(), self.update_meta(), + return_exceptions=True, + ) + for name, res in zip(("update_prices", "update_meta"), results): + if isinstance(res, BaseException): + logger.warning("[scanner] %s failed: %r", name, res) # Discover new addresses: always on first run, then every 30 minutes import time as _time @@ -102,12 +119,18 @@ async def discover_addresses(self, limit: int = 100) -> set[str]: }) if isinstance(data, list): for trade in data: + # Validate at the boundary: exchange payloads are + # untrusted, and a junk identifier persisted here gets + # re-scanned (one API call per cycle) forever. + candidates: list[object] = [] for side_key in ("buyer", "seller", "users"): if side_key in trade and isinstance(trade[side_key], str): - new_addresses.add(trade[side_key]) + candidates.append(trade[side_key]) if "users" in trade and isinstance(trade["users"], list): - for addr in trade["users"]: - new_addresses.add(addr) + candidates.extend(trade["users"]) + for addr in candidates: + if address_store.is_valid_address(addr): + new_addresses.add(address_store.normalize_address(addr)) if len(new_addresses) >= limit: break except Exception: @@ -279,6 +302,7 @@ async def _post(self, payload: dict) -> dict | list | None: API_URL, json=payload, headers={"Content-Type": "application/json"}, + timeout=HTTP_TIMEOUT, ) as resp: resp.raise_for_status() return await resp.json() @@ -294,6 +318,10 @@ def _save_discovered_addresses(self): address_store.add_addresses(self.discovered_addresses, source="position_scanner") def add_addresses(self, addresses: list[str]): - """Manually add addresses to track.""" - self.discovered_addresses.update(addresses) - address_store.add_addresses(addresses, source="position_scanner_manual") + """Manually add addresses to track (validated + normalized).""" + valid = [ + address_store.normalize_address(a) + for a in addresses if address_store.is_valid_address(a) + ] + self.discovered_addresses.update(valid) + address_store.add_addresses(valid, source="position_scanner_manual") diff --git a/src/data_layer/smart_money.py b/src/data_layer/smart_money.py index a45859f..fde584d 100644 --- a/src/data_layer/smart_money.py +++ b/src/data_layer/smart_money.py @@ -62,6 +62,7 @@ class WalletProfile: pnl_score: float = 0.0 # log-scaled PnL sharpe_ratio: float = 0.0 # risk-adjusted returns composite_score: float = 0.0 # final weighted score + confidence: float = 0.0 # 0-1 sample-size confidence (see _compute_confidence) # Classification rank: int = 0 # 1 = best performer @@ -101,8 +102,13 @@ class SmartMoneyEngine: BETA = 0.40 # PnL weight (log-scaled) GAMMA = 0.25 # Sharpe weight - # Thresholds - MIN_TRADES_FOR_RANKING = 3 # Low for early data collection; tighten later + # Thresholds. Three closed trades says nothing about skill — a coin flip + # "wins" three in a row 12.5% of the time — so ranking requires a + # minimally meaningful sample plus real volume, and every profile carries + # a sample-size confidence that consumers must surface alongside tiers. + MIN_TRADES_FOR_RANKING = 10 + MIN_VOLUME_FOR_RANKING = 50_000 # Total traded volume (USD) + FULL_CONFIDENCE_TRADES = 50 # Trades at which confidence saturates SMART_MONEY_TOP_N = 100 # Top 100 = smart money DUMB_MONEY_BOTTOM_N = 100 # Bottom 100 = dumb money ANALYSIS_INTERVAL = 300 # Analyze wallets every 5 minutes @@ -200,8 +206,8 @@ async def fetch_leaderboard_wallets(self, top_n: int = 50) -> list[str]: or row.get("user") or "" ) - if isinstance(addr, str) and addr.startswith("0x"): - addresses.append(addr.lower()) + if address_store.is_valid_address(addr): + addresses.append(address_store.normalize_address(addr)) if addresses: logger.info( "[smart_money] Fetched %d leaderboard wallets from %s", @@ -229,8 +235,8 @@ async def seed_from_leaderboard(self) -> int: raw = json.loads(ANCHOR_PATH.read_text()) for entry in raw.get("anchors", []): addr = entry.get("address", "") - if isinstance(addr, str) and addr.startswith("0x"): - seed_addrs.append(addr.lower()) + if address_store.is_valid_address(addr): + seed_addrs.append(address_store.normalize_address(addr)) anchor_count += 1 except Exception: logger.debug("[smart_money] Failed to load anchor_wallets.json") @@ -337,14 +343,19 @@ async def _discovery_loop(self) -> None: if data.get("channel") == "trades": for trade in data.get("data", []): for addr in trade.get("users", []): - if addr and addr not in self.wallets: + # Untrusted payload: only well-formed + # wallet addresses become profiles. + if not address_store.is_valid_address(addr): + continue + addr = address_store.normalize_address(addr) + if addr not in self.wallets: self.wallets[addr] = WalletProfile( address=addr, discovered_at=time.time(), last_seen=time.time(), last_analyzed=0, ) - elif addr and addr in self.wallets: + else: self.wallets[addr].last_seen = time.time() except asyncio.CancelledError: return @@ -376,7 +387,10 @@ async def discover_from_trades(self, symbols: list[str] | None = None, duration: if data.get("channel") == "trades": for trade in data.get("data", []): for addr in trade.get("users", []): - if addr and addr not in self.wallets: + if not address_store.is_valid_address(addr): + continue + addr = address_store.normalize_address(addr) + if addr not in self.wallets: self.wallets[addr] = WalletProfile( address=addr, discovered_at=time.time(), @@ -425,7 +439,7 @@ async def _analysis_loop(self) -> None: # Re-rank after each batch self.rank_all() - ranked_count = sum(1 for w in self.wallets.values() if w.total_trades >= self.MIN_TRADES_FOR_RANKING) + ranked_count = sum(1 for w in self.wallets.values() if self._qualifies_for_ranking(w)) logger.info( "[smart_money] Analyzed %d wallets, %d total tracked, %d ranked", len(batch), len(self.wallets), ranked_count, @@ -519,6 +533,7 @@ async def analyze_wallet(self, address: str) -> WalletProfile: wallet.sharpe_ratio = self._compute_sharpe(close_pnls) wallet.pnl_score = self._compute_pnl_score(total_pnl) wallet.composite_score = self._compute_composite(wallet) + wallet.confidence = self._compute_confidence(wallet) # 5. Fetch clearinghouse state for account value + open positions ch = await self._fetch_clearinghouse(address) @@ -578,16 +593,36 @@ def _compute_composite(self, w: WalletProfile) -> float: return self.ALPHA * wr + self.BETA * pnl + self.GAMMA * sharpe + def _compute_confidence(self, w: WalletProfile) -> float: + """Sample-size confidence in [0, 1]. + + Linear in closed-trade count up to FULL_CONFIDENCE_TRADES. This is a + coverage heuristic (recent fills only, no confidence interval on win + rate) — consumers must show it next to any smart/dumb label rather + than presenting tiers as certainty. + """ + return min(1.0, w.total_trades / float(self.FULL_CONFIDENCE_TRADES)) + + def _qualifies_for_ranking(self, w: WalletProfile) -> bool: + return (w.total_trades >= self.MIN_TRADES_FOR_RANKING + and w.total_volume_usd >= self.MIN_VOLUME_FOR_RANKING) + # ── Ranking ─────────────────────────────────────────────────────── def rank_all(self) -> None: """Re-rank all wallets by composite_score.""" - qualified = [ - w for w in self.wallets.values() - if w.total_trades >= self.MIN_TRADES_FOR_RANKING - ] + qualified = [w for w in self.wallets.values() if self._qualifies_for_ranking(w)] qualified.sort(key=lambda w: w.composite_score, reverse=True) + qualified_set = {w.address for w in qualified} + # Wallets that no longer qualify must lose their old rank/tier, or a + # stale "smart" label survives after the sample stops qualifying. + for w in self.wallets.values(): + if w.address not in qualified_set: + w.rank = 0 + if w.tier in ("smart", "average", "dumb"): + w.tier = "unknown" + for i, w in enumerate(qualified, 1): w.rank = i if i <= self.SMART_MONEY_TOP_N: @@ -707,7 +742,7 @@ def get_wallet(self, address: str) -> WalletProfile | None: def get_stats(self) -> dict: """Summary stats: total wallets, ranked wallets, signals generated.""" - ranked = sum(1 for w in self.wallets.values() if w.total_trades >= self.MIN_TRADES_FOR_RANKING) + ranked = sum(1 for w in self.wallets.values() if self._qualifies_for_ranking(w)) smart = sum(1 for w in self.wallets.values() if w.tier == "smart") dumb = sum(1 for w in self.wallets.values() if w.tier == "dumb") return { @@ -716,4 +751,15 @@ def get_stats(self) -> dict: "smart_wallets": smart, "dumb_wallets": dumb, "total_signals": len(self.signals), + # Coverage caveats: rankings come from recent fills only and small + # samples — consumers should show these limits, not just tiers. + "ranking_criteria": { + "min_trades": self.MIN_TRADES_FOR_RANKING, + "min_volume_usd": self.MIN_VOLUME_FOR_RANKING, + "note": ( + "Performance is computed from recent fills only; tiers are " + "heuristic. Check each wallet's `confidence` (0-1 sample-" + "size score) before acting on smart/dumb labels." + ), + }, } diff --git a/src/strategies/__init__.py b/src/strategies/__init__.py index 0f1644e..6cc6a66 100644 --- a/src/strategies/__init__.py +++ b/src/strategies/__init__.py @@ -1,2 +1,4 @@ -from .base import Strategy, Signal +from .base import Signal, Strategy from .paper_trader import PaperTrader + +__all__ = ["Signal", "Strategy", "PaperTrader"] diff --git a/src/strategies/base.py b/src/strategies/base.py index 78b305c..7a08212 100644 --- a/src/strategies/base.py +++ b/src/strategies/base.py @@ -8,8 +8,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from dataclasses import dataclass, field - +from dataclasses import dataclass # --------------------------------------------------------------------------- # Signal — the output of every strategy evaluation diff --git a/src/strategies/examples/__init__.py b/src/strategies/examples/__init__.py index a0ac70a..33099ee 100644 --- a/src/strategies/examples/__init__.py +++ b/src/strategies/examples/__init__.py @@ -2,3 +2,5 @@ from .funding_rate_arb import FundingRateArb from .liquidation_cascade import LiquidationCascade from .whale_follow import WhaleFollow + +__all__ = ["CVDMomentum", "FundingRateArb", "LiquidationCascade", "WhaleFollow"] diff --git a/src/strategies/examples/cvd_momentum.py b/src/strategies/examples/cvd_momentum.py index e3fc78c..6700b9c 100644 --- a/src/strategies/examples/cvd_momentum.py +++ b/src/strategies/examples/cvd_momentum.py @@ -11,7 +11,7 @@ from __future__ import annotations -from src.strategies.base import Strategy, Signal +from src.strategies.base import Signal, Strategy class CVDMomentum(Strategy): diff --git a/src/strategies/examples/funding_rate_arb.py b/src/strategies/examples/funding_rate_arb.py index b684716..23d6274 100644 --- a/src/strategies/examples/funding_rate_arb.py +++ b/src/strategies/examples/funding_rate_arb.py @@ -12,7 +12,7 @@ from __future__ import annotations -from src.strategies.base import Strategy, Signal +from src.strategies.base import Signal, Strategy class FundingRateArb(Strategy): diff --git a/src/strategies/examples/liquidation_cascade.py b/src/strategies/examples/liquidation_cascade.py index c33c368..9485b9e 100644 --- a/src/strategies/examples/liquidation_cascade.py +++ b/src/strategies/examples/liquidation_cascade.py @@ -12,7 +12,7 @@ import time -from src.strategies.base import Strategy, Signal +from src.strategies.base import Signal, Strategy class LiquidationCascade(Strategy): @@ -42,8 +42,11 @@ def evaluate(self, hub) -> Signal | None: if not stats: return None - # Look for a spike in long liquidations (longs getting wiped = price dropping) - long_liq_usd = getattr(stats, "total_long_usd", 0) or 0 + # Look for a spike in long liquidations (longs getting wiped = price + # dropping). get_stats() returns a dict; the long-side dollar volume + # key is long_volume_usd (getattr on a dict always returned 0 and + # silently disabled this strategy). + long_liq_usd = stats.get("long_volume_usd", 0) or 0 if long_liq_usd >= self.cascade_threshold_usd: self._last_signal_time = now diff --git a/src/strategies/examples/whale_follow.py b/src/strategies/examples/whale_follow.py index 00390be..fe08e19 100644 --- a/src/strategies/examples/whale_follow.py +++ b/src/strategies/examples/whale_follow.py @@ -10,7 +10,7 @@ """ from __future__ import annotations -from src.strategies.base import Strategy, Signal +from src.strategies.base import Signal, Strategy class WhaleFollow(Strategy): diff --git a/src/strategies/llm_agent.py b/src/strategies/llm_agent.py index 357350d..7859e4d 100644 --- a/src/strategies/llm_agent.py +++ b/src/strategies/llm_agent.py @@ -14,9 +14,12 @@ from __future__ import annotations import asyncio +import concurrent.futures import json import logging import os +import time +from collections import deque import aiohttp @@ -42,6 +45,10 @@ class LLMAgent(Strategy): """Strategy that delegates trading decisions to a language model.""" + # Budget guardrail: an LLM call per check interval adds up. Configurable + # via LLM_MAX_EVALS_PER_HOUR; evaluations beyond the budget are skipped. + DEFAULT_MAX_EVALS_PER_HOUR = 60 + def __init__(self, symbol: str = "BTC") -> None: self.symbol = symbol @@ -49,17 +56,49 @@ def __init__(self, symbol: str = "BTC") -> None: self.base_url = os.environ.get("LLM_BASE_URL", "http://localhost:11434/v1") self.model = os.environ.get("LLM_MODEL", "llama3") self.api_key = os.environ.get("LLM_API_KEY", "") + try: + self.max_evals_per_hour = int( + os.environ.get("LLM_MAX_EVALS_PER_HOUR", self.DEFAULT_MAX_EVALS_PER_HOUR) + ) + except ValueError: + self.max_evals_per_hour = self.DEFAULT_MAX_EVALS_PER_HOUR + self._eval_times: deque[float] = deque(maxlen=max(self.max_evals_per_hour, 1)) + # One long-lived worker thread — not a new executor per evaluation. + self._pool = concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix="llm-agent" + ) @property def name(self) -> str: return "llm_agent" - def evaluate(self, hub) -> Signal | None: + def _within_budget(self, now: float | None = None) -> bool: + """Sliding-window cap on LLM calls per hour.""" + now = time.time() if now is None else now + while self._eval_times and now - self._eval_times[0] > 3600: + self._eval_times.popleft() + if len(self._eval_times) >= self.max_evals_per_hour: + return False + self._eval_times.append(now) + return True + + def _refund_eval_slot(self) -> None: + """Return the most recent budget slot. + + Called when the call failed at the TRANSPORT level (timeout, refused + connection, HTTP error) — no tokens were consumed, so a flaky + provider must not exhaust the hourly budget. Parse failures keep + their slot: the provider did the work and billed for it. + """ + if self._eval_times: + self._eval_times.pop() + + async def evaluate(self, hub) -> Signal | None: """Build a market summary and ask the LLM for a decision. - NOTE: This calls an async HTTP endpoint. Since evaluate() is - synchronous, we use asyncio to run the coroutine. If you're - already in an async context, see _async_evaluate() directly. + Async: the blocking HTTP call runs in the persistent worker thread + and is awaited, so a slow LLM response cannot stall the paper + trader's event loop (other strategies keep evaluating). """ # If no API key and not using a local model, warn and skip if not self.api_key and "localhost" not in self.base_url: @@ -69,12 +108,23 @@ def evaluate(self, hub) -> Signal | None: ) return None + if not self._within_budget(): + logger.warning( + "LLM eval budget exhausted (%d/hour) — skipping evaluation", + self.max_evals_per_hour, + ) + return None + + loop = asyncio.get_running_loop() try: - # Run blocking LLM call in a thread so it doesn't stall the async loop - import concurrent.futures - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - future = pool.submit(self._sync_evaluate, hub) - return future.result(timeout=20) + return await asyncio.wait_for( + loop.run_in_executor(self._pool, self._sync_evaluate, hub), + timeout=20, + ) + except asyncio.TimeoutError: + self._refund_eval_slot() + logger.warning("LLM evaluation timed out after 20s") + return None except Exception: logger.exception("LLM agent error") return None @@ -113,6 +163,9 @@ def _sync_evaluate(self, hub) -> Signal | None: data = _json.loads(resp.read()) text = data["choices"][0]["message"]["content"].strip() except Exception as e: + # Transport-level failure: no tokens consumed — give the budget + # slot back so a down provider can't burn the hourly allowance. + self._refund_eval_slot() logger.warning("LLM API call failed: %s", e) return None @@ -161,50 +214,30 @@ async def _async_evaluate(self, hub) -> Signal | None: # ---- Parse response ---- try: text = data["choices"][0]["message"]["content"].strip() - except (KeyError, IndexError): + except (KeyError, IndexError, TypeError, AttributeError): logger.warning("Unexpected LLM response format: %s", json.dumps(data)[:200]) return None - lines = text.split("\n", 1) - action_word = lines[0].strip().upper() - reason = lines[1].strip() if len(lines) > 1 else "" + return self._parse_response(text) - # Validate action - if action_word not in ("BUY", "SELL", "HOLD"): - # Try to find the action word somewhere in the first line - for word in ("BUY", "SELL", "HOLD"): - if word in action_word: - action_word = word - break - else: - logger.warning("LLM returned unparseable action: %s", lines[0]) - return None + def _parse_response(self, text: str) -> Signal | None: + """Parse LLM response text into a Signal — deterministic, reject-on-ambiguous. - if action_word == "HOLD": + The first NON-EMPTY line must be exactly BUY, SELL, or HOLD + (case-insensitive, surrounding punctuation tolerated; leading blank + lines are ignored). Substring matching is deliberately NOT done: + "I would not BUY here" must never resolve to a BUY. + """ + lines = [ln.strip() for ln in text.splitlines() if ln.strip()] + if not lines: + logger.warning("LLM returned empty response") return None - - return Signal( - symbol=self.symbol, - action=action_word, - size_usd=100.0, - confidence=0.6, - reason=f"[LLM] {reason}", - ) - - def _parse_response(self, text: str) -> Signal | None: - """Parse LLM response text into a Signal.""" - lines = text.split("\n", 1) - action_word = lines[0].strip().upper() - reason = lines[1].strip() if len(lines) > 1 else "" + action_word = lines[0].upper().strip(".!:*# ") + reason = " ".join(lines[1:]) if len(lines) > 1 else "" if action_word not in ("BUY", "SELL", "HOLD"): - for word in ("BUY", "SELL", "HOLD"): - if word in action_word: - action_word = word - break - else: - logger.warning("LLM returned unparseable action: %s", lines[0]) - return None + logger.warning("LLM returned ambiguous action, rejecting: %r", lines[0][:100]) + return None if action_word == "HOLD": return None diff --git a/src/strategies/paper_trader.py b/src/strategies/paper_trader.py index aa9ef38..9d75f8a 100644 --- a/src/strategies/paper_trader.py +++ b/src/strategies/paper_trader.py @@ -19,7 +19,9 @@ from __future__ import annotations import asyncio +import inspect import logging +import math import sqlite3 import time from pathlib import Path @@ -27,7 +29,7 @@ from rich.console import Console -from .base import Strategy, Signal +from .base import Signal, Strategy logger = logging.getLogger(__name__) console = Console() @@ -130,12 +132,18 @@ async def stop(self) -> None: # ------------------------------------------------------------------ async def _loop(self) -> None: - """Evaluate all strategies every check_interval seconds.""" + """Evaluate all strategies every check_interval seconds. + + Strategies may implement evaluate() as sync or async; async ones + (e.g. the LLM agent) are awaited so a slow evaluation never blocks + the event loop for the other strategies. + """ while self._running: try: for strategy in self.strategies: try: - signal = strategy.evaluate(self.hub) + result = strategy.evaluate(self.hub) + signal = await result if inspect.isawaitable(result) else result if signal is None: continue if signal.action in ("BUY", "SELL"): @@ -153,19 +161,52 @@ async def _loop(self) -> None: # Trade execution # ------------------------------------------------------------------ + @staticmethod + def _signal_is_valid(signal: Signal) -> bool: + """Reject malformed signals before they can corrupt the books.""" + try: + size = float(signal.size_usd) + except (TypeError, ValueError): + return False + return ( + signal.action in ("BUY", "SELL") + and isinstance(signal.symbol, str) and bool(signal.symbol) + and size > 0 + and math.isfinite(size) + ) + def _execute_trade(self, strategy_name: str, signal: Signal) -> None: - """Execute a paper trade: update positions, log to SQLite, print.""" + """Execute a paper trade: update positions, log to SQLite, print. + + Accounting invariants: + - balance never goes negative (adds to a position are balance-checked + exactly like opens); + - adding to a position updates the size-weighted average entry price; + - an opposite-side signal closes the whole position (explicit + close-all semantics; partial reduction is not modeled). + """ + if not self._signal_is_valid(signal): + logger.warning( + "Rejected invalid signal from %s: action=%r symbol=%r size_usd=%r", + strategy_name, signal.action, signal.symbol, signal.size_usd, + ) + return + # Get current market price for the symbol asset = self.hub.market.assets.get(signal.symbol) - if asset is None: + if asset is None or not asset.price or asset.price <= 0: logger.warning( "Cannot execute trade for %s — no market data", signal.symbol ) return price = asset.price - # Calculate PnL if closing/reducing an existing position + # Plan the state mutation WITHOUT applying it yet: the trade is + # persisted to SQLite first, and only a logged trade mutates the + # books. Otherwise a DB error silently diverges get_portfolio() + # from the audit trail. pnl = 0.0 + apply_mutation: Any if signal.symbol in self.positions: pos = self.positions[signal.symbol] # Closing a long (SELL) or closing a short (BUY) @@ -175,12 +216,32 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: if pos["side"] == "short": price_change_pct = -price_change_pct pnl = pos["size_usd"] * price_change_pct - self.balance += pos["size_usd"] + pnl - del self.positions[signal.symbol] + credit = pos["size_usd"] + pnl + + def apply_mutation() -> None: + # A loss beyond the margin posted would take the account + # negative; a real venue liquidates first. Floor at zero + # (position is wiped, balance cannot go below broke). + self.balance = max(0.0, self.balance + credit) + del self.positions[signal.symbol] else: - # Adding to position in same direction — just increase size - pos["size_usd"] += signal.size_usd - self.balance -= signal.size_usd + # Adding in the same direction: balance-checked like an open, + # entry price becomes the size-weighted average. + if signal.size_usd > self.balance: + logger.warning( + "Insufficient balance to add to %s (need $%.2f, have $%.2f)", + signal.symbol, signal.size_usd, self.balance, + ) + return + new_size = pos["size_usd"] + signal.size_usd + new_entry = ( + pos["entry_price"] * pos["size_usd"] + price * signal.size_usd + ) / new_size + + def apply_mutation() -> None: + pos["entry_price"] = new_entry + pos["size_usd"] = new_size + self.balance -= signal.size_usd else: # Open a new position if signal.size_usd > self.balance: @@ -190,13 +251,15 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: ) return side = "long" if signal.action == "BUY" else "short" - self.positions[signal.symbol] = { - "side": side, - "entry_price": price, - "size_usd": signal.size_usd, - "opened_at": time.time(), - } - self.balance -= signal.size_usd + + def apply_mutation() -> None: + self.positions[signal.symbol] = { + "side": side, + "entry_price": price, + "size_usd": signal.size_usd, + "opened_at": time.time(), + } + self.balance -= signal.size_usd # Build trade record trade = { @@ -210,9 +273,8 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: "reason": signal.reason, "pnl": pnl, } - self.trades.append(trade) - # Persist to SQLite + # Persist FIRST; a trade that cannot be logged is not executed. if self._db: try: self._db.execute( @@ -227,7 +289,14 @@ def _execute_trade(self, strategy_name: str, signal: Signal) -> None: ) self._db.commit() except sqlite3.Error: - logger.exception("Failed to persist trade to SQLite") + logger.exception( + "Failed to persist trade to SQLite — trade NOT executed " + "(books stay consistent with the audit log)" + ) + return + + apply_mutation() + self.trades.append(trade) # Print to console with Rich color = "green" if signal.action == "BUY" else "red" diff --git a/tests/conftest.py b/tests/conftest.py index 5842a73..4677a82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,8 @@ """Pytest configuration and shared fixtures for HyperData tests.""" from __future__ import annotations -import sys import os +import sys # Ensure project root and src/ are importable _PROJECT_ROOT = os.path.join(os.path.dirname(__file__), "..") diff --git a/tests/test_deribit.py b/tests/test_deribit.py index 693ef7b..309c9d3 100644 --- a/tests/test_deribit.py +++ b/tests/test_deribit.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.deribit import DeribitIVSnapshot, DeribitFeed + +from data_layer.deribit import DeribitFeed, DeribitIVSnapshot def test_snapshot_fields(): diff --git a/tests/test_funding_rates.py b/tests/test_funding_rates.py index 0707153..d27d3f9 100644 --- a/tests/test_funding_rates.py +++ b/tests/test_funding_rates.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.funding_rates import FundingRateSnapshot, FundingRateCollector, normalise_fr_symbol + +from data_layer.funding_rates import FundingRateCollector, FundingRateSnapshot, normalise_fr_symbol def test_normalise_fr_symbol(): diff --git a/tests/test_hardening.py b/tests/test_hardening.py new file mode 100644 index 0000000..518f5b9 --- /dev/null +++ b/tests/test_hardening.py @@ -0,0 +1,862 @@ +"""Tests for the adversarial-review hardening pass. + +Covers: API bind guard / auth / CORS / rate limiting, WebSocket abuse limits, +liquidation dedup + cascade bypass, malformed exchange payloads, degraded hub +startup, paper-trader accounting invariants, LLM response parsing, persistence +corruption quarantine + schema versioning, and alert log redaction. +""" +from __future__ import annotations + +import inspect +import json +import sqlite3 +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer + +from data_layer.liquidation_feed import ( + BinanceConnection, + BybitConnection, + LiquidationEvent, + LiquidationFeed, + OKXConnection, +) +from data_layer.orderbook import OrderBookEngine +from data_layer.persistence import DataStore +from src.api_server import ( + WS_BAD_MSG_LIMIT, + HyperDataAPI, + _is_loopback_host, + _make_auth_middleware, + _make_cors_middleware, + _make_rate_limit_middleware, + _RateLimiter, + _WSClient, +) +from src.strategies.base import Signal +from src.strategies.llm_agent import LLMAgent +from src.strategies.paper_trader import PaperTrader + +# ── Helpers ────────────────────────────────────────────────────── + +def _liq_event(**overrides) -> LiquidationEvent: + defaults = dict( + timestamp=time.time(), + exchange="binance", + symbol="BTC", + side="long", + size_usd=25_000.0, + price=70_000.0, + quantity=0.357, + ) + defaults.update(overrides) + return LiquidationEvent(**defaults) + + +def _api(monkeypatch=None, host="127.0.0.1") -> HyperDataAPI: + return HyperDataAPI(hub=MagicMock(), host=host) + + +async def _client_for(middlewares) -> TestClient: + app = web.Application(middlewares=middlewares) + + async def ok(request): + return web.json_response({"ok": True}) + + app.router.add_get("/v1/live", ok) + app.router.add_get("/v1/health", ok) + app.router.add_get("/v1/whales", ok) + client = TestClient(TestServer(app)) + await client.start_server() + return client + + +# ── Bind guard / security resolution ───────────────────────────── + +class TestBindGuard: + def test_loopback_hosts(self): + assert _is_loopback_host("127.0.0.1") + assert _is_loopback_host("localhost") + assert _is_loopback_host("::1") + assert not _is_loopback_host("0.0.0.0") + assert not _is_loopback_host("192.168.1.10") + assert not _is_loopback_host("") + + def test_loopback_needs_no_key(self, monkeypatch): + monkeypatch.delenv("HYPERDATA_API_KEY", raising=False) + monkeypatch.delenv("HYPERDATA_UNSAFE_PUBLIC_API", raising=False) + monkeypatch.delenv("HYPERDATA_CORS_ORIGINS", raising=False) + key, origins = _api(host="127.0.0.1")._resolve_security() + assert key == "" + assert origins is None # wildcard CORS allowed on loopback + + def test_public_bind_refused_without_key_or_ack(self, monkeypatch): + monkeypatch.delenv("HYPERDATA_API_KEY", raising=False) + monkeypatch.delenv("HYPERDATA_UNSAFE_PUBLIC_API", raising=False) + with pytest.raises(RuntimeError, match="Refusing to bind"): + _api(host="0.0.0.0")._resolve_security() + + def test_public_bind_allowed_with_key(self, monkeypatch): + monkeypatch.setenv("HYPERDATA_API_KEY", "sekrit") + monkeypatch.delenv("HYPERDATA_CORS_ORIGINS", raising=False) + key, origins = _api(host="0.0.0.0")._resolve_security() + assert key == "sekrit" + assert origins == set() # never wildcard CORS off loopback + + def test_public_bind_allowed_with_explicit_ack(self, monkeypatch): + monkeypatch.delenv("HYPERDATA_API_KEY", raising=False) + monkeypatch.setenv("HYPERDATA_UNSAFE_PUBLIC_API", "1") + key, origins = _api(host="0.0.0.0")._resolve_security() + assert key == "" + assert origins == set() + + def test_cors_allowlist_parsed(self, monkeypatch): + monkeypatch.setenv("HYPERDATA_API_KEY", "k") + monkeypatch.setenv("HYPERDATA_CORS_ORIGINS", "https://a.example, https://b.example") + _, origins = _api(host="0.0.0.0")._resolve_security() + assert origins == {"https://a.example", "https://b.example"} + + +# ── Middlewares over a live test server ────────────────────────── + +class TestAuthMiddleware: + @pytest.mark.asyncio + async def test_key_required_except_liveness(self): + client = await _client_for([ + _make_auth_middleware("sekrit"), + _make_cors_middleware(None), + ]) + try: + # Only the minimal liveness probe is exempt; the detailed health + # payload is operational recon and requires the key. + assert (await client.get("/v1/live")).status == 200 + assert (await client.get("/v1/health")).status == 401 + assert (await client.get("/v1/whales")).status == 401 + ok_bearer = await client.get( + "/v1/whales", headers={"Authorization": "Bearer sekrit"}) + assert ok_bearer.status == 200 + ok_health = await client.get( + "/v1/health", headers={"X-API-Key": "sekrit"}) + assert ok_health.status == 200 + bad = await client.get( + "/v1/whales", headers={"Authorization": "Bearer wrong"}) + assert bad.status == 401 + finally: + await client.close() + + @pytest.mark.asyncio + async def test_cors_allowlist_echoes_only_allowed_origin(self): + client = await _client_for([ + _make_cors_middleware({"https://ok.example"}), + ]) + try: + allowed = await client.get( + "/v1/health", headers={"Origin": "https://ok.example"}) + assert allowed.headers.get("Access-Control-Allow-Origin") == "https://ok.example" + denied = await client.get( + "/v1/health", headers={"Origin": "https://evil.example"}) + assert "Access-Control-Allow-Origin" not in denied.headers + finally: + await client.close() + + @pytest.mark.asyncio + async def test_rate_limit_returns_429(self): + limiter = _RateLimiter(max_requests=3, window_s=60) + client = await _client_for([_make_rate_limit_middleware(limiter)]) + try: + for _ in range(3): + assert (await client.get("/v1/whales")).status == 200 + assert (await client.get("/v1/whales")).status == 429 + finally: + await client.close() + + @pytest.mark.asyncio + async def test_health_probes_exempt_from_rate_limit(self): + """LBs/monitors behind one NAT IP poll health continuously — a 429 + there makes the balancer mark the backend down.""" + limiter = _RateLimiter(max_requests=2, window_s=60) + client = await _client_for([_make_rate_limit_middleware(limiter)]) + try: + for _ in range(10): + assert (await client.get("/v1/health")).status == 200 + assert (await client.get("/v1/live")).status == 200 + # Non-exempt routes still consume the budget normally. + assert (await client.get("/v1/whales")).status == 200 + assert (await client.get("/v1/whales")).status == 200 + assert (await client.get("/v1/whales")).status == 429 + finally: + await client.close() + + +class TestRateLimiterUnit: + def test_sliding_window(self): + limiter = _RateLimiter(max_requests=2, window_s=10) + assert limiter.allow("ip", now=100.0) + assert limiter.allow("ip", now=101.0) + assert not limiter.allow("ip", now=102.0) + # Window slides: the first hit expires. + assert limiter.allow("ip", now=110.5) + + def test_per_key_isolation(self): + limiter = _RateLimiter(max_requests=1, window_s=10) + assert limiter.allow("a", now=1.0) + assert limiter.allow("b", now=1.0) + assert not limiter.allow("a", now=2.0) + + +# ── WebSocket abuse limits ─────────────────────────────────────── + +class TestWSLimits: + def _client(self) -> _WSClient: + ws = MagicMock() + ws.closed = False + return _WSClient(ws) + + @pytest.mark.asyncio + async def test_bad_messages_disconnect(self): + api = _api() + client = self._client() + for i in range(WS_BAD_MSG_LIMIT - 1): + assert api._ws_msg_violates_limits(client, "{not json") is False + assert api._ws_msg_violates_limits(client, "{not json") is True + + @pytest.mark.asyncio + async def test_subscribe_still_works(self): + api = _api() + client = self._client() + raw = json.dumps({"subscribe": ["trade", "bogus_channel"]}) + assert api._ws_msg_violates_limits(client, raw) is False + assert client.subscriptions == {"trade"} + # Confirmation got queued for the writer task. + assert client.queue.qsize() == 1 + + @pytest.mark.asyncio + async def test_message_flood_disconnects(self): + api = _api() + client = self._client() + raw = json.dumps({"subscribe": ["trade"]}) + violated = False + for _ in range(50): + if api._ws_msg_violates_limits(client, raw): + violated = True + break + assert violated + + @pytest.mark.asyncio + async def test_broadcast_drops_when_queue_full(self): + api = _api() + client = self._client() + client.subscriptions = {"trade"} + api._ws_clients.append(client) + # No writer task draining -> queue fills to maxsize then drops. + for _ in range(client.queue.maxsize + 10): + api._broadcast("trade", {"x": 1}) + assert client.queue.qsize() == client.queue.maxsize + assert client.dropped_msgs == 10 + + +# ── Liquidation dedup + cascade bypass ─────────────────────────── + +class TestLiquidationDedup: + def test_same_event_deduped(self): + api = _api() + ev = _liq_event() + assert api._is_duplicate_liq(ev) is False + assert api._is_duplicate_liq(ev) is True + + def test_different_exchanges_not_deduped(self): + api = _api() + ts = time.time() + assert api._is_duplicate_liq(_liq_event(exchange="binance", timestamp=ts)) is False + assert api._is_duplicate_liq(_liq_event(exchange="okx", timestamp=ts)) is False + + def test_dedup_uses_exchange_timestamp_not_local_clock(self): + api = _api() + base = 1_700_000_001.0 + # Two records of the same event in different dedup buckets by + # exchange time are distinct regardless of local arrival time. + assert api._is_duplicate_liq(_liq_event(timestamp=base)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=base + 9.0)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=base + 0.5)) is True + + def test_implausible_timestamp_skips_dedup_never_local_clock(self): + """ts=0 (parse fallback) or ms-scale ts must not be hashed — falling + back to the local clock would collide distinct events that merely + arrived together.""" + api = _api() + # Missing/zero timestamp: identical-looking events both broadcast. + assert api._is_duplicate_liq(_liq_event(timestamp=0.0)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=0.0)) is False + # Millisecond-scale (connector forgot /1000): not safely dedupable. + assert api._is_duplicate_liq(_liq_event(timestamp=1.7e12)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=1.7e12)) is False + + def test_similar_but_distinct_sizes_not_deduped(self): + """$99,950 and $100,050 both round to $100k at round(-2) — the hash + must use the exact size so distinct events never collapse.""" + api = _api() + ts = 1_700_000_001.0 + assert api._is_duplicate_liq(_liq_event(timestamp=ts, size_usd=99_950.0)) is False + assert api._is_duplicate_liq(_liq_event(timestamp=ts, size_usd=100_050.0)) is False + # An exact replay (identical payload) still dedups. + assert api._is_duplicate_liq(_liq_event(timestamp=ts, size_usd=99_950.0)) is True + + def test_cascade_bypass_has_absolute_cap(self): + """Continuous cascade re-triggers must not extend the dedup bypass + forever (replayed duplicates would keep the floodgate open).""" + api = _api() + ev = _liq_event() + key = f"{ev.symbol}_{ev.side}_{ev.exchange}" + for _ in range(30): + api._check_cascade(ev) + started = api._cascade_bypass_started[key] + cap = started + 2 * api._CASCADE_BYPASS_DURATION + assert api._cascade_bypass[key] <= cap + 1e-6 + + def test_cascade_bypass_lifts_dedup_for_own_venue_only(self): + api = _api() + ts = time.time() + binance = _liq_event(exchange="binance", timestamp=ts) + # Three rapid events trigger the cascade bypass for binance/BTC/long. + for _ in range(3): + api._check_cascade(binance) + # A duplicate binance record now passes (cascade mode)... + assert api._is_duplicate_liq(binance) is False + assert api._is_duplicate_liq(binance) is False + # ...but hyperliquid's heuristic stream still dedups normally. + hl = _liq_event(exchange="hyperliquid", timestamp=ts) + assert api._is_duplicate_liq(hl) is False + assert api._is_duplicate_liq(hl) is True + + +# ── Malformed exchange payloads ────────────────────────────────── + +class TestMalformedPayloads: + @pytest.mark.asyncio + async def test_binance_malformed_dropped_not_raised(self): + feed = LiquidationFeed() + conn = BinanceConnection(feed) + received = [] + feed.on_liquidation(received.append) + + await conn._on_message({"e": "forceOrder", "o": {"p": "", "q": "1", "S": "SELL", + "s": "BTCUSDT", "T": 1}}) + await conn._on_message({"e": "forceOrder", "o": {}}) + await conn._on_message({"e": "forceOrder"}) + assert received == [] + assert feed.parse_errors["binance"] == 3 + + # A valid message still parses after the bad ones. + await conn._on_message({"e": "forceOrder", "o": { + "p": "70000", "q": "0.5", "S": "SELL", "s": "BTCUSDT", "T": 1700000000000, + }}) + assert len(received) == 1 + assert received[0].symbol == "BTC" + assert received[0].side == "long" + + @pytest.mark.asyncio + async def test_okx_malformed_detail_dropped_individually(self): + feed = LiquidationFeed() + conn = OKXConnection(feed) + received = [] + feed.on_liquidation(received.append) + + await conn._on_message({"data": [{ + "instId": "BTC-USDT-SWAP", + "details": [ + {"bkPx": "", "sz": "bogus", "side": "sell", "ts": "x"}, # bad + {"bkPx": "70000", "sz": "1", "side": "sell", "ts": "1700000000000"}, + ], + }]}) + assert len(received) == 1 + assert received[0].symbol == "BTC" + assert feed.parse_errors["okx"] == 1 + + @pytest.mark.asyncio + async def test_okx_non_list_shapes(self): + feed = LiquidationFeed() + conn = OKXConnection(feed) + await conn._on_message({"data": "not-a-list"}) + await conn._on_message({"data": [{"instId": "X", "details": "not-a-list"}]}) + assert feed.parse_errors["okx"] == 2 + + def test_orderbook_malformed_levels_dropped(self): + engine = OrderBookEngine(symbols=["BTC"]) + engine._update_book("BTC", {"levels": [ + [{"px": "70000", "sz": "1"}, {"px": "", "sz": "zzz"}, "garbage"], + [{"px": "70010", "sz": "2"}], + ]}) + book = engine.books["BTC"] + assert len(book["bids"]) == 1 + assert book["bids"][0].price == 70000.0 + assert len(book["asks"]) == 1 + + def test_orderbook_non_list_levels_ignored(self): + engine = OrderBookEngine(symbols=["BTC"]) + engine._update_book("BTC", {"levels": {"bad": "shape"}}) + assert engine.books["BTC"]["bids"] == [] + + +# ── Degraded hub startup ───────────────────────────────────────── + +class TestHubDegradedStartup: + @pytest.mark.asyncio + async def test_failed_component_recorded_not_swallowed(self, tmp_path, monkeypatch): + from src.data_layer import address_store, persistence + monkeypatch.setattr(persistence, "DB_PATH", tmp_path / "hub.db") + monkeypatch.setattr(address_store, "DATA_DIR", tmp_path) + monkeypatch.setattr(address_store, "DB_PATH", tmp_path / "hub.db") + monkeypatch.setattr(address_store, "LEGACY_JSON", tmp_path / "legacy.json") + monkeypatch.setattr(address_store, "_initialized", False) + + from src.data_layer.hub import HyperDataHub + hub = HyperDataHub() + # Every component start is stubbed: one fails, the rest succeed. + hub.liquidations.start = AsyncMock(side_effect=ConnectionError("down")) + for comp in (hub.orderflow, hub.smart_money, hub.hlp, hub.funding, + hub.lsr, hub.orderbook, hub.deribit): + comp.start = AsyncMock() + hub.spot.start = AsyncMock() + + await hub._start_live() + + assert hub.status.failed_components == ["liquidation_feed"] + assert hub.status.liquidation_feed == "error" + assert hub.status.orderflow_engine == "connected" + # Loop-driven components are 'starting', never a blind 'ready'. + assert hub.status.position_scanner == "starting" + assert hub.status.market_data == "starting" + + hub.store.close() + + +# ── Paper trader accounting invariants ─────────────────────────── + +class TestPaperTraderAccounting: + def _trader(self, price=100.0, balance=10_000.0) -> PaperTrader: + hub = MagicMock() + hub.market.assets = {"BTC": SimpleNamespace(price=price)} + return PaperTrader(hub, [], starting_balance=balance) + + def test_add_to_position_is_balance_checked(self): + trader = self._trader(balance=10_000.0) + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=6_000.0)) + assert trader.balance == 4_000.0 + # Second same-direction BUY exceeds the remaining balance -> rejected. + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=6_000.0)) + assert trader.balance == 4_000.0 + assert trader.positions["BTC"]["size_usd"] == 6_000.0 + assert trader.balance >= 0 + + def test_repeated_buys_never_go_negative(self): + trader = self._trader(balance=1_000.0) + for _ in range(50): + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=400.0)) + assert trader.balance >= 0 + assert trader.positions["BTC"]["size_usd"] == 800.0 + + def test_weighted_average_entry_price(self): + trader = self._trader(price=100.0) + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=1_000.0)) + trader.hub.market.assets["BTC"].price = 200.0 + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=1_000.0)) + pos = trader.positions["BTC"] + # (100*1000 + 200*1000) / 2000 = 150 + assert abs(pos["entry_price"] - 150.0) < 1e-9 + assert pos["size_usd"] == 2_000.0 + + def test_close_realizes_pnl(self): + trader = self._trader(price=100.0, balance=1_000.0) + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=500.0)) + trader.hub.market.assets["BTC"].price = 110.0 + trader._execute_trade("t", Signal("BTC", "SELL", size_usd=500.0)) + # +10% on 500 = +50 + assert abs(trader.balance - 1_050.0) < 1e-9 + assert "BTC" not in trader.positions + + def test_invalid_signals_rejected(self): + trader = self._trader() + for bad in [ + Signal("BTC", "BUY", size_usd=-5.0), + Signal("BTC", "BUY", size_usd=0.0), + Signal("BTC", "BUY", size_usd=float("nan")), + Signal("BTC", "BUY", size_usd=float("inf")), + Signal("BTC", "HOLD", size_usd=100.0), + Signal("", "BUY", size_usd=100.0), + ]: + trader._execute_trade("t", bad) + assert trader.positions == {} + assert trader.balance == 10_000.0 + + +# ── LLM response parsing ───────────────────────────────────────── + +class TestLLMParsing: + def _agent(self) -> LLMAgent: + return LLMAgent(symbol="BTC") + + def test_exact_actions(self): + agent = self._agent() + assert agent._parse_response("BUY\nmomentum").action == "BUY" + assert agent._parse_response("SELL\nfunding").action == "SELL" + assert agent._parse_response("HOLD\nchop") is None + + def test_case_and_punctuation_tolerated(self): + agent = self._agent() + assert agent._parse_response("buy.").action == "BUY" + assert agent._parse_response("**SELL**\nx").action == "SELL" + + def test_ambiguous_rejected_not_substring_matched(self): + agent = self._agent() + assert agent._parse_response("I would not BUY here") is None + assert agent._parse_response("BUY or SELL depending on funding") is None + assert agent._parse_response("Definitely bullish") is None + assert agent._parse_response("") is None + + def test_eval_budget(self): + agent = self._agent() + agent.max_evals_per_hour = 2 + agent._eval_times.clear() + assert agent._within_budget(now=100.0) + assert agent._within_budget(now=101.0) + assert not agent._within_budget(now=102.0) + # Window slides after an hour. + assert agent._within_budget(now=100.0 + 3601) + + +# ── Persistence: quarantine + schema version ───────────────────── + +class TestPersistence: + def test_corrupted_db_quarantined_not_deleted(self, tmp_path): + db_path = tmp_path / "hyperdata.db" + db_path.write_bytes(b"this is not a sqlite database " * 100) + + store = DataStore(db_path) + try: + quarantine = tmp_path / "corrupted" + quarantined = list(quarantine.glob("hyperdata.db.*")) + assert len(quarantined) == 1 + # Original bytes preserved for postmortem. + assert quarantined[0].read_bytes().startswith(b"this is not") + # Fresh DB works. + assert store.get_db_stats()["liquidations_stored"] == 0 + finally: + store.close() + + def test_schema_version_recorded(self, tmp_path): + store = DataStore(tmp_path / "fresh.db") + try: + assert store.get_schema_version() == DataStore.SCHEMA_VERSION + finally: + store.close() + + def test_reopen_keeps_schema_version(self, tmp_path): + path = tmp_path / "reopen.db" + DataStore(path).close() + store = DataStore(path) + try: + assert store.get_schema_version() == DataStore.SCHEMA_VERSION + finally: + store.close() + + +# ── Alert redaction ────────────────────────────────────────────── + +class TestAlertRedaction: + @pytest.mark.asyncio + async def test_alert_payload_not_logged(self, caplog): + from data_layer.alerts import AlertManager + mgr = AlertManager() + mgr.telegram_token = "" # no channels configured + mgr.discord_webhook = "" + secret_wallet = "0x" + "ab" * 20 + message = f"whale alert\nwallet {secret_wallet} is near liquidation" + + with caplog.at_level("WARNING", logger="data_layer.alerts"): + await mgr._send(message) + log_text = caplog.text + assert secret_wallet not in log_text + assert "ALERT sent" in log_text + await mgr.stop() + + +# ── Smart money ranking thresholds ─────────────────────────────── + +class TestSmartMoneyThresholds: + def _wallet(self, engine, addr, trades, volume, score): + from data_layer.smart_money import WalletProfile + w = WalletProfile( + address=addr, discovered_at=0, last_seen=0, last_analyzed=0, + total_trades=trades, total_volume_usd=volume, composite_score=score, + ) + engine.wallets[addr] = w + return w + + def test_small_samples_not_ranked(self): + from data_layer.smart_money import SmartMoneyEngine + engine = SmartMoneyEngine() + tiny = self._wallet(engine, "0x" + "1" * 40, trades=3, volume=1e6, score=0.9) + thin = self._wallet(engine, "0x" + "2" * 40, trades=50, volume=100.0, score=0.9) + solid = self._wallet(engine, "0x" + "3" * 40, + trades=SmartMoneyEngine.MIN_TRADES_FOR_RANKING, + volume=SmartMoneyEngine.MIN_VOLUME_FOR_RANKING, score=0.5) + engine.rank_all() + assert tiny.rank == 0 and tiny.tier == "unknown" + assert thin.rank == 0 and thin.tier == "unknown" + assert solid.rank == 1 and solid.tier == "smart" + + def test_disqualified_wallet_loses_stale_tier(self): + from data_layer.smart_money import SmartMoneyEngine + engine = SmartMoneyEngine() + w = self._wallet(engine, "0x" + "4" * 40, trades=20, volume=1e6, score=0.8) + engine.rank_all() + assert w.tier == "smart" + w.total_trades = 2 # sample no longer qualifies + engine.rank_all() + assert w.rank == 0 + assert w.tier == "unknown" + + def test_confidence_scales_with_sample(self): + from data_layer.smart_money import SmartMoneyEngine, WalletProfile + engine = SmartMoneyEngine() + w = WalletProfile(address="0x" + "5" * 40, discovered_at=0, + last_seen=0, last_analyzed=0, total_trades=25) + assert engine._compute_confidence(w) == 0.5 + w.total_trades = 500 + assert engine._compute_confidence(w) == 1.0 + + +# ── Per-venue orderflow freshness ──────────────────────────────── + +class TestPerVenueFreshness: + def test_dead_venue_visible_while_combined_fresh(self): + from data_layer.orderflow_engine import OrderFlowEngine + e = OrderFlowEngine(symbols=["BTC"]) + now = time.time() + e.last_hl_message_at = now - 1000 # HL dead + e.last_binance_message_at = now - 1 # Binance fresh + assert e.is_stale() is False # combined follows freshest + assert e.venue_is_stale("hyperliquid") is True + assert e.venue_is_stale("binance") is False + fresh = e.venue_freshness() + assert fresh["hyperliquid"]["stale"] is True + assert fresh["binance"]["stale"] is False + + def test_no_data_reports_none_age(self): + from data_layer.orderflow_engine import OrderFlowEngine + e = OrderFlowEngine(symbols=["BTC"]) + fresh = e.venue_freshness() + assert fresh["hyperliquid"]["data_age_seconds"] is None + assert fresh["hyperliquid"]["stale"] is True + + +# ── Review round 2: WS origin, balance floor, persist-first, feeds ── + +class TestWSOriginCheck: + @pytest.mark.asyncio + async def test_browser_origin_rejected_by_default(self): + """Browser WS is not gated by SOP: any webpage can open + ws://127.0.0.1 — an unlisted Origin must be refused.""" + api = _api() + api._cors_origins = None # loopback wildcard REST CORS + request = MagicMock() + request.headers = {"Origin": "https://evil.example"} + resp = await api.handle_ws(request) + assert resp.status == 403 + + @pytest.mark.asyncio + async def test_allowlisted_origin_passes_the_gate(self): + api = _api() + api._cors_origins = {"https://ok.example"} + request = MagicMock() + request.headers = {"Origin": "https://bad.example"} + assert (await api.handle_ws(request)).status == 403 + # An allowlisted origin proceeds past the origin gate (the next + # check is the connection cap, exercised here by filling it). + api._ws_clients = [MagicMock()] * 100 + request.headers = {"Origin": "https://ok.example"} + assert (await api.handle_ws(request)).status == 429 + + +class TestPaperTraderRound2: + def _trader(self, price=100.0, balance=10_000.0) -> PaperTrader: + hub = MagicMock() + hub.market.assets = {"BTC": SimpleNamespace(price=price)} + return PaperTrader(hub, [], starting_balance=balance) + + def test_catastrophic_close_floors_at_zero(self): + """A short losing far more than the posted margin must not drive + the account balance negative.""" + trader = self._trader(price=100.0, balance=1_000.0) + trader._execute_trade("t", Signal("BTC", "SELL", size_usd=1_000.0)) # short + trader.hub.market.assets["BTC"].price = 10_000.0 # +9900% against us + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=1_000.0)) # close + assert trader.balance == 0.0 + assert "BTC" not in trader.positions + + def test_db_error_means_trade_not_executed(self): + """Persist-first: a trade that cannot be logged must not mutate the + books, or the portfolio silently diverges from the audit trail.""" + trader = self._trader(balance=5_000.0) + db = MagicMock() + db.execute.side_effect = sqlite3.OperationalError("disk full") + trader._db = db + trader._execute_trade("t", Signal("BTC", "BUY", size_usd=1_000.0)) + assert trader.balance == 5_000.0 + assert trader.positions == {} + assert trader.trades == [] + + +class TestLLMRound2: + def test_blank_lines_before_action_tolerated(self): + agent = LLMAgent(symbol="BTC") + assert agent._parse_response("\n\nBUY\nmomentum").action == "BUY" + assert agent._parse_response(" \nSELL") .action == "SELL" + assert agent._parse_response("\n\n") is None + + def test_transport_failure_refunds_budget_slot(self, monkeypatch): + """A down provider must not exhaust the hourly budget: transport + failures consumed no tokens, so their slots are returned.""" + agent = LLMAgent(symbol="BTC") + agent.api_key = "k" + agent.base_url = "https://llm.example/v1" + assert agent._within_budget(now=100.0) + assert len(agent._eval_times) == 1 + + def boom(*a, **kw): + raise OSError("connection refused") + + monkeypatch.setattr("urllib.request.urlopen", boom) + hub = MagicMock() + hub.market.assets = {"BTC": SimpleNamespace(price=100.0, funding_rate=0.0)} + assert agent._sync_evaluate(hub) is None + assert len(agent._eval_times) == 0 # slot refunded + + @pytest.mark.asyncio + async def test_evaluate_is_async_and_skips_without_key(self): + """evaluate() is awaited by the paper trader so a slow LLM cannot + block the event loop for other strategies.""" + agent = LLMAgent(symbol="BTC") + agent.api_key = "" + agent.base_url = "https://api.example.com/v1" # non-local, no key + result = agent.evaluate(MagicMock()) + assert inspect.isawaitable(result) + assert await result is None + + +class TestFeedsRound2: + @pytest.mark.asyncio + async def test_binance_array_frame_parsed(self): + """@arr frames may batch events into a JSON array — an array frame + must parse instead of silently failing an isinstance-dict check.""" + feed = LiquidationFeed() + conn = BinanceConnection(feed) + received = [] + feed.on_liquidation(received.append) + frame = [ + {"e": "forceOrder", "o": {"p": "70000", "q": "0.5", "S": "SELL", + "s": "BTCUSDT", "T": 1700000000000}}, + {"e": "forceOrder", "o": {"p": "3500", "q": "2", "S": "BUY", + "s": "ETHUSDT", "T": 1700000000001}}, + ] + await conn._on_message(frame) + assert [ev.symbol for ev in received] == ["BTC", "ETH"] + + @pytest.mark.asyncio + async def test_bybit_v5_list_payload_with_short_keys(self): + """Bybit v5 allLiquidation sends data as a LIST of records with + short keys (p/v/S/s/T) — both shapes must parse without raising.""" + feed = LiquidationFeed() + conn = BybitConnection(feed) + received = [] + feed.on_liquidation(received.append) + await conn._on_message({ + "topic": "allLiquidation.BTCUSDT", + "data": [ + {"p": "70000", "v": "0.5", "S": "Sell", "s": "BTCUSDT", + "T": 1700000000000}, + {"p": "", "v": None, "S": None, "s": None, "T": "x"}, # malformed + ], + }) + assert len(received) == 1 + assert received[0].symbol == "BTC" + assert received[0].side == "long" + assert feed.parse_errors["bybit"] == 1 + + @pytest.mark.asyncio + async def test_bybit_non_dict_data_does_not_raise(self): + feed = LiquidationFeed() + conn = BybitConnection(feed) + await conn._on_message({"topic": "allLiquidation.BTCUSDT", "data": "junk"}) + await conn._on_message({"topic": "allLiquidation.BTCUSDT", "data": [None, 42]}) + assert feed.parse_errors["bybit"] >= 2 + + +class TestPersistenceRound2: + def test_locked_db_raises_instead_of_quarantining(self, tmp_path, monkeypatch): + """'database is locked' is contention, not corruption — a healthy DB + held by another process must never be quarantined.""" + import data_layer.persistence as persistence_mod + + real_connect = persistence_mod.sqlite3.connect + + def locked_connect(*args, **kwargs): + raise sqlite3.OperationalError("database is locked") + + monkeypatch.setattr(persistence_mod.sqlite3, "connect", locked_connect) + with pytest.raises(sqlite3.OperationalError, match="locked"): + DataStore(tmp_path / "locked.db") + monkeypatch.setattr(persistence_mod.sqlite3, "connect", real_connect) + assert not (tmp_path / "corrupted").exists() + + +class TestAlertsRound2: + @pytest.mark.asyncio + async def test_send_failure_does_not_leak_token(self, caplog): + """aiohttp error messages can embed the request URL — which contains + the bot token — so failure logs carry the exception type only.""" + from data_layer.alerts import AlertManager + mgr = AlertManager() + mgr.telegram_token = "123456:SECRET-TOKEN-VALUE" + mgr.telegram_chat_id = "42" + mgr.discord_webhook = "" + session = MagicMock() + session.post = MagicMock(side_effect=RuntimeError( + f"cannot connect to https://api.telegram.org/bot{mgr.telegram_token}/sendMessage" + )) + mgr._session = session + + with caplog.at_level("DEBUG", logger="data_layer.alerts"): + await mgr._send("test message") + assert "SECRET-TOKEN-VALUE" not in caplog.text + assert "Telegram send failed" in caplog.text + + @pytest.mark.asyncio + async def test_wallet_on_first_line_still_redacted(self, caplog): + from data_layer.alerts import AlertManager + mgr = AlertManager() + mgr.telegram_token = "" + mgr.discord_webhook = "" + wallet = "0x" + "cd" * 20 + with caplog.at_level("WARNING", logger="data_layer.alerts"): + await mgr._send(f"whale {wallet} near liquidation") + assert wallet not in caplog.text + assert "ALERT sent" in caplog.text + await mgr.stop() + + +class TestCascadeExampleStrategy: + def test_cascade_strategy_actually_fires(self): + """getattr on the stats dict always returned 0 and silently disabled + this strategy — dict access must read the real key.""" + from src.strategies.examples import LiquidationCascade + hub = MagicMock() + hub.liquidations.get_stats.return_value = {"long_volume_usd": 2_000_000.0} + strat = LiquidationCascade(symbol="BTC", cascade_threshold_usd=1_000_000) + signal = strat.evaluate(hub) + assert signal is not None + assert signal.action == "BUY" diff --git a/tests/test_long_short_ratio.py b/tests/test_long_short_ratio.py index 279e422..bdab713 100644 --- a/tests/test_long_short_ratio.py +++ b/tests/test_long_short_ratio.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.long_short_ratio import LongShortSnapshot, LongShortCollector + +from data_layer.long_short_ratio import LongShortCollector, LongShortSnapshot def test_snapshot_fields(): diff --git a/tests/test_market_data.py b/tests/test_market_data.py index 9625e9a..5988706 100644 --- a/tests/test_market_data.py +++ b/tests/test_market_data.py @@ -2,6 +2,7 @@ from __future__ import annotations import pytest + from data_layer.market_data import AssetInfo, _timeframe_to_ms diff --git a/tests/test_orderbook.py b/tests/test_orderbook.py index 6a46f82..35274fd 100644 --- a/tests/test_orderbook.py +++ b/tests/test_orderbook.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.orderbook import OrderBookLevel, OrderBookSnapshot, OrderBookEngine, compute_imbalance + +from data_layer.orderbook import OrderBookEngine, OrderBookLevel, compute_imbalance def test_order_book_level(): diff --git a/tests/test_orderflow.py b/tests/test_orderflow.py index f78c687..b1f853a 100644 --- a/tests/test_orderflow.py +++ b/tests/test_orderflow.py @@ -6,9 +6,9 @@ import pytest from data_layer.orderflow_engine import ( - Trade, - TimeframeBucket, OrderFlowEngine, + TimeframeBucket, + Trade, classify_signal, ) diff --git a/tests/test_position_scanner.py b/tests/test_position_scanner.py index ed0205f..01e4eb8 100644 --- a/tests/test_position_scanner.py +++ b/tests/test_position_scanner.py @@ -1,23 +1,38 @@ +"""PositionScanner unit tests. + +All network calls are mocked; the SQLite-backed address store is redirected to +a per-test temp directory so tests never touch the repo's data/ files. +""" from __future__ import annotations -import asyncio -import json -from unittest.mock import AsyncMock, MagicMock, patch +import sqlite3 +from unittest.mock import AsyncMock, MagicMock import pytest -from src.data_layer.position_scanner import ( - DISCOVERED_ADDRESSES_PATH, - PositionScanner, - TrackedPosition, -) - +from data_layer.position_scanner import PositionScanner, TrackedPosition +from src.data_layer import address_store # ── Fixtures ───────────────────────────────────────────────────── +@pytest.fixture(autouse=True) +def isolated_address_store(tmp_path, monkeypatch): + """Point the address store at a temp SQLite DB for every test.""" + monkeypatch.setattr(address_store, "DATA_DIR", tmp_path) + monkeypatch.setattr(address_store, "DB_PATH", tmp_path / "hyperdata.db") + monkeypatch.setattr(address_store, "LEGACY_JSON", tmp_path / "discovered_addresses.json") + monkeypatch.setattr(address_store, "_initialized", False) + yield tmp_path + + +def _addr(seed: str) -> str: + """Build a valid 0x + 40-hex wallet address from a short seed.""" + return "0x" + (seed * 40)[:40] + + def _make_position(**overrides) -> TrackedPosition: defaults = dict( - address="0xabc", + address=_addr("abc"), symbol="BTC", side="long", size_usd=50_000.0, @@ -33,6 +48,17 @@ def _make_position(**overrides) -> TrackedPosition: return TrackedPosition(**defaults) +def _mock_session(json_payload) -> MagicMock: + session = MagicMock() + response = AsyncMock() + response.json = AsyncMock(return_value=json_payload) + response.raise_for_status = MagicMock() + response.__aenter__ = AsyncMock(return_value=response) + response.__aexit__ = AsyncMock(return_value=False) + session.post = MagicMock(return_value=response) + return session + + MOCK_ALL_MIDS = {"BTC": "71000.0", "ETH": "3500.0", "SOL": "150.0"} MOCK_META = { @@ -77,7 +103,7 @@ def _make_position(**overrides) -> TrackedPosition: class TestTrackedPosition: def test_creation(self): p = _make_position() - assert p.address == "0xabc" + assert p.address == _addr("abc") assert p.side == "long" assert p.leverage == 10.0 @@ -137,9 +163,9 @@ def test_unknown_coin_defaults(self): class TestFilterMethods: - def setup_method(self): - self.scanner = PositionScanner() - self.scanner.positions = [ + def setup_positions(self): + scanner = PositionScanner() + scanner.positions = [ _make_position(side="long", distance_pct=0.5, size_usd=100_000), _make_position(side="long", distance_pct=1.5, size_usd=50_000), _make_position(side="short", distance_pct=0.8, size_usd=200_000), @@ -147,79 +173,111 @@ def setup_method(self): _make_position(side="long", distance_pct=4.5, size_usd=30_000), _make_position(side="short", distance_pct=10.0, size_usd=10_000), ] + return scanner def test_get_danger_zone_default(self): - danger = self.scanner.get_danger_zone() + danger = self.setup_positions().get_danger_zone() assert len(danger) == 3 assert all(p.distance_pct <= 2.0 for p in danger) def test_get_danger_zone_custom_threshold(self): - danger = self.scanner.get_danger_zone(threshold_pct=1.0) + danger = self.setup_positions().get_danger_zone(threshold_pct=1.0) assert len(danger) == 2 def test_get_closest_longs(self): - longs = self.scanner.get_closest_longs(n=2) + longs = self.setup_positions().get_closest_longs(n=2) assert len(longs) == 2 assert all(p.side == "long" for p in longs) assert longs[0].distance_pct < longs[1].distance_pct def test_get_closest_shorts(self): - shorts = self.scanner.get_closest_shorts(n=2) + shorts = self.setup_positions().get_closest_shorts(n=2) assert len(shorts) == 2 assert all(p.side == "short" for p in shorts) assert shorts[0].distance_pct < shorts[1].distance_pct def test_get_closest_longs_more_than_available(self): - longs = self.scanner.get_closest_longs(n=100) + longs = self.setup_positions().get_closest_longs(n=100) assert len(longs) == 3 def test_get_zone_summary(self): - summary = self.scanner.get_zone_summary() + summary = self.setup_positions().get_zone_summary() assert summary["within_1pct"]["count"] == 2 assert summary["within_1pct"]["total_value"] == 300_000 assert summary["within_2pct"]["count"] == 3 assert summary["within_5pct"]["count"] == 5 def test_get_zone_summary_empty(self): - self.scanner.positions = [] - summary = self.scanner.get_zone_summary() + scanner = self.setup_positions() + scanner.positions = [] + summary = scanner.get_zone_summary() assert summary["within_1pct"]["count"] == 0 assert summary["within_5pct"]["total_value"] == 0.0 class TestAddressPersistence: - def test_add_addresses(self, tmp_path, monkeypatch): - addr_file = tmp_path / "discovered_addresses.json" - monkeypatch.setattr( - "src.data_layer.position_scanner.DISCOVERED_ADDRESSES_PATH", addr_file - ) + """Address persistence is SQLite-backed (data_layer.address_store).""" + + def test_add_addresses_persists_to_sqlite(self, isolated_address_store): scanner = PositionScanner() - scanner.add_addresses(["0xaaa", "0xbbb"]) - assert "0xaaa" in scanner.discovered_addresses - assert addr_file.exists() - - loaded = json.loads(addr_file.read_text()) - assert "0xaaa" in loaded - assert "0xbbb" in loaded - - def test_load_existing_addresses(self, tmp_path, monkeypatch): - addr_file = tmp_path / "discovered_addresses.json" - addr_file.write_text(json.dumps(["0x111", "0x222"])) - monkeypatch.setattr( - "src.data_layer.position_scanner.DISCOVERED_ADDRESSES_PATH", addr_file - ) + a1, a2 = _addr("aaa"), _addr("bbb") + scanner.add_addresses([a1, a2]) + + assert a1 in scanner.discovered_addresses + assert a2 in scanner.discovered_addresses + + # Rows actually landed in the discovered_addresses table. + conn = sqlite3.connect(str(isolated_address_store / "hyperdata.db")) + rows = { + r[0] for r in + conn.execute("SELECT address FROM discovered_addresses").fetchall() + } + conn.close() + assert {a1, a2} <= rows + + def test_load_existing_addresses(self): + a1, a2 = _addr("111"), _addr("222") + address_store.add_addresses([a1, a2], source="test") + + scanner = PositionScanner() + assert a1 in scanner.discovered_addresses + assert a2 in scanner.discovered_addresses + + def test_invalid_addresses_rejected(self, isolated_address_store): scanner = PositionScanner() - assert "0x111" in scanner.discovered_addresses - assert "0x222" in scanner.discovered_addresses - - def test_corrupted_file_handled(self, tmp_path, monkeypatch): - addr_file = tmp_path / "discovered_addresses.json" - addr_file.write_text("NOT VALID JSON {{{") - monkeypatch.setattr( - "src.data_layer.position_scanner.DISCOVERED_ADDRESSES_PATH", addr_file - ) + scanner.add_addresses([ + "not-an-address", + "0xTOOSHORT", + "0x" + "g" * 40, # non-hex + _addr("c0ffee"), # the only valid one + ]) + assert scanner.discovered_addresses == {_addr("c0ffee")} + + conn = sqlite3.connect(str(isolated_address_store / "hyperdata.db")) + count = conn.execute("SELECT COUNT(*) FROM discovered_addresses").fetchone()[0] + conn.close() + assert count == 1 + + def test_addresses_normalized_to_lowercase(self): + mixed = "0x" + "AbCdEf0123456789aBcDeF0123456789ABCDEF01" scanner = PositionScanner() - assert len(scanner.discovered_addresses) == 0 + scanner.add_addresses([mixed]) + assert mixed.lower() in scanner.discovered_addresses + assert mixed not in scanner.discovered_addresses + + def test_retention_cap_expires_oldest(self, monkeypatch): + monkeypatch.setattr(address_store, "MAX_TRACKED_ADDRESSES", 3) + for i in range(5): + address_store.add_addresses([_addr(f"{i}{i}{i}")], source="test") + remaining = address_store.get_all_addresses() + assert len(remaining) == 3 + + def test_store_validator(self): + assert address_store.is_valid_address(_addr("abc")) + assert not address_store.is_valid_address("0xabc") + assert not address_store.is_valid_address(None) + assert not address_store.is_valid_address(42) + assert not address_store.is_valid_address("0x" + "z" * 40) class TestGetPositionsForAddress: @@ -228,18 +286,9 @@ async def test_parses_clearinghouse_state(self): scanner = PositionScanner() scanner.market_prices = {"BTC": 71_000.0, "ETH": 3_500.0} scanner.market_meta = MOCK_META + scanner._session = _mock_session(MOCK_CLEARINGHOUSE_STATE) - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=MOCK_CLEARINGHOUSE_STATE) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) - - scanner._session = mock_session - - positions = await scanner.get_positions_for_address("0xtest") + positions = await scanner.get_positions_for_address(_addr("fed")) assert len(positions) == 2 @@ -259,18 +308,9 @@ async def test_empty_positions(self): scanner = PositionScanner() scanner.market_prices = {} scanner.market_meta = MOCK_META + scanner._session = _mock_session({"assetPositions": [], "marginSummary": {}}) - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value={"assetPositions": [], "marginSummary": {}}) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) - - scanner._session = mock_session - - positions = await scanner.get_positions_for_address("0xempty") + positions = await scanner.get_positions_for_address(_addr("e")) assert positions == [] @pytest.mark.asyncio @@ -295,36 +335,38 @@ async def test_fallback_liq_price_when_missing(self): scanner = PositionScanner() scanner.market_prices = {"BTC": 70_000.0} scanner.market_meta = MOCK_META + scanner._session = _mock_session(state) - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=state) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) - - scanner._session = mock_session - - positions = await scanner.get_positions_for_address("0xfallback") + positions = await scanner.get_positions_for_address(_addr("f")) assert len(positions) == 1 assert abs(positions[0].liq_price - 63_210.0) < 0.01 -class TestUpdatePrices: +class TestDiscoverAddresses: @pytest.mark.asyncio - async def test_update_prices(self): + async def test_discovery_validates_payload_addresses(self): + """Junk strings in exchange trade payloads must never be persisted.""" + good = _addr("dead") + trades = [ + {"buyer": good, "seller": "junk-string"}, + {"users": [good, "0xshort", 12345, None]}, + {"users": "not-a-list"}, + ] scanner = PositionScanner() + scanner._session = _mock_session(trades) - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=MOCK_ALL_MIDS) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) + discovered = await scanner.discover_addresses(limit=10) + assert good in discovered + assert "junk-string" not in discovered + assert "0xshort" not in discovered + assert address_store.get_all_addresses() == {good} - scanner._session = mock_session + +class TestUpdatePrices: + @pytest.mark.asyncio + async def test_update_prices(self): + scanner = PositionScanner() + scanner._session = _mock_session(MOCK_ALL_MIDS) await scanner.update_prices() assert scanner.market_prices["BTC"] == 71_000.0 @@ -336,16 +378,7 @@ class TestUpdateMeta: @pytest.mark.asyncio async def test_update_meta(self): scanner = PositionScanner() - - mock_session = MagicMock() - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=MOCK_META) - mock_response.raise_for_status = MagicMock() - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - mock_session.post = MagicMock(return_value=mock_response) - - scanner._session = mock_session + scanner._session = _mock_session(MOCK_META) await scanner.update_meta() assert scanner.market_meta == MOCK_META @@ -367,10 +400,6 @@ async def test_meta_caching(self): class TestDistanceCalculation: def test_distance_for_long(self): - scanner = PositionScanner() - scanner.market_prices = {"BTC": 70_000.0} - scanner.market_meta = MOCK_META - # liq at 63500, current at 70000 # distance = |70000 - 63500| / 70000 * 100 = 9.2857% current = 70_000.0 @@ -399,3 +428,16 @@ async def test_rate_limit_tracks_requests(self): for _ in range(5): await scanner._rate_limit() assert len(scanner._request_times) == 5 + + +class TestPostTimeout: + @pytest.mark.asyncio + async def test_post_sends_explicit_timeout(self): + """Every outbound request must carry an explicit deadline (High 4).""" + scanner = PositionScanner() + session = _mock_session({}) + scanner._session = session + await scanner._post({"type": "allMids"}) + _, kwargs = session.post.call_args + assert kwargs.get("timeout") is not None + assert kwargs["timeout"].total == 10 diff --git a/tests/test_spot_prices.py b/tests/test_spot_prices.py index d1cad11..63c3a4a 100644 --- a/tests/test_spot_prices.py +++ b/tests/test_spot_prices.py @@ -2,8 +2,10 @@ from __future__ import annotations import time + import pytest -from data_layer.spot_prices import SpotPriceSnapshot, SpotPriceCollector + +from data_layer.spot_prices import SpotPriceCollector, SpotPriceSnapshot def test_snapshot_fields():