diff --git a/.gitignore b/.gitignore index 50bd30e..bd69e27 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,9 @@ __pycache__/ .coverage htmlcov/ .pytest_cache/ + +wata_app*.zip + +.DS_Store + +execution_timing_*.jsonl \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8823670 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,273 @@ +# Changelog + +## v0.7.0 - Async Architecture & Real-Time Streaming + +**Release date:** 2026-03-29 + +This is a major architectural release that migrates WATA from a synchronous, polling-based system to a fully asynchronous, real-time streaming architecture. Every service-critical path is now `async/await`, the database backend moves from DuckDB to PostgreSQL, and position monitoring switches from 7-second REST API polling to Saxo's WebSocket Streaming API. + +--- + +### Breaking Changes + +- **Database: DuckDB → PostgreSQL.** The embedded DuckDB database is replaced by a PostgreSQL 16 instance. A migration tool is provided (see Migration Guide below). The `duckdb` config section is no longer used at runtime. +- **New service: `position_monitor`.** Position monitoring is now a dedicated container (`WATA_APP_ROLE=position_monitor`) that must be deployed alongside the existing services. +- **Scheduler no longer polls positions.** The `job_check_positions_on_saxo_api` job (every 7 seconds) has been removed. Position checks are now driven by real-time WebSocket streaming in the Position Monitor. +- **RabbitMQ queue split.** Trading signals now go to `trading-signals` (consumed by Trader); operational commands go to `trading-ops` (consumed by Position Monitor). Previously everything went to a single queue. +- **New config sections required.** `postgresql` and `trade.config.general.streaming` must be added to `config.json` (see Migration Guide). + +--- + +### New Features + +#### Real-Time WebSocket Streaming (Phase 2) +- **`src/saxo_streaming/client.py`** - New persistent async WebSocket client for Saxo Bank's streaming API. + - Subscribes to position updates via `POST port/v1/positions/subscriptions`. + - Parses binary frames using the existing `decode_ws_msg()` utility. + - Applies delta-compressed updates to an in-memory position snapshot. + - Handles all Saxo control messages: `_heartbeat`, `_resetsubscriptions`, `_disconnect`. + - Auto-reconnects with exponential backoff and `messageid` resume. + - Periodically re-authorises the WebSocket connection (default: every 15 minutes). + +#### Async Architecture (Phase 1) +- **`src/saxo_openapi/async_client.py`** - Async HTTP client using `httpx.AsyncClient` with HTTP/2, async rate limiting, and hot-swappable tokens. +- **`src/trade/async_services.py`** - Async reimplementation of all trade services: + - `AsyncSaxoApiClient` - token-refreshing facade. + - `AsyncInstrumentService` - parallel turbo search. + - `AsyncOrderService` - non-blocking order placement. + - `AsyncPositionService` - concurrent position queries. + - `AsyncTradingOrchestrator` - parallel instrument search + spending-power fetch. + - `AsyncPerformanceMonitor` - SL/TP/trailing-stop checks with bounded-concurrency closures. +- **`src/database/postgres.py`** - Full async PostgreSQL layer with connection pooling (`asyncpg`), auto-schema creation, and equivalent managers for orders, positions, and trade performance. +- **`src/mq_telegram/async_tools.py`** - Async Telegram message sender using `aio-pika`. +- **`src/trader/__init__.py`** - Fully async Trader service consuming from `trading-signals` queue via `aio-pika`. +- **`src/position_monitor/__init__.py`** - New dedicated service that combines WebSocket streaming for real-time position monitoring with queue consumption for time-triggered events (`daily_stats`). +- **`src/database/migration.py`** - One-time DuckDB→PostgreSQL migration tool. + +--- + +### Changed + +- **`deploy/docker-compose.yml`** + - Added `postgres1` service (PostgreSQL 16 Alpine) with health checks and volume mapping. + - Added `position_monitor1` service as a new container. + - Trader and web server now depend on `postgres1` health check. + +- **`src/start_python_script.sh`** + - Added `position_monitor` and `trader_legacy` cases. + - The new `trader` role runs the async Trader; `trader_legacy` runs the old sync `main.py`. + +- **`src/scheduler/__init__.py`** + - Removed `job_check_positions_on_saxo_api` (7-second polling loop). + - Added smart queue routing: `daily_stats` and `check_positions_on_saxo_api` → `trading-ops`; trade signals → `trading-signals`. + - `job_daily_stats` and `job_close_position` are unchanged. + +- **`src/web_server/__init__.py`** + - Webhook now publishes to `trading-signals` queue instead of the generic queue. + +- **`src/trade/async_services.py`** + - `AsyncPerformanceMonitor` gains `check_positions_from_stream(streamed_positions)` - accepts pre-fetched position data from the WebSocket stream instead of polling the REST API. + - Internal logic refactored into shared `_evaluate_positions()` used by both stream-fed and REST-polled paths. + +- **`etc/config_example.json`** + - Added `postgresql` section with DSN and pool sizing. + - Added `trade.config.general.streaming` section with `refresh_rate_ms`, `reconnect_delay_seconds`, `max_reconnect_delay_seconds`, `reauth_interval_seconds`. + - Existing `websocket` and `position_check` sections kept for backward compatibility. + +- **`requirements.txt`** + - Added: `asyncpg==0.30.0`, `aio-pika==9.5.4`. + - Already present: `websockets==12.0`, `httpx==0.27.0`, `uvloop==0.19.0`. + +--- + +### Architecture Diagram (Before → After) + +**Before (v0.6.x):** +``` +TradingView → Webhook → RabbitMQ (single queue) → Trader (sync, requests) + ↕ +Scheduler (every 7s) → RabbitMQ → Trader → Saxo REST API (3+ calls per check) + ↕ + DuckDB +``` + +**After (v0.7.0):** +``` +TradingView → Webhook → RabbitMQ [trading-signals] → Async Trader (httpx/HTTP2) + ↕ + PostgreSQL + ↕ + Saxo WebSocket Stream ←→ Position Monitor (real-time) + ↕ + Scheduler → RabbitMQ [trading-ops] → Position Monitor (daily_stats) +``` + +--- + +### Performance Impact + +| Metric | Before (v0.6.x) | After (v0.7.0) | +|--------|-----------------|-----------------| +| Position check latency | Up to 7s (poll interval) | Sub-second (stream push) | +| API calls per check cycle | 3+ REST calls | 0 (data pushed via WebSocket) | +| HTTP protocol | HTTP/1.1 (requests) | HTTP/2 (httpx) | +| Database | DuckDB (embedded, single-writer) | PostgreSQL (pooled, concurrent) | +| Message broker pattern | Single queue | Split queues (signals vs ops) | +| I/O model | Synchronous (blocking) | Fully async (asyncio + uvloop) | + +--- + +# Migration Guide - v0.6.x → v0.7.0 + +## Prerequisites + +- Docker and Docker Compose +- Access to your current `config.json` and DuckDB database file +- A brief maintenance window (services will be restarted) + +## Step 1: Set PostgreSQL Password + +In `/app/wata/.env` in your server, set a secure postgres password: + +```bash +POSTGRES_PASSWORD="your_secure_password_here" +``` + +## Step 2: Update Configuration + +Add the following sections to your `etc/config.json`: + +### 2a. PostgreSQL connection + +Add at the top level (next to `duckdb`): + +```json +"postgresql": { + "dsn": "postgresql://wata:YOUR_PASSWORD@postgres1:5432/wata", + "pool_min_size": 2, + "pool_max_size": 10 +} +``` + +> **Important:** Replace `YOUR_PASSWORD` with a strong password. Set the same password as the `POSTGRES_PASSWORD` environment variable in `/app/wata/.env`. + +### 2b. Streaming configuration + +Add inside `trade.config.general`: + +```json +"streaming": { + "refresh_rate_ms": 1000, + "reconnect_delay_seconds": 1.0, + "max_reconnect_delay_seconds": 30.0, + "reauth_interval_seconds": 900 +} +``` + +| Key | Description | Default | +|-----|-------------|---------| +| `refresh_rate_ms` | How often Saxo pushes position updates (milliseconds) | `1000` | +| `reconnect_delay_seconds` | Initial delay before reconnecting after a WS drop | `1.0` | +| `max_reconnect_delay_seconds` | Cap for exponential-backoff reconnection | `30.0` | +| `reauth_interval_seconds` | How often to re-authorise the WS connection | `900` (15 min) | + +### 2c. Keep existing sections + +The `duckdb`, `websocket`, and `position_check` config sections can remain - they are not used by the new services but won't cause errors. + +## Step 3: Rebuild the Docker Image + +```bash +./docker_build.sh +``` + +This rebuilds `wata-base:latest` with the new code and dependencies. + +## Step 4: Start PostgreSQL First + +```bash +cd deploy +docker compose up -d postgres1 +``` + +Wait for it to be healthy: + +```bash +docker compose ps # should show postgres1 as "healthy" +``` + +## Step 5: Migrate Data from DuckDB + +Run the migration tool from inside a container: + +```bash +docker compose run --rm \ + -e WATA_APP_ROLE=trader \ + -e WATA_CONFIG_PATH=/app/etc/config.json \ + trader1 \ + python -m src.database.migration +``` + +This reads all data from your existing DuckDB file and inserts it into PostgreSQL. The tool is idempotent - running it twice won't duplicate data. + +Verify the migration: + +```bash +docker compose exec postgres1 psql -U wata -c "SELECT count(*) FROM turbo_data_order;" +docker compose exec postgres1 psql -U wata -c "SELECT count(*) FROM turbo_data_position;" +``` + +## Step 6: Deploy All Services + +```bash +docker compose up -d +``` + +This starts all services including the new `position_monitor1` container. + +## Step 7: Verify + +1. **Check logs** for the Position Monitor: + ```bash + docker compose logs -f position_monitor1 + ``` + You should see: + ``` + WebSocket connected. + Position subscription created (refId=pos-xxxx). Snapshot: N positions. + ``` + +2. **Check Telegram** - you should receive: + ``` + WATA Position Monitor vX.X.X is running (WebSocket streaming + trading-ops queue). + ``` + +3. **Verify the scheduler** no longer polls: + ```bash + docker compose logs scheduler1 | grep check_positions + ``` + Should show no new `check_positions_on_saxo_api` messages. + +4. **Test a webhook** to verify the full signal flow still works end-to-end. + +## Rollback + +If you need to roll back: + +1. Revert to the previous Docker image (tag your images before upgrading). +2. The DuckDB database file is untouched - the migration tool only reads from it. +3. Remove the `postgresql` and `streaming` config sections from `config.json`. +4. Bring down the new containers: `docker compose down`. +5. Restart with the old image. + +## New Service Map + +| Container | Role | Queue | Purpose | +|-----------|------|-------|---------| +| `web_server1` | web_server | publishes to `trading-signals` | Webhook endpoint | +| `trader1` | trader | consumes `trading-signals` | Executes trades | +| `position_monitor1` | position_monitor | WebSocket + consumes `trading-ops` | Real-time SL/TP/trailing-stop + daily stats | +| `scheduler1` | scheduler | publishes to `trading-ops` / `trading-signals` | Timed events (daily_stats, close-position) | +| `telegram1` | telegram | consumes `telegram` queue | Notifications | +| `rabbitmq1` | - | - | Message broker | +| `postgres1` | - | - | Database | diff --git a/VERSION b/VERSION index 7ceb040..bcaffe1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1 \ No newline at end of file +0.7.0 \ No newline at end of file diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 108c22f..bc2c14b 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -2,6 +2,34 @@ x-common: rabbitmq-password: &rabbitmq_password pykvys-9nixqo-cuqvYt services: + postgres1: + image: postgres:16-alpine + hostname: postgres1 + container_name: 'postgres' + environment: + - POSTGRES_USER=wata + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-wata_secret_change_me} + - POSTGRES_DB=wata + volumes: + - ../../var/lib/postgresql/data:/var/lib/postgresql/data + networks: + - trade-net + healthcheck: + test: ["CMD-SHELL", "pg_isready -U wata"] + interval: 10s + timeout: 5s + retries: 5 + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: always + delay: 5s + window: 120s + resources: + limits: + memory: 512M + rabbitmq1: image: rabbitmq:3-management hostname: rabbitmq1 @@ -60,6 +88,8 @@ services: depends_on: rabbitmq1: condition: service_healthy + postgres1: + condition: service_healthy telegram1: image: wata-base:latest @@ -133,6 +163,36 @@ services: depends_on: rabbitmq1: condition: service_healthy + postgres1: + condition: service_healthy + + position_monitor1: + image: wata-base:latest + hostname: position_monitor1 + container_name: 'position_monitor1' + environment: + - WATA_APP_ROLE=position_monitor + volumes: + - ../../var/log:/app/var/log/ + - ../../var/lib:/app/var/lib/ + - ../../etc/config.json:/app/etc/config.json + networks: + - trade-net + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: always + delay: 5s + window: 120s + resources: + limits: + memory: 1024M + depends_on: + rabbitmq1: + condition: service_healthy + postgres1: + condition: service_healthy networks: trade-net: diff --git a/deploy/tools/deploy_app_to_your_server.sh b/deploy/tools/deploy_app_to_your_server.sh old mode 100644 new mode 100755 diff --git a/etc/config_example.json b/etc/config_example.json index a8be183..a8ddfe7 100644 --- a/etc/config_example.json +++ b/etc/config_example.json @@ -43,6 +43,11 @@ "db_path": "/app/var/lib/duckdb/trading_data.duckdb" } }, + "postgresql": { + "dsn": "postgresql://wata:wata_secret_change_me@postgres1:5432/wata", + "pool_min_size": 2, + "pool_max_size": 10 + }, "trade": { "rules": [ { @@ -110,6 +115,14 @@ "risky_trading_start_hour": 21, "risky_trading_start_minute": 54 } + }, + { + "rule_name": "risk_management", + "rule_type": "risk_management", + "rule_config": { + "cooldown_after_loss_minutes": 10, + "max_trades_per_day": 8 + } } ], "config": { @@ -122,16 +135,82 @@ }, "buying_power": { "max_account_funds_to_use_percentage": 100, + "reserve_cash_percent": 0, "safety_margins": { "bid_calculation": 1 } }, "position_management": { "performance_thresholds": { - "stoploss_percent": -20, - "max_profit_percent": 60 + "stoploss_percent": -15, + "max_profit_percent": 60, + "trailing_stop": { + "enabled": true, + "activation_percent": 5, + "drawdown_percent": 8 + } } }, + "position_sizing": { + "time_of_day_scaling": { + "enabled": true, + "periods": [ + { + "start_hour": 8, + "start_minute": 0, + "end_hour": 14, + "end_minute": 30, + "scale_percent": 100, + "label": "European Session" + }, + { + "start_hour": 14, + "start_minute": 30, + "end_hour": 16, + "end_minute": 0, + "scale_percent": 50, + "label": "US Open (volatile)" + }, + { + "start_hour": 16, + "start_minute": 0, + "end_hour": 22, + "end_minute": 0, + "scale_percent": 75, + "label": "US Session" + } + ] + }, + "confidence_scaling": { + "enabled": true, + "default_confidence": 1.0, + "min_confidence_threshold": 0.3, + "scaling_rules": [ + { + "min_confidence": 0.0, + "max_confidence": 0.5, + "scale_percent": 50, + "label": "Low confidence" + }, + { + "min_confidence": 0.5, + "max_confidence": 0.8, + "scale_percent": 75, + "label": "Medium confidence" + }, + { + "min_confidence": 0.8, + "max_confidence": 2.0, + "scale_percent": 100, + "label": "High confidence" + } + ] + } + }, + "turbo_cache": { + "enabled": true, + "ttl_seconds": 30 + }, "general": { "api_limits": { "top_instruments": 200, @@ -149,6 +228,12 @@ "websocket": { "refresh_rate_ms": 10000 }, + "streaming": { + "refresh_rate_ms": 1000, + "reconnect_delay_seconds": 1.0, + "max_reconnect_delay_seconds": 30.0, + "reauth_interval_seconds": 900 + }, "timezone": "Europe/Paris" }, "trading_mode": "day_trading" diff --git a/package.sh b/package.sh old mode 100644 new mode 100755 diff --git a/reporting/build-report-site.sh b/reporting/build-report-site.sh new file mode 100644 index 0000000..315a094 --- /dev/null +++ b/reporting/build-report-site.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Get the directory where this script is located +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# Define the dashboard directory +DASHBOARD_DIR="trading-dashboard/hello-framework" + +cd $SCRIPT_DIR/$DASHBOARD_DIR + +# Start the build process +npm run build \ No newline at end of file diff --git a/reporting/day_trading_percent_simulated.py b/reporting/day_trading_percent_simulated.py index 402e9fc..7634445 100644 --- a/reporting/day_trading_percent_simulated.py +++ b/reporting/day_trading_percent_simulated.py @@ -1,16 +1,9 @@ -import json import sys from datetime import datetime, timedelta import pandas as pd import pyarrow.parquet as pq import pyarrow as pa -# Check if output path was provided as command-line argument -if len(sys.argv) > 1: - output_path = sys.argv[1] -else: - output_path = '.' # Default to current directory - # Default config data default_config_data = { "trade": { @@ -33,21 +26,187 @@ "rule_name": "day_trading", "rule_type": "day_trading", "rule_config": { - "percent_profit_wanted_per_days": 1.7 + "percent_profit_wanted_per_days": 1.2 } } ] }, "reporting": { "money_expectation_indicator": { - "trading_start_date": "30/03/2025", - "initial_money": 200.00, + "trading_start_date": "25/05/2026", + "initial_money": 1000.00, "weekday_without_trading": ["saturday", "sunday"], "trading_date_to_generate": 1000 } } } +def format_currency(amount): + return f"${amount:,.2f}" + + +def format_milestone(target, milestone_info): + status = "Reached" if milestone_info["within_simulation"] else "Projected" + extra_days_text = "" + if milestone_info["extra_trading_days"] > 0: + extra_days_text = f", +{milestone_info['extra_trading_days']} more trading days" + + return ( + f"{status} {format_currency(target)} on {milestone_info['date']:%Y-%m-%d} " + f"(trading day {milestone_info['trading_day']:,}, " + f"{milestone_info['calendar_days']:,} calendar days from start{extra_days_text})" + ) + + +def format_growth_event(label, milestone_info): + status = "Reached" if milestone_info["within_simulation"] else "Projected" + extra_days_text = "" + if milestone_info["extra_trading_days"] > 0: + extra_days_text = f", +{milestone_info['extra_trading_days']} more trading days" + + return ( + f"{status} {label} on {milestone_info['date']:%Y-%m-%d} " + f"(trading day {milestone_info['trading_day']:,}, " + f"{milestone_info['calendar_days']:,} calendar days from start{extra_days_text})" + ) + + +def find_milestone(df, target_balance, start_date, percent_profit_per_day, excluded_weekdays, market_closed_dates): + milestone_rows = df[df["balance"] >= target_balance] + if not milestone_rows.empty: + milestone_row = milestone_rows.iloc[0] + milestone_date = milestone_row["date"].to_pydatetime() + return { + "date": milestone_date, + "trading_day": int(milestone_row["trading_day"]), + "calendar_days": (milestone_date - start_date).days, + "within_simulation": True, + "extra_trading_days": 0, + } + + projected_date = df.iloc[-1]["date"].to_pydatetime() + projected_balance = float(df.iloc[-1]["balance"]) + projected_trading_day = int(df.iloc[-1]["trading_day"]) + + while projected_balance < target_balance: + projected_date += timedelta(days=1) + if projected_date.weekday() in excluded_weekdays or projected_date in market_closed_dates: + continue + + projected_balance *= (1 + percent_profit_per_day) + projected_trading_day += 1 + + return { + "date": projected_date, + "trading_day": projected_trading_day, + "calendar_days": (projected_date - start_date).days, + "within_simulation": False, + "extra_trading_days": projected_trading_day - len(df), + } + + +def calculate_average_period_pl_percent(df, frequency): + period_returns = ( + df.assign( + opening_balance=df["balance"] - df["daily_gain"], + period=df["date"].dt.to_period(frequency), + ) + .groupby("period") + .agg(period_open=("opening_balance", "first"), period_close=("balance", "last")) + ) + period_returns["return_percent"] = ( + (period_returns["period_close"] - period_returns["period_open"]) + / period_returns["period_open"] + * 100 + ) + return float(period_returns["return_percent"].mean()) + + +def print_money_lover_stats(df, start_date, initial_money, percent_profit_per_day, excluded_weekdays, market_closed_dates): + final_row = df.iloc[-1] + final_balance = float(final_row["balance"]) + total_profit = float(final_row["profit"]) + last_daily_gain = float(final_row["daily_gain"]) + total_return_percent = (total_profit / initial_money) * 100 + multiplier = final_balance / initial_money + + doubling_info = find_milestone( + df, + initial_money * 2, + start_date, + percent_profit_per_day, + excluded_weekdays, + market_closed_dates, + ) + + ten_bagger_info = find_milestone( + df, + initial_money * 10, + start_date, + percent_profit_per_day, + excluded_weekdays, + market_closed_dates, + ) + + single_day_initial_break = df[df["daily_gain"] >= initial_money] + single_day_initial_break_text = "Not reached during simulation." + if not single_day_initial_break.empty: + first_break_row = single_day_initial_break.iloc[0] + first_break_date = first_break_row["date"].to_pydatetime() + single_day_initial_break_text = ( + f"One trading day profit first beat the starting bankroll on {first_break_date:%Y-%m-%d} " + f"(trading day {int(first_break_row['trading_day']):,})" + ) + + monthly_profit = ( + df.assign(month=df["date"].dt.to_period("M")) + .groupby("month")["daily_gain"] + .sum() + ) + average_weekly_pl_percent = calculate_average_period_pl_percent(df, "W-FRI") + average_monthly_pl_percent = calculate_average_period_pl_percent(df, "M") + average_yearly_pl_percent = calculate_average_period_pl_percent(df, "Y") + best_month = monthly_profit.idxmax() + best_month_profit = float(monthly_profit.loc[best_month]) + + milestone_targets = [100000, 500000, 1000000] + milestone_summaries = [] + for target in milestone_targets: + milestone_info = find_milestone( + df, + target, + start_date, + percent_profit_per_day, + excluded_weekdays, + market_closed_dates, + ) + milestone_summaries.append(format_milestone(target, milestone_info)) + + print("\nMoney lover stats:") + print(f"Simulation window: {start_date:%Y-%m-%d} -> {final_row['date']:%Y-%m-%d}") + print(f"Trading days generated: {len(df):,}") + print(f"Final bankroll: {format_currency(final_balance)}") + print(f"Net profit: {format_currency(total_profit)} ({total_return_percent:,.2f}% total return)") + print(f"Bankroll multiplier: {multiplier:,.2f}x") + print(format_growth_event("money doubled", doubling_info)) + print(format_growth_event("10x bankroll", ten_bagger_info)) + print(single_day_initial_break_text) + print(f"Average gain per trading day: {format_currency(df['daily_gain'].mean())}") + print(f"Average P/L per week: {average_weekly_pl_percent:,.2f}%") + print(f"Average P/L per month: {average_monthly_pl_percent:,.2f}%") + print(f"Average P/L per year: {average_yearly_pl_percent:,.2f}%") + print(f"Last simulated trading day gain: {format_currency(last_daily_gain)}") + print(f"Best month: {best_month} with {format_currency(best_month_profit)} in gains") + print("Big milestones:") + for summary in milestone_summaries: + print(f"- {summary}") + +# Check if output path was provided as command-line argument +if len(sys.argv) > 1: + output_path = sys.argv[1] +else: + output_path = '.' # Default to current directory + # Ask the user if they want to use the default configuration print("Current configuration:") print(f"Trading start date: {default_config_data['reporting']['money_expectation_indicator']['trading_start_date']}") @@ -89,6 +248,7 @@ trading_data = [] current_date = start_date current_money = initial_money +previous_money = initial_money # Loop through the specified number of trading dates generated_days = 0 @@ -96,13 +256,37 @@ if (current_date.weekday() not in excluded_weekdays) and (current_date not in market_closed_dates): # Calculate new money amount for this trading day current_money *= (1 + percent_profit_per_day) - trading_data.append({"date": current_date, "money": current_money - initial_money}) + daily_gain = current_money - previous_money + trading_data.append( + { + "date": current_date, + "trading_day": generated_days + 1, + "balance": current_money, + "profit": current_money - initial_money, + "daily_gain": daily_gain, + "money": current_money - initial_money, + } + ) + previous_money = current_money generated_days += 1 # Only count valid trading days # Move to the next day current_date += timedelta(days=1) # Create a DataFrame and save as Parquet file df = pd.DataFrame(trading_data) +if df.empty: + print("No trading days were generated with the current configuration.") + sys.exit(1) + +print_money_lover_stats( + df, + start_date, + initial_money, + percent_profit_per_day, + excluded_weekdays, + market_closed_dates, +) + table = pa.Table.from_pandas(df) # Save to specified output path diff --git a/requirements.txt b/requirements.txt index 45a499b..e60ab31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ fastapi-cli==0.0.3 h11==0.16.0 httpcore==1.0.9 httptools==0.6.1 -httpx==0.27.0 +httpx[http2]==0.27.0 idna==3.7 Jinja2==3.1.6 jsonschema==4.22.0 @@ -55,5 +55,11 @@ websockets==12.0 wsproto==1.2.0 websocket-client~=1.8.0 cryptography==44.0.2 +# --- Async & PostgreSQL stack --- +asyncpg==0.30.0 +aio-pika==9.5.4 +psycopg[binary]==3.2.6 +# --------------------------------- pytest pytest-cov +pytest-asyncio==0.25.3 diff --git a/src/database/__init__.py b/src/database/__init__.py index f14afac..04ad001 100644 --- a/src/database/__init__.py +++ b/src/database/__init__.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta import duckdb import os @@ -276,6 +276,43 @@ def get_open_positions_ids_actions(self): result_list.append(result_schema) return result_list + def get_today_trade_count(self): + """ + Returns the number of trades (positions opened) today. + Counts both open and closed positions opened today. + """ + formatted_date = datetime.now().strftime('%Y/%m/%d') + result = self.conn.execute( + """ + SELECT COUNT(*) + FROM turbo_data_position + WHERE strftime(execution_time_open, '%Y/%m/%d') = ? + """, + (formatted_date,), + ).fetchone() + return result[0] if result else 0 + + def get_last_closed_position_performance(self): + """ + Returns the performance percent and close time of the last closed position today. + Returns None if no position was closed today. + """ + formatted_date = datetime.now().strftime('%Y/%m/%d') + result = self.conn.execute( + """ + SELECT position_total_performance_percent, execution_time_close + FROM turbo_data_position + WHERE position_status = 'Closed' + AND strftime(execution_time_close, '%Y/%m/%d') = ? + ORDER BY execution_time_close DESC + LIMIT 1 + """, + (formatted_date,), + ).fetchone() + if result: + return {"performance_percent": result[0], "close_time": result[1]} + return None + def get_max_position_percent(self, position_id): """ Retrieves the maximum position percentage for a position. @@ -362,6 +399,78 @@ def get_stats_of_the_day(self): final = {"general": stats_list, "detail_stats": detail_stats_list} return final + def get_closed_trade_history(self, start_date=None, end_date=None): + """Return closed trades for reporting over an optional date range.""" + conditions = ["position_status = 'Closed'", "execution_time_close IS NOT NULL"] + params = [] + + if start_date is not None: + conditions.append("CAST(execution_time_close AS DATE) >= ?") + params.append(start_date) + if end_date is not None: + conditions.append("CAST(execution_time_close AS DATE) <= ?") + params.append(end_date) + + rows = self.conn.execute( + f""" + SELECT + action, + position_id, + position_total_performance_percent AS performance_percent, + position_max_performance_percent AS max_performance_percent, + position_profit_loss AS profit_loss, + execution_time_close + FROM turbo_data_position + WHERE {' AND '.join(conditions)} + ORDER BY execution_time_close ASC, position_id ASC + """, + params, + ).fetchall() + + return [ + { + "action": row[0], + "position_id": row[1], + "performance_percent": row[2], + "max_performance_percent": row[3], + "profit_loss": row[4], + "execution_time_close": row[5], + } + for row in rows + ] + + def get_daily_profit_history(self, end_date=None): + """Return daily realized P/L history for winning-streak calculations.""" + conditions = ["position_status = 'Closed'", "execution_time_close IS NOT NULL"] + params = [] + + if end_date is not None: + conditions.append("CAST(execution_time_close AS DATE) <= ?") + params.append(end_date) + + rows = self.conn.execute( + f""" + SELECT + CAST(execution_time_close AS DATE) AS day_date, + COALESCE(SUM(position_profit_loss), 0.0) AS sum_profit, + COUNT(*) AS trade_count + FROM turbo_data_position + WHERE {' AND '.join(conditions)} + GROUP BY day_date + ORDER BY day_date ASC + """, + params, + ).fetchall() + + return [ + { + "day_date": row[0], + "sum_profit": row[1], + "trade_count": row[2], + } + for row in rows + ] + def _apply_percentage_change(self, value, p_percentage): """ Apply a percentage change to a given value. diff --git a/src/database/migration.py b/src/database/migration.py new file mode 100644 index 0000000..368ce25 --- /dev/null +++ b/src/database/migration.py @@ -0,0 +1,209 @@ +""" +DuckDB → PostgreSQL migration tool. + +Reads all data from the existing DuckDB database and inserts it into PostgreSQL. +Designed to be run once during the migration from the old stack to the new async stack. + +Usage: + WATA_CONFIG_PATH=/app/etc/config.json python -m src.database.migration +""" + +import asyncio +import json +import logging +import os +import sys + +import duckdb + +from src.configuration import ConfigurationManager +from src.database.postgres import PostgresConnectionManager, init_schema +from src.logging_helper import setup_logging + +logger = logging.getLogger(__name__) + + +def read_duckdb_table(conn: duckdb.DuckDBPyConnection, table_name: str) -> list[dict]: + """Read all rows from a DuckDB table as a list of dicts.""" + try: + result = conn.execute(f"SELECT * FROM {table_name}").fetchdf() + records = result.to_dict(orient="records") + logger.info("Read %d rows from DuckDB table '%s'", len(records), table_name) + return records + except Exception as e: + logger.warning("Could not read DuckDB table '%s': %s", table_name, e) + return [] + + +async def migrate_orders(pg: PostgresConnectionManager, rows: list[dict]): + """Migrate turbo_data_order rows to PostgreSQL.""" + for row in rows: + try: + await pg.execute( + """INSERT INTO turbo_data_order + (action, buy_sell, order_id, order_amount, order_type, order_kind, + order_submit_time, related_order_id, position_id, + instrument_name, instrument_symbol, instrument_uic, + instrument_price, instrument_currency, order_cost) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) + ON CONFLICT (order_id) DO NOTHING""", + row.get("action"), row.get("buy_sell"), row.get("order_id"), + row.get("order_amount"), row.get("order_type"), row.get("order_kind"), + row.get("order_submit_time"), + row.get("related_order_id", []), + row.get("position_id"), + row.get("instrument_name"), row.get("instrument_symbol"), + row.get("instrument_uic"), row.get("instrument_price"), + row.get("instrument_currency"), row.get("order_cost"), + ) + except Exception as e: + logger.error("Failed to migrate order row %s: %s", row.get("order_id"), e) + + +async def migrate_positions(pg: PostgresConnectionManager, rows: list[dict]): + """Migrate turbo_data_position rows to PostgreSQL.""" + for row in rows: + try: + await pg.execute( + """INSERT INTO turbo_data_position + (action, position_id, position_amount, position_open_price, + position_total_open_price, position_close_price, + position_total_close_price, position_profit_loss, + position_total_performance_percent, + position_max_performance_percent, + position_status, position_kind, position_close_reason, + execution_time_open, execution_time_close, + order_id, related_order_id, + instrument_name, instrument_symbol, + instrument_uic, instrument_currency) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21) + ON CONFLICT (position_id) DO NOTHING""", + row.get("action"), row.get("position_id"), + row.get("position_amount"), row.get("position_open_price"), + row.get("position_total_open_price"), row.get("position_close_price"), + row.get("position_total_close_price"), row.get("position_profit_loss"), + row.get("position_total_performance_percent"), + row.get("position_max_performance_percent"), + row.get("position_status"), row.get("position_kind"), + row.get("position_close_reason"), + row.get("execution_time_open"), row.get("execution_time_close"), + row.get("order_id"), + row.get("related_order_id", []), + row.get("instrument_name"), row.get("instrument_symbol"), + row.get("instrument_uic"), row.get("instrument_currency"), + ) + except Exception as e: + logger.error("Failed to migrate position row %s: %s", row.get("position_id"), e) + + +async def migrate_trade_performance(pg: PostgresConnectionManager, rows: list[dict]): + """Migrate trade_performance rows to PostgreSQL.""" + for row in rows: + try: + await pg.execute( + """INSERT INTO trade_performance + (date, total_performance_percent, total_performance_percent_on_max, + best_performance_percent, best_performance_percent_on_max, + trade_count, stoploss_count, takeprofit_count, trailing_stop_count) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) + ON CONFLICT (date) DO NOTHING""", + row.get("date"), row.get("total_performance_percent"), + row.get("total_performance_percent_on_max"), + row.get("best_performance_percent"), + row.get("best_performance_percent_on_max"), + row.get("trade_count"), row.get("stoploss_count"), + row.get("takeprofit_count"), row.get("trailing_stop_count"), + ) + except Exception as e: + logger.error("Failed to migrate trade_performance row %s: %s", row.get("date"), e) + + +async def migrate_tokens(pg: PostgresConnectionManager, rows: list[dict]): + """Migrate auth_tokens rows to PostgreSQL.""" + for row in rows: + try: + await pg.execute( + """INSERT INTO auth_tokens (token_id, encrypted_data, updated_at) + VALUES ($1, $2, NOW()) + ON CONFLICT (token_id) DO UPDATE SET + encrypted_data = EXCLUDED.encrypted_data, + updated_at = NOW()""", + row.get("token_id"), + row.get("encrypted_data"), + ) + except Exception as e: + logger.error("Failed to migrate token row %s: %s", row.get("token_id"), e) + + +async def run_migration(config_manager: ConfigurationManager): + """Main migration logic.""" + # Connect to DuckDB + duckdb_path = config_manager.get_config_value("duckdb.persistant.db_path") + if not os.path.exists(duckdb_path): + logger.error("DuckDB file not found: %s", duckdb_path) + print(f"ERROR: DuckDB file not found at {duckdb_path}") + return + + logger.info("Opening DuckDB: %s", duckdb_path) + duck = duckdb.connect(duckdb_path, read_only=True) + + # Connect to PostgreSQL + logger.info("Connecting to PostgreSQL...") + pg = PostgresConnectionManager.from_config(config_manager) + await pg.connect() + await init_schema(pg) + + try: + # Read all DuckDB tables + orders = read_duckdb_table(duck, "turbo_data_order") + positions = read_duckdb_table(duck, "turbo_data_position") + performances = read_duckdb_table(duck, "trade_performance") + tokens = read_duckdb_table(duck, "auth_tokens") + + # Migrate to PostgreSQL + logger.info("Migrating orders...") + await migrate_orders(pg, orders) + + logger.info("Migrating positions...") + await migrate_positions(pg, positions) + + logger.info("Migrating trade_performance...") + await migrate_trade_performance(pg, performances) + + logger.info("Migrating tokens...") + await migrate_tokens(pg, tokens) + + # Verify counts + pg_orders = await pg.fetchval("SELECT COUNT(*) FROM turbo_data_order") + pg_positions = await pg.fetchval("SELECT COUNT(*) FROM turbo_data_position") + pg_perf = await pg.fetchval("SELECT COUNT(*) FROM trade_performance") + pg_tokens = await pg.fetchval("SELECT COUNT(*) FROM auth_tokens") + + logger.info("=== Migration Complete ===") + logger.info("Orders: DuckDB=%d PostgreSQL=%d", len(orders), pg_orders) + logger.info("Positions: DuckDB=%d PostgreSQL=%d", len(positions), pg_positions) + logger.info("Performance: DuckDB=%d PostgreSQL=%d", len(performances), pg_perf) + logger.info("Tokens: DuckDB=%d PostgreSQL=%d", len(tokens), pg_tokens) + + print("\n=== Migration Complete ===") + print(f"Orders: DuckDB={len(orders)} PostgreSQL={pg_orders}") + print(f"Positions: DuckDB={len(positions)} PostgreSQL={pg_positions}") + print(f"Performance: DuckDB={len(performances)} PostgreSQL={pg_perf}") + print(f"Tokens: DuckDB={len(tokens)} PostgreSQL={pg_tokens}") + + finally: + duck.close() + await pg.close() + + +if __name__ == "__main__": + config_path = os.getenv("WATA_CONFIG_PATH") + if not config_path: + print("FATAL: WATA_CONFIG_PATH not set", file=sys.stderr) + sys.exit(1) + + config_manager = ConfigurationManager(config_path) + setup_logging(config_manager, "wata-migration") + + print("Starting DuckDB → PostgreSQL migration...") + asyncio.run(run_migration(config_manager)) diff --git a/src/database/postgres.py b/src/database/postgres.py new file mode 100644 index 0000000..b0b7330 --- /dev/null +++ b/src/database/postgres.py @@ -0,0 +1,636 @@ +# src/database/postgres.py +""" +Async PostgreSQL database layer for WATA. +Replaces DuckDB for the write path (Trader, Position Monitor). +Uses asyncpg connection pool for high-performance async I/O. +""" + +import asyncio +import logging +from datetime import date, datetime, timedelta +from typing import Any + +import asyncpg + +from src.configuration import ConfigurationManager + +logger = logging.getLogger(__name__) + +_DATETIME_FIELDS = frozenset({ + "order_submit_time", + "order_time", + "execution_time_open", + "execution_time_close", +}) +_DATE_FIELDS = frozenset({"date_day"}) + + +def _parse_datetime_value(value: Any, field_name: str) -> Any: + if value is None or isinstance(value, datetime): + return value + if isinstance(value, date): + return datetime.combine(value, datetime.min.time()) + if not isinstance(value, str): + return value + + candidate = value.strip() + if not candidate: + return None + if candidate.endswith("Z"): + candidate = f"{candidate[:-1]}+00:00" + + try: + return datetime.fromisoformat(candidate) + except ValueError: + pass + + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"): + try: + return datetime.strptime(candidate, fmt) + except ValueError: + continue + + raise ValueError(f"Unsupported datetime value for {field_name}: {value!r}") + + +def _parse_date_value(value: Any, field_name: str) -> Any: + if value is None: + return value + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + if not isinstance(value, str): + return value + + candidate = value.strip() + if not candidate: + return None + + try: + return date.fromisoformat(candidate) + except ValueError: + parsed_datetime = _parse_datetime_value(candidate, field_name) + if isinstance(parsed_datetime, datetime): + return parsed_datetime.date() + raise ValueError(f"Unsupported date value for {field_name}: {value!r}") + + +def _normalize_temporal_fields( + data: dict[str, Any], + *, + datetime_fields: frozenset[str] = frozenset(), + date_fields: frozenset[str] = frozenset(), +) -> dict[str, Any]: + normalized = dict(data) + for field_name in datetime_fields: + if field_name in normalized: + normalized[field_name] = _parse_datetime_value(normalized[field_name], field_name) + for field_name in date_fields: + if field_name in normalized: + normalized[field_name] = _parse_date_value(normalized[field_name], field_name) + return normalized + + +# ────────────────────────────────────────────── +# Connection Pool Manager +# ────────────────────────────────────────────── + +class PostgresConnectionManager: + """Manages an asyncpg connection pool (singleton-friendly).""" + + def __init__(self, dsn: str, min_size: int = 2, max_size: int = 10): + self._dsn = dsn + self._min_size = min_size + self._max_size = max_size + self._pool: asyncpg.Pool | None = None + + @classmethod + def from_config(cls, config_manager: ConfigurationManager) -> "PostgresConnectionManager": + pg_cfg = config_manager.get_config_value("postgresql", {}) + dsn = pg_cfg.get("dsn", "postgresql://wata:wata@localhost:5432/wata") + min_size = pg_cfg.get("pool_min_size", 2) + max_size = pg_cfg.get("pool_max_size", 10) + return cls(dsn=dsn, min_size=min_size, max_size=max_size) + + async def connect(self): + if self._pool is None: + logger.info("Creating asyncpg connection pool (min=%d, max=%d)…", self._min_size, self._max_size) + self._pool = await asyncpg.create_pool( + self._dsn, + min_size=self._min_size, + max_size=self._max_size, + command_timeout=30, + ) + logger.info("asyncpg connection pool created.") + return self._pool + + @property + def pool(self) -> asyncpg.Pool: + if self._pool is None: + raise RuntimeError("PostgresConnectionManager.connect() has not been awaited yet.") + return self._pool + + async def close(self): + if self._pool: + await self._pool.close() + self._pool = None + logger.info("asyncpg connection pool closed.") + + async def execute(self, query: str, *args) -> str: + return await self.pool.execute(query, *args) + + async def fetch(self, query: str, *args) -> list[asyncpg.Record]: + return await self.pool.fetch(query, *args) + + async def fetchrow(self, query: str, *args) -> asyncpg.Record | None: + return await self.pool.fetchrow(query, *args) + + async def fetchval(self, query: str, *args) -> Any: + return await self.pool.fetchval(query, *args) + + +# ────────────────────────────────────────────── +# Schema Initialization +# ────────────────────────────────────────────── + +SCHEMA_SQL = """ +-- Orders table +CREATE TABLE IF NOT EXISTS turbo_data_order ( + action VARCHAR(16), + buy_sell VARCHAR(8), + order_id VARCHAR(64) PRIMARY KEY, + order_amount INTEGER, + order_type VARCHAR(32), + order_kind VARCHAR(32), + order_time TIMESTAMPTZ, + related_order_id TEXT[], + position_id VARCHAR(64), + instrument_name TEXT, + instrument_symbol VARCHAR(64), + instrument_uic INTEGER, + instrument_price DOUBLE PRECISION, + instrument_currency VARCHAR(8), + order_cost DOUBLE PRECISION +); + +-- Positions table +CREATE TABLE IF NOT EXISTS turbo_data_position ( + action VARCHAR(16), + position_id VARCHAR(64) PRIMARY KEY, + position_amount INTEGER, + position_open_price DOUBLE PRECISION, + position_close_price DOUBLE PRECISION, + position_close_reason VARCHAR(128), + position_profit_loss DOUBLE PRECISION, + position_total_open_price DOUBLE PRECISION, + position_total_close_price DOUBLE PRECISION, + position_total_performance_percent DOUBLE PRECISION, + position_max_performance_percent DOUBLE PRECISION, + position_status VARCHAR(16) DEFAULT 'Open', + position_kind VARCHAR(32), + execution_time_open TIMESTAMPTZ, + execution_time_close TIMESTAMPTZ, + order_id VARCHAR(64), + related_order_id TEXT[], + instrument_name TEXT, + instrument_symbol VARCHAR(64), + instrument_uic INTEGER, + instrument_currency VARCHAR(8) +); + +-- Trade performance (daily aggregates) +CREATE TABLE IF NOT EXISTS trade_performance ( + date_day DATE PRIMARY KEY, + perf_day_real DOUBLE PRECISION, + money_made_real DOUBLE PRECISION, + trade_number_real INTEGER, + max_perf_day_simulated DOUBLE PRECISION +); + +-- Encrypted token storage +CREATE TABLE IF NOT EXISTS auth_tokens ( + token_id VARCHAR(64) PRIMARY KEY, + token_type VARCHAR(32), + encrypted_data BYTEA, + creation_time TIMESTAMPTZ, + last_update TIMESTAMPTZ, + metadata TEXT +); + +-- Indexes for common queries +CREATE INDEX IF NOT EXISTS idx_position_status ON turbo_data_position (position_status); +CREATE INDEX IF NOT EXISTS idx_position_close_time ON turbo_data_position (execution_time_close); +CREATE INDEX IF NOT EXISTS idx_position_open_time ON turbo_data_position (execution_time_open); +""" + + +async def init_schema(conn_mgr: PostgresConnectionManager): + """Creates all tables and indexes if they don't exist.""" + logger.info("Initializing PostgreSQL schema…") + await conn_mgr.execute(SCHEMA_SQL) + logger.info("PostgreSQL schema initialized.") + + +# ────────────────────────────────────────────── +# Async DB Managers +# ────────────────────────────────────────────── + +class AsyncDbOrderManager: + """Async order persistence using PostgreSQL.""" + + def __init__(self, conn_mgr: PostgresConnectionManager): + self.db = conn_mgr + + async def insert_turbo_order_data(self, data: dict): + normalized_data = _normalize_temporal_fields(data, datetime_fields=_DATETIME_FIELDS) + await self.db.execute( + """ + INSERT INTO turbo_data_order + (action, buy_sell, order_id, order_amount, order_type, order_kind, + order_time, related_order_id, position_id, instrument_name, + instrument_symbol, instrument_uic, instrument_price, + instrument_currency, order_cost) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) + ON CONFLICT (order_id) DO NOTHING + """, + normalized_data["action"], + normalized_data["buy_sell"], + normalized_data["order_id"], + normalized_data["order_amount"], + normalized_data["order_type"], + normalized_data["order_kind"], + normalized_data.get("order_submit_time") or normalized_data.get("order_time"), + normalized_data.get("related_order_id", []), + normalized_data["position_id"], + normalized_data["instrument_name"], + normalized_data["instrument_symbol"], + normalized_data["instrument_uic"], + normalized_data["instrument_price"], + normalized_data["instrument_currency"], + normalized_data.get("order_cost"), + ) + + +class AsyncDbPositionManager: + """Async position persistence using PostgreSQL.""" + + def __init__(self, conn_mgr: PostgresConnectionManager): + self.db = conn_mgr + + @staticmethod + def _today() -> date: + return date.today() + + async def insert_turbo_open_position_data(self, data: dict): + normalized_data = _normalize_temporal_fields(data, datetime_fields=_DATETIME_FIELDS) + await self.db.execute( + """ + INSERT INTO turbo_data_position + (action, position_id, position_amount, position_open_price, + position_total_open_price, position_status, position_kind, + execution_time_open, order_id, related_order_id, + instrument_name, instrument_symbol, instrument_uic, + instrument_currency) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) + ON CONFLICT (position_id) DO NOTHING + """, + normalized_data["action"], + normalized_data["position_id"], + normalized_data["position_amount"], + normalized_data["position_open_price"], + normalized_data["position_total_open_price"], + normalized_data.get("position_status", "Open"), + normalized_data["position_kind"], + normalized_data.get("execution_time_open"), + normalized_data["order_id"], + normalized_data.get("related_order_id", []), + normalized_data["instrument_name"], + normalized_data["instrument_symbol"], + normalized_data["instrument_uic"], + normalized_data["instrument_currency"], + ) + + async def update_turbo_position_data(self, position_id: str, update_data: dict): + if not update_data: + return + normalized_update_data = _normalize_temporal_fields(update_data, datetime_fields=_DATETIME_FIELDS) + set_parts = [] + values = [] + for idx, (key, val) in enumerate(normalized_update_data.items(), start=1): + set_parts.append(f"{key} = ${idx}") + values.append(val) + values.append(position_id) + query = f"UPDATE turbo_data_position SET {', '.join(set_parts)} WHERE position_id = ${len(values)}" + await self.db.execute(query, *values) + + async def get_open_positions_ids(self) -> list[str]: + rows = await self.db.fetch( + "SELECT position_id FROM turbo_data_position WHERE position_status = 'Open'" + ) + return [r["position_id"] for r in rows] + + async def get_open_positions_ids_actions(self) -> list[dict]: + rows = await self.db.fetch( + "SELECT position_id, action FROM turbo_data_position WHERE position_status = 'Open'" + ) + return [{"position_id": r["position_id"], "action": r["action"]} for r in rows] + + async def get_today_trade_count(self) -> int: + today = self._today() + val = await self.db.fetchval( + """ + SELECT COUNT(*) FROM turbo_data_position + WHERE execution_time_open::date = $1::date + """, + today, + ) + return val or 0 + + async def get_percent_of_the_day(self) -> float: + today = self._today() + rows = await self.db.fetch( + """ + SELECT position_total_performance_percent + FROM turbo_data_position + WHERE position_status = 'Closed' + AND execution_time_close::date = $1::date + """, + today, + ) + return self._calculate_final_percentage(rows) + + async def get_max_position_percent(self, position_id: str) -> float: + val = await self.db.fetchval( + "SELECT position_max_performance_percent FROM turbo_data_position WHERE position_id = $1", + position_id, + ) + return val if val is not None else 0.0 + + async def get_last_closed_position_performance(self) -> dict | None: + today = self._today() + row = await self.db.fetchrow( + """ + SELECT position_total_performance_percent, execution_time_close + FROM turbo_data_position + WHERE position_status = 'Closed' + AND execution_time_close::date = $1::date + ORDER BY execution_time_close DESC + LIMIT 1 + """, + today, + ) + if row: + return {"performance_percent": row[0], "close_time": row[1]} + return None + + async def check_position_ids_exist(self, position_ids: list[str]) -> dict: + result = {"position_ids_in_db": [], "position_ids_not_found": []} + for pid in position_ids: + row = await self.db.fetchrow( + "SELECT position_id, order_id, action FROM turbo_data_position WHERE position_id = $1", + pid, + ) + if row: + result["position_ids_in_db"].append(dict(row)) + else: + result["position_ids_not_found"].append(pid) + return result + + async def get_stats_of_the_day(self) -> dict: + today = self._today() + + general = await self.db.fetch( + """ + SELECT + to_char(execution_time_close, 'YYYY/MM/DD') AS day_date, + COUNT(*) AS position_count, + AVG(position_total_performance_percent) AS avg_percent, + MAX(position_total_performance_percent) AS max_percent, + MIN(position_total_performance_percent) AS min_percent, + SUM(position_profit_loss) AS sum_profit + FROM turbo_data_position + WHERE position_status = 'Closed' + AND execution_time_close::date = $1::date + GROUP BY day_date + ORDER BY day_date DESC + """, + today, + ) + detail = await self.db.fetch( + """ + SELECT + to_char(execution_time_close, 'YYYY/MM/DD') AS day_date, + action, + COUNT(*) AS position_count, + AVG(position_total_performance_percent) AS avg_percent, + MAX(position_total_performance_percent) AS max_percent, + MIN(position_total_performance_percent) AS min_percent, + SUM(position_profit_loss) AS sum_profit + FROM turbo_data_position + WHERE position_status = 'Closed' + AND execution_time_close::date = $1::date + GROUP BY day_date, action + ORDER BY day_date DESC, action ASC + """, + today, + ) + return { + "general": [dict(r) for r in general], + "detail_stats": [dict(r) for r in detail], + } + + async def get_closed_trade_history( + self, + start_date: date | None = None, + end_date: date | None = None, + ) -> list[dict]: + conditions = ["position_status = 'Closed'"] + args: list[Any] = [] + + if start_date is not None: + args.append(start_date) + conditions.append(f"execution_time_close::date >= ${len(args)}::date") + if end_date is not None: + args.append(end_date) + conditions.append(f"execution_time_close::date <= ${len(args)}::date") + + rows = await self.db.fetch( + f""" + SELECT + action, + position_id, + position_total_performance_percent AS performance_percent, + position_max_performance_percent AS max_performance_percent, + position_profit_loss AS profit_loss, + execution_time_close + FROM turbo_data_position + WHERE {' AND '.join(conditions)} + ORDER BY execution_time_close ASC, position_id ASC + """, + *args, + ) + return [dict(r) for r in rows] + + async def get_daily_profit_history(self, end_date: date | None = None) -> list[dict]: + args: list[Any] = [] + end_date_clause = "" + if end_date is not None: + args.append(end_date) + end_date_clause = f" AND execution_time_close::date <= ${len(args)}::date" + + rows = await self.db.fetch( + f""" + SELECT + execution_time_close::date AS day_date, + SUM(position_profit_loss) AS sum_profit, + COUNT(*) AS trade_count + FROM turbo_data_position + WHERE position_status = 'Closed'{end_date_clause} + GROUP BY execution_time_close::date + ORDER BY day_date ASC + """, + *args, + ) + return [dict(r) for r in rows] + + async def get_percent_of_last_n_days(self, n: int) -> dict: + return await self._get_percentages_for_n_days(n, "position_total_performance_percent", self._calculate_final_percentage) + + async def get_best_percent_of_last_n_days(self, n: int) -> dict: + return await self._get_percentages_for_n_days(n, "position_total_performance_percent", self._calculate_best_percentage) + + async def get_theoretical_percent_of_last_n_days_on_max(self, n: int) -> dict: + return await self._get_percentages_for_n_days(n, "position_max_performance_percent", self._calculate_final_percentage) + + async def get_best_theoretical_percent_of_last_n_days_on_max(self, n: int) -> dict: + return await self._get_percentages_for_n_days(n, "position_max_performance_percent", self._calculate_best_percentage) + + # ── helpers ── + + async def _get_percentages_for_n_days(self, n: int, column: str, calc_fn) -> dict: + results = {} + for i in range(n): + d = date.today() - timedelta(days=i) + display_date = d.strftime("%Y/%m/%d") + rows = await self.db.fetch( + f""" + SELECT {column} + FROM turbo_data_position + WHERE position_status = 'Closed' + AND execution_time_close::date = $1::date + """, + d, + ) + results[display_date] = calc_fn(rows) if rows else 0.0 + return results + + @staticmethod + def _calculate_final_percentage(rows) -> float: + val = 1.0 + for r in rows: + pct = r[0] if r[0] is not None else 0 + val *= 1 + pct / 100.0 + return round((val - 1) * 100, 2) + + @staticmethod + def _calculate_best_percentage(rows) -> float: + val = 1.0 + intermediates = [] + for r in rows: + pct = r[0] if r[0] is not None else 0 + val *= 1 + pct / 100.0 + intermediates.append(val) + return round((max(intermediates) - 1) * 100, 2) if intermediates else 0.0 + + +class AsyncDbTradePerformanceManager: + """Async trade performance persistence.""" + + def __init__(self, conn_mgr: PostgresConnectionManager): + self.db = conn_mgr + + async def insert_trade_performance_data(self, data: dict): + normalized_data = _normalize_temporal_fields(data, date_fields=_DATE_FIELDS) + await self.db.execute( + """ + INSERT INTO trade_performance (date_day, perf_day_real, money_made_real, trade_number_real, max_perf_day_simulated) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (date_day) DO NOTHING + """, + normalized_data["date_day"], + normalized_data["perf_day_real"], + normalized_data["money_made_real"], + normalized_data["trade_number_real"], + normalized_data.get("max_perf_day_simulated"), + ) + + async def create_last_day_trade_performance_data(self): + today = date.today() + row = await self.db.fetchrow( + """ + SELECT + COALESCE(SUM(position_profit_loss), 0.0) AS money_made_real, + COUNT(position_id) AS trade_number_real + FROM turbo_data_position + WHERE position_status = 'Closed' + AND execution_time_close::date = $1::date + """, + today, + ) + if row: + # Calculate perf_day_real from sequential percentages + pct_rows = await self.db.fetch( + """ + SELECT position_total_performance_percent + FROM turbo_data_position + WHERE position_status = 'Closed' + AND execution_time_close::date = $1::date + """, + today, + ) + perf = AsyncDbPositionManager._calculate_final_percentage(pct_rows) if pct_rows else 0.0 + await self.insert_trade_performance_data({ + "date_day": today, + "perf_day_real": perf, + "money_made_real": row["money_made_real"], + "trade_number_real": row["trade_number_real"], + "max_perf_day_simulated": None, + }) + + +class AsyncDbTokenManager: + """Async encrypted token storage.""" + + def __init__(self, conn_mgr: PostgresConnectionManager): + self.db = conn_mgr + + async def store_token(self, token_id: str, token_type: str, encrypted_data: bytes, metadata: str | None = None): + now = datetime.now() + await self.db.execute( + """ + INSERT INTO auth_tokens (token_id, token_type, encrypted_data, creation_time, last_update, metadata) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (token_id) DO UPDATE SET + encrypted_data = EXCLUDED.encrypted_data, + last_update = EXCLUDED.last_update, + metadata = EXCLUDED.metadata + """, + token_id, token_type, encrypted_data, now, now, metadata, + ) + + async def get_token(self, token_id: str) -> bytes | None: + return await self.db.fetchval( + "SELECT encrypted_data FROM auth_tokens WHERE token_id = $1", + token_id, + ) + + async def token_exists(self, token_id: str) -> bool: + val = await self.db.fetchval( + "SELECT COUNT(*) FROM auth_tokens WHERE token_id = $1", + token_id, + ) + return (val or 0) > 0 + + async def delete_token(self, token_id: str): + await self.db.execute("DELETE FROM auth_tokens WHERE token_id = $1", token_id) diff --git a/src/main.py b/src/main.py deleted file mode 100644 index cc798f6..0000000 --- a/src/main.py +++ /dev/null @@ -1,712 +0,0 @@ -import os -import traceback -import json -import logging -import jsonschema -import pika -from functools import partial -import sys - -# --- Configuration and Core Components --- -from configuration import ConfigurationManager -# --- Import Saxo Services --- -from trade.api_actions import ( - SaxoApiClient, - InstrumentService, - OrderService, - PositionService, - TradingOrchestrator, - PerformanceMonitor -) -from src.saxo_authen import SaxoAuth -# --- Other necessary imports --- -from trade.rules import TradingRule -from schema import SchemaLoader -from database import DbOrderManager, DbPositionManager, DbTradePerformanceManager -from mq_telegram.tools import send_message_to_mq_for_telegram -# --- Use the Updated Message Helper --- -from message_helper import ( - generate_daily_stats_message, - generate_performance_stats_message, - TelegramMessageComposer -) -from logging_helper import setup_logging - -# --- Specific Exceptions --- -from trade.exceptions import ( - TradingRuleViolation, - NoMarketAvailableException, - NoTurbosAvailableException, - PositionNotFoundException, - InsufficientFundsException, - ApiRequestException, - TokenAuthenticationException, - DatabaseOperationException, - PositionCloseException, - WebSocketConnectionException, - SaxoApiError, - OrderPlacementError, - ConfigurationError -) - -# --- Global for Version --- -APP_VERSION = "unknown" - -# --- Utility Functions --- -def get_version(): - """Reads the application version from the VERSION file.""" - try: - version_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'VERSION') - with open(version_file, 'r') as file: - return file.read().strip() - except Exception as e: - logging.error(f"Could not read VERSION file: {e}") - return "unknown" - -# --- Error Handling Helpers --- -def handle_exception( - e, composer, ch, method, body, is_critical=False, exit_code=None, log_level=logging.ERROR -): - """Centralized function to handle exceptions.""" - error_type = type(e).__name__ - log_message = f"{'CRITICAL' if is_critical else 'ERROR'} ({error_type}): {e}" - detail_message = f"{log_message}\n{traceback.format_exc()}" # Include traceback - - logging.log(log_level, detail_message) - - # Try to compose a message using the updated composer - if composer: - # Let the composer handle detailed formatting based on exception type - composer.add_generic_error(error_type, e, is_critical=is_critical) - # Optionally add traceback snippet for severe errors - if is_critical or log_level >= logging.ERROR: - # Add more specific details if available from custom exceptions - extra_details = {} - if hasattr(e, 'status_code'): extra_details['Status'] = e.status_code - if hasattr(e, 'saxo_error_details'): extra_details['Saxo Details'] = json.dumps(e.saxo_error_details, indent=2) - if hasattr(e, 'order_details'): extra_details['Order Payload'] = json.dumps(e.order_details, indent=2) - if hasattr(e, 'request_details'): extra_details['Request Details'] = json.dumps(e.request_details, indent=2) - if hasattr(e, 'order_id'): extra_details['Order ID'] = e.order_id - if extra_details: - composer.add_dict_section("Error Details", extra_details) - composer.add_text_section("Traceback Snippet", traceback.format_exc(limit=5)) - telegram_message = composer.get_message() - else: - # Fallback if composer failed early - raw_body_str = body.decode(errors='ignore') - telegram_message = f"CRITICAL ERROR ({error_type}) processing raw message: {raw_body_str}\n\nError: {e}\n\n{traceback.format_exc()}" - - try: - send_message_to_mq_for_telegram(rabbit_connection, telegram_message) - except Exception as mq_err: - logging.error(f"Failed to send error notification to Telegram MQ: {mq_err}") - - # Acknowledge non-critical messages to avoid reprocessing loops for recoverable errors - # or errors specific to the message content (like rule violations). - # Critical errors might lead to exit *without* acking, depending on infrastructure setup - # (e.g., dead-letter queue). Current logic acks most handled errors. - if not is_critical: - try: - ch.basic_ack(delivery_tag=method.delivery_tag) - logging.debug(f"Non-critical error ({error_type}) handled, message {method.delivery_tag} ACKed.") - except Exception as ack_err: - logging.error(f"Failed to ACK message {method.delivery_tag} after handling error: {ack_err}") - - # Exit for critical errors - if is_critical and exit_code is not None: - logging.critical(f"Terminating service due to critical error ({error_type}). Exit code: {exit_code}") - sys.exit(exit_code) - -def handle_validation_error(e, body, ch, method): - """Handles JSON schema validation errors.""" - logging.error(f"Schema Validation Error: {e}") - raw_body_str = body.decode(errors='ignore') - error_msg = f"SCHEMA ERROR: Invalid message format received.\n\nError: {e}\n\nRaw Body:\n{raw_body_str}" - send_message_to_mq_for_telegram(rabbit_connection, error_msg) - ch.basic_ack(delivery_tag=method.delivery_tag) # Ack invalid message - -def handle_rule_violation(trv, composer, ch, method): - """Handles expected trading rule violations.""" - logging.warning(f"Trading Rule Violation: {trv}") - # TODO: Add a config param for notification delivery - #composer.add_rule_violation(trv) # Uses updated composer method - #send_message_to_mq_for_telegram(rabbit_connection, composer.get_message()) - ch.basic_ack(delivery_tag=method.delivery_tag) - -def handle_trade_setup_issue(trade_issue, composer, ch, method): - """Handles issues like NoTurbos, NoMarket, InsufficientFunds (pre-order).""" - logging.warning(f"Trade Setup Issue ({type(trade_issue).__name__}): {trade_issue}") - # Use composer methods which now handle specific exceptions better - if isinstance(trade_issue, (NoMarketAvailableException, NoTurbosAvailableException)): - composer.add_turbo_search_result(error=trade_issue, search_context=getattr(trade_issue, 'search_context', None)) - elif isinstance(trade_issue, InsufficientFundsException): - composer.add_position_result(error=trade_issue) # Composer formats this now - else: - composer.add_generic_error(type(trade_issue).__name__, trade_issue) - - send_message_to_mq_for_telegram(rabbit_connection, composer.get_message()) - ch.basic_ack(delivery_tag=method.delivery_tag) - -def handle_order_placement_error(ope, composer, ch, method): - """Handles specific order rejection errors from Saxo.""" - logging.error(f"Order Placement Rejected by Saxo: {ope}") - # Composer's add_position_result or add_generic_error now handles detailed formatting - composer.add_position_result(error=ope) # Let composer format it - send_message_to_mq_for_telegram(rabbit_connection, composer.get_message()) - ch.basic_ack(delivery_tag=method.delivery_tag) # Ack, Saxo rejected it - -def handle_unknown_action(action, composer, ch, method): - """Handles messages with unrecognized actions.""" - logging.error(f"Unknown action received: {action}") - composer.add_generic_error("Unknown Action", ValueError(f"Action '{action}' not recognized.")) - send_message_to_mq_for_telegram(rabbit_connection, composer.get_message()) - ch.basic_ack(delivery_tag=method.delivery_tag) - - -# --- Action Handlers --- - -def handle_trading_action( - data, composer: TelegramMessageComposer, ch, method, - # --- Injected Services --- - trading_orchestrator: TradingOrchestrator, - performance_monitor: PerformanceMonitor, - trading_rule: TradingRule, - db_position_manager: DbPositionManager, - # --- Other Args --- - trade_turbo_exchange_id: str, - rabbit_connection -): - """Handles 'long' and 'short' trading actions using new services.""" - action = data.get("action") - indice = data.get("indice") - logging.info(f"Processing trading action: {action} for {indice}") - - execution_result = None - - try: - # 1. Rule Checks - logging.debug("Checking trading rules...") - trading_rule.check_signal_timestamp(action, data.get("alert_timestamp")) - trading_rule.check_market_hours(data.get("signal_timestamp")) - indice_id = trading_rule.get_allowed_indice_id(indice) - TradingRule.check_if_open_position_is_same_signal(action, db_position_manager) - trading_rule.check_profit_per_day() - logging.debug("Trading rules passed.") - - # 2. Close existing positions - logging.info("Attempting to close any existing managed positions before opening new one...") - close_result = performance_monitor.close_managed_positions_by_criteria(action_filter=None) # None closes all - logging.info(f"Attempted closure of existing positions. Initiated: {close_result.get('closed_initiated_count', 0)}, Errors: {close_result.get('errors_count', 0)}") - # Optionally add summary (consider if monitor's own messages are sufficient) - composer.add_text_section("Pre-Trade Closure", f"Attempted closing existing positions. Initiated: {close_result['closed_initiated_count']}, Errors: {close_result['errors_count']}") - - - # 3. Execute Trade Signal - logging.info(f"Executing trade signal: Exchange {trade_turbo_exchange_id}, IndiceID {indice_id}, Action {action}") - execution_result = trading_orchestrator.execute_trade_signal( - exchange_id=trade_turbo_exchange_id, - underlying_uics=indice_id, - keywords=action - ) - # execution_result contains: {'order_details': {...}, 'position_details': {...}, 'selected_turbo_info': {...}, 'message': '...'} - - # 4. Compose Success Message using updated composer methods - composer.add_turbo_search_result(founded_turbo=execution_result['selected_turbo_info']) - composer.add_position_result(buy_details=execution_result) # Pass the whole result - - logging.info(f"Successfully executed and recorded trade action: {action}. OrderID: {execution_result['order_details']['order_id']}, PositionID: {execution_result['position_details']['position_id']}") - - # 5. Final Success Message & Ack - send_message_to_mq_for_telegram(rabbit_connection, composer.get_message()) - ch.basic_ack(delivery_tag=method.delivery_tag) - logging.info(f"Message {method.delivery_tag} ACKed for successful trade action: {action}") - - # --- Expected/Handled Errors during Trading --- - except TradingRuleViolation as trv: - handle_rule_violation(trv, composer, ch, method) - except (NoMarketAvailableException, NoTurbosAvailableException, InsufficientFundsException) as setup_err: - # Let the specific handler format the message via the composer - handle_trade_setup_issue(setup_err, composer, ch, method) - except OrderPlacementError as ope: - # Let the specific handler format the message via the composer - handle_order_placement_error(ope, composer, ch, method) - - # --- Critical Errors (Need to bubble up for main handler) --- - except PositionNotFoundException as pnfe: - logging.critical(f"CRITICAL: Position not found after placing order {pnfe.order_id}: {pnfe}") - # Composer updated by handle_exception before exit - raise - except DatabaseOperationException as dbe: - logging.critical(f"CRITICAL DB ERROR during trade action '{action}': {dbe}") - # Composer updated by handle_exception before exit - raise - except ValueError as ve: - logging.error(f"ValueError during trading action '{action}': {ve}", exc_info=True) - if "CRITICAL" in str(ve).upper(): - raise # Re-raise critical ValueErrors for main handler - else: - # Treat as non-critical, let composer format and ACK - composer.add_generic_error(f"ValueError in {action}", ve) - send_message_to_mq_for_telegram(rabbit_connection, composer.get_message()) - ch.basic_ack(delivery_tag=method.delivery_tag) - - # Let other unexpected API/Config/Token errors bubble up - -def handle_close_action( - data, composer: TelegramMessageComposer, ch, method, - # --- Injected Services --- - performance_monitor: PerformanceMonitor, - # --- Other Args --- - rabbit_connection -): - """Handles 'close-long', 'close-short', 'close-position' using PerformanceMonitor.""" - action = data.get("action") - logging.info(f"Processing close action: {action}") - - try: - close_action_filter = None - if action == "close-long": close_action_filter = "long" - elif action == "close-short": close_action_filter = "short" - - logging.info(f"Attempting closure via Performance Monitor. Filter: {close_action_filter}") - close_result = performance_monitor.close_managed_positions_by_criteria( - action_filter=close_action_filter - ) - closed_count = close_result.get('closed_initiated_count', 0) - error_count = close_result.get('errors_count', 0) - logging.info(f"Close action '{action}' processed. Positions closed/attempted: {closed_count}, Errors: {error_count}") - - composer.add_text_section( - f"{action.upper()} ACTION", # Simpler title - f"Processed signal. Attempted closure for {closed_count} position(s)." - f"{f' Encountered {error_count} error(s).' if error_count > 0 else ''}" - f" Check other messages for details." - ) - - if closed_count > 0: - send_message_to_mq_for_telegram(rabbit_connection, composer.get_message()) - - ch.basic_ack(delivery_tag=method.delivery_tag) - - except (PositionCloseException, DatabaseOperationException, ApiRequestException, SaxoApiError) as e: - # Catch errors from monitor/services if they bubble up critically - logging.error(f"Error during '{action}' execution: {e}", exc_info=True) - # Let main callback handler manage critical error message and exit - raise - except Exception as e: - logging.error(f"Unexpected error during '{action}': {e}", exc_info=True) - raise # Let main callback handler manage critical error message and exit - - -def handle_check_positions( - data, composer: TelegramMessageComposer, ch, method, - # --- Injected Services --- - performance_monitor: PerformanceMonitor, - db_position_manager: DbPositionManager, # Still needed to apply sync updates - # --- Other Args --- - rabbit_connection -): - """Handles 'check_positions_on_saxo_api' using PerformanceMonitor.""" - action = "check_positions_on_saxo_api" - logging.info(f"Processing action: {action}") - sync_updates_applied = 0 - sync_errors = 0 - perf_check_errors = 0 - perf_closed_count = 0 - - try: - # 1. Check Performance - logging.info("Checking positions performance...") - perf_result = performance_monitor.check_all_positions_performance() - perf_check_errors = perf_result.get('errors', 0) - perf_closed_count = len(perf_result.get('closed_positions_processed', [])) - logging.info(f"Performance check results: Closed={perf_closed_count}, DB Updates={len(perf_result.get('db_updates',[]))}, Errors={perf_check_errors}") - - # 2. Sync DB state with API - logging.info("Syncing DB positions with API closed positions...") - sync_result = performance_monitor.sync_db_positions_with_api() - updates_to_apply = sync_result.get("updates_for_db", []) - - # 3. Apply DB updates from sync result - if updates_to_apply: - logging.info(f"Applying {len(updates_to_apply)} DB updates from API sync...") - for position_id, update_data in updates_to_apply: - try: - db_position_manager.update_turbo_position_data(position_id, update_data) - sync_updates_applied += 1 - except Exception as db_err: - sync_errors += 1 - logging.critical(f"CRITICAL SYNC ERROR: Failed DB update for Pos {position_id}: {db_err}", exc_info=True) - # Send critical notification directly - send_message_to_mq_for_telegram(rabbit_connection, f"🚨 CRITICAL SYNC ERROR: Failed DB update for Pos {position_id}: {db_err}") - # This might warrant raising DatabaseOperationException if critical - # raise DatabaseOperationException(f"Failed sync update for {position_id}", operation="sync_update", entity_id=position_id) from db_err - logging.info(f"Sync DB updates applied: {sync_updates_applied}, Errors: {sync_errors}") - else: - logging.info("No DB updates required from API sync.") - - total_errors = perf_check_errors + sync_errors - logging.info(f"{action}: Completed. Perf Closed={perf_closed_count}, Sync Updates={sync_updates_applied}, Total Errors={total_errors}") - - # Acknowledge ONLY if sync DB updates were successful (or none needed) - # If perf check had non-critical errors, we might still ACK. - if sync_errors == 0: - ch.basic_ack(delivery_tag=method.delivery_tag) - logging.debug(f"{action}: Message {method.delivery_tag} ack'd successfully.") - else: - # Do not ACK if sync failed, message might need retry or dead-lettering - logging.error(f"{action}: Sync DB update errors occurred. Message {method.delivery_tag} NOT ACKed.") - # Optionally NACK: ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) - - - except (DatabaseOperationException, ApiRequestException, SaxoApiError) as e: - logging.error(f"Critical Error during '{action}': {e}", exc_info=True) - # Let main handler format and potentially exit - raise - except Exception as e: - logging.error(f"Unexpected error during '{action}': {e}", exc_info=True) - raise # Let main handler deal with unexpected - - -def handle_daily_stats(data, composer: TelegramMessageComposer, ch, method, db_position_manager, table_trade_performance_manager, rabbit_connection): - """Handles 'daily_stats' action. """ - logging.info("Processing action: daily_stats") - try: - days = 7 # Number of days for performance stats - logging.debug("Fetching daily stats...") - stats_of_the_day = db_position_manager.get_stats_of_the_day() - message = generate_daily_stats_message(stats_of_the_day) - - logging.debug(f"Fetching performance stats for last {days} days...") - last_days_percentages = db_position_manager.get_percent_of_last_n_days(days) - last_best_days_percentages = db_position_manager.get_best_percent_of_last_n_days(days) - last_days_percentages_on_max = db_position_manager.get_theoretical_percent_of_last_n_days_on_max(days) - last_best_days_percentages_on_max = db_position_manager.get_best_theoretical_percent_of_last_n_days_on_max(days) - - message = generate_performance_stats_message( - message, days, last_days_percentages, last_best_days_percentages, - last_days_percentages_on_max, last_best_days_percentages_on_max - ) - logging.debug("Sending daily stats message...") - send_message_to_mq_for_telegram(rabbit_connection, message) - - logging.debug("Inserting daily trade performance data...") - table_trade_performance_manager.create_last_day_trade_performance_data() - - logging.info("Daily stats processed and sent successfully.") - ch.basic_ack(delivery_tag=method.delivery_tag) - - except DatabaseOperationException as dbe: - logging.error(f"Database error during daily stats processing: {dbe}") - # Let main handler manage critical error - raise - except Exception as e: - logging.error(f"Failed to process daily stats: {e}", exc_info=True) - # Send general error message and ACK (non-critical) - composer.add_generic_error("Daily Stats Processing", e) - send_message_to_mq_for_telegram(rabbit_connection, composer.get_message()) - ch.basic_ack(delivery_tag=method.delivery_tag) - - -# --- Action Dispatcher --- -ACTION_HANDLERS = { - "long": handle_trading_action, - "short": handle_trading_action, - "close-long": handle_close_action, - "close-short": handle_close_action, - "close-position": handle_close_action, - "check_positions_on_saxo_api": handle_check_positions, - "daily_stats": handle_daily_stats, -} - -# --- Main Callback --- - -def callback(ch, method, properties, body, # RabbitMQ args first - # --- Injected dependencies follow: --- - trading_orchestrator: TradingOrchestrator, - performance_monitor: PerformanceMonitor, - trading_rule: TradingRule, - db_position_manager: DbPositionManager, - table_trade_performance_manager: DbTradePerformanceManager, - trade_turbo_exchange_id: str, - rabbit_conn): - """Main callback function executed for each message.""" - composer = None - data_from_mq = None - - try: - logging.debug(f"Received message (delivery_tag={method.delivery_tag})") - # 1. Decode Body - try: - raw_body_str = body.decode('utf-8') - data_from_mq = json.loads(raw_body_str) - logging.info(f"Received action: {data_from_mq.get('action', 'N/A')}, Signal ID: {data_from_mq.get('signal_id', 'N/A')}") - except (json.JSONDecodeError, TypeError, Exception) as e: # Broadened catch - logging.error(f"Error decoding message body: {e}", exc_info=True) - error_msg = f"CRITICAL: Error decoding message body: {e}\n\nBody:\n{body.decode(errors='ignore')}" - # Cannot use composer here as data_from_mq might be None - send_message_to_mq_for_telegram(rabbit_conn, error_msg) - ch.basic_ack(delivery_tag=method.delivery_tag) # Ack invalid message - return - - # 2. Schema Validation - try: - jsonschema.validate( - instance=data_from_mq, schema=SchemaLoader.get_trading_action_schema() - ) - except jsonschema.exceptions.ValidationError as e: - handle_validation_error(e, body, ch, method) # Uses global rabbit_connection - return - - # 3. Initialize Composer - composer = TelegramMessageComposer(data_from_mq) - - # 4. Dispatch to Handler - action = data_from_mq.get("action") - handler = ACTION_HANDLERS.get(action) - - if handler: - # --- Argument Construction Logic (Updated) --- - handler_args = { - "data": data_from_mq, "composer": composer, "ch": ch, - "method": method, "rabbit_connection": rabbit_conn - } - if action in ["long", "short"]: - handler_args.update({ - "trading_orchestrator": trading_orchestrator, - "performance_monitor": performance_monitor, - "trading_rule": trading_rule, - "db_position_manager": db_position_manager, - "trade_turbo_exchange_id": trade_turbo_exchange_id - }) - elif action in ["close-long", "close-short", "close-position"]: - handler_args["performance_monitor"] = performance_monitor - elif action == "check_positions_on_saxo_api": - handler_args.update({ - "performance_monitor": performance_monitor, - "db_position_manager": db_position_manager - }) - elif action == "daily_stats": - handler_args.update({ - "db_position_manager": db_position_manager, - "table_trade_performance_manager": table_trade_performance_manager - }) - - logging.debug(f"Dispatching action '{action}' to handler {handler.__name__} with args: {list(handler_args.keys())}") - handler(**handler_args) - else: - handle_unknown_action(action, composer, ch, method) # Uses global rabbit_connection - - # --- Outer Exception Handling --- - except (ConfigurationError, TokenAuthenticationException) as critical_config_err: - handle_exception(critical_config_err, composer, ch, method, body, is_critical=True, exit_code=2, log_level=logging.CRITICAL) - except (DatabaseOperationException) as critical_db_err: - # Specific handling for critical DB errors bubbled up - handle_exception(critical_db_err, composer, ch, method, body, is_critical=True, exit_code=12, log_level=logging.CRITICAL) - except PositionNotFoundException as critical_runtime_err: - handle_exception(critical_runtime_err, composer, ch, method, body, is_critical=True, exit_code=3, log_level=logging.CRITICAL) - except PositionCloseException as position_err: - # If this bubbles up, treat as critical needing investigation - handle_exception(position_err, composer, ch, method, body, is_critical=True, exit_code=4, log_level=logging.ERROR) - except (SaxoApiError, ApiRequestException) as api_err: - # Treat persistent API errors as critical - handle_exception(api_err, composer, ch, method, body, is_critical=True, exit_code=5, log_level=logging.ERROR) - except ValueError as val_err: - # Only treat as critical if explicitly marked - if "CRITICAL" in str(val_err).upper(): - handle_exception(val_err, composer, ch, method, body, is_critical=True, exit_code=6, log_level=logging.CRITICAL) - else: - # Non-critical ValueErrors should ideally be handled and ACKed lower down. - # If one bubbles up here, log it but don't exit. Assume prior handler failed to ACK. - handle_exception(val_err, composer, ch, method, body, is_critical=False, log_level=logging.ERROR) - except WebSocketConnectionException as ws_err: - # Non-critical - handle_exception(ws_err, composer, ch, method, body, is_critical=False, log_level=logging.WARNING) - except Exception as e: - # Catch-all for truly unexpected errors - handle_exception(e, composer, ch, method, body, is_critical=True, exit_code=1, log_level=logging.CRITICAL) - - -# --- Main Execution Block --- - -if __name__ == "__main__": - global rabbit_connection - rabbit_connection = None - channel = None - - try: - APP_VERSION = get_version() - - # --- 1. Configuration --- - config_path = os.getenv("WATA_CONFIG_PATH") - if not config_path: - print("FATAL: WATA_CONFIG_PATH environment variable not set", file=sys.stderr) - sys.exit(10) - try: - config_manager = ConfigurationManager(config_path) - print("Configuration loaded and validated successfully.") - except Exception as e: - print(f"FATAL: Configuration validation failed: {e}", file=sys.stderr) - print(traceback.format_exc(), file=sys.stderr) - sys.exit(11) - - # --- 2. Logging --- - setup_logging(config_manager, "wata-trader") - logging.info(f"--- Starting WATA Trader v{APP_VERSION} ---") - - # --- 3. Database Initialization --- - logging.info("Initializing database managers...") - try: - db_order_manager = DbOrderManager(config_manager) - db_position_manager = DbPositionManager(config_manager) - table_trade_performance_manager = DbTradePerformanceManager(config_manager) - logging.info("Database managers initialized.") - except Exception as e: - logging.critical(f"Failed to initialize database managers: {e}", exc_info=True) - sys.exit(12) - - # --- 4. Trading Rules --- - logging.info("Initializing trading rules...") - try: - trading_rule = TradingRule(config_manager, db_position_manager) - trade_turbo_exchange_id = config_manager.get_config_value( - "trade.config.turbo_preference.exchange_id" - ) - logging.info(f"Trading rules initialized. Preferred exchange: {trade_turbo_exchange_id}") - except Exception as e: - logging.critical(f"Failed to initialize TradingRule: {e}", exc_info=True) - sys.exit(13) - - # --- 5. RabbitMQ Connection --- - logging.info("Connecting to RabbitMQ...") - try: - rabbitmq_config = config_manager.get_rabbitmq_config() - credentials = pika.PlainCredentials( - rabbitmq_config["authentication"]["username"], - rabbitmq_config["authentication"]["password"], - ) - parameters = pika.ConnectionParameters( - host=rabbitmq_config["hostname"], - credentials=credentials, - heartbeat=600, - blocked_connection_timeout=300 - ) - rabbit_connection = pika.BlockingConnection(parameters) - channel = rabbit_connection.channel() - channel.queue_declare(queue="trading-action") - logging.info("RabbitMQ connection established and queue declared.") - except Exception as e: - logging.critical(f"Failed to connect to RabbitMQ: {e}", exc_info=True) - sys.exit(14) - - # --- 6. Saxo Components Initialization --- - logging.info("Initializing Saxo components...") - account_key = None - client_key = None - from src.saxo_openapi.contrib.session import account_info - - try: - # Initialize Auth and API Client first - saxo_auth = SaxoAuth(config_manager, rabbit_connection) - api_client = SaxoApiClient(config_manager, saxo_auth) - - # Fetch Account/Client keys using the utility - logging.info("Fetching AccountKey and ClientKey using account_info utility...") - try: - acc_info = account_info(api_client) - account_key = acc_info.AccountKey - client_key = acc_info.ClientKey - logging.info( - f"Successfully retrieved keys. Using AccountKey: {account_key}, ClientKey: {client_key}") - except (ApiRequestException, SaxoApiError, TokenAuthenticationException) as api_err: - # Catch errors specifically from the API call within account_info - logging.critical(f"Failed API call within account_info utility: {api_err}", exc_info=True) - raise ConfigurationError( - f"Could not retrieve Account/Client Keys via account_info: {api_err}") from api_err - except (IndexError, KeyError, AttributeError) as data_err: - # Catch potential errors if the response structure is unexpected - logging.critical( - f"Unexpected data structure received from account_info's underlying API call: {data_err}", - exc_info=True) - raise ConfigurationError( - f"Failed to parse account details from API response: {data_err}") from data_err - except Exception as e: - # Catch any other unexpected error during the utility call - logging.critical(f"Unexpected error calling account_info utility: {e}", exc_info=True) - raise ConfigurationError(f"Failed to get account details via account_info: {e}") from e - - # Instantiate domain services with the retrieved keys - instrument_service = InstrumentService(api_client, config_manager, account_key) - order_service = OrderService(api_client, account_key, client_key) - position_service = PositionService(api_client, order_service, config_manager, account_key, client_key) - - # Instantiate high-level orchestrators/monitors - trading_orchestrator = TradingOrchestrator(instrument_service, order_service, position_service, - config_manager, db_order_manager, db_position_manager) - performance_monitor = PerformanceMonitor(position_service, order_service, config_manager, - db_position_manager, trading_rule, rabbit_connection) - - logging.info("Saxo services initialized successfully.") - - # Keep the outer exception handling for critical init failures - except (ConfigurationError, TokenAuthenticationException, SaxoApiError, ApiRequestException) as e: - logging.critical(f"Failed to initialize Saxo components: {e}", exc_info=True) - if rabbit_connection and rabbit_connection.is_open: - try: - send_message_to_mq_for_telegram(rabbit_connection, - f"🚨 CRITICAL FAILURE: Trader failed to initialize Saxo components: {e}") - except Exception as mq_err: - logging.error(f"Failed to send Saxo init failure to Telegram MQ: {mq_err}") - sys.exit(15) - except Exception as e: # Catch unexpected init errors - logging.critical(f"Unexpected error initializing Saxo components: {e}", exc_info=True) - if rabbit_connection and rabbit_connection.is_open: - try: - send_message_to_mq_for_telegram(rabbit_connection, - f"🚨 CRITICAL FAILURE: Unexpected error initializing Trader Saxo components: {e}") - except Exception as mq_err: - logging.error(f"Failed to send Saxo init failure to Telegram MQ: {mq_err}") - sys.exit(16) - - - # --- 7. Setup Consumer Callback with Dependencies --- - callback_with_deps = partial( - callback, - trading_orchestrator=trading_orchestrator, - performance_monitor=performance_monitor, - trading_rule=trading_rule, - db_position_manager=db_position_manager, - table_trade_performance_manager=table_trade_performance_manager, - trade_turbo_exchange_id=trade_turbo_exchange_id, - rabbit_conn=rabbit_connection - ) - - # --- 8. Start Consuming --- - channel.basic_consume( - queue="trading-action", - on_message_callback=callback_with_deps, - auto_ack=False - ) - - logging.info("Trader service startup complete. Waiting for messages...") - send_message_to_mq_for_telegram(rabbit_connection, f"✅📈 WATA Trader v{APP_VERSION} is running and ready for orders.") - - channel.start_consuming() - - # --- Global Exception Handling for Startup --- - except Exception as e: - logging.critical(f"Unhandled exception during startup: {e}", exc_info=True) - if rabbit_connection and rabbit_connection.is_open: - try: send_message_to_mq_for_telegram(rabbit_connection, f"🚨 CRITICAL STARTUP FAILURE: WATA Trader failed: {e}") - except Exception as mq_err: logging.error(f"Failed to send final startup error to Telegram MQ: {mq_err}") - else: - print(f"FATAL: Unhandled exception during startup: {e}", file=sys.stderr) - print(traceback.format_exc(), file=sys.stderr) - sys.exit(1) # General critical failure exit code - - finally: - # --- Cleanup --- - if rabbit_connection and rabbit_connection.is_open: - logging.info("Closing RabbitMQ connection.") - rabbit_connection.close() - logging.info(f"--- WATA Trader v{APP_VERSION} Shutting Down ---") \ No newline at end of file diff --git a/src/message_helper/__init__.py b/src/message_helper/__init__.py index 678164e..6e00630 100644 --- a/src/message_helper/__init__.py +++ b/src/message_helper/__init__.py @@ -1,9 +1,11 @@ import logging import textwrap -from datetime import datetime -import pytz +from collections import defaultdict +from datetime import date, datetime, timedelta import json # Import json for formatting details +import pytz + from src.trade.exceptions import ( TradingRuleViolation, NoMarketAvailableException, @@ -407,10 +409,14 @@ def add_rule_violation(self, error: TradingRuleViolation): full_section = f"{section_title}\n{textwrap.dedent(message_body)}" self.sections.append(full_section) - def add_text_section(self, title: str, text: str): - """Adds a custom text section.""" + def add_text_section(self, title: str, text): + """Adds a custom section, coercing structured values safely.""" + if isinstance(text, dict): + self.add_dict_section(title, text) + return + section_title = f"--- {title.upper()} ---" # Standardize title format - message_body = textwrap.dedent(text) + message_body = "" if text is None else textwrap.dedent(str(text)) full_section = f"{section_title}\n{message_body}" self.sections.append(full_section) @@ -430,6 +436,610 @@ def get_message(self) -> str: return "\n\n".join(self.sections).strip() +def _coerce_date(value) -> date | None: + if value is None: + return None + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + if isinstance(value, str): + candidate = value.strip() + if not candidate: + return None + try: + return datetime.fromisoformat(candidate.replace("Z", "+00:00")).date() + except ValueError: + pass + for fmt in ("%Y/%m/%d", "%Y-%m-%d"): + try: + return datetime.strptime(candidate, fmt).date() + except ValueError: + continue + return None + + +def _coerce_datetime(value) -> datetime | None: + if value is None: + return None + if isinstance(value, datetime): + return value + if isinstance(value, str): + candidate = value.strip() + if not candidate: + return None + try: + return datetime.fromisoformat(candidate.replace("Z", "+00:00")) + except ValueError: + pass + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"): + try: + return datetime.strptime(candidate, fmt) + except ValueError: + continue + return None + + +def _as_float(value, default: float = 0.0) -> float: + if value is None: + return default + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _get_trade_value(trade: dict, *keys: str, default=None): + for key in keys: + if key in trade and trade[key] is not None: + return trade[key] + return default + + +def _get_trade_close_dt(trade: dict) -> datetime | None: + return _coerce_datetime(_get_trade_value(trade, "execution_time_close")) + + +def _get_trade_close_date(trade: dict) -> date | None: + close_dt = _get_trade_close_dt(trade) + if close_dt is not None: + return close_dt.date() + return _coerce_date(_get_trade_value(trade, "day_date")) + + +def _get_trade_percent(trade: dict) -> float: + return _as_float( + _get_trade_value(trade, "performance_percent", "position_total_performance_percent"), + 0.0, + ) + + +def _get_trade_max_percent(trade: dict) -> float: + return _as_float( + _get_trade_value(trade, "max_performance_percent", "position_max_performance_percent"), + 0.0, + ) + + +def _get_trade_profit(trade: dict) -> float: + return _as_float(_get_trade_value(trade, "profit_loss", "position_profit_loss"), 0.0) + + +def _sort_trades_chronologically(trades: list[dict]) -> list[dict]: + return sorted( + trades, + key=lambda trade: ( + _get_trade_close_dt(trade) or datetime.min, + str(_get_trade_value(trade, "position_id", default="")), + ), + ) + + +def format_signed_percent(value: float | None) -> str: + if value is None: + return "N/A" + numeric = _as_float(value) + sign = "+" if numeric > 0 else "" + return f"{sign}{numeric:.2f}%" + + +def format_percent(value: float | None) -> str: + if value is None: + return "N/A" + return f"{_as_float(value):.2f}%" + + +def format_signed_currency(value: float | None) -> str: + if value is None: + return "N/A" + numeric = _as_float(value) + sign = "+" if numeric > 0 else "" + return f"{sign}{numeric:.2f}" + + +def format_profit_factor(value: float | None) -> str: + if value is None: + return "N/A" + if value == float("inf"): + return "∞" + return f"{_as_float(value):.2f}" + + +def classify_trade_outcome(trade: dict) -> str: + performance_percent = _get_trade_percent(trade) + if performance_percent > 0: + return "win" + if performance_percent < 0: + return "loss" + return "break_even" + + +def calculate_win_loss_break_even_counts(trades: list[dict]) -> dict: + counts = { + "wins": 0, + "losses": 0, + "break_even": 0, + "total": len(trades), + } + for trade in trades: + outcome = classify_trade_outcome(trade) + if outcome == "win": + counts["wins"] += 1 + elif outcome == "loss": + counts["losses"] += 1 + else: + counts["break_even"] += 1 + + counts["win_rate"] = (counts["wins"] / counts["total"] * 100.0) if counts["total"] else 0.0 + return counts + + +def calculate_profit_factor(trades: list[dict]) -> float: + gross_profit = sum(max(_get_trade_profit(trade), 0.0) for trade in trades) + gross_loss = abs(sum(min(_get_trade_profit(trade), 0.0) for trade in trades)) + if gross_loss == 0: + return float("inf") if gross_profit > 0 else 0.0 + return gross_profit / gross_loss + + +def calculate_average_win_vs_loss(trades: list[dict]) -> dict: + performance_values = [_get_trade_percent(trade) for trade in trades] + win_values = [value for value in performance_values if value > 0] + loss_values = [value for value in performance_values if value < 0] + + return { + "avg_trade": sum(performance_values) / len(performance_values) if performance_values else 0.0, + "avg_win": sum(win_values) / len(win_values) if win_values else None, + "avg_loss": sum(loss_values) / len(loss_values) if loss_values else None, + "best_trade": max(performance_values) if performance_values else 0.0, + "worst_trade": min(performance_values) if performance_values else 0.0, + } + + +def calculate_compounded_return(trades: list[dict], *, use_max_performance: bool = False) -> float: + equity = 1.0 + ordered_trades = _sort_trades_chronologically(trades) + for trade in ordered_trades: + performance_percent = _get_trade_max_percent(trade) if use_max_performance else _get_trade_percent(trade) + multiplier = 1 + performance_percent / 100.0 + if multiplier <= 0: + return -100.0 + equity *= multiplier + return round((equity - 1.0) * 100.0, 2) + + +def calculate_best_case_return(trades: list[dict]) -> float: + ordered_trades = _sort_trades_chronologically(trades) + equity = 1.0 + peak_return = 0.0 + for trade in ordered_trades: + multiplier = 1 + _get_trade_percent(trade) / 100.0 + if multiplier <= 0: + return -100.0 + equity *= multiplier + peak_return = max(peak_return, (equity - 1.0) * 100.0) + return round(peak_return, 2) + + +def calculate_daily_max_drawdown(trades: list[dict]) -> float: + equity = 1.0 + equity_peak = 1.0 + max_drawdown = 0.0 + + for trade in _sort_trades_chronologically(trades): + multiplier = 1 + _get_trade_percent(trade) / 100.0 + if multiplier <= 0: + return -100.0 + equity *= multiplier + equity_peak = max(equity_peak, equity) + drawdown = ((equity / equity_peak) - 1.0) * 100.0 if equity_peak else 0.0 + max_drawdown = min(max_drawdown, drawdown) + + return round(max_drawdown, 2) + + +def calculate_winning_streak(daily_profit_history: list[dict]) -> dict: + normalized_rows = [] + for row in daily_profit_history: + day_date = _coerce_date(_get_trade_value(row, "day_date")) + if day_date is None: + continue + profit_value = _as_float(_get_trade_value(row, "sum_profit", "net_profit"), 0.0) + normalized_rows.append({"day_date": day_date, "sum_profit": profit_value}) + + normalized_rows.sort(key=lambda row: row["day_date"]) + best_streak = 0 + running_streak = 0 + last_win_date = None + + for row in normalized_rows: + if row["sum_profit"] > 0: + running_streak += 1 + best_streak = max(best_streak, running_streak) + last_win_date = row["day_date"] + else: + running_streak = 0 + + current_streak = 0 + for row in reversed(normalized_rows): + if row["sum_profit"] > 0: + current_streak += 1 + else: + break + + return { + "current_streak": current_streak, + "best_streak": best_streak, + "last_win_date": last_win_date, + } + + +def calculate_trade_summary(trades: list[dict]) -> dict: + counts = calculate_win_loss_break_even_counts(trades) + averages = calculate_average_win_vs_loss(trades) + net_profit = round(sum(_get_trade_profit(trade) for trade in trades), 2) + + return { + **counts, + **averages, + "trade_count": len(trades), + "net_profit": net_profit, + "profit_factor": calculate_profit_factor(trades), + "max_drawdown": calculate_daily_max_drawdown(trades), + "compounded_percent": calculate_compounded_return(trades), + } + + +def calculate_timeframe_aggregation(trades: list[dict], report_date: date, days: int) -> dict: + start_date = report_date - timedelta(days=days - 1) + period_trades = [ + trade for trade in trades + if (trade_date := _get_trade_close_date(trade)) is not None and start_date <= trade_date <= report_date + ] + summary = calculate_trade_summary(period_trades) + return { + **summary, + "days": days, + "start_date": start_date, + "end_date": report_date, + } + + +def calculate_weekly_aggregations(trades: list[dict], report_date: date, weeks: int = 10) -> list[dict]: + trades_by_week: dict[tuple[int, int], list[dict]] = defaultdict(list) + for trade in trades: + trade_date = _get_trade_close_date(trade) + if trade_date is None: + continue + iso_year, iso_week, _ = trade_date.isocalendar() + trades_by_week[(iso_year, iso_week)].append(trade) + + current_week_start = report_date - timedelta(days=report_date.weekday()) + weekly_rows = [] + for offset in range(weeks): + week_start = current_week_start - timedelta(weeks=offset) + iso_year, iso_week, _ = week_start.isocalendar() + week_trades = trades_by_week.get((iso_year, iso_week), []) + summary = calculate_trade_summary(week_trades) + weekly_rows.append( + { + **summary, + "week_start": week_start, + "iso_year": iso_year, + "iso_week": iso_week, + "week_label": f"W{iso_week:02d}", + } + ) + return weekly_rows + + +def _month_start(value: date) -> date: + return value.replace(day=1) + + +def _shift_month(value: date, months: int) -> date: + year = value.year + ((value.month - 1 + months) // 12) + month = ((value.month - 1 + months) % 12) + 1 + return date(year, month, 1) + + +def calculate_monthly_aggregations(trades: list[dict], report_date: date, months: int = 12) -> list[dict]: + trades_by_month: dict[tuple[int, int], list[dict]] = defaultdict(list) + for trade in trades: + trade_date = _get_trade_close_date(trade) + if trade_date is None: + continue + trades_by_month[(trade_date.year, trade_date.month)].append(trade) + + current_month_start = _month_start(report_date) + monthly_rows = [] + for offset in range(months): + month_start = _shift_month(current_month_start, -offset) + month_trades = trades_by_month.get((month_start.year, month_start.month), []) + if not month_trades: + continue + summary = calculate_trade_summary(month_trades) + monthly_rows.append( + { + **summary, + "month_start": month_start, + "year": month_start.year, + "month": month_start.month, + "month_label": month_start.strftime("%B"), + } + ) + return monthly_rows + + +def calculate_yearly_aggregations(trades: list[dict], report_date: date, years: int = 5) -> list[dict]: + trades_by_year: dict[int, list[dict]] = defaultdict(list) + for trade in trades: + trade_date = _get_trade_close_date(trade) + if trade_date is None: + continue + trades_by_year[trade_date.year].append(trade) + + yearly_rows = [] + for year in range(report_date.year, report_date.year - years, -1): + year_trades = trades_by_year.get(year, []) + if not year_trades: + continue + summary = calculate_trade_summary(year_trades) + yearly_rows.append( + { + **summary, + "year": year, + } + ) + return yearly_rows + + +def merge_daily_performance_series( + daily_real: dict, + daily_best: dict, + daily_max: dict, + *, + report_date: date, + days: int = 7, +) -> list[dict]: + rows = [] + for offset in range(days): + day = report_date - timedelta(days=offset) + day_key = day.strftime("%Y/%m/%d") + rows.append( + { + "day_date": day, + "day_key": day_key, + "real": _as_float(daily_real.get(day_key), 0.0), + "best": _as_float(daily_best.get(day_key), 0.0), + "max": _as_float(daily_max.get(day_key), 0.0), + } + ) + return rows + + +def build_daily_trading_report_payload( + *, + report_date: date | datetime | str | None, + closed_trades: list[dict], + daily_profit_history: list[dict], + daily_real: dict, + daily_best: dict, + daily_max: dict, +) -> dict: + normalized_report_date = _coerce_date(report_date) or datetime.utcnow().date() + ordered_closed_trades = _sort_trades_chronologically(closed_trades) + daily_trades = [ + trade for trade in ordered_closed_trades + if _get_trade_close_date(trade) == normalized_report_date + ] + + return { + "report_date": normalized_report_date, + "streak": calculate_winning_streak(daily_profit_history), + "daily_summary": calculate_trade_summary(daily_trades), + "by_direction": { + "long": calculate_trade_summary([trade for trade in daily_trades if str(trade.get("action", "")).lower() == "long"]), + "short": calculate_trade_summary([trade for trade in daily_trades if str(trade.get("action", "")).lower() == "short"]), + }, + "cumulative": { + days: calculate_timeframe_aggregation(ordered_closed_trades, normalized_report_date, days) + for days in (7, 30, 60) + }, + "last_7_days": merge_daily_performance_series( + daily_real, + daily_best, + daily_max, + report_date=normalized_report_date, + days=7, + ), + "last_10_weeks": calculate_weekly_aggregations(ordered_closed_trades, normalized_report_date, weeks=10), + "last_12_months": calculate_monthly_aggregations(ordered_closed_trades, normalized_report_date, months=12), + "last_5_years": calculate_yearly_aggregations(ordered_closed_trades, normalized_report_date, years=5), + } + + +def _format_direction_title(direction: str) -> str: + if direction == "long": + return "🟢 LONGS" + return "🔴 SHORTS" + + +def _format_direction_counts(summary: dict) -> str: + trade_count = summary["trade_count"] + trade_label = "trade" if trade_count == 1 else "trades" + base = f"{trade_count} {trade_label} | {summary['wins']}W - {summary['losses']}L" + if summary["break_even"]: + base += f" - {summary['break_even']}BE" + return base + + +def _format_date_label(value: date | None) -> str: + if value is None: + return "N/A" + return value.strftime("%Y/%m/%d") + + +def format_daily_trading_report(payload: dict) -> str: + report_date = payload["report_date"] + streak = payload["streak"] + daily_summary = payload["daily_summary"] + long_summary = payload["by_direction"]["long"] + short_summary = payload["by_direction"]["short"] + + cumulative_lines = [] + for days in (7, 30, 60): + summary = payload["cumulative"][days] + period_label = f"{days:>2} Days" + percent_text = format_signed_percent(summary["compounded_percent"]).rjust(8) + money_text = f"{format_signed_currency(summary['net_profit'])} €".rjust(10) + win_rate_text = format_percent(summary["win_rate"]).rjust(7) + cumulative_lines.append( + f"⏱ {period_label}: {percent_text} | {money_text} (🎯 {win_rate_text} WR)" + ) + + last_seven_day_lines = [] + for row in payload["last_7_days"]: + day_label = row["day_date"].strftime("%m/%d") + real_text = format_signed_percent(row["real"]).rjust(8) + best_text = format_signed_percent(row["best"]).rjust(8) + max_text = format_signed_percent(row["max"]).rjust(8) + last_seven_day_lines.append(f"{day_label}: {real_text} | {best_text} | {max_text}") + + weekly_lines = [] + for row in payload["last_10_weeks"]: + performance_text = format_signed_percent(row["compounded_percent"]).rjust(8) + money_text = f"{format_signed_currency(row['net_profit'])} €".rjust(10) + win_rate_text = format_percent(row["win_rate"]).rjust(7) + weekly_lines.append( + f"{row['week_label']}: {performance_text} ({money_text}) | 🎯 {win_rate_text}" + ) + + monthly_lines = [] + for row in payload["last_12_months"]: + month_label = f"{row['month_label']}:".ljust(11) + performance_text = format_signed_percent(row["compounded_percent"]).rjust(8) + money_text = f"{format_signed_currency(row['net_profit'])} €".rjust(10) + win_rate_text = format_percent(row["win_rate"]).rjust(7) + monthly_lines.append( + f"{month_label} {performance_text} ({money_text}) | 🎯 {win_rate_text}" + ) + + yearly_lines = [] + for row in payload["last_5_years"]: + year_label = f"{row['year']}:" + performance_text = format_signed_percent(row["compounded_percent"]).rjust(8) + money_text = f"{format_signed_currency(row['net_profit'])} €".rjust(10) + win_rate_text = format_percent(row["win_rate"]).rjust(7) + yearly_lines.append( + f"{year_label} {performance_text} ({money_text}) | 🎯 {win_rate_text}" + ) + + lines = [ + f"📊 Trading Report | {report_date.strftime('%Y/%m/%d')}", + f"🔥 Winning Streak: {streak['current_streak']} Days (Last win: {_format_date_label(streak['last_win_date'])})", + "", + "--- 📝 DAILY SUMMARY ---", + f"💰 Net Profit: {format_signed_currency(daily_summary['net_profit'])} €", + ( + f"🎯 Win Rate: {format_percent(daily_summary['win_rate'])} " + f"({daily_summary['wins']}W | {daily_summary['losses']}L | {daily_summary['break_even']}BE)" + ), + f"⚖️ Profit Factor: {format_profit_factor(daily_summary['profit_factor'])}", + f"📉 Daily Max Drawdown: {format_signed_percent(daily_summary['max_drawdown'])}", + "", + f"📊 Avg Trade: {format_signed_percent(daily_summary['avg_trade'])}", + ( + f"🟩 Avg Win: {format_signed_percent(daily_summary['avg_win'])} | " + f"🟥 Avg Loss: {format_signed_percent(daily_summary['avg_loss'])}" + ), + ( + f"📈 Best: {format_signed_percent(daily_summary['best_trade'])} | " + f"📉 Worst: {format_signed_percent(daily_summary['worst_trade'])}" + ), + "", + "--- 🔍 BY DIRECTION ---", + f"{_format_direction_title('long')} ({_format_direction_counts(long_summary)})", + ( + f"Avg: {format_signed_percent(long_summary['avg_trade'])} | " + f"Max: {format_signed_percent(long_summary['best_trade'])} | " + f"Min: {format_signed_percent(long_summary['worst_trade'])}" + ), + "", + f"{_format_direction_title('short')} ({_format_direction_counts(short_summary)})", + ( + f"Avg: {format_signed_percent(short_summary['avg_trade'])} | " + f"Max: {format_signed_percent(short_summary['best_trade'])} | " + f"Min: {format_signed_percent(short_summary['worst_trade'])}" + ), + "", + "--- 🗓 CUMULATIVE P/L ---", + *cumulative_lines, + "", + "--- 📊 LAST 7 DAYS (Real | Best | Max) ---", + *last_seven_day_lines, + "", + "--- 📅 LAST 10 WEEKS ---", + *weekly_lines, + ] + + if monthly_lines: + lines.extend([ + "", + "--- 📅 LAST 12 MONTHS ---", + *monthly_lines, + ]) + + if yearly_lines: + lines.extend([ + "", + "--- 📅 LAST 5 YEARS ---", + *yearly_lines, + ]) + + return "\n".join(lines).strip() + + +def build_daily_trading_report_message( + *, + report_date: date | datetime | str | None, + closed_trades: list[dict], + daily_profit_history: list[dict], + daily_real: dict, + daily_best: dict, + daily_max: dict, +) -> str: + payload = build_daily_trading_report_payload( + report_date=report_date, + closed_trades=closed_trades, + daily_profit_history=daily_profit_history, + daily_real=daily_real, + daily_best=daily_best, + daily_max=daily_max, + ) + return format_daily_trading_report(payload) + + # --- Standalone Helper Functions (No changes needed for these) --- def append_performance_message(p_message, title, percentages): diff --git a/src/mq_telegram/async_tools.py b/src/mq_telegram/async_tools.py new file mode 100644 index 0000000..b1d2233 --- /dev/null +++ b/src/mq_telegram/async_tools.py @@ -0,0 +1,58 @@ +""" +Async Telegram message publishing via aio-pika. + +Usage: + sender = AsyncTelegramSender(config_manager) + await sender.connect() + await sender.send("Hello from WATA") + await sender.close() +""" + +import json +import logging +import aio_pika + + +logger = logging.getLogger(__name__) + + +class AsyncTelegramSender: + """Publishes messages to the ``telegram_channel`` RabbitMQ queue via aio-pika.""" + + def __init__(self, config_manager): + self.config_manager = config_manager + self._connection: aio_pika.abc.AbstractRobustConnection | None = None + self._channel: aio_pika.abc.AbstractChannel | None = None + + async def connect(self): + """Establish a persistent connection to RabbitMQ.""" + if self._connection and not self._connection.is_closed: + return + + rabbitmq_config = self.config_manager.get_rabbitmq_config() + host = rabbitmq_config["hostname"] + user = rabbitmq_config["authentication"]["username"] + password = rabbitmq_config["authentication"]["password"] + + url = f"amqp://{user}:{password}@{host}/" + self._connection = await aio_pika.connect_robust(url) + self._channel = await self._connection.channel() + await self._channel.declare_queue("telegram_channel", durable=False) + logger.info("AsyncTelegramSender connected to RabbitMQ.") + + async def send(self, message: str): + """Publish a message to the telegram queue.""" + if not self._channel or self._channel.is_closed: + await self.connect() + + body = json.dumps({"message": message}).encode() + await self._channel.default_exchange.publish( + aio_pika.Message(body=body), + routing_key="telegram_channel", + ) + logger.debug("Telegram message published (%d chars)", len(message)) + + async def close(self): + if self._connection and not self._connection.is_closed: + await self._connection.close() + logger.info("AsyncTelegramSender connection closed.") diff --git a/src/position_monitor/__init__.py b/src/position_monitor/__init__.py new file mode 100644 index 0000000..9f8bb12 --- /dev/null +++ b/src/position_monitor/__init__.py @@ -0,0 +1,304 @@ +""" +Position Monitor Service — real-time WebSocket streaming + ``trading-ops`` queue. + +Uses Saxo's WebSocket Streaming API for instant position monitoring +(replaces the 7-second polling approach). Still consumes ``trading-ops`` +for time-triggered events like ``daily_stats``. + +Runs as a separate Docker container (WATA_APP_ROLE=position_monitor). +""" + +import asyncio +import json +import logging +import os +import sys +from datetime import date, timedelta + +import aio_pika + +from src.configuration import ConfigurationManager +from src.logging_helper import setup_logging +from src.trade.async_services import ( + AsyncSaxoApiClient, + AsyncOrderService, + AsyncPositionService, + AsyncPerformanceMonitor, +) +from src.database.postgres import ( + PostgresConnectionManager, + AsyncDbPositionManager, + AsyncDbTradePerformanceManager, + init_schema, +) +from src.mq_telegram.async_tools import AsyncTelegramSender +from src.saxo_authen import SaxoAuth +from src.saxo_streaming.client import SaxoStreamClient +from src.trade.rules import TradingRule +from src.message_helper import build_daily_trading_report_message + +logger = logging.getLogger(__name__) + +APP_VERSION = "unknown" + + +def get_version() -> str: + try: + vf = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "VERSION") + with open(vf, "r") as f: + return f.read().strip() + except Exception: + return "unknown" + + +# ───────────────────────────────────────────────────── +# Streaming callback +# ───────────────────────────────────────────────────── + +async def on_positions_update( + positions: dict[str, dict], + performance_monitor: AsyncPerformanceMonitor, + db_position_manager: AsyncDbPositionManager, + telegram: AsyncTelegramSender, +): + """ + Called by :class:`SaxoStreamClient` every time the position snapshot + changes. Runs the same SL/TP/trailing-stop checks that the old + ``handle_check_positions`` did, but without any API polling. + """ + if not positions: + return + + try: + perf_result = await performance_monitor.check_positions_from_stream(positions) + perf_errors = perf_result.get("errors", 0) + perf_closed = len(perf_result.get("closed_positions_processed", [])) + + # DB sync is less critical with streaming (we see closures in real-time) + # but we still reconcile to catch anything done externally + sync_result = await performance_monitor.sync_db_positions_with_api() + updates = sync_result.get("updates_for_db", []) + + sync_applied = 0 + sync_errors = 0 + for position_id, update_data in updates: + try: + await db_position_manager.update_turbo_position_data(position_id, update_data) + sync_applied += 1 + except Exception as e: + sync_errors += 1 + logger.critical("SYNC ERROR: Failed DB update for Pos %s: %s", position_id, e, exc_info=True) + await telegram.send(f"CRITICAL SYNC ERROR: Failed DB update for Pos {position_id}: {e}") + + total_errors = perf_errors + sync_errors + if total_errors > 0: + logger.warning("Stream position check: closed=%d, synced=%d, errors=%d", perf_closed, sync_applied, total_errors) + else: + logger.debug("Stream position check: closed=%d, synced=%d, errors=%d", perf_closed, sync_applied, total_errors) + + except Exception as e: + logger.error("Error in streaming position callback: %s", e, exc_info=True) + + +async def handle_daily_stats( + db_position_manager: AsyncDbPositionManager, + db_perf_manager: AsyncDbTradePerformanceManager, + telegram: AsyncTelegramSender, +): + """Generate and send daily performance stats.""" + days = 7 + report_date = date.today() + history_start = date(report_date.year - 4, 1, 1) + + closed_trades, daily_profit_history, real_daily, best_daily, max_daily = await asyncio.gather( + db_position_manager.get_closed_trade_history(start_date=history_start, end_date=report_date), + db_position_manager.get_daily_profit_history(end_date=report_date), + db_position_manager.get_percent_of_last_n_days(days), + db_position_manager.get_best_percent_of_last_n_days(days), + db_position_manager.get_theoretical_percent_of_last_n_days_on_max(days), + ) + + message = build_daily_trading_report_message( + report_date=report_date, + closed_trades=closed_trades, + daily_profit_history=daily_profit_history, + daily_real=real_daily, + daily_best=best_daily, + daily_max=max_daily, + ) + await telegram.send(message) + await db_perf_manager.create_last_day_trade_performance_data() + logger.info("Daily stats sent.") + + +# ───────────────────────────────────────────────────── +# Message dispatcher +# ───────────────────────────────────────────────────── + +async def dispatch_ops_message( + message: aio_pika.IncomingMessage, + performance_monitor: AsyncPerformanceMonitor, + db_position_manager: AsyncDbPositionManager, + db_perf_manager: AsyncDbTradePerformanceManager, + telegram: AsyncTelegramSender, +): + async with message.process(requeue=False): + try: + body = json.loads(message.body.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + logger.error("Cannot decode message: %s", e) + return + + action = body.get("action") + logger.debug("Ops dispatch: action=%s", action) + + try: + if action == "daily_stats": + await handle_daily_stats(db_position_manager, db_perf_manager, telegram) + elif action == "check_positions_on_saxo_api": + # Legacy — streaming handles this now; run a one-off check as fallback + logger.info("Legacy check_positions_on_saxo_api received — running REST fallback.") + result = await performance_monitor.check_all_positions_performance() + logger.info("REST fallback completed: %s", result) + else: + logger.warning("Unknown ops action: %s", action) + + except Exception as e: + logger.error("Error processing ops action %s: %s", action, e, exc_info=True) + await telegram.send(f"Position Monitor ERROR ({action}): {type(e).__name__}: {e}") + + +# ───────────────────────────────────────────────────── +# Main +# ───────────────────────────────────────────────────── + +async def main(): + global APP_VERSION + APP_VERSION = get_version() + + config_path = os.getenv("WATA_CONFIG_PATH") + if not config_path: + print("FATAL: WATA_CONFIG_PATH not set", file=sys.stderr) + sys.exit(10) + + config_manager = ConfigurationManager(config_path) + setup_logging(config_manager, "wata-position-monitor") + logger.info("--- Starting WATA Position Monitor v%s ---", APP_VERSION) + + telegram = AsyncTelegramSender(config_manager) + await telegram.connect() + + stream_client: SaxoStreamClient | None = None + pg = None + api_client = None + + try: + # PostgreSQL + pg = PostgresConnectionManager.from_config(config_manager) + await pg.connect() + await init_schema(pg) + db_position_manager = AsyncDbPositionManager(pg) + db_perf_manager = AsyncDbTradePerformanceManager(pg) + + # Saxo API client + saxo_auth = SaxoAuth(config_manager) + api_client = AsyncSaxoApiClient(config_manager, saxo_auth) + await api_client.ensure_ready() + + # Fetch account keys + from src.trader import get_account_info_async + acct = await get_account_info_async(api_client) + account_key = acct.AccountKey + client_key = acct.ClientKey + + # Services + order_service = AsyncOrderService(api_client, account_key, client_key) + position_service = AsyncPositionService(api_client, order_service, config_manager, account_key, client_key) + trading_rule = TradingRule(config_manager, None) + performance_monitor = AsyncPerformanceMonitor( + position_service, order_service, config_manager, + db_position_manager, trading_rule, telegram.send, + ) + + # ── Streaming configuration ── + environment = config_manager.get_config_value("saxo_auth.env", "live") + streaming_config = config_manager.get_config_value("trade.config.general.streaming", {}) + refresh_rate_ms = streaming_config.get("refresh_rate_ms", 1000) + reconnect_delay = streaming_config.get("reconnect_delay_seconds", 1.0) + max_reconnect_delay = streaming_config.get("max_reconnect_delay_seconds", 30.0) + reauth_interval_seconds = streaming_config.get("reauth_interval_seconds", 900) + + # Build the streaming callback (closes over service objects) + async def _stream_callback(positions: dict[str, dict]): + await on_positions_update( + positions, performance_monitor, db_position_manager, telegram, + ) + + # Create streaming client + stream_client = SaxoStreamClient( + api_client=api_client, + account_key=account_key, + client_key=client_key, + on_positions_update=_stream_callback, + environment=environment, + access_token_getter=lambda: saxo_auth.get_token(), + refresh_rate_ms=refresh_rate_ms, + reconnect_delay=reconnect_delay, + max_reconnect_delay=max_reconnect_delay, + reauth_interval_seconds=reauth_interval_seconds, + ) + + # ── RabbitMQ — still consume trading-ops for daily_stats etc. ── + rmq_config = config_manager.get_rabbitmq_config() + rmq_url = f"amqp://{rmq_config['authentication']['username']}:{rmq_config['authentication']['password']}@{rmq_config['hostname']}/" + connection = await aio_pika.connect_robust(rmq_url) + channel = await connection.channel() + await channel.set_qos(prefetch_count=1) + ops_queue = await channel.declare_queue("trading-ops", durable=True) + + async def on_message(msg: aio_pika.IncomingMessage): + await dispatch_ops_message( + msg, performance_monitor, db_position_manager, + db_perf_manager, telegram, + ) + + await ops_queue.consume(on_message) + + await telegram.send( + f"WATA Position Monitor v{APP_VERSION} is running " + f"(WebSocket streaming + trading-ops queue)." + ) + logger.info( + "Position Monitor startup complete. Streaming to %s, " + "also consuming trading-ops queue.", + environment, + ) + + # Run the streaming client (blocks until stop() or fatal error) + await stream_client.start() + + except Exception as e: + logger.critical("Unhandled startup error: %s", e, exc_info=True) + try: + await telegram.send(f"Position Monitor CRITICAL FAILURE: {e}") + except Exception: + pass + sys.exit(1) + finally: + logger.info("--- Shutting down WATA Position Monitor ---") + if stream_client: + await stream_client.stop() + if api_client is not None: + await api_client.close() + if pg is not None: + await pg.close() + await telegram.close() + + +if __name__ == "__main__": + try: + import uvloop + uvloop.install() + except ImportError: + pass + asyncio.run(main()) diff --git a/src/saxo_authen/__init__.py b/src/saxo_authen/__init__.py index 52d65a8..e096e00 100644 --- a/src/saxo_authen/__init__.py +++ b/src/saxo_authen/__init__.py @@ -7,6 +7,7 @@ import os import logging import base64 +import pika from cryptography.fernet import Fernet from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC @@ -14,7 +15,6 @@ from src.configuration import ConfigurationManager from src.mq_telegram.tools import send_message_to_mq_for_telegram -from src.database import DbTokenManager logger = logging.getLogger(__name__) @@ -41,10 +41,6 @@ def __init__(self, config_manager, rabbit_connection=None): # Initialize the encryption key self._initialize_encryption() - - # Initialize token database manager - self.token_db = DbTokenManager(config_manager) - self.token_id = "saxo_token" # Unique identifier for Saxo tokens def _initialize_encryption(self): """ @@ -90,6 +86,63 @@ def _decrypt_data(self, encrypted_data): logger.error(f"Error decrypting data: {e}") return None + def _write_encrypted_token_file(self, encrypted_data): + """Persist encrypted token bytes to the configured token file.""" + with open(self.token_file_path, "wb") as token_file: + token_file.write(encrypted_data) + os.chmod(self.token_file_path, stat.S_IRUSR | stat.S_IWUSR) + + def _read_encrypted_token_file(self): + """Read encrypted token bytes from the configured token file.""" + if not os.path.exists(self.token_file_path): + return None + + with open(self.token_file_path, "rb") as token_file: + return token_file.read() + + def _send_telegram_notification(self, message): + """ + Send a Telegram notification via RabbitMQ. + + If a pre-existing blocking RabbitMQ connection was provided, reuse it. + Otherwise open a short-lived synchronous connection from config so + notifications still work during async service startup. + """ + if self.rabbit_connection is not None: + try: + send_message_to_mq_for_telegram(self.rabbit_connection, message) + return True + except Exception as e: + logger.error(f"Failed to send Telegram notification via existing RabbitMQ connection: {e}") + + connection = None + try: + rabbitmq_config = self.config_manager.get_rabbitmq_config() + connection = pika.BlockingConnection( + pika.ConnectionParameters( + host=rabbitmq_config["hostname"], + credentials=pika.PlainCredentials( + rabbitmq_config["authentication"]["username"], + rabbitmq_config["authentication"]["password"], + ), + ) + ) + channel = connection.channel() + channel.queue_declare(queue="telegram_channel") + payload = json.dumps({"message": message}) + channel.basic_publish( + exchange="", + routing_key="telegram_channel", + body=payload, + ) + return True + except Exception as e: + logger.error(f"Failed to send Telegram notification: {e}") + return False + finally: + if connection is not None and connection.is_open: + connection.close() + def get_authorization_url(self): """ Generate and return the authorization URL for the user to visit in their browser. @@ -145,17 +198,13 @@ def get_authorization_code(self): auth_instructions += "\n\n⏳ Waiting for authorization code..." print(auth_instructions) - - # Send the instructions to Telegram - if hasattr(self, 'rabbit_connection'): - try: - send_message_to_mq_for_telegram(self.rabbit_connection, - f"--- 🔐 SAXO AUTHORIZATION REQUIRED ---\n{auth_instructions}") - logger.info("Authorization instructions sent to Telegram") - except Exception as e: - logger.error(f"Failed to send authorization instructions to Telegram: {e}") + + if self._send_telegram_notification( + f"--- 🔐 SAXO AUTHORIZATION REQUIRED ---\n{auth_instructions}" + ): + logger.info("Authorization instructions sent to Telegram") else: - logger.warning("No rabbit_connection available, can't send message to Telegram") + logger.warning("Unable to send authorization instructions to Telegram") # Wait for the auth code file to appear (with timeout) max_wait_time = 300 # 5 minutes @@ -164,18 +213,16 @@ def get_authorization_code(self): while time.time() - start_time < max_wait_time: code = self.read_auth_code_from_file() if code: - if hasattr(self, 'rabbit_connection'): - send_message_to_mq_for_telegram(self.rabbit_connection, - "✅ Authorization code received successfully!") + self._send_telegram_notification( + "✅ Authorization code received successfully!" + ) return code time.sleep(5) error_message = "Timeout waiting for authorization code" logger.error(error_message) - - if hasattr(self, 'rabbit_connection'): - send_message_to_mq_for_telegram(self.rabbit_connection, - f"❌ ERROR: {error_message}") + + self._send_telegram_notification(f"❌ ERROR: {error_message}") raise TimeoutError(error_message) @@ -257,28 +304,16 @@ def ask_new_token(self): def save_token_data(self, token_data): """ - Save token data to database in encrypted format. + Save token data to the configured file in encrypted format. """ token_data["date_saved"] = datetime.datetime.now().isoformat() # Encrypt the token data encrypted_data = self._encrypt_data(token_data) - - # Store in database - metadata = json.dumps({ - "expires_in": token_data["expires_in"], - "refresh_token_expires_in": token_data.get("refresh_token_expires_in", 0), - "date_saved": token_data["date_saved"] - }) - - self.token_db.store_token( - token_id=self.token_id, - token_type="saxo_oauth", - encrypted_data=encrypted_data, - metadata=metadata - ) - - logger.info("Token data securely saved to database with encryption") + + self._write_encrypted_token_file(encrypted_data) + + logger.info("Token data securely saved to encrypted file") def is_token_expired(self, token_data): @@ -325,12 +360,11 @@ def get_token(self): """ try: token_data = {} - - # Try to get token from database first - encrypted_data = self.token_db.get_token(self.token_id) + + encrypted_data = self._read_encrypted_token_file() if encrypted_data: token_data = self._decrypt_data(encrypted_data) or {} - logger.debug("Token data retrieved from database") + logger.debug("Token data retrieved from encrypted file") if self.is_token_expired(token_data): if self.is_refresh_token_expired(token_data): @@ -370,19 +404,8 @@ def get_token(self): print("Configuring") # Create an instance of ConfigurationManager config_manager = ConfigurationManager(config_path) - - # Try to initialize rabbit connection if available - rabbit_connection = None - try: - from src.mq_telegram.rabbit_connection import RabbitMQConnection - rabbit_config = config_manager.get_config_value("mq_telegram") - rabbit_connection = RabbitMQConnection(rabbit_config) - print("RabbitMQ connection established") - except Exception as rabbit_error: - print(f"Could not establish RabbitMQ connection: {rabbit_error}") - logger.warning(f"Could not establish RabbitMQ connection: {rabbit_error}") - - saxo_auth = SaxoAuth(config_manager, rabbit_connection) + + saxo_auth = SaxoAuth(config_manager) token = saxo_auth.get_token() print(f"Access Token: {token}") except Exception as e: diff --git a/src/saxo_openapi/async_client.py b/src/saxo_openapi/async_client.py new file mode 100644 index 0000000..d3e41d2 --- /dev/null +++ b/src/saxo_openapi/async_client.py @@ -0,0 +1,188 @@ +# src/saxo_openapi/async_client.py +""" +Async Saxo OpenAPI client using httpx.AsyncClient. +Drop-in replacement for the synchronous requests-based client, +designed for the async Trader and Position Monitor services. +""" + +import json +import logging +import time +import asyncio +from threading import Lock + +import httpx + +from .exceptions import OpenAPIError + +logger = logging.getLogger(__name__) + +TRADING_ENVIRONMENTS = { + "simulation": { + "stream": "https://sim-streaming.saxobank.com", + "api": "https://gateway.saxobank.com", + "prefix": "sim", + }, + "live": { + "stream": "https://live-streaming.saxobank.com", + "api": "https://gateway.saxobank.com", + }, +} + +DEFAULT_HEADERS = {"Accept-Encoding": "gzip, deflate"} + + +def _mk_endpoint(endpoint, env: str, ep_type: str) -> str: + if env == "live": + path = str(endpoint) + elif env == "simulation": + path = f"{TRADING_ENVIRONMENTS[env]['prefix']}/{endpoint}" + else: + raise ValueError(f"Unknown environment: {env}") + return f"{TRADING_ENVIRONMENTS[env][ep_type]}/{path}" + + +class AsyncRateLimiter: + """Async-safe rate-limiter that reads Saxo response headers.""" + + def __init__(self): + self.session_remaining: int = 120 + self.session_reset: int = 0 + self._lock = asyncio.Lock() + self.LOW_REQUESTS_THRESHOLD = 10 + + def update_limits(self, headers: httpx.Headers): + # No lock needed – called right after an await in the same task + if "X-RateLimit-Session-Remaining" in headers: + self.session_remaining = int(headers["X-RateLimit-Session-Remaining"]) + else: + self.session_remaining = 120 + if "X-RateLimit-Session-Reset" in headers: + self.session_reset = int(headers["X-RateLimit-Session-Reset"]) + else: + self.session_reset = 0 + + async def wait_if_needed(self): + async with self._lock: + if self.session_remaining <= 1: + wait_time = max(self.session_reset, 1) + logger.info("Rate limit near threshold. Waiting %d seconds", wait_time) + await asyncio.sleep(wait_time) + elif self.session_remaining <= self.LOW_REQUESTS_THRESHOLD: + logger.info( + "Rate limit below %d (%d remaining). Adding 1s delay", + self.LOW_REQUESTS_THRESHOLD, + self.session_remaining, + ) + await asyncio.sleep(1) + + +class AsyncAPI: + """Async Saxo OpenAPI client powered by httpx.AsyncClient.""" + + def __init__( + self, + access_token: str, + environment: str = "live", + headers: dict | None = None, + timeout: float = 30.0, + ): + if environment not in TRADING_ENVIRONMENTS: + raise ValueError(f"Unknown environment: {environment}") + + self.environment = environment + self.access_token = access_token + self.rate_limiter = AsyncRateLimiter() + + _headers = {**DEFAULT_HEADERS, "Authorization": f"Bearer {access_token}"} + if headers: + _headers.update(headers) + + client_kwargs = { + "headers": _headers, + "timeout": httpx.Timeout(timeout), + "limits": httpx.Limits( + max_connections=20, + max_keepalive_connections=10, + keepalive_expiry=60, + ), + } + + try: + self._client = httpx.AsyncClient( + http2=True, + **client_kwargs, + ) + except ImportError: + logger.warning( + "HTTP/2 support is unavailable because the optional 'h2' dependency " + "is not installed. Falling back to HTTP/1.1." + ) + self._client = httpx.AsyncClient( + http2=False, + **client_kwargs, + ) + + def update_token(self, new_token: str): + """Hot-swap the access token without recreating the client.""" + self.access_token = new_token + self._client.headers["Authorization"] = f"Bearer {new_token}" + + async def close(self): + await self._client.aclose() + + async def request(self, endpoint): + """ + Execute an APIRequest endpoint object asynchronously. + + Mirrors the synchronous API.request() interface so that + existing endpoint objects (rd.instruments.Instruments, etc.) + work without modification. + """ + method: str = endpoint.method.lower() + params = getattr(endpoint, "params", {}) + ep_headers = getattr(endpoint, "HEADERS", {}) if hasattr(endpoint, "HEADERS") else {} + + url = _mk_endpoint(endpoint, self.environment, "api") + + kwargs: dict = {} + if method in ("get", "delete", "patch"): + kwargs["params"] = params + if hasattr(endpoint, "data") and endpoint.data: + kwargs["json"] = endpoint.data + + await self.rate_limiter.wait_if_needed() + logger.debug("AsyncAPI: %s %s", method.upper(), url) + + try: + response = await self._client.request(method, url, headers=ep_headers, **kwargs) + except httpx.RequestError as err: + logger.error("AsyncAPI: request to %s failed: %s", url, err) + raise err + + self.rate_limiter.update_limits(response.headers) + + # Handle 429 rate-limit with one retry + if response.status_code == 429: + reset_time = int(response.headers.get("X-RateLimit-Session-Reset", 60)) + logger.warning("Rate limit exceeded (429). Waiting %d seconds…", reset_time) + await asyncio.sleep(reset_time) + response = await self._client.request(method, url, headers=ep_headers, **kwargs) + self.rate_limiter.update_limits(response.headers) + + if response.status_code >= 400: + content = response.text + logger.error("AsyncAPI: %s %s → %d %s", method.upper(), url, response.status_code, content[:200]) + raise OpenAPIError(response.status_code, response.reason_phrase, content) + + # Parse response + if hasattr(endpoint, "RESPONSE_DATA") and getattr(endpoint, "RESPONSE_DATA") is None: + content = None + elif hasattr(endpoint, "RESPONSE_DATA") and getattr(endpoint, "RESPONSE_DATA") == "text": + content = response.text + else: + content = response.json() if response.content else None + + endpoint.response = content + endpoint.status_code = response.status_code + return content diff --git a/src/saxo_openapi/contrib/orders/helper.py b/src/saxo_openapi/contrib/orders/helper.py index ee934ed..b6fae07 100644 --- a/src/saxo_openapi/contrib/orders/helper.py +++ b/src/saxo_openapi/contrib/orders/helper.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from datetime import datetime -import saxo_openapi.definitions.orders as OD +from ...definitions import orders as OD def direction_from_amount(Amount): diff --git a/src/saxo_openapi/contrib/orders/limitorder.py b/src/saxo_openapi/contrib/orders/limitorder.py index cc10cd6..6866b62 100644 --- a/src/saxo_openapi/contrib/orders/limitorder.py +++ b/src/saxo_openapi/contrib/orders/limitorder.py @@ -2,7 +2,7 @@ from .baseorder import BaseOrder from .helper import direction_from_amount, order_duration_spec -import saxo_openapi.definitions.orders as OD +from ...definitions import orders as OD from .mixin import OnFillHnd diff --git a/src/saxo_openapi/contrib/orders/marketorder.py b/src/saxo_openapi/contrib/orders/marketorder.py index 02ea1fd..2549a4e 100644 --- a/src/saxo_openapi/contrib/orders/marketorder.py +++ b/src/saxo_openapi/contrib/orders/marketorder.py @@ -2,7 +2,7 @@ from .baseorder import BaseOrder from .helper import direction_from_amount -import saxo_openapi.definitions.orders as OD +from ...definitions import orders as OD from .mixin import OnFillHnd diff --git a/src/saxo_openapi/contrib/orders/onfill.py b/src/saxo_openapi/contrib/orders/onfill.py index f0a18d4..62a61b0 100644 --- a/src/saxo_openapi/contrib/orders/onfill.py +++ b/src/saxo_openapi/contrib/orders/onfill.py @@ -160,7 +160,7 @@ from abc import abstractmethod from .baseorder import BaseOrder -import saxo_openapi.definitions.orders as OD +from ...definitions import orders as OD from .helper import order_duration_spec diff --git a/src/saxo_openapi/contrib/orders/stoporder.py b/src/saxo_openapi/contrib/orders/stoporder.py index b4ca82a..61e11a2 100644 --- a/src/saxo_openapi/contrib/orders/stoporder.py +++ b/src/saxo_openapi/contrib/orders/stoporder.py @@ -2,7 +2,7 @@ from .baseorder import BaseOrder from .helper import direction_from_amount, order_duration_spec -import saxo_openapi.definitions.orders as OD +from ...definitions import orders as OD from .mixin import OnFillHnd diff --git a/src/saxo_openapi/contrib/session.py b/src/saxo_openapi/contrib/session.py index effdd6f..d1ac078 100644 --- a/src/saxo_openapi/contrib/session.py +++ b/src/saxo_openapi/contrib/session.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import saxo_openapi.endpoints.portfolio as pf +from ..endpoints import portfolio as pf from collections import namedtuple diff --git a/src/saxo_openapi/contrib/util/instrument_to_uic.py b/src/saxo_openapi/contrib/util/instrument_to_uic.py index df8a071..9d690a7 100644 --- a/src/saxo_openapi/contrib/util/instrument_to_uic.py +++ b/src/saxo_openapi/contrib/util/instrument_to_uic.py @@ -2,8 +2,8 @@ """Utility classes and/or functions.""" -from saxo_openapi.definitions.orders import AssetType -import saxo_openapi.endpoints.referencedata as rd +from ...definitions.orders import AssetType +from ...endpoints import referencedata as rd def InstrumentToUic(client, AccountKey, spec, assettype=AssetType.FxSpot): diff --git a/src/saxo_openapi/definitions/__init__.py b/src/saxo_openapi/definitions/__init__.py index bb2e3fb..d1e9d93 100644 --- a/src/saxo_openapi/definitions/__init__.py +++ b/src/saxo_openapi/definitions/__init__.py @@ -38,7 +38,7 @@ class provides the ID and the description of the definitions. def make_definition_classes(mod): """Dynamically create the definition classes from module 'mod'.""" - rootpath = "saxo_openapi" + rootpath = __name__.rpartition(".")[0] PTH = "{}.definitions.{}".format(rootpath, mod) M = import_module(PTH) diff --git a/src/saxo_openapi/saxo_openapi.py b/src/saxo_openapi/saxo_openapi.py index 0e9203d..e0d1b2d 100644 --- a/src/saxo_openapi/saxo_openapi.py +++ b/src/saxo_openapi/saxo_openapi.py @@ -13,12 +13,12 @@ TRADING_ENVIRONMENTS = { 'simulation': { - 'stream': 'https://streaming.saxotrader.com', + 'stream': 'https://sim-streaming.saxobank.com', 'api': 'https://gateway.saxobank.com', 'prefix': 'sim' }, 'live': { - 'stream': 'https://streaming.saxotrader.com', + 'stream': 'https://live-streaming.saxobank.com', 'api': 'https://gateway.saxobank.com' } } diff --git a/src/saxo_streaming/__init__.py b/src/saxo_streaming/__init__.py new file mode 100644 index 0000000..dac2262 --- /dev/null +++ b/src/saxo_streaming/__init__.py @@ -0,0 +1,2 @@ +# src/saxo_streaming/__init__.py +"""Saxo Bank WebSocket Streaming — real-time position & price monitoring.""" diff --git a/src/saxo_streaming/client.py b/src/saxo_streaming/client.py new file mode 100644 index 0000000..d0f81e7 --- /dev/null +++ b/src/saxo_streaming/client.py @@ -0,0 +1,549 @@ +# src/saxo_streaming/client.py +""" +Persistent async WebSocket client for Saxo Bank's streaming API. + +This module maintains a WebSocket connection to Saxo's streaming endpoint +and manages position + price subscriptions. When a position or price update +arrives, a user-supplied callback is invoked with the merged (snapshot + +delta) data. + +Architecture +~~~~~~~~~~~~ +1. ``SaxoStreamClient.start()`` opens the WS connection and creates the + initial subscriptions via REST (the snapshot is returned in the + subscription response). +2. Binary frames arriving over the WS are parsed with + ``saxo_openapi.contrib.ws.stream.decode_ws_msg``. +3. Delta updates are merged into the cached snapshot (deep merge). +4. Control messages (_heartbeat, _resetsubscriptions, _disconnect) are + handled according to the Saxo protocol. +5. Re-authorisation happens automatically before the token expires. +6. Reconnection preserves the ``last_message_id`` so the server can + resume from where it left off. +""" + +from __future__ import annotations + +import asyncio +import copy +import json +import logging +import time +from urllib.parse import urlencode +import uuid +from typing import Any, Callable, Coroutine + +import websockets +import websockets.exceptions + +from src.saxo_openapi.async_client import AsyncAPI, TRADING_ENVIRONMENTS +from src.saxo_openapi.contrib.ws.stream import decode_ws_msg +import src.saxo_openapi.endpoints.portfolio as pf +import src.saxo_openapi.endpoints.trading as tr + +logger = logging.getLogger(__name__) + +# Type alias for the user-supplied position-update callback. +# Signature: async def on_update(positions: dict[str, dict]) -> None +PositionUpdateCallback = Callable[[dict[str, dict]], Coroutine[Any, Any, None]] + + +def _deep_merge(base: dict, delta: dict) -> dict: + """ + Recursively merge *delta* into *base* **in place** and return *base*. + + - Lists in *delta* replace lists in *base* entirely (Saxo convention). + - ``__count`` fields are ignored in the delta (recalculated from Data). + - A value of ``None`` in *delta* deletes the key from *base*. + """ + for key, value in delta.items(): + if key == "__count": + continue + if value is None: + base.pop(key, None) + elif isinstance(value, dict) and isinstance(base.get(key), dict): + _deep_merge(base[key], value) + else: + base[key] = value + return base + + +class SaxoStreamClient: + """ + High-level async streaming client for Saxo Bank's plain WebSocket API. + + Parameters + ---------- + api_client : AsyncAPI | compatible wrapper + An already-initialised async HTTP client or wrapper exposing + ``request()``. The position monitor passes ``AsyncSaxoApiClient`` so + token refreshes are handled consistently for REST subscription calls. + account_key : str + Saxo account key. + client_key : str + Saxo client key. + on_positions_update : PositionUpdateCallback + Async callback invoked with {position_id: merged_position_dict} + every time a position or price delta arrives. + environment : str + ``"live"`` or ``"simulation"``. + access_token_getter : Callable[[], str] + Synchronous callable that returns the current bearer token (e.g. + ``saxo_auth.get_token``). + refresh_rate_ms : int + Desired refresh rate for the subscriptions (milliseconds). + reconnect_delay : float + Base delay in seconds before attempting to reconnect after a WS drop. + max_reconnect_delay : float + Cap for exponential-backoff reconnection delay. + reauth_interval_seconds : int + Interval between WebSocket re-authorisation calls. + """ + + # ── Construction ────────────────────────────────────────────── + + def __init__( + self, + api_client, + account_key: str, + client_key: str, + on_positions_update: PositionUpdateCallback, + environment: str = "live", + access_token_getter: Callable[[], str] | None = None, + refresh_rate_ms: int = 1000, + reconnect_delay: float = 1.0, + max_reconnect_delay: float = 30.0, + reauth_interval_seconds: int = 15 * 60, + ): + self._api = api_client + self._account_key = account_key + self._client_key = client_key + self._on_positions_update = on_positions_update + self._environment = environment + self._get_token = access_token_getter + self._refresh_rate_ms = refresh_rate_ms + self._reconnect_delay = reconnect_delay + self._max_reconnect_delay = max_reconnect_delay + self._reauth_interval_seconds = reauth_interval_seconds + + # Connection state + self._context_id: str = "" + self._pos_ref_id: str = "" + self._ws: websockets.WebSocketClientProtocol | None = None + self._last_message_id: int | None = None + self._running = False + self._receive_task: asyncio.Task | None = None + self._reauth_task: asyncio.Task | None = None + self._subscription_ready = False + + # Snapshot cache: {position_id: full_position_dict} + self._positions: dict[str, dict] = {} + + # ── Public API ──────────────────────────────────────────────── + + async def start(self): + """Start the streaming loop (run forever until ``stop()`` is called).""" + self._running = True + backoff = self._reconnect_delay + + while self._running: + try: + await self._connect_and_subscribe() + backoff = self._reconnect_delay # reset on success + await self._receive_loop() + except asyncio.CancelledError: + break + except websockets.exceptions.ConnectionClosed as e: + logger.warning("WebSocket closed: code=%s reason=%s", e.code, e.reason) + except Exception as e: + logger.error("Stream error: %s", e, exc_info=True) + + if not self._running: + break + + logger.info("Reconnecting in %.1fs …", backoff) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, self._max_reconnect_delay) + + async def stop(self): + """Gracefully tear down the connection & subscriptions.""" + self._running = False + if self._reauth_task and not self._reauth_task.done(): + self._reauth_task.cancel() + if self._receive_task and not self._receive_task.done(): + self._receive_task.cancel() + await self._cleanup_subscriptions() + if self._ws and not self._ws.closed: + await self._ws.close() + logger.info("WebSocket closed gracefully.") + + @property + def positions(self) -> dict[str, dict]: + """Return a **deep copy** of the current position snapshot cache.""" + return copy.deepcopy(self._positions) + + # ── Connection ──────────────────────────────────────────────── + + async def _connect_and_subscribe(self): + """Open WS, create position subscription, start reauth timer.""" + # 1. Generate IDs once and reuse them across reconnects. + if not self._context_id: + self._context_id = _short_id("ctx") + if not self._pos_ref_id: + self._pos_ref_id = _short_id("pos") + + # 2. Build WS URL + token = self._current_token() + ws_url = self._build_ws_url(token) + + # 3. Open WebSocket + logger.info("Connecting to Saxo WebSocket (contextId=%s) …", self._context_id) + self._ws = await websockets.connect( + ws_url, + ping_interval=20, + ping_timeout=20, + close_timeout=5, + max_size=2 ** 22, # 4 MiB + ) + logger.info("WebSocket connected.") + + # 4. Position subscription (REST call, snapshot returned) — only when needed. + if not self._subscription_ready: + await self._create_position_subscription() + self._subscription_ready = True + + # 5. Periodic re-authorisation in background + if self._reauth_task and not self._reauth_task.done(): + self._reauth_task.cancel() + self._reauth_task = asyncio.create_task(self._reauth_loop()) + + async def _create_position_subscription(self): + """Create a position list subscription via REST and seed the cache.""" + self._sync_api_token() + + data = { + "Arguments": { + "AccountKey": self._account_key, + "ClientKey": self._client_key, + "FieldGroups": [ + "PositionBase", + "PositionView", + "DisplayAndFormat", + "ExchangeInfo", + ], + }, + "ContextId": self._context_id, + "ReferenceId": self._pos_ref_id, + "RefreshRate": self._refresh_rate_ms, + "Format": "application/json", + } + + req = pf.positions.PositionListSubscription(data=data) + resp = await self._request(req) + + # Seed snapshot cache + self._positions.clear() + snapshot_data = resp.get("Snapshot", {}).get("Data", []) if resp else [] + for pos in snapshot_data: + pid = pos.get("PositionId") + if pid: + self._positions[pid] = pos + + logger.info( + "Position subscription created (refId=%s). Snapshot: %d positions.", + self._pos_ref_id, + len(self._positions), + ) + + # Fire initial callback with snapshot + if self._positions: + await self._fire_callback() + + async def _cleanup_subscriptions(self): + """Delete active subscriptions (best-effort).""" + if not self._context_id: + return + try: + self._sync_api_token() + req = pf.positions.PositionSubscriptionRemoveMultiple( + ContextId=self._context_id + ) + await self._request(req) + logger.info("Subscriptions for context %s removed.", self._context_id) + self._subscription_ready = False + except Exception as e: + logger.warning("Failed to clean up subscriptions: %s", e) + + # ── Receive loop ────────────────────────────────────────────── + + async def _receive_loop(self): + """Read binary frames from the WS, parse, dispatch.""" + assert self._ws is not None + + async for raw in self._ws: + if not self._running: + break + + if isinstance(raw, str): + # Saxo should always send binary, but handle text just in case + logger.debug("Text frame: %s", raw[:200]) + continue + + try: + for message in decode_ws_msg(raw): + self._last_message_id = message.get("msgId") + ref_id = message.get("refid", "") + payload = message.get("msg") + + if ref_id.startswith("_"): + await self._handle_control_message(ref_id, payload) + elif ref_id == self._pos_ref_id: + await self._handle_position_update(payload) + else: + logger.debug("Ignoring message with unknown refId=%s", ref_id) + + except Exception as e: + logger.error("Error parsing WS frame: %s", e, exc_info=True) + + # ── Position updates ────────────────────────────────────────── + + async def _handle_position_update(self, payload: dict | list | Any): + """ + Apply a position delta update to the snapshot cache. + + Saxo sends either: + - A single position update dict (with ``PositionId``). + - A list of position update dicts. + - A wrapper with ``Data`` key containing a list. + """ + updates: list[dict] = [] + + if isinstance(payload, dict): + if "Data" in payload: + updates = payload["Data"] + elif "PositionId" in payload: + updates = [payload] + else: + # Could be a position-removed notification + updates = [payload] + elif isinstance(payload, list): + updates = payload + else: + logger.warning("Unexpected position payload type: %s", type(payload)) + return + + changed = False + for delta in updates: + pid = delta.get("PositionId") + if not pid: + continue + + status = ( + delta.get("PositionBase", {}).get("Status") + or (self._positions.get(pid, {}).get("PositionBase", {}).get("Status")) + ) + + if status in ("Closed", "Closing"): + # Remove from cache when position is closed + if pid in self._positions: + logger.info("Position %s removed from stream cache (status=%s).", pid, status) + del self._positions[pid] + changed = True + continue + + if pid in self._positions: + _deep_merge(self._positions[pid], delta) + else: + # New position appeared + self._positions[pid] = delta + + changed = True + + if changed: + await self._fire_callback() + + async def _fire_callback(self): + """Invoke the user callback with the full position snapshot.""" + try: + await self._on_positions_update(copy.deepcopy(self._positions)) + except Exception as e: + logger.error("Error in positions-update callback: %s", e, exc_info=True) + + # ── Control messages ────────────────────────────────────────── + + async def _handle_control_message(self, ref_id: str, payload: Any): + """ + Handle Saxo streaming control messages. + + _heartbeat → log, no action + _resetsubscriptions → re-create affected subscriptions + _disconnect → stop (user must re-authenticate) + """ + if ref_id == "_heartbeat": + self._handle_heartbeat(payload) + elif ref_id == "_resetsubscriptions": + await self._handle_reset(payload) + elif ref_id == "_disconnect": + logger.critical("Received _disconnect from Saxo — service must re-login.") + self._running = False + else: + logger.debug("Unknown control message: ref=%s", ref_id) + + def _handle_heartbeat(self, payload: Any): + """Process heartbeat — check for SubscriptionTemporarilyDisabled.""" + if not isinstance(payload, dict): + return + heartbeats = payload.get("Heartbeats", []) + for hb in heartbeats: + reason = hb.get("Reason", "") + origin = hb.get("OriginatingReferenceId", "") + if reason == "SubscriptionTemporarilyDisabled": + logger.warning( + "Heartbeat: subscription %s temporarily disabled.", origin + ) + else: + logger.debug("Heartbeat: ref=%s reason=%s", origin, reason) + + async def _handle_reset(self, payload: Any): + """ + Re-create subscriptions listed in TargetReferenceIds. + If the list is empty, reset ALL subscriptions. + """ + target_refs = [] + if isinstance(payload, dict): + target_refs = payload.get("TargetReferenceIds", []) + + should_reset_positions = ( + not target_refs or self._pos_ref_id in target_refs + ) + + if should_reset_positions: + logger.warning("Resetting position subscription (requested by server).") + # Delete old subscription, create new one + try: + self._sync_api_token() + old_ref = self._pos_ref_id + self._pos_ref_id = _short_id("pos") + self._subscription_ready = False + + req = pf.positions.PositionSubscriptionRemove( + ContextId=self._context_id, + ReferenceId=old_ref, + ) + await self._request(req) + except Exception as e: + logger.warning("Failed to delete old subscription: %s", e) + + await self._create_position_subscription() + self._subscription_ready = True + + # ── Re-authorisation loop ───────────────────────────────────── + + async def _reauth_loop(self): + """ + Periodically re-authorise the WS connection so it survives token + refreshes. Saxo's access tokens typically last 20 minutes, so we + re-authorise every 15 minutes. + """ + while self._running: + await asyncio.sleep(self._reauth_interval_seconds) + if not self._running: + break + try: + token = self._current_token() + url = self._build_reauth_url() + headers = {"Authorization": f"Bearer {token}"} + + # PUT /streamingws/authorize?contextid={contextId} + import httpx + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.put(url, headers=headers) + if resp.status_code == 202: + logger.info("WebSocket re-authorised successfully.") + # Also update the REST client's token + self._update_rest_client_token(token) + else: + logger.warning( + "Re-auth returned %d: %s", resp.status_code, resp.text[:200] + ) + except asyncio.CancelledError: + break + except Exception as e: + logger.error("Re-auth failed: %s", e, exc_info=True) + + # ── URL helpers ─────────────────────────────────────────────── + + def _build_ws_url(self, token: str) -> str: + """ + Build the WebSocket connection URL. + + Format: + wss://sim-streaming.saxobank.com/sim/oapi/streaming/ws/connect?authorization=BEARER%20TOKEN&contextId=X + """ + env = TRADING_ENVIRONMENTS[self._environment] + stream_base = env["stream"].replace("https://", "wss://") + + if self._environment == "simulation": + prefix = env.get("prefix", "sim") + path = f"/{prefix}/oapi/streaming/ws/connect" + else: + path = "/oapi/streaming/ws/connect" + + params = { + "authorization": f"BEARER {token}", + "contextId": self._context_id, + } + if self._last_message_id is not None: + params["messageid"] = str(self._last_message_id) + + return f"{stream_base}{path}?{urlencode(params)}" + + def _build_reauth_url(self) -> str: + """Build the re-authorisation PUT URL.""" + env = TRADING_ENVIRONMENTS[self._environment] + base = env["stream"] + if self._environment == "simulation": + prefix = env.get("prefix", "sim") + return f"{base}/{prefix}/oapi/streaming/ws/authorize?contextid={self._context_id}" + return f"{base}/oapi/streaming/ws/authorize?contextid={self._context_id}" + + def _current_token(self) -> str: + token = self._get_token() if self._get_token else self._rest_client_access_token + if token and token != self._rest_client_access_token: + self._update_rest_client_token(token) + logger.info("Updated Saxo streaming REST client with refreshed access token.") + return token + + def _sync_api_token(self) -> str: + """Ensure the underlying AsyncAPI client uses the freshest access token.""" + return self._current_token() + + async def _request(self, endpoint): + """Dispatch a REST request through the configured async API client/wrapper.""" + return await self._api.request(endpoint) + + @property + def _rest_client_access_token(self) -> str | None: + if hasattr(self._api, "access_token"): + return self._api.access_token + nested_api = getattr(self._api, "_api", None) + if nested_api is not None and hasattr(nested_api, "access_token"): + return nested_api.access_token + return None + + def _update_rest_client_token(self, token: str): + """Update the underlying REST client's bearer token when supported.""" + if hasattr(self._api, "update_token"): + self._api.update_token(token) + return + + nested_api = getattr(self._api, "_api", None) + if nested_api is not None and hasattr(nested_api, "update_token"): + nested_api.update_token(token) + + +# ── Helpers ─────────────────────────────────────────────────────── + +def _short_id(prefix: str) -> str: + """Generate a short alphanumeric ID suitable for Saxo context/ref IDs.""" + return f"{prefix}-{uuid.uuid4().hex[:12]}" diff --git a/src/scheduler/__init__.py b/src/scheduler/__init__.py index f52e142..83b0aac 100644 --- a/src/scheduler/__init__.py +++ b/src/scheduler/__init__.py @@ -28,19 +28,11 @@ # Get trading rule trading_rule = TradingRule(config_manager, None) -# Function to send 'ping_saxo_api' every 1 minutes -@repeat(every(15).seconds) -def job_check_positions_on_saxo_api(): - # Get the current time in UTC - now_utc = datetime.now(pytz.utc) - message = { - "action": "check_positions_on_saxo_api", - "indice": "n/a", - "signal_timestamp": "2024-05-09T12:26:00Z", - "alert_timestamp": now_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), - "mqsend_timestamp": now_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), - } - send_message_to_trading(message) +# NOTE: Position monitoring is now handled by the Position Monitor's +# WebSocket streaming connection. The old 7-second polling job +# (job_check_positions_on_saxo_api) has been removed. +# A fallback REST check can still be triggered manually via +# the trading-ops queue if needed. @repeat(every().day.at(time_str="22:00", tz=timezone)) @@ -118,6 +110,17 @@ def job_close_position(): def send_message_to_trading(message): + """Send a message to the appropriate RabbitMQ queue based on action type.""" + action = message.get("action", "") + + # Route operational messages to trading-ops, trade signals to trading-signals + if action in ("check_positions_on_saxo_api", "daily_stats"): + queue_name = "trading-ops" + elif action in ("long", "short", "close-long", "close-short", "close-position"): + queue_name = "trading-signals" + else: + queue_name = "trading-signals" # Default + try: # Retrieve RabbitMQ credentials from the configuration rabbitmq_config = config_manager.get_rabbitmq_config() @@ -133,11 +136,11 @@ def send_message_to_trading(message): ) ) channel = connection.channel() - channel.queue_declare(queue="trading-action") + channel.queue_declare(queue=queue_name, durable=True) body = json.dumps(message) - channel.basic_publish(exchange="", routing_key="trading-action", body=body) - logging.info(f"Send message to channel trading-action, message {body}") + channel.basic_publish(exchange="", routing_key=queue_name, body=body) + logging.info(f"Send message to channel {queue_name}, message {body}") except pika.exceptions.AMQPConnectionError as e: logging.error(f"Failed to connect to RabbitMQ: {e}") except Exception as e: diff --git a/src/schema/__init__.py b/src/schema/__init__.py index 3c55600..f5776ee 100644 --- a/src/schema/__init__.py +++ b/src/schema/__init__.py @@ -14,6 +14,11 @@ "type": "string", "pattern": r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 2.0, + }, }, "required": ["action", "indice", "signal_timestamp", "alert_timestamp"], } @@ -48,6 +53,11 @@ "type": "string", "pattern": r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 2.0, + }, }, "required": ["action", "indice", "signal_timestamp", "alert_timestamp"], } diff --git a/src/start_python_script.sh b/src/start_python_script.sh index 06a5687..9efcacf 100755 --- a/src/start_python_script.sh +++ b/src/start_python_script.sh @@ -12,7 +12,10 @@ case $WATA_APP_ROLE in python -u src/web_server/__init__.py ;; "trader") - python -u /app/src/main.py + python -u /app/src/trader/__init__.py + ;; + "position_monitor") + python -u /app/src/position_monitor/__init__.py ;; "scheduler") python -u /app/src/scheduler/__init__.py diff --git a/src/trade/api_actions.py b/src/trade/api_actions.py deleted file mode 100644 index 0fa1321..0000000 --- a/src/trade/api_actions.py +++ /dev/null @@ -1,1608 +0,0 @@ -import os -import uuid -import time -# Import the original API class -from src.saxo_openapi.saxo_openapi import API as SaxoOpenApiLib -from src.saxo_openapi.exceptions import OpenAPIError as SaxoOpenApiLibError - -import logging -import re -import json -import math -from copy import deepcopy -from datetime import datetime -import pytz -from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type, RetryError -from collections import defaultdict - -# --- Saxo OpenApi Components --- -import src.saxo_openapi.endpoints.referencedata as rd -import src.saxo_openapi.endpoints.trading as tr -import src.saxo_openapi.endpoints.portfolio as pf -from src.saxo_openapi.contrib.orders import MarketOrder, tie_account_to_order, direction_from_amount -from src.saxo_openapi.contrib.orders.helper import direction_invert -import requests # Import requests exceptions if needed for translation - -# --- Local Imports --- -from src.saxo_authen import SaxoAuth -from .exceptions import ( - NoMarketAvailableException, - NoTurbosAvailableException, - PositionNotFoundException, - InsufficientFundsException, - ApiRequestException, - TokenAuthenticationException, - DatabaseOperationException, - SaxoApiError, - OrderPlacementError -) -from src.mq_telegram.tools import send_message_to_mq_for_telegram -from src.configuration import ConfigurationManager -# --- Import DB Managers and TradingRule for injection --- -from src.database import DbOrderManager, DbPositionManager # Needed for new responsibilities -from src.trade.rules import TradingRule # Needed for PerformanceMonitor - -# --- Constants --- -DEFAULT_RETRY_ATTEMPTS = 5 -DEFAULT_RETRY_WAIT_SECONDS = 2 - -# --- Utilities --- - -def parse_saxo_turbo_description(description): - pattern = r"(.*) (\w+) (\w+) (\d+(?:\.\d+)?) (\w+)$" - match = re.match(pattern, description) - if match: - return { - "name": match.group(1), "kind": match.group(2), - "buysell": match.group(3), "price": match.group(4), - "from": match.group(5), - } - return None - -# === Low-Level API Client Wrapper === - -class SaxoApiClient: - """ - Acts as a Facade/Wrapper around the existing `src.saxo_openapi.saxo_openapi.API` - class. It handles token management via SaxoAuth and translates exceptions - from the underlying library into WATA-specific exceptions. - """ - - def __init__(self, config_manager: ConfigurationManager, saxo_auth: SaxoAuth): - self.config_manager = config_manager - self.saxo_auth = saxo_auth - self.environment = config_manager.get_config_value("saxo_auth.env", "live") - self._saxo_api_instance: SaxoOpenApiLib | None = None - self._current_token: str | None = None - self._ensure_valid_token_and_api_instance() # Initialize on creation - - def _ensure_valid_token_and_api_instance(self): - """ - Ensures the underlying SaxoOpenApiLib instance exists and uses the latest token. - Re-initializes the SaxoOpenApiLib instance if the token changes. - """ - try: - latest_token = self.saxo_auth.get_token() - if latest_token != self._current_token or self._saxo_api_instance is None: - logging.info(f"SaxoApiClient: Token changed or API instance missing. Re-initializing SaxoOpenApiLib for env '{self.environment}'.") - # Configure request parameters if needed (e.g., timeouts) - request_params = {"timeout": 30} - self._saxo_api_instance = SaxoOpenApiLib( - access_token=latest_token, - environment=self.environment, - request_params=request_params - # Add headers if needed from config - ) - self._current_token = latest_token - logging.info("SaxoApiClient: SaxoOpenApiLib instance refreshed.") - - except TokenAuthenticationException: - logging.critical("SaxoApiClient: Failed to obtain/refresh token during API instance setup.") - raise # Propagate critical auth errors - - def request(self, endpoint_request_obj): - """ - Makes an API request using the underlying SaxoOpenApiLib instance. - - Args: - endpoint_request_obj: An instance representing the API endpoint - (e.g., rd.instruments.Instruments). - - Returns: - The response content from the API (typically dict or None). - - Raises: - ApiRequestException: For connection issues or request setup problems. - TokenAuthenticationException: If authentication fails. - SaxoApiError: For general Saxo API errors (>=400 status codes). - OrderPlacementError: For errors specifically related to order placement. - InsufficientFundsException: For insufficient funds errors reported by API. - Various specific exceptions based on error content. - """ - self._ensure_valid_token_and_api_instance() # Check/refresh token and API instance - - if self._saxo_api_instance is None: - # This should ideally be caught by _ensure_valid_token..., but defensive check - logging.critical("SaxoApiClient: SaxoOpenApiLib instance is not available.") - raise ApiRequestException("Saxo API client instance not initialized.", endpoint=str(endpoint_request_obj)) - - try: - # Delegate the actual request to the underlying library instance - logging.debug(f"SaxoApiClient: Forwarding request to SaxoOpenApiLib for endpoint: {type(endpoint_request_obj).__name__}") - response_content = self._saxo_api_instance.request(endpoint_request_obj) - logging.debug(f"SaxoApiClient: Received response content (type: {type(response_content).__name__})") - return response_content - - except SaxoOpenApiLibError as e: - # Handle errors raised by the underlying saxo_openapi.API library - status_code = e.code - content_str = e.content - - # Ensure content_str is actually a string before trying to parse JSON - # The saxo_openapi library might pass None or other types for content - if not isinstance(content_str, str): - content_str = str(content_str) # Convert to string representation if not already - - saxo_error_details = None - error_message = content_str # Default error message is the raw content - error_code = None - - try: - # Attempt to parse the error content as JSON for more details - saxo_error_details = json.loads(content_str) - # Use e.reason as a fallback if 'Message' is not in the JSON - error_message = saxo_error_details.get('Message', e.reason) # Use e.reason as fallback - error_code = saxo_error_details.get('ErrorCode') - except json.JSONDecodeError: - # If content is not JSON, use the original content_str or e.reason - error_message = content_str if content_str else e.reason - saxo_error_details = content_str # Keep the raw string if not JSON - except Exception as json_parse_err: # Catch other potential parsing errors - logging.warning(f"Error parsing Saxo error content: {json_parse_err}. Content was: {content_str}", - exc_info=False) - error_message = content_str if content_str else e.reason # Fallback message - saxo_error_details = content_str # Keep raw content - - # Log using the correct attributes and potentially e.reason - logging.error( - f"Saxo API Error Wrapper: Caught OpenAPIError (Status: {status_code}, Reason: {e.reason}, Code: {error_code}, Msg Content: {error_message}), Endpoint: {type(endpoint_request_obj).__name__}" - ) - - # Specific Error Mapping based on status code and potentially error_code/message - # (The rest of the logic using status_code, error_code, error_message, saxo_error_details remains the same) - if status_code == 400 and error_code == "InsufficientFunds": # Confirm the actual ErrorCode from Saxo docs/testing - raise InsufficientFundsException( - message=error_message or "Insufficient funds reported by API", - saxo_error_details=saxo_error_details - ) from e - # Check if it looks like an order placement error - endpoint_path = getattr(endpoint_request_obj, 'path', 'unknown') - is_order_endpoint = "/trade/v2/orders" in endpoint_path - if (status_code in [400, 403, 409] or error_code) and is_order_endpoint: - order_payload = getattr(endpoint_request_obj, 'data', None) - raise OrderPlacementError( - f"Saxo rejected order ({status_code}): {error_message}", - status_code=status_code, - saxo_error_details=saxo_error_details, - order_details=order_payload - ) from e - if status_code == 401: # Should ideally be handled by token refresh, but catch if it leaks - raise TokenAuthenticationException(f"API returned 401 Unauthorized: {error_message}", - saxo_error_details=saxo_error_details) from e - if status_code == 429: # Rate limit exceeded despite underlying library's retry - logging.warning(f"Persistent Rate Limit Error (429) received from API: {error_message}") - raise SaxoApiError(f"Persistent Rate Limit Error (429): {error_message}", status_code=status_code, - saxo_error_details=saxo_error_details) from e - - # Default to general SaxoApiError - raise SaxoApiError( - f"Saxo API Error ({status_code} - {e.reason}): {error_message}", # Include reason for context - status_code=status_code, - saxo_error_details=saxo_error_details, - request_details={"endpoint_type": type(endpoint_request_obj).__name__, - "params": getattr(endpoint_request_obj, 'params', {}), - "data": getattr(endpoint_request_obj, 'data', None)} - ) from e - - except requests.RequestException as e: - # Handle connection errors, timeouts etc. from the underlying requests library - logging.error(f"API Request Exception Wrapper for endpoint {type(endpoint_request_obj).__name__}: {e}", exc_info=True) - raise ApiRequestException(f"Underlying request failed: {e}", endpoint=str(endpoint_request_obj)) from e - - except TokenAuthenticationException: - # Re-raise if caught during _ensure_valid_token... - logging.critical("Token authentication failed during request sequence.") - raise - - except Exception as e: - # Catch any other unexpected errors during the process - logging.exception(f"Unexpected error during Saxo API request wrapper for {type(endpoint_request_obj).__name__}: {e}") # Use logging.exception - raise ApiRequestException(f"Unexpected wrapper error: {e}", endpoint=str(endpoint_request_obj)) from e - - -# === Domain-Specific Services === - -class InstrumentService: - """Handles finding and retrieving instrument details.""" - - def __init__(self, api_client: SaxoApiClient, config_manager: ConfigurationManager, account_key: str): - self.api_client = api_client - self.config = config_manager # Keep ref if needed for multiple values - self.account_key = account_key - self.api_limits = self.config.get_config_value("trade.config.general.api_limits", {"top_instruments": 200}) - self.turbo_price_range = self.config.get_config_value("trade.config.turbo_preference.price_range", {"min": 4, "max": 15}) - # Add retry config for the specific Bid retry - self.retry_config = self.config.get_config_value("trade.config.general.retry_config", {"max_retries": 3, "retry_sleep_seconds": 1}) # Use specific or general config - self.websocket_config = self.config.get_config_value("trade.config.general.websocket", {"refresh_rate_ms": 10000}) - - - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1), retry=retry_if_exception_type(ApiRequestException)) - def _get_infoprices_for_asset_type(self, identifiers_string: str, exchange_id: str, asset_type: str): - """Helper to get InfoPrices for a specific asset type with retry.""" - logging.debug(f"Fetching InfoPrices for {asset_type} on {exchange_id} for UICs: {identifiers_string[:100]}...") # Log sample - req = tr.infoprices.InfoPrices( - params={ - "$top": self.api_limits["top_instruments"], - "AccountKey": self.account_key, - "ExchangeId": exchange_id, - "FieldGroups": "Commissions,DisplayAndFormat,Greeks,HistoricalChanges,InstrumentPriceDetails,MarketDepth,PriceInfo,PriceInfoDetails,Quote", - "Uics": identifiers_string, - "AssetType": asset_type, - } - ) - try: - return self.api_client.request(req) - except ApiRequestException as e: - logging.warning(f"ApiRequestException during _get_infoprices_for_asset_type (will retry): {e}") - raise # Re-raise for tenacity - - def find_turbos(self, exchange_id: str, underlying_uics: str, keywords: str): - """Finds suitable turbo warrants based on criteria.""" - logging.info(f"Finding turbos: Exchange={exchange_id}, Underlying={underlying_uics}, Keywords={keywords}") - - # 1. Initial Instrument Search - req_instruments = rd.instruments.Instruments( - params={ - "$top": self.api_limits["top_instruments"], - "AccountKey": self.account_key, - "ExchangeId": exchange_id, - "Keywords": keywords, - "IncludeNonTradable": False, - "UnderlyingUics": underlying_uics, - "AssetTypes": "WarrantKnockOut,WarrantOpenEndKnockOut,MiniFuture,WarrantDoubleKnockOut", - } - ) - response_instruments = self.api_client.request(req_instruments) - - if not response_instruments or not response_instruments.get("Data"): - logging.warning("No instruments found in initial search.") - raise NoTurbosAvailableException("No instruments found in initial search.", search_context=req_instruments.params) - - logging.debug(f"Found {len(response_instruments['Data'])} instruments in initial search for keywords '{keywords}'.") - logging.debug(f"Phase 1 : Initial search response: {json.dumps(response_instruments)}") - - # 2. Parse and Filter Initial List - valid_items = [] - for item in response_instruments["Data"]: - parsed_data = parse_saxo_turbo_description(item.get("Description", "")) - if parsed_data: - item["appParsedData"] = parsed_data - valid_items.append(item) - else: - logging.warning(f"Failed to parse description: {item.get('Description')}") - - if not valid_items: - logging.warning("No instruments remaining after parsing descriptions.") - raise NoTurbosAvailableException("No instruments found with parsable descriptions.", search_context=req_instruments.params) - - logging.debug(f"Found {len(valid_items)} instruments with valid descriptions.") - logging.debug(f"Phase 2 : Valid items after parsing: {json.dumps(valid_items)}") - - # 3. Sort by Knock-out Price (from parsed data) - sort_reverse = keywords.lower() != "short" # True for long (higher price first), False for short (lower price first) - try: - sorted_instruments = sorted( - valid_items, - key=lambda x: float(x["appParsedData"]["price"]), - reverse=sort_reverse, - ) - except (KeyError, ValueError) as e: - logging.error(f"Error sorting instruments by parsed price: {e}") - raise ValueError("Could not sort instruments by parsed price.") from e - - # 4. Group instruments by AssetType to handle multiple types correctly - instrument_groups = defaultdict(list) - for item in sorted_instruments: - # We only care about the top N instruments in total - if len(instrument_groups) < self.api_limits["top_instruments"]: - instrument_groups[item['AssetType']].append(item) - - if not instrument_groups: - raise NoTurbosAvailableException("No identifiers found after sorting.", search_context=req_instruments.params) - - logging.debug(f"Phase 3 : Sorted instruments grouped by AssetType: {json.dumps(instrument_groups)}") - - # 5. Get Detailed Price Info for Sorted Instruments - response_infoprices = None - bid_data_missing = True # Initial state for the loop, indicates we need to check/retry for bid data - bid_retries = 0 - max_bid_retries = self.retry_config["max_retries"] - retry_sleep = self.retry_config["retry_sleep_seconds"] - - while bid_data_missing and bid_retries < max_bid_retries: - current_attempt = bid_retries + 1 - logging.debug(f"Bid check loop: Attempt {current_attempt}/{max_bid_retries}") - - # This list will hold aggregated price data from all groups - all_infoprices_data = [] - - try: - for asset_type, instruments_in_group in instrument_groups.items(): - identifiers = [item["Identifier"] for item in instruments_in_group] - identifiers_string = ",".join(map(str, identifiers)) - - logging.debug(f"Attempting to fetch InfoPrices for AssetType '{asset_type}' ({len(instruments_in_group)} instruments)") - group_response = self._get_infoprices_for_asset_type(identifiers_string, exchange_id, asset_type) - - if group_response and group_response.get("Data"): - all_infoprices_data.extend(group_response["Data"]) - else: - logging.warning(f"No InfoPrice data received for group {asset_type}.") - - # After fetching for all groups, create a single response object to check - if all_infoprices_data: - response_infoprices = {"Data": all_infoprices_data} - else: - response_infoprices = None # Ensure it's None if all groups fail - - if not response_infoprices or not response_infoprices.get("Data"): - logging.warning( - f"No InfoPrice data received from any group. " - f"Bid check attempt {current_attempt}/{max_bid_retries}." - ) - # This situation means we couldn't get data to check bids. Consume a bid_retry. - bid_retries += 1 - if bid_retries < max_bid_retries: - time.sleep(retry_sleep) - # bid_data_missing remains True. Loop will re-evaluate. - continue # Go to next iteration of the while loop - - # Filter items that have a "Quote" field, as only these are candidates for having a "Bid" - items_with_quote_field = [ - item for item in response_infoprices["Data"] if "Quote" in item - ] - - if not items_with_quote_field: - # If no items have a "Quote" field, then no "Bid" can be missing from within a "Quote". - # Thus, the specific condition for retrying (missing Bids) is not met. - logging.debug( - "No instruments found with a 'Quote' field in InfoPrices response. " - "Stopping Bid-specific retries as no Bids can be considered missing." - ) - bid_data_missing = False # Condition for bid-specific retry not met. - break # Exit the while loop - - num_total_items_with_quote = len(items_with_quote_field) - - # Count how many of these items (that have a "Quote") are missing the "Bid" attribute - items_missing_bid_attr = [ - item for item in items_with_quote_field if "Bid" not in item["Quote"] - ] - num_items_missing_bid_attr = len(items_missing_bid_attr) - - # Calculate percentage of items missing "Bid" out of those that have "Quote" - # Avoid division by zero if num_total_items_with_quote is somehow 0 (though caught by 'if not items_with_quote_field') - percentage_missing_bid = 0.0 - if num_total_items_with_quote > 0: - percentage_missing_bid = (num_items_missing_bid_attr / num_total_items_with_quote) * 100 - - if percentage_missing_bid > 50: - logging.warning( - f"{percentage_missing_bid:.2f}% ({num_items_missing_bid_attr}/{num_total_items_with_quote}) of " - f"items with a 'Quote' field are missing the 'Bid' attribute. " - f"Retrying InfoPrices. Bid check attempt {current_attempt}/{max_bid_retries}." - ) - bid_retries += 1 # Consume a retry for the "missing bid" condition - if bid_retries < max_bid_retries: - time.sleep(retry_sleep) - # bid_data_missing remains True. Loop will re-evaluate. - else: - # Percentage missing is <= 50% (or 0% if all bids present) - if num_items_missing_bid_attr > 0: - logging.info( - f"{percentage_missing_bid:.2f}% ({num_items_missing_bid_attr}/{num_total_items_with_quote}) of " - f"items with 'Quote' field are missing 'Bid' attribute. This is within tolerance. Proceeding." - ) - else: - logging.debug( - "All items with a 'Quote' field have the 'Bid' attribute. Bid data check passed." - ) - bid_data_missing = False # Condition met, stop retrying for *this specific reason*. - # Loop will terminate as bid_data_missing is False. - - except RetryError as e: # Raised by tenacity in _get_instrument_details if it exhausts its retries - logging.error( - f"Persistent failure in _get_infoprices_for_asset_type after its internal retries during bid check " - f"(Attempt {current_attempt}/{max_bid_retries}): {e}" - ) - response_infoprices = None # Ensure no stale data is used - # This is a critical failure to get data; re-raise appropriately. - # Exiting the bid_data_missing loop. - raise ApiRequestException( - "Failed to get instrument details for bid checking after underlying API call retries.", cause=e - ) from e - except Exception as e: # Catch other unexpected errors during the bid check logic itself - logging.error( - f"Unexpected error during Bid check logic (Attempt {current_attempt}/{max_bid_retries}): {e}", - exc_info=True - ) - bid_retries += 1 # Consume a retry for this unexpected error - if bid_retries < max_bid_retries: - logging.info(f"Retrying bid check loop after unexpected error. Sleeping for {retry_sleep}s.") - time.sleep(retry_sleep) - # bid_data_missing remains True. Loop will re-evaluate. - continue - else: - logging.error("Max retries reached for bid check loop due to unexpected errors.") - # bid_data_missing remains True, loop will terminate due to bid_retries condition. - # Re-raise the last error to indicate failure of this stage. - raise # Or wrap in a custom exception like ApiRequestException - - # Case 1: Loop terminated because bid_retries >= max_bid_retries AND bid_data_missing is still True. - # This means the condition (>50% missing bids, or no data from API) persisted through all retries. - if bid_data_missing and bid_retries >= max_bid_retries: - logging.error( - f"After {max_bid_retries} retries, the condition for fetching Bid data " - f"(e.g., >50% missing 'Bid' or no data from API) was not resolved." - ) - # If response_infoprices is None here, it means the last attempt also failed to get data. - # If it has data, it's data where the >50% condition was met. - - # Case 2: Loop terminated because bid_data_missing became False. - # This means either all Bids were present, or <=50% were missing, or no items had "Quote". - - # Case 3: An exception (like RetryError from _get_instrument_details) caused an early exit by re-raising. - # In this case, the code below won't be reached if the exception wasn't caught and suppressed by find_turbos. - # Our RetryError catch re-raises, so this part is skipped for that. - - # --- Consistently handle response_infoprices state and filter items --- - - if not response_infoprices or not response_infoprices.get("Data"): - # This covers: - # 1. _get_instrument_details consistently failed to return data through all bid_retries. - # 2. An exception within the loop (not re-raised out of find_turbos) led to response_infoprices being None. - logging.error("No InfoPrice data is available after all attempts to fetch and check Bid attributes.") - raise NoMarketAvailableException( - "Failed to obtain valid InfoPrice data with Bid attributes after all retries." - ) - - # Filter out any remaining items that do not have a 'Bid' attribute in their 'Quote'. - # This is crucial regardless of why the loop exited, to ensure downstream code only gets items with 'Bid'. - logging.debug("Final filtering of InfoPrices items to ensure 'Bid' attribute is present in 'Quote'.") - - initial_item_count_before_final_filter = len(response_infoprices["Data"]) - valid_items_with_bid = [] - for item in response_infoprices["Data"]: - # Check for 'Quote' and then 'Bid' within 'Quote' - if item.get("Quote") and item["Quote"].get("Bid") is not None: - valid_items_with_bid.append(item) - else: - logging.debug( - f"Item Uic:{item.get('Uic', 'N/A')} (Identifier: {item.get('Identifier', 'N/A')}) " - f"is being filtered out due to missing 'Bid' in 'Quote' after retry loop." - ) - - response_infoprices["Data"] = valid_items_with_bid - num_filtered_out_in_final_step = initial_item_count_before_final_filter - len(valid_items_with_bid) - - if num_filtered_out_in_final_step > 0: - logging.info( - f"Filtered out an additional {num_filtered_out_in_final_step} items from InfoPrices " - f"due to missing 'Bid' attribute in the final filtering step." - ) - - # After final filtering, if no items remain, it's an issue. - if not response_infoprices.get("Data"): - logging.error( - "No instruments with a valid 'Bid' attribute found after all retries and final filtering." - ) - raise NoMarketAvailableException( - "No instruments with Bid data available after retries and final filtering." - ) - - logging.info( - f"Proceeding with {len(response_infoprices['Data'])} instruments that have 'Bid' data " - f"after retry and filtering logic." - ) - - logging.debug(f"Phase 5 : Final InfoPrices response after bid checks: {json.dumps(response_infoprices)}") - - # 6. Filter by Market State and Availability - available_items = [ - item for item in response_infoprices["Data"] - if item["Quote"].get("PriceTypeAsk") != "NoMarket" and - item["Quote"].get("PriceTypeBid") != "NoMarket" and - item["Quote"].get("MarketState") != "Closed" - ] - - if not available_items: - logging.warning("No instruments available after filtering market state/price types.") - raise NoMarketAvailableException(f"No markets available for {keywords} turbo in {exchange_id}.") - - logging.debug(f"{len(available_items)} instruments available after market state filtering.") - logging.debug(f"Phase 6 : Available items after market state filtering: {json.dumps(available_items)}") - - # 7. Filter by Price Range (using Bid price for selection consistency) - min_price = self.turbo_price_range["min"] - max_price = self.turbo_price_range["max"] - price_filtered_items = [ - item for item in available_items - if min_price <= item["Quote"]["Bid"] <= max_price - ] - - if not price_filtered_items: - logging.warning(f"No turbos found within price range {min_price}-{max_price}.") - raise NoTurbosAvailableException( - f"No turbos found in price range {min_price}-{max_price}.", - search_context={'PriceRange': (min_price, max_price), 'AvailableCount': len(available_items)} - ) - - logging.debug(f"{len(price_filtered_items)} instruments available after price filtering.") - logging.debug(f"Phase 7 : Price filtered items: {json.dumps(price_filtered_items)}") - - # 8. Select the Best Match (first one after filtering) - final_candidates = sorted(price_filtered_items, key=lambda x: x["Quote"]["Bid"]) - - selected_turbo_info = deepcopy(final_candidates[0]) # Use the first candidate - - # --- 9. Create Price Subscription to get the latest snapshot --- - context_id = str(uuid.uuid1()) # Generate unique IDs per call like original - reference_id = str(uuid.uuid1()) - selected_uic = selected_turbo_info["Uic"] - selected_asset_type = selected_turbo_info["AssetType"] - refresh_rate = self.websocket_config["refresh_rate_ms"] # Get from config - - logging.debug( - f"Creating price subscription for Uic {selected_uic}, ContextId: {context_id}, RefId: {reference_id}") - - req_price_sub = tr.prices.CreatePriceSubscription( - data={ - "Arguments": { - "Uic": selected_uic, - "AccountKey": self.account_key, - "AssetType": selected_asset_type, - "Amount": 1, - "FieldGroups": [ - "Commissions", "DisplayAndFormat", "Greeks", "HistoricalChanges", - "InstrumentPriceDetails", "MarketDepth", "PriceInfo", - "PriceInfoDetails", "Quote", "Timestamps", - ], - }, - "ContextId": context_id, - "ReferenceId": reference_id, - "RefreshRate": refresh_rate, - "Format": "application/json", - } - ) - - final_snapshot_data = None - sub_context_id = None - sub_reference_id = None - - try: - response_price_sub = self.api_client.request(req_price_sub) - snapshot = response_price_sub.get("Snapshot") - if not snapshot: - logging.warning( - f"Price subscription response for {selected_uic} missing 'Snapshot'. Falling back. Response: {response_price_sub}") - final_snapshot_data = selected_turbo_info # Fallback to InfoPrice data - else: - logging.debug(f"Successfully obtained snapshot via price subscription for {selected_uic}") - final_snapshot_data = snapshot - # Store IDs only if subscription successful - sub_context_id = context_id - sub_reference_id = reference_id - - except (ApiRequestException, SaxoApiError) as e: - logging.warning( - f"Failed to create price subscription for {selected_uic} (Context: {context_id}), proceeding with InfoPrice data: {e}", - exc_info=False) # Don't need full trace usually - final_snapshot_data = selected_turbo_info # Fallback - except Exception as e: - logging.error(f"Unexpected error during price subscription for {selected_uic} (Context: {context_id}): {e}", - exc_info=True) - final_snapshot_data = selected_turbo_info # Fallback - - - # --- 10. Prepare and Return Result --- - if final_snapshot_data is None: - # This should only happen if the fallback above fails unexpectedly - logging.error(f"Critical error: No price data available for selected turbo {selected_uic}") - raise ValueError(f"Could not retrieve final price data for {selected_uic}") - - result = { - "input_criteria": { - "exchange_id": exchange_id, - "underlying_uics": underlying_uics, - "keywords": keywords, - }, - "selected_instrument": { - "uic": selected_turbo_info["Uic"], # Uic/AssetType are from selection - "asset_type": selected_turbo_info["AssetType"], - # Use data from the final snapshot source (subscription or fallback) - "description": final_snapshot_data.get("DisplayAndFormat", {}).get("Description", "N/A"), - "symbol": final_snapshot_data.get("DisplayAndFormat", {}).get("Symbol", "N/A"), - "currency": final_snapshot_data.get("DisplayAndFormat", {}).get("Currency", "N/A"), - "decimals": final_snapshot_data.get("DisplayAndFormat", {}).get("OrderDecimals", 2), - "parsed_data": parse_saxo_turbo_description( - final_snapshot_data.get("DisplayAndFormat", {}).get("Description", "")), - "quote": final_snapshot_data.get("Quote", {}), - "commissions": final_snapshot_data.get("Commissions", {}), - # Keep explicit latest price fields used downstream - "latest_ask": final_snapshot_data.get("Quote", {}).get("Ask"), - "latest_bid": final_snapshot_data.get("Quote", {}).get("Bid"), - # --- Include Subscription IDs --- - "subscription_context_id": sub_context_id, - "subscription_reference_id": sub_reference_id, - } - } - logging.info(f"Selected Turbo: {result['selected_instrument']['description']} (Sub Ctx: {sub_context_id})") - return result - - -class OrderService: - """Handles placing, retrieving, and cancelling orders.""" - - def __init__(self, api_client: SaxoApiClient, account_key: str, client_key: str): - self.api_client = api_client - self.account_key = account_key - self.client_key = client_key # Needed for some order endpoints - - def place_market_order(self, uic: int, asset_type: str, amount: int, buy_sell: str, order_duration: str = "DayOrder"): - """Places a market order.""" - logging.info(f"Placing Market Order: {buy_sell} {amount} of {uic} ({asset_type})") - pre_order = MarketOrder( - Uic=uic, - AssetType=asset_type, - Amount=amount, - BuySell=buy_sell - # TODO : Saxo API return error if StopLossOnFill and TakeProfitOnFill are set - # StopLossOnFill=onfill.StopLossDetails(stop_loss_price), - # TakeProfitOnFill=onfill.TakeProfitDetails(profit_price), - ) - - # Inject AccountKey using the utility - final_order_payload = tie_account_to_order(self.account_key, pre_order) - logging.debug(f"Final order payload: {json.dumps(final_order_payload)}") - - request_order = tr.orders.Order(data=final_order_payload) - try: - validated_order = self.api_client.request(request_order) - except OrderPlacementError as e: - # Add context if possible (re-raised from api_client) - e.order_details = final_order_payload # Ensure order details are attached - logging.error(f"Order placement rejected by API: {e}") - raise e # Re-raise the specific error - except SaxoApiError as e: - logging.error(f"API error during order placement: {e}") - # Potentially wrap in OrderPlacementError if context suggests it - raise OrderPlacementError(f"API error during order placement: {e}", saxo_error_details=e.saxo_error_details, order_details=final_order_payload) from e - - if not validated_order or not validated_order.get("OrderId"): - logging.error(f"Order placement response missing OrderId: {validated_order}") - raise OrderPlacementError("Order placement response missing OrderId.", order_details=final_order_payload, saxo_error_details=validated_order) - - logging.info(f"Order placed successfully. OrderId: {validated_order['OrderId']}") - return validated_order # Return the full response from Saxo - - def get_single_order(self, order_id: str): - """Retrieves details for a single open order.""" - logging.debug(f"Getting details for order: {order_id}") - req_single_order = pf.orders.GetOpenOrder( - ClientKey=self.client_key, - OrderId=order_id - ) - return self.api_client.request(req_single_order) - - def cancel_order(self, order_id: str): - """Cancels a specific order. Returns True on success, False on failure.""" - logging.info(f"Attempting to cancel order: {order_id}") - request_cancel = tr.orders.CancelOrders( - OrderIds=order_id, - params={"AccountKey": self.account_key} - ) - try: - # API might return 200 OK with details or potentially 204 No Content - response = self.api_client.request(request_cancel) - # Check response - Saxo might return 200 OK with details, - # or potentially 204 No Content on success. Assume 2xx is success. - # If specific success criteria are needed, adjust check here. - logging.info(f"Order cancellation request successful for {order_id}. Response: {response}") - return True # Indicate success - except (ApiRequestException, SaxoApiError) as e: - # Includes 404 Not Found if order already cancelled/filled - logging.error(f"Failed to cancel order {order_id}: {e}") - # You might want to check e.status_code == 404 and treat it differently - return False # Indicate failure - except Exception as e: - logging.error(f"Unexpected error cancelling order {order_id}: {e}", exc_info=True) - return False # Indicate failure - - -class PositionService: - """Handles retrieving position and balance information.""" - - def __init__(self, api_client: SaxoApiClient, order_service: OrderService, config_manager: ConfigurationManager, account_key: str, client_key: str): - self.api_client = api_client - self.order_service = order_service - self.config = config_manager - self.account_key = account_key - self.client_key = client_key - self.api_limits = self.config.get_config_value("trade.config.general.api_limits", {"top_positions": 200, "top_closed_positions": 500}) - self.retry_config = self.config.get_config_value("trade.config.general.retry_config", {"max_retries": DEFAULT_RETRY_ATTEMPTS, "retry_sleep_seconds": DEFAULT_RETRY_WAIT_SECONDS}) - - - def get_open_positions(self): - """Retrieves all open positions for the account.""" - logging.debug("Getting open positions...") - req_positions = pf.positions.PositionsMe( - params={ - # "$top": self.api_limits["top_positions"], # Be careful with $top if pagination needed - "ClientKey": self.client_key, # Often required - "AccountKey": self.account_key, # Sometimes required - "FieldGroups": "PositionBase,PositionView,DisplayAndFormat,ExchangeInfo", # Adjust fields as needed - } - ) - response = self.api_client.request(req_positions) - # Add __count if not present for compatibility, or adjust usage upstream - if response and 'Data' in response and '__count' not in response: - response['__count'] = len(response['Data']) - elif not response: - return {'__count': 0, 'Data': []} # Return empty structure - return response - - - def get_closed_positions(self, top: int | None = None, skip: int = 0): - """Retrieves closed positions.""" - if top is None: - top = self.api_limits["top_closed_positions"] - logging.debug(f"Getting closed positions (Top={top}, Skip={skip})...") - req_positions = pf.closedpositions.ClosedPositionsMe( - params={ - "$top": top, - "$skip": skip, - "AccountKey": self.account_key, # Often required - "FieldGroups": "ClosedPosition,ClosedPositionDetails,DisplayAndFormat,ExchangeInfo", # Adjust as needed - } - ) - response = self.api_client.request(req_positions) - if not response: - return {'__count': 0, 'Data': []} - return response - - def get_single_position(self, position_id: str): - """Retrieves details for a single position.""" - logging.debug(f"Getting single position details for: {position_id}") - request_single_position = pf.positions.SinglePosition( - PositionId=position_id, - params={ - "ClientKey": self.client_key, - "AccountKey": self.account_key, # Often needed - "FieldGroups": "PositionBase,PositionView,DisplayAndFormat,Costs,ExchangeInfo", # Adjust as needed - } - ) - return self.api_client.request(request_single_position) - - @retry(stop=stop_after_attempt(DEFAULT_RETRY_ATTEMPTS), wait=wait_fixed(DEFAULT_RETRY_WAIT_SECONDS), retry=retry_if_exception_type(PositionNotFoundException)) - def _find_position_attempt(self, order_id: str): - """Single attempt to find the position, wrapped by tenacity.""" - logging.debug(f"Attempting to find position for OrderId: {order_id}") - all_positions = self.get_open_positions() - - if all_positions and 'Data' in all_positions: - for position in all_positions["Data"]: - if position.get("PositionBase", {}).get("SourceOrderId") == order_id: - logging.info(f"Position {position.get('PositionId')} found for order ID {order_id}.") - return position # Return the found position - - logging.warning(f"Position not found for OrderId {order_id} in current open positions. Retrying...") - # Raise specific exception for tenacity to catch and retry - raise PositionNotFoundException(f"Position for order {order_id} not found yet.", order_id=order_id) - - def find_position_by_order_id_with_retry(self, order_id: str): - """ - Finds an open position matching a source order ID, with retries. - Attempts to cancel the order if the position is not found after all retries. - """ - try: - # Call the internal method that tenacity decorates - found_position = self._find_position_attempt(order_id) - return found_position - except RetryError as e: - # This means all retry attempts failed - error_message_base = f"Position not found after {self.retry_config['max_retries']} retries for order ID {order_id}" - logging.critical(f"{error_message_base}. Attempting order cancellation.") - - # Attempt to cancel the order - cancellation_succeeded = self.order_service.cancel_order(order_id) - - if cancellation_succeeded: - cancel_msg = f"✅ Successfully cancelled potentially orphan order {order_id}" - logging.info(cancel_msg) - # Raise the specific exception noting successful cancellation - raise PositionNotFoundException(f"{error_message_base}. {cancel_msg}", order_id=order_id, cancellation_attempted=True, cancellation_succeeded=True) from e - else: - cancel_fail_msg = f"❌ Failed to cancel potentially orphan order {order_id}" - logging.error(cancel_fail_msg) - # Raise the specific exception noting failed cancellation - raise PositionNotFoundException(f"{error_message_base}. {cancel_fail_msg}", order_id=order_id, cancellation_attempted=True, cancellation_succeeded=False) from e - except Exception as e: - # Catch other unexpected errors during the find process - logging.error(f"Unexpected error finding position for order {order_id}: {e}", exc_info=True) - # Re-raise without attempting cancellation here, as the state is unknown - raise - - def get_spending_power(self): - """Gets the current account spending power.""" - logging.debug("Getting account balance/spending power...") - # Assuming balance endpoint provides this. Adjust if needed. - req_balance = pf.balances.AccountBalances( - params={"ClientKey": self.client_key} - ) - resp_balance = self.api_client.request(req_balance) - if not resp_balance or "SpendingPower" not in resp_balance: - logging.error(f"Invalid balance response: {resp_balance}") - raise SaxoApiError("Invalid balance response received, missing SpendingPower.") - spending_power = resp_balance["SpendingPower"] - if not isinstance(spending_power, (int, float)): - logging.error(f"Invalid SpendingPower value: {spending_power}") - raise SaxoApiError(f"Invalid SpendingPower value received: {spending_power}") - - logging.info(f"Spending Power retrieved: {spending_power}") - return spending_power - - -# === High-Level Orchestration & Monitoring === - -class TradingOrchestrator: - """Orchestrates the process of executing a trade signal.""" - - def __init__(self, instrument_service: InstrumentService, order_service: OrderService, position_service: PositionService, config_manager: ConfigurationManager, db_order_manager: DbOrderManager, db_position_manager: DbPositionManager): - self.instrument_service = instrument_service - self.order_service = order_service - self.position_service = position_service - self.config = config_manager - self.db_order_manager = db_order_manager - self.db_position_manager = db_position_manager - self.buying_power_config = self.config.get_config_value("trade.config.buying_power", {}) - self.safety_margins = self.buying_power_config.get("safety_margins", {"bid_calculation": 1}) - self.retry_config = self.config.get_config_value("trade.config.general.retry_config", {"max_retries": DEFAULT_RETRY_ATTEMPTS, "retry_sleep_seconds": DEFAULT_RETRY_WAIT_SECONDS}) - - - def _calculate_bid_amount(self, turbo_info: dict, spending_power: float): - """Calculates the amount to buy based on turbo price and spending power.""" - # Use the latest snapshot Ask price if available, otherwise fallback - ask_price = turbo_info['selected_instrument'].get('latest_ask') - if ask_price is None: - ask_price = turbo_info['selected_instrument'].get('quote', {}).get('Ask') - - if ask_price is None or not isinstance(ask_price, (int, float)) or ask_price <= 0: - raise ValueError(f"Invalid ask price for bid calculation: {ask_price}") - - logging.info(f"Calculating amount: SpendingPower={spending_power}, AskPrice={ask_price}") - - max_account_percent = self.buying_power_config.get("max_account_funds_to_use_percentage", 100) - available_funds = spending_power * (max_account_percent / 100.0) - logging.info(f"Available funds for trading ({max_account_percent}% of {spending_power}): {available_funds:.2f}") - - safety_margin_units = self.safety_margins.get("bid_calculation", 1) - cost_per_unit = ask_price # Add estimated commission per unit if significant and available - - # Check if funds cover at least one unit + margin - # Ensure calculation safety margin applies correctly - required_funds = cost_per_unit * (1 + safety_margin_units) # Funds needed for 1 unit + margin units equivalent - if available_funds < required_funds: - pre_amount = 0 # Cannot afford even one unit with margin - else: - # Calculate max units affordable - max_units = available_funds / cost_per_unit - # Subtract safety margin (as units) - pre_amount = max_units - safety_margin_units - - amount = int(math.floor(pre_amount)) - - if amount <= 0: - raise InsufficientFundsException( - message=f"Insufficient funds to buy required units @ {ask_price:.{turbo_info['selected_instrument']['decimals']}f}", - available_funds=available_funds, - required_price=ask_price, - calculated_amount=amount - ) - - logging.info(f"Calculated bid amount: {amount}") - return amount - - - def execute_trade_signal(self, exchange_id: str, underlying_uics: str, keywords: str): - """ - Full workflow: Find -> Calculate -> Place Order -> Confirm Position -> **Persist Order/Position**. - Returns details for logging/notification, not for DB persistence by caller. - """ - logging.info(f"--- Executing & Recording Trade Signal: {keywords} on {underlying_uics} ---") - confirmed_position = None # Initialize - validated_order = None # Initialize - turbo_info = None # Initialize - - try: - # 1. Find Turbo - # Exceptions (NoTurbos, NoMarket, Api) handled by caller or bubble up - turbo_info = self.instrument_service.find_turbos(exchange_id, underlying_uics, keywords) - - # 2. Get Spending Power - # Exceptions (Api, SaxoApiError) handled by caller or bubble up - spending_power = self.position_service.get_spending_power() - - # 3. Calculate Amount - # Raises InsufficientFundsException, ValueError - amount = self._calculate_bid_amount(turbo_info, spending_power) - - # 4. Place Buy Order - # Raises OrderPlacementError, SaxoApiError, ApiRequestException - validated_order = self.order_service.place_market_order( - uic=turbo_info['selected_instrument']['uic'], - asset_type=turbo_info['selected_instrument']['asset_type'], - amount=amount, - buy_sell="Buy" - ) - order_id = validated_order['OrderId'] - - # 5. Confirm Position Creation (with retry) - confirmed_position = self.position_service.find_position_by_order_id_with_retry(order_id) - - # --- *** 6. Persist to Database *** --- - now_utc = datetime.now(pytz.utc) - # Prepare Order Data for DB - order_data_for_db = { - "action": keywords, "buy_sell": "Buy", "order_id": order_id, "order_amount": amount, - "order_type": "Market", - "order_kind": "main", "order_submit_time": now_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), - "related_order_id": [], - "position_id": confirmed_position.get("PositionId"), - "instrument_name": turbo_info['selected_instrument']['description'], - "instrument_symbol": turbo_info['selected_instrument']['symbol'], - "instrument_uic": turbo_info['selected_instrument']['uic'], - "instrument_price": turbo_info['selected_instrument'].get('latest_ask'), - "instrument_currency": turbo_info['selected_instrument']['currency'], - "order_cost": turbo_info['selected_instrument'].get('commissions', {}).get('CostBuy'), - } - # Prepare Position Data for DB - pos_base = confirmed_position.get("PositionBase", {}) - pos_disp = confirmed_position.get("DisplayAndFormat", {}) - position_data_for_db = { - "action": keywords, "position_id": confirmed_position.get("PositionId"), - "position_amount": pos_base.get("Amount"), - "position_open_price": pos_base.get("OpenPrice"), - "position_total_open_price": (pos_base.get("Amount", 0) * pos_base.get("OpenPrice", 0)), - "position_status": pos_base.get("Status", "Open"), "position_kind": "main", - "execution_time_open": pos_base.get("ExecutionTimeOpen"), - "order_id": pos_base.get("SourceOrderId"), "related_order_id": pos_base.get("RelatedOpenOrders", []), - "instrument_name": pos_disp.get("Description"), "instrument_symbol": pos_disp.get("Symbol"), - "instrument_uic": pos_base.get("Uic"), "instrument_currency": pos_disp.get("Currency"), - } - - # Perform DB Inserts - try: - logging.info(f"Persisting order {order_id} to database...") - self.db_order_manager.insert_turbo_order_data(order_data_for_db) - # TODO: Handle potential sub-orders if OCO/related orders are implemented - logging.info(f"Persisting position {position_data_for_db['position_id']} to database...") - self.db_position_manager.insert_turbo_open_position_data(position_data_for_db) - logging.info("Order and position persisted successfully.") - except Exception as db_err: - # CRITICAL: Trade executed but failed to record in DB! - logging.critical( - f"CRITICAL DB ERROR: Failed to persist order/position after execution! OrderID: {order_id}, PositionID: {position_data_for_db['position_id']}. Error: {db_err}", - exc_info=True) - # Raise a specific error indicating this critical state - raise DatabaseOperationException(f"CRITICAL: Failed to persist executed trade OrderID {order_id}", - operation="insert_trade_data", entity_id=order_id) from db_err - - logging.info( - f"Trade execution & recording successful for OrderId {order_id}, PositionId {confirmed_position.get('PositionId')}") - - # --- *** 7. Return Execution Details (for logging/notification) *** --- - # Return details that might be useful for the caller (e.g., for composer) - return { - "order_details": order_data_for_db, # Return the prepared DB data - "position_details": position_data_for_db, - "selected_turbo_info": turbo_info, - "message": f"Successfully executed and recorded trade for {keywords}." - } - - # --- Exception Handling during Execution --- - except PositionNotFoundException as e: - # This specific error already attempted order cancellation inside find_position... - logging.critical( - f"CRITICAL: Position not found for OrderId {e.order_id} after retries. Order cancellation attempted.") - # Re-raise the critical exception for the main callback handler - raise e - except Exception as e: - # Catch other errors (NoTurbos, InsufficientFunds, OrderPlacementError, DB errors, etc.) - logging.error(f"Trade execution failed during '{keywords}' signal: {e}", exc_info=True) - # Attempt cleanup if order was placed but position failed *before* confirmation loop - if validated_order and not confirmed_position: - order_id_to_cancel = validated_order.get('OrderId') - if order_id_to_cancel: - logging.warning( - f"Attempting to cancel potentially orphan order {order_id_to_cancel} due to execution failure.") - try: - self.order_service.cancel_order(order_id_to_cancel) - logging.info(f"Successfully cancelled potentially orphan order {order_id_to_cancel}") - except Exception as cancel_err: - logging.error(f"Failed to cancel potentially orphan order {order_id_to_cancel}: {cancel_err}") - # Re-raise the original error for the main callback handler - raise e - - -class PerformanceMonitor: - """Monitors open positions, checks performance, triggers closures, syncs DB.""" - - def __init__(self, position_service: PositionService, order_service: OrderService, config_manager: ConfigurationManager, db_position_manager: DbPositionManager, trading_rule: TradingRule, rabbit_connection): - self.position_service = position_service - self.order_service = order_service - self.config = config_manager - self.db_position_manager = db_position_manager # Needed for daily profit check, max performance - self.trading_rule = trading_rule # Needed for daily profit target - self.rabbit_connection = rabbit_connection # For notifications - self.perf_config = self.config.get_config_value("trade.config.position_management", {}) - self.thresholds = self.perf_config.get("performance_thresholds", {"stoploss_percent": -20, "max_profit_percent": 60}) - self.general_config = self.config.get_config_value("trade.config.general", {}) - self.timezone = self.general_config.get("timezone", "Europe/Paris") - self.logging_config = self.config.get_logging_config() - # Get daily profit target from trading_rule config - try: - day_trading_rules = self.trading_rule.get_rule_config("day_trading") - self.percent_profit_wanted_per_days = day_trading_rules.get("percent_profit_wanted_per_days", 1.0) # Default 1% - except Exception as e: - logging.warning(f"Could not get day_trading rules for profit target, defaulting: {e}") - self.percent_profit_wanted_per_days = 1.0 - - def _fetch_and_update_closed_position_in_db(self, opening_position_id: str, closed_from_reason: str) -> bool | None: - """ - Fetches closed position details from API after a delay, finds the matching one, - calculates performance, updates the database, and sends a notification. - Mimics the logic from the original `act_on_db_closed_position`. - - Args: - opening_position_id: The ID of the position *before* it was closed. - closed_from_reason: A string indicating why the position was closed (e.g., "Performance", "Explicit"). - - Returns: - True if the position was found and updated successfully, False otherwise. - """ - logging.info( - f"Processing DB update for closed position {opening_position_id}. Reason: {closed_from_reason}. Waiting briefly...") - time.sleep(2) # Replicate original delay to allow API to update - - try: - # Fetch recent closed positions - # Increase 'top' slightly to improve chances of finding it if multiple closed quickly - all_closed_positions = self.position_service.get_closed_positions(top=50) - if not all_closed_positions or not all_closed_positions.get("Data"): - logging.warning(f"No closed positions found in API when checking for {opening_position_id}.") - return False - - position_found_in_api = False - for api_closed_position in all_closed_positions["Data"]: - closed_info = api_closed_position.get("ClosedPosition", {}) - if closed_info.get("OpeningPositionId") == opening_position_id: - position_found_in_api = True - logging.info(f"Found matching closed position in API for {opening_position_id}.") - - # Extract data - display_info = api_closed_position.get("DisplayAndFormat", {}) - close_price = closed_info.get("ClosingPrice") - open_price = closed_info.get("OpenPrice") - amount = closed_info.get("Amount") - profit_loss = closed_info.get("ProfitLossOnTrade") - exec_time_close = closed_info.get("ExecutionTimeClose") - description = display_info.get("Description", "N/A") - - # Calculate derived fields - position_total_close_price = None - performance_percent = None - if close_price is not None and amount is not None: - position_total_close_price = float(close_price * amount) - if close_price is not None and open_price is not None and open_price != 0: - performance_percent = round(((close_price * 100) / open_price) - 100, 2) - - # Prepare DB update data - turbo_position_data_at_close = { - "position_close_price": close_price, - "position_profit_loss": profit_loss, - "position_total_close_price": position_total_close_price, - "position_status": "Closed", - "position_total_performance_percent": performance_percent, - "position_close_reason": closed_from_reason, # Use the provided reason - "execution_time_close": exec_time_close, - } - - # Update Database - try: - logging.debug( - f"Updating DB for position {opening_position_id} with data: {turbo_position_data_at_close}") - self.db_position_manager.update_turbo_position_data( - opening_position_id, turbo_position_data_at_close - ) - logging.info( - f"Successfully updated database for closed position {opening_position_id} ({description}).") - except Exception as e: - # Use the specific DatabaseOperationException logic from original - error_message = f"CRITICAL: Failed to update DB for closed position {opening_position_id} ({description}): {e}. Manual update needed." - logging.critical(error_message, exc_info=True) - self.db_position_manager.mark_database_as_corrupted( - error_message) - db_exception = DatabaseOperationException( - error_message, - operation="update_turbo_position_data", - entity_id=opening_position_id - ) - send_message_to_mq_for_telegram(self.rabbit_connection, - f"CRITICAL DB UPDATE FAILED: {db_exception}") - # Don't re-raise here, just report failure - return False - - # Send Notification - try: - # Get max position % and today % for notification (optional, based on original) - max_position_percent = self.db_position_manager.get_max_position_percent( - opening_position_id) - today_percent = self.db_position_manager.get_percent_of_the_day() - - message = f""" ---- CLOSED POSITION --- -Instrument : {description} -Open Price : {open_price} -Close Price : {close_price} -Amount : {amount} -Total Close Price : {position_total_close_price} -Profit/Loss : {profit_loss} -Performance % : {performance_percent} -Close Time : {exec_time_close} -Closed Reason : {closed_from_reason} -Opening Position ID : {opening_position_id} -Max position % during trade : {max_position_percent} -------- -Today's Realized Profit % (after close) : {today_percent}% -""" - send_message_to_mq_for_telegram(self.rabbit_connection, message) - except Exception as notify_err: - logging.error( - f"Failed to send notification for closed position {opening_position_id}: {notify_err}") - - return True # Successfully found and processed - - # If loop finishes without finding the position - if not position_found_in_api: - logging.warning( - f"Abnormal: Position {opening_position_id} was expected to be closed, but not found in recent API closed positions.") - # The sync mechanism might catch it later if it appears. - return False - - except (ApiRequestException, SaxoApiError) as api_err: - logging.error(f"API error fetching closed positions for {opening_position_id}: {api_err}") - return False - except Exception as e: - logging.error(f"Unexpected error processing closed position {opening_position_id}: {e}", exc_info=True) - return False - - def check_all_positions_performance(self): - """Checks performance, closes if needed, and **immediately updates DB**.""" - logging.info("--- Checking Performance of Open Positions ---") - # --- Use the correct method --- - db_open_positions = self.db_position_manager.get_open_positions_ids_actions() - # ----------------------------- - if not db_open_positions: - logging.info("No manageable open positions found in database to check.") - return {"closed_positions_processed": [], "db_updates": [], "errors": 0} - - db_position_ids = [p['position_id'] for p in db_open_positions] - logging.debug(f"Checking DB positions: {db_position_ids}") - - try: - api_positions_response = self.position_service.get_open_positions() - api_positions_dict = {p["PositionId"]: p for p in api_positions_response.get("Data", [])} - except Exception as e: - logging.error(f"Failed to get open positions from API during performance check: {e}") - return {"closed_positions_processed": [], "db_updates": [], "errors": 1} - - positions_to_close = [] # Still collect first to avoid modifying list while iterating - db_updates = [] - processed_positions = [] # Track positions processed by this run - errors_count = 0 - - for db_pos in db_open_positions: - position_id = db_pos['position_id'] - if position_id not in api_positions_dict: - logging.warning(f"Position {position_id} open in DB but not found in API. Skipping perf check (will be synced).") - continue - - api_pos = api_positions_dict[position_id] - open_price = api_pos.get("PositionBase", {}).get("OpenPrice") - current_bid = api_pos.get("PositionView", {}).get("Bid") - performance_percent = None - - # Calculate Performance & Log - if open_price and current_bid and open_price != 0: - performance_percent = round(((current_bid * 100) / open_price) - 100, 2) - logging.info(f"Pos {position_id}: Open={open_price}, Bid={current_bid}, Perf={performance_percent}%") - self._log_performance_detail(position_id, api_pos, performance_percent) - max_perf = self.db_position_manager.get_max_position_percent(position_id) - if max_perf is None: max_perf = -float('inf') # Handle initial None case - if performance_percent > max_perf: - db_updates.append((position_id, {"position_max_performance_percent": performance_percent})) - else: - logging.warning(f"Could not calculate performance for {position_id}. Open={open_price}, Bid={current_bid}. Skipping checks.") - continue # Cannot check thresholds without performance - - # Check Thresholds & Daily Profit - close_reason = None - if performance_percent <= self.thresholds["stoploss_percent"]: - close_reason = f"Stoploss ({self.thresholds['stoploss_percent']}%) hit at {performance_percent}%" - elif performance_percent >= self.thresholds["max_profit_percent"]: - close_reason = f"Takeprofit ({self.thresholds['max_profit_percent']}%) hit at {performance_percent}%" - - # Check daily profit target only if no other close reason yet - if not close_reason: - try: - # --- Refined Daily Profit Check --- - # Get today's *realized* profit percentage so far - today_realized_percent = self.db_position_manager.get_percent_of_the_day() - - # Calculate the *potential* total realized profit if this position is closed *now* - # This needs careful calculation, especially with multiple open positions. - # Simplistic approach: Check if current position's performance PLUS today's realized meets target. - # Better approach: Calculate total potential profit based on current open positions' value changes. - # For now, using a simpler check: If closing this position *alone* would reach the target from current realized profit. - # Let initial_capital = 1.0 - # current_capital = 1.0 * (1 + today_realized_percent / 100.0) - # potential_capital_after_close = current_capital * (1 + performance_percent / 100.0) # This assumes the *entire* capital is in this one trade - likely WRONG - # Let's stick to the original simpler check for now, acknowledging its limitation: - today_profit_factor = 1.0 + (today_realized_percent / 100.0) - position_profit_factor = 1.0 + (performance_percent / 100.0) - # This combined factor doesn't accurately reflect portfolio impact unless only 1 position open. - potential_today_percent = round((today_profit_factor * position_profit_factor - 1) * 100, 2) - - # Let's use a direct comparison: Is today's realized + this position's gain >= target? - # This is still not quite right without knowing position size relative to portfolio. - if potential_today_percent >= self.percent_profit_wanted_per_days: - close_reason = f"Daily profit target ({self.percent_profit_wanted_per_days}%) potentially met (Combined factor: {potential_today_percent}%)" - logging.info(f"Daily profit check triggered closure for {position_id}. Realized today: {today_realized_percent}%, Position perf: {performance_percent}%.") - # ------------------------------- - - except Exception as e: - logging.error(f"Error checking daily profit target for {position_id}: {e}") - - - if close_reason: - logging.info(f"Marking position {position_id} for closure. Reason: {close_reason}") - positions_to_close.append({"position_id": position_id, "api_details": api_pos, "reason": close_reason}) - - - # 5. Execute Closures - for pos_to_close in positions_to_close: - api_pos = pos_to_close["api_details"] - position_id = pos_to_close["position_id"] - close_reason = pos_to_close["reason"] - - if not api_pos.get("PositionBase", {}).get("CanBeClosed", False): - logging.warning(f"Position {position_id} flagged for closure but CanBeClosed is False. Skipping.") - processed_positions.append({"id": position_id, "close_reason": close_reason, "status": "Skipped (Cannot Be Closed)"}) - continue - - try: - direction = direction_from_amount(api_pos["PositionBase"]["Amount"]) - order_direction = direction_invert(direction) # Sell to close Buy, Buy to close Sell - logging.info(f"Attempting to close {position_id} ({order_direction} {api_pos['PositionBase']['Amount']}). Reason: {close_reason}") - - # Place closing order - close_order_result = self.order_service.place_market_order( - uic=api_pos["PositionBase"]["Uic"], - asset_type=api_pos["PositionBase"]["AssetType"], - amount=api_pos["PositionBase"]["Amount"], - buy_sell=order_direction - ) - logging.info(f"Close order placed for {position_id}. OrderId: {close_order_result.get('OrderId')}. Now attempting immediate DB update.") - - # --- Call the helper for immediate update --- - update_success = self._fetch_and_update_closed_position_in_db(position_id, f"Performance ({close_reason})") - processed_positions.append({ - "id": position_id, - "close_reason": close_reason, - "db_update_attempted": True, - "db_update_success": update_success, - "status": "Closed" if update_success else "Closed (DB Update Failed)" - }) - if not update_success: - logging.error(f"Immediate DB update failed for closed position {position_id}. Sync mechanism will retry later.") - errors_count += 1 # Count DB update failure as an error - - - except (OrderPlacementError, SaxoApiError, ApiRequestException) as e: - logging.error(f"Failed to place close order for position {position_id}: {e}") - # Send error notification - error_message = f"ERROR: Failed closing {position_id}. Reason: {close_reason}. Error: {e}" - send_message_to_mq_for_telegram(self.rabbit_connection, error_message) - errors_count += 1 - processed_positions.append({"id": position_id, "close_reason": close_reason, "error": str(e), "status": "Close Order Failed"}) - except Exception as e: - logging.error(f"Unexpected error closing position {position_id}: {e}", exc_info=True) - error_message = f"CRITICAL ERROR: Unexpected error closing {position_id}. Error: {e}" - send_message_to_mq_for_telegram(self.rabbit_connection, error_message) - errors_count += 1 - processed_positions.append({"id": position_id, "close_reason": close_reason, "error": str(e), "status": "Close Failed (Unexpected)"}) - - # Apply Max Performance DB Updates (collected earlier) - if db_updates: - logging.info(f"Applying {len(db_updates)} max performance updates to DB.") - for pos_id, update_data in db_updates: - try: - self.db_position_manager.update_turbo_position_data(pos_id, update_data) - except Exception as e: - logging.error(f"Failed to update max performance for {pos_id}: {e}") - # This is less critical than a closure update failure - errors_count += 1 # Optionally track this minor error type - - logging.info(f"Performance check finished. Positions processed/closed: {len(processed_positions)}, Max Perf Updates: {len(db_updates)}, Errors: {errors_count}") - return {"closed_positions_processed": processed_positions, "db_updates": db_updates, "errors": errors_count} - - def sync_db_positions_with_api(self): - """Compares DB open positions with API closed positions and returns updates.""" - logging.info("--- Syncing DB Positions with API Closed Positions ---") - db_open_positions = self.db_position_manager.get_open_positions_ids() # Get only IDs - - try: - api_open_positions_response = self.position_service.get_open_positions() - api_open_position_ids = {p["PositionId"] for p in api_open_positions_response.get("Data", [])} - except Exception as e: - logging.error(f"Failed to get API open positions during sync: {e}") - return {"updates_for_db": []} # Cannot proceed - - # This statement can be made before the API request, but WATA need to maintain the auth token up to date (with the request) - if not db_open_positions: - logging.info("No open positions in DB to sync.") - return {"updates_for_db": []} - - potential_closed_in_db = [] - for db_pos_id in db_open_positions: - if db_pos_id not in api_open_position_ids: - logging.info(f"Position {db_pos_id} is open in DB but not in API open list. Checking closed API positions.") - potential_closed_in_db.append(db_pos_id) - - if not potential_closed_in_db: - logging.info("All DB open positions found in API open positions. Sync complete.") - return {"updates_for_db": []} - - # Fetch recent closed positions from API - try: - # Fetch a decent number to increase chance of finding the match - api_closed_positions_response = self.position_service.get_closed_positions(top=len(potential_closed_in_db) + 50) - # Create dict mapping OpeningPositionId to closed position data - api_closed_map = { - p["ClosedPosition"]["OpeningPositionId"]: p - for p in api_closed_positions_response.get("Data", []) - if p and "ClosedPosition" in p and "OpeningPositionId" in p["ClosedPosition"] - } - except Exception as e: - logging.error(f"Failed to get API closed positions during sync: {e}") - return {"updates_for_db": []} # Cannot proceed - - - updates_for_db = [] - for position_id_to_check in potential_closed_in_db: - if position_id_to_check in api_closed_map: - api_closed_pos = api_closed_map[position_id_to_check] - logging.info(f"Found match for DB open position {position_id_to_check} in API closed positions. Preparing DB update.") - - closed_pos_data = api_closed_pos.get("ClosedPosition", {}) - display_data = api_closed_pos.get("DisplayAndFormat", {}) - - close_price = closed_pos_data.get("ClosingPrice") - open_price = closed_pos_data.get("OpenPrice") - amount = closed_pos_data.get("Amount") - pl = closed_pos_data.get("ProfitLossOnTrade") - close_time = closed_pos_data.get("ExecutionTimeClose") - description = display_data.get("Description", "N/A") - - - performance_percent = None - total_close_price = None - if close_price is not None and amount is not None: - total_close_price = close_price * amount - if open_price is not None and open_price != 0 and close_price is not None: - performance_percent = round(((close_price * 100) / open_price) - 100, 2) - - update_data = { - "position_close_price": close_price, - "position_profit_loss": pl, - "position_total_close_price": total_close_price, - "position_status": "Closed", - "position_total_performance_percent": performance_percent, - "position_close_reason": "SaxoAPI", # Indicates found closed via API sync - "execution_time_close": close_time, - } - updates_for_db.append((position_id_to_check, update_data)) - - # Send notification - message = f"""SYNC CLOSE: Position {position_id_to_check} ({description}) closed on API. -Open: {open_price}, Close: {close_price}, Amount: {amount} -P/L: {pl}, Perf: {performance_percent}% -Close Time: {close_time}""" - send_message_to_mq_for_telegram(self.rabbit_connection, message) - - else: - # Position is open in DB, not in API open, not in recent API closed. - # This is an anomaly. Maybe closed long ago, or error state. - logging.warning(f"ANOMALY: Position {position_id_to_check} open in DB, not found in API open or recent closed positions.") - # TODO: Consider marking it as 'Unknown' or 'SyncError' in DB? For now, just log. - # updates_for_db.append((position_id_to_check, {"position_status": "SyncError", "position_close_reason": "SyncAnomaly"})) - - - logging.info(f"Sync check complete. Found {len(updates_for_db)} positions closed on API to update in DB.") - return {"updates_for_db": updates_for_db} - - - def _log_performance_detail(self, position_id, api_pos, performance_percent): - """Writes detailed performance data to a JSONL file.""" - try: - current_time = datetime.now(pytz.timezone(self.timezone)) - pos_base = api_pos.get("PositionBase", {}) - pos_view = api_pos.get("PositionView", {}) - open_time_str = pos_base.get("ExecutionTimeOpen") - open_time = None - open_hour = None - open_minute = None - if open_time_str: - try: - # Ensure timezone handling is robust - if open_time_str.endswith('Z'): - open_time_dt = datetime.fromisoformat(open_time_str.replace('Z', '+00:00')) - else: - open_time_dt = datetime.fromisoformat(open_time_str) # Assume UTC if no Z - - # Convert to local timezone - local_tz = pytz.timezone(self.timezone) - open_time = open_time_dt.astimezone(local_tz) - open_hour = open_time.hour - open_minute = open_time.minute - except (ValueError, TypeError) as parse_err: - logging.warning(f"Could not parse open time {open_time_str}: {parse_err}") - - - performance_json = { - "position_id": position_id, - "performance": performance_percent, - "open_price": pos_base.get("OpenPrice"), - "bid": pos_view.get("Bid"), # Use PositionView for current bid - "time": current_time.strftime("%Y-%m-%d %H:%M:%S"), - "current_hour": current_time.hour, - "current_minute": current_time.minute, - "open_hour": open_hour, - "open_minute": open_minute - } - - # Construct the filename using today's date - today_date = current_time.strftime("%Y-%m-%d") - log_path = self.logging_config.get('persistant', {}).get('log_path', '.') # Get log path safely - if not os.path.exists(log_path): os.makedirs(log_path) # Ensure log dir exists - filename = os.path.join(log_path, f"performance_{today_date}.jsonl") - - # Write the performance_json to the JSON Lines file - with open(filename, 'a') as file: - file.write(json.dumps(performance_json) + '\n') - - except Exception as e: - logging.error(f"Failed to write performance log for position {position_id}: {e}") - - def close_managed_positions_by_criteria(self, action_filter: str | None = None): - """ - Closes open positions managed by the app, optionally filtered by action ('long'/'short'). - Initiates closure and attempts immediate DB update. - - Args: - action_filter: If 'long' or 'short', closes only positions matching that action. - If None, closes all managed open positions. - """ - logging.info(f"--- Closing Managed Positions by Criteria (Filter: {action_filter}) ---") - closed_initiated_count = 0 - errors_count = 0 - processed_positions = [] # Track positions processed - - # 1. Get currently open positions managed by the app from DB - try: - db_open_positions = self.db_position_manager.get_open_positions_ids_actions() - if not db_open_positions: - logging.info("No managed positions open in DB to close.") - return {"closed_initiated_count": 0, "errors_count": 0} - except Exception as e: - logging.error(f"Failed to get open positions from DB for closure: {e}") - raise # Re-raise as we cannot proceed - - # 2. Get currently open positions from API - try: - api_positions_response = self.position_service.get_open_positions() - api_positions_dict = {p["PositionId"]: p for p in api_positions_response.get("Data", [])} - except Exception as e: - logging.error(f"Failed to get open positions from API for closure: {e}") - raise # Re-raise as we cannot compare - - # 3. Filter and Initiate Closure - for db_pos in db_open_positions: - position_id = db_pos.get('position_id') - db_action = db_pos.get('action') - if not position_id: continue # Skip if somehow ID is missing - - # Apply filter - if action_filter and db_action != action_filter: - logging.debug(f"Skipping pos {position_id}: Action '{db_action}' != Filter '{action_filter}'") - continue # Skip if action doesn't match filter - - # Check if position exists and is closable on API - if position_id not in api_positions_dict: - logging.warning(f"Position {position_id} (Action: {db_action}) to be closed is not open on API. Skipping (will be synced later).") - processed_positions.append({"id": position_id, "action": db_action, "filter": action_filter, "status": "Skipped (Not in API)"}) - continue - - api_pos = api_positions_dict[position_id] - pos_base = api_pos.get("PositionBase", {}) - if not pos_base.get("CanBeClosed", False): - logging.warning(f"Position {position_id} (Action: {db_action}) cannot be closed via API (CanBeClosed=False). Skipping.") - processed_positions.append({"id": position_id, "action": db_action, "filter": action_filter, "status": "Skipped (Cannot Be Closed)"}) - continue - - # Initiate closure - try: - amount_to_close = pos_base.get("Amount", 0) - direction = direction_from_amount(pos_base.get("Amount", 0)) - order_direction = direction_invert(direction) - logging.info(f"Initiating explicit close for {position_id} (Action: {db_action}), Filter: {action_filter}. Order: {order_direction} {amount_to_close}") - - close_order_result = self.order_service.place_market_order( - uic=pos_base.get("Uic"), - asset_type=pos_base.get("AssetType"), - amount=amount_to_close, # Use positive amount - buy_sell=order_direction - ) - closed_initiated_count += 1 - logging.info(f"Close order placed for {position_id}. OrderId: {close_order_result.get('OrderId')}. Attempting immediate DB update.") - - # --- Call the helper for immediate update --- - close_reason_str = f"Explicit Close ({action_filter or 'All'})" - update_success = self._fetch_and_update_closed_position_in_db(position_id, close_reason_str) - processed_positions.append({ - "id": position_id, - "action": db_action, - "filter": action_filter, - "db_update_attempted": True, - "db_update_success": update_success, - "status": "Closed" if update_success else "Closed (DB Update Failed)" - }) - if not update_success: - logging.error(f"Immediate DB update failed for explicitly closed position {position_id}. Sync mechanism will retry.") - errors_count += 1 # Count DB update failure as an error - - except (OrderPlacementError, SaxoApiError, ApiRequestException) as e: - logging.error(f"Failed to place explicit close order for position {position_id}: {e}") - error_message = f"ERROR: Failed explicit close for {position_id} (Action: {db_action}). Error: {e}" - send_message_to_mq_for_telegram(self.rabbit_connection, error_message) - errors_count += 1 - processed_positions.append({"id": position_id, "action": db_action, "filter": action_filter, "error": str(e), "status":"Close Order Failed"}) - except Exception as e: - logging.error(f"Unexpected error during explicit close for position {position_id}: {e}", exc_info=True) - error_message = f"CRITICAL ERROR: Unexpected error during explicit close for {position_id}. Error: {e}" - send_message_to_mq_for_telegram(self.rabbit_connection, error_message) - errors_count += 1 - processed_positions.append({"id": position_id, "action": db_action, "filter": action_filter, "error": str(e), "status": "Close Failed (Unexpected)"}) - - logging.info(f"Explicit closure process finished. Initiated: {closed_initiated_count}, Errors: {errors_count}. Processed: {len(processed_positions)}") - return {"closed_initiated_count": closed_initiated_count, "errors_count": errors_count, "processed_positions": processed_positions} \ No newline at end of file diff --git a/src/trade/async_services.py b/src/trade/async_services.py new file mode 100644 index 0000000..2e40470 --- /dev/null +++ b/src/trade/async_services.py @@ -0,0 +1,1203 @@ +# src/trade/async_services.py +""" +Async versions of SaxoApiClient, InstrumentService, OrderService, +PositionService, TradingOrchestrator, and PerformanceMonitor. + +Key improvements over the sync versions: + - Non-blocking HTTP via httpx.AsyncClient (AsyncAPI) + - Parallel API calls where independent (find_turbos ‖ get_spending_power) + - asyncio.sleep instead of blocking time.sleep + - Async PostgreSQL persistence + - Bounded concurrency for multi-position operations +""" + +import asyncio +import json +import logging +import math +import os +import re +import time +from collections import defaultdict +from datetime import datetime + +import pytz + +# --- Saxo OpenApi Components --- +import src.saxo_openapi.endpoints.referencedata as rd +import src.saxo_openapi.endpoints.trading as tr +import src.saxo_openapi.endpoints.portfolio as pf +from src.saxo_openapi.contrib.orders import MarketOrder, tie_account_to_order, direction_from_amount +from src.saxo_openapi.contrib.orders.helper import direction_invert +from src.saxo_openapi.async_client import AsyncAPI +from src.saxo_openapi.exceptions import OpenAPIError as SaxoOpenApiLibError + +# --- Local Imports --- +from src.saxo_authen import SaxoAuth +from src.configuration import ConfigurationManager +from src.database.postgres import ( + AsyncDbOrderManager, + AsyncDbPositionManager, +) +from .exceptions import ( + NoMarketAvailableException, + NoTurbosAvailableException, + PositionNotFoundException, + InsufficientFundsException, + ApiRequestException, + TokenAuthenticationException, + DatabaseOperationException, + SaxoApiError, + OrderPlacementError, +) + +logger = logging.getLogger(__name__) + +TURBO_DESCRIPTION_PATTERN = re.compile(r"(.*) (\w+) (\w+) (\d+(?:\.\d+)?) (\w+)$") +INITIAL_TURBO_SEARCH_LIMIT = 60 +INFO_PRICE_CANDIDATE_LIMIT = 20 +INFO_PRICE_RETRY_BACKOFFS = (0.2, 0.4, 0.8) + + +# ────────────────────────────────────────────── +# Utilities +# ────────────────────────────────────────────── + +def parse_saxo_turbo_description(description: str) -> dict | None: + match = TURBO_DESCRIPTION_PATTERN.match(description) + if match: + return { + "name": match.group(1), + "kind": match.group(2), + "buysell": match.group(3), + "price": match.group(4), + "from": match.group(5), + } + return None + + +# ────────────────────────────────────────────── +# Async SaxoApiClient +# ────────────────────────────────────────────── + +class AsyncSaxoApiClient: + """ + Async facade around AsyncAPI (httpx-based). + Handles token refresh and translates exceptions. + """ + + def __init__(self, config_manager: ConfigurationManager, saxo_auth: SaxoAuth): + self.config_manager = config_manager + self.saxo_auth = saxo_auth + self.environment = config_manager.get_config_value("saxo_auth.env", "live") + self._api: AsyncAPI | None = None + self._current_token: str | None = None + + async def ensure_ready(self): + """Initialize or refresh the underlying AsyncAPI instance.""" + latest_token = self.saxo_auth.get_token() + if latest_token != self._current_token or self._api is None: + if self._api: + await self._api.close() + logger.info("AsyncSaxoApiClient: (re)initializing AsyncAPI for env '%s'", self.environment) + self._api = AsyncAPI( + access_token=latest_token, + environment=self.environment, + timeout=30.0, + ) + self._current_token = latest_token + + async def close(self): + if self._api: + await self._api.close() + self._api = None + + async def request(self, endpoint_request_obj): + """ + Async API request with exception translation. + Mirrors the sync SaxoApiClient.request() interface. + """ + await self.ensure_ready() + + try: + return await self._api.request(endpoint_request_obj) + + except SaxoOpenApiLibError as e: + status_code = e.code + content_str = str(e.content) if not isinstance(e.content, str) else e.content + + error_message = content_str + error_code = None + saxo_error_details = None + try: + saxo_error_details = json.loads(content_str) + error_message = saxo_error_details.get("Message", e.reason) + error_code = saxo_error_details.get("ErrorCode") + except (json.JSONDecodeError, Exception): + saxo_error_details = content_str + + logger.error( + "Saxo API Error (async): Status=%s, Code=%s, Msg=%s, Endpoint=%s", + status_code, error_code, error_message, + type(endpoint_request_obj).__name__, + ) + + if status_code == 400 and error_code == "InsufficientFunds": + raise InsufficientFundsException( + message=error_message or "Insufficient funds", + saxo_error_details=saxo_error_details, + ) from e + + endpoint_path = getattr(endpoint_request_obj, "path", "unknown") + is_order_endpoint = "/trade/v2/orders" in endpoint_path + if (status_code in [400, 403, 409] or error_code) and is_order_endpoint: + raise OrderPlacementError( + f"Saxo rejected order ({status_code}): {error_message}", + status_code=status_code, + saxo_error_details=saxo_error_details, + order_details=getattr(endpoint_request_obj, "data", None), + ) from e + + if status_code == 401: + raise TokenAuthenticationException( + f"API returned 401: {error_message}", + saxo_error_details=saxo_error_details, + ) from e + + raise SaxoApiError( + f"Saxo API Error ({status_code}): {error_message}", + status_code=status_code, + saxo_error_details=saxo_error_details, + ) from e + + except Exception as e: + if isinstance(e, (InsufficientFundsException, OrderPlacementError, + TokenAuthenticationException, SaxoApiError)): + raise + logger.exception("Unexpected error in async Saxo request: %s", e) + raise ApiRequestException( + f"Unexpected request error: {e}", + endpoint=str(endpoint_request_obj), + ) from e + + +# ────────────────────────────────────────────── +# Async InstrumentService +# ────────────────────────────────────────────── + +class AsyncInstrumentService: + """Finds and retrieves turbo instruments asynchronously.""" + + def __init__(self, api_client: AsyncSaxoApiClient, config_manager: ConfigurationManager, account_key: str): + self.api_client = api_client + self.config = config_manager + self.account_key = account_key + self.api_limits = self.config.get_config_value("trade.config.general.api_limits", {"top_instruments": 200}) + self.turbo_price_range = self.config.get_config_value("trade.config.turbo_preference.price_range", {"min": 4, "max": 15}) + self.retry_config = self.config.get_config_value("trade.config.general.retry_config", {"max_retries": 3, "retry_sleep_seconds": 1}) + self.cache_config = self.config.get_config_value("trade.config.turbo_cache", {"enabled": False, "ttl_seconds": 30}) + self._turbo_cache: dict = {} + + async def _get_infoprices_for_asset_type( + self, + identifiers_string: str, + exchange_id: str, + asset_type: str, + field_groups: str, + top: int | None = None, + ): + req = tr.infoprices.InfoPrices( + params={ + "$top": top or len(identifiers_string.split(",")), + "AccountKey": self.account_key, + "ExchangeId": exchange_id, + "FieldGroups": field_groups, + "Uics": identifiers_string, + "AssetType": asset_type, + } + ) + for attempt in range(3): + try: + return await self.api_client.request(req) + except ApiRequestException: + if attempt < 2: + await asyncio.sleep(1) + else: + raise + + async def find_turbos(self, exchange_id: str, underlying_uics: str, keywords: str) -> dict: + """Finds turbos — uses cache if enabled, otherwise fetches fresh.""" + if self.cache_config.get("enabled", False): + cache_key = (exchange_id, underlying_uics, keywords) + cached = self._turbo_cache.get(cache_key) + if cached: + age = time.time() - cached["timestamp"] + ttl = self.cache_config.get("ttl_seconds", 30) + if age < ttl: + logger.info("Turbo cache HIT (age=%.1fs)", age) + return cached["result"] + + result = await self._find_turbos_uncached(exchange_id, underlying_uics, keywords) + + if self.cache_config.get("enabled", False): + self._turbo_cache[(exchange_id, underlying_uics, keywords)] = { + "result": result, + "timestamp": time.time(), + } + return result + + async def _find_turbos_uncached(self, exchange_id: str, underlying_uics: str, keywords: str) -> dict: + logger.info("Finding turbos: Exchange=%s, Underlying=%s, Keywords=%s", exchange_id, underlying_uics, keywords) + + # 1. Instrument search + initial_search_limit = min(self.api_limits.get("top_instruments", INITIAL_TURBO_SEARCH_LIMIT), INITIAL_TURBO_SEARCH_LIMIT) + req = rd.instruments.Instruments( + params={ + "$top": initial_search_limit, + "AccountKey": self.account_key, + "ExchangeId": exchange_id, + "Keywords": keywords, + "IncludeNonTradable": False, + "UnderlyingUics": underlying_uics, + "AssetTypes": "WarrantKnockOut,WarrantOpenEndKnockOut,MiniFuture,WarrantDoubleKnockOut", + } + ) + response = await self.api_client.request(req) + if not response or not response.get("Data"): + raise NoTurbosAvailableException("No instruments found.", search_context=req.params) + + # 2. Parse & filter + valid_items = [] + for item in response["Data"]: + parsed = parse_saxo_turbo_description(item.get("Description", "")) + if parsed: + item["appParsedData"] = parsed + valid_items.append(item) + + if not valid_items: + raise NoTurbosAvailableException("No instruments with parsable descriptions.", search_context=req.params) + + initial_instruments_by_identifier = { + item["Identifier"]: item + for item in valid_items + if item.get("Identifier") is not None + } + + # 3. Sort + sort_reverse = keywords.lower() != "short" + sorted_instruments = sorted(valid_items, key=lambda x: float(x["appParsedData"]["price"]), reverse=sort_reverse) + + candidate_instruments = sorted_instruments[:INFO_PRICE_CANDIDATE_LIMIT] + if not candidate_instruments: + raise NoTurbosAvailableException("No identifiers found after sorting.", search_context=req.params) + + # 4. Group by AssetType + instrument_groups: dict[str, list] = defaultdict(list) + for item in candidate_instruments: + instrument_groups[item["AssetType"]].append(item) + + # 5. Fetch quote-only InfoPrices for the top candidates + max_retries = min(self.retry_config.get("max_retries", len(INFO_PRICE_RETRY_BACKOFFS)), len(INFO_PRICE_RETRY_BACKOFFS)) + retry_backoffs = INFO_PRICE_RETRY_BACKOFFS[:max_retries] + min_price = self.turbo_price_range["min"] + max_price = self.turbo_price_range["max"] + + response_infoprices = None + for attempt in range(max_retries): + tasks = [] + for asset_type, instruments in instrument_groups.items(): + ids_str = ",".join(str(i["Identifier"]) for i in instruments) + tasks.append( + self._get_infoprices_for_asset_type( + ids_str, + exchange_id, + asset_type, + field_groups="Quote", + top=len(instruments), + ) + ) + + results = await asyncio.gather(*tasks, return_exceptions=True) + all_data = [] + for r in results: + if isinstance(r, Exception): + logger.warning("InfoPrices group failed: %s", r) + elif r and r.get("Data"): + all_data.extend(r["Data"]) + + if not all_data: + logger.warning("No InfoPrice data (attempt %d/%d)", attempt + 1, max_retries) + if attempt < max_retries - 1: + await asyncio.sleep(retry_backoffs[attempt]) + continue + + has_valid_candidate = any( + item.get("Quote") + and item["Quote"].get("Bid") is not None + and item["Quote"].get("PriceTypeAsk") != "NoMarket" + and item["Quote"].get("PriceTypeBid") != "NoMarket" + and item["Quote"].get("MarketState") != "Closed" + and min_price <= item["Quote"]["Bid"] <= max_price + for item in all_data + ) + + if has_valid_candidate: + response_infoprices = {"Data": all_data} + break + + # Check bid availability + with_quote = [i for i in all_data if "Quote" in i] + if not with_quote: + break # No quotes at all — nothing to retry for + + missing_bid = [i for i in with_quote if "Bid" not in i["Quote"]] + pct_missing = (len(missing_bid) / len(with_quote)) * 100 if with_quote else 0 + + if pct_missing > 50: + logger.warning("%.1f%% missing Bid (attempt %d/%d)", pct_missing, attempt + 1, max_retries) + if attempt < max_retries - 1: + await asyncio.sleep(retry_backoffs[attempt]) + continue + + response_infoprices = {"Data": all_data} + break + + if not response_infoprices or not response_infoprices.get("Data"): + raise NoMarketAvailableException("Failed to obtain InfoPrice data after retries.") + + # Filter items with valid Bid + valid_bid_items = [ + i for i in response_infoprices["Data"] + if i.get("Quote") and i["Quote"].get("Bid") is not None + ] + if not valid_bid_items: + raise NoMarketAvailableException("No instruments with Bid data after filtering.") + + # 6. Market state filter + available_items = [ + i for i in valid_bid_items + if i["Quote"].get("PriceTypeAsk") != "NoMarket" + and i["Quote"].get("PriceTypeBid") != "NoMarket" + and i["Quote"].get("MarketState") != "Closed" + ] + if not available_items: + raise NoMarketAvailableException(f"No markets available for {keywords} in {exchange_id}.") + + # 7. Price range filter + price_filtered = [i for i in available_items if min_price <= i["Quote"]["Bid"] <= max_price] + if not price_filtered: + raise NoTurbosAvailableException( + f"No turbos in price range {min_price}-{max_price}.", + search_context={"PriceRange": (min_price, max_price), "AvailableCount": len(available_items)}, + ) + + # 8. Select best + final_candidates = sorted(price_filtered, key=lambda x: x["Quote"]["Bid"]) + selected = final_candidates[0] + selected_source = initial_instruments_by_identifier.get(selected.get("Identifier"), {}) + detail_response = await self._get_infoprices_for_asset_type( + str(selected["Uic"]), + exchange_id, + selected["AssetType"], + field_groups="Commissions,DisplayAndFormat,InstrumentPriceDetails", + top=1, + ) + detail_rows = (detail_response or {}).get("Data") or [] + detail_snapshot = detail_rows[0] if detail_rows else {} + display_and_format = detail_snapshot.get("DisplayAndFormat", {}) + commissions = detail_snapshot.get("Commissions", {}) + description = display_and_format.get("Description", selected_source.get("Description", "N/A")) + final_snapshot = { + "Quote": selected.get("Quote", {}), + "DisplayAndFormat": display_and_format, + "Commissions": commissions, + } + + return { + "input_criteria": {"exchange_id": exchange_id, "underlying_uics": underlying_uics, "keywords": keywords}, + "selected_instrument": { + "uic": selected["Uic"], + "asset_type": selected["AssetType"], + "description": description, + "symbol": display_and_format.get("Symbol", "N/A"), + "currency": display_and_format.get("Currency", "N/A"), + "decimals": display_and_format.get("OrderDecimals", 2), + "parsed_data": parse_saxo_turbo_description( + description + ), + "quote": final_snapshot.get("Quote", {}), + "commissions": commissions, + "latest_ask": final_snapshot.get("Quote", {}).get("Ask"), + "latest_bid": final_snapshot.get("Quote", {}).get("Bid"), + "subscription_context_id": None, + "subscription_reference_id": None, + }, + } + + +# ────────────────────────────────────────────── +# Async OrderService +# ────────────────────────────────────────────── + +class AsyncOrderService: + def __init__(self, api_client: AsyncSaxoApiClient, account_key: str, client_key: str): + self.api_client = api_client + self.account_key = account_key + self.client_key = client_key + + async def place_market_order(self, uic: int, asset_type: str, amount: int, buy_sell: str) -> dict: + logger.info("Placing Market Order: %s %d of %d (%s)", buy_sell, amount, uic, asset_type) + pre_order = MarketOrder(Uic=uic, AssetType=asset_type, Amount=amount, BuySell=buy_sell) + final_payload = tie_account_to_order(self.account_key, pre_order) + req = tr.orders.Order(data=final_payload) + + try: + result = await self.api_client.request(req) + except SaxoApiError as e: + raise OrderPlacementError( + f"API error placing order: {e}", + saxo_error_details=e.saxo_error_details, + order_details=final_payload, + ) from e + + if not result or not result.get("OrderId"): + raise OrderPlacementError("Response missing OrderId.", order_details=final_payload, saxo_error_details=result) + + logger.info("Order placed — OrderId: %s", result["OrderId"]) + return result + + async def cancel_order(self, order_id: str) -> bool: + logger.info("Cancelling order: %s", order_id) + req = tr.orders.CancelOrders(OrderIds=order_id, params={"AccountKey": self.account_key}) + try: + await self.api_client.request(req) + logger.info("Order %s cancelled.", order_id) + return True + except Exception as e: + logger.error("Failed to cancel order %s: %s", order_id, e) + return False + + +# ────────────────────────────────────────────── +# Async PositionService +# ────────────────────────────────────────────── + +class AsyncPositionService: + def __init__(self, api_client: AsyncSaxoApiClient, order_service: AsyncOrderService, + config_manager: ConfigurationManager, account_key: str, client_key: str): + self.api_client = api_client + self.order_service = order_service + self.config = config_manager + self.account_key = account_key + self.client_key = client_key + self.api_limits = self.config.get_config_value("trade.config.general.api_limits", {"top_positions": 200, "top_closed_positions": 500}) + retry_cfg = self.config.get_config_value("trade.config.general.retry_config", {"max_retries": 5, "retry_sleep_seconds": 2}) + self.max_retries = retry_cfg["max_retries"] + self.retry_sleep = retry_cfg["retry_sleep_seconds"] + + async def get_open_positions(self) -> dict: + req = pf.positions.PositionsMe( + params={ + "ClientKey": self.client_key, + "AccountKey": self.account_key, + "FieldGroups": "PositionBase,PositionView,DisplayAndFormat,ExchangeInfo", + } + ) + response = await self.api_client.request(req) + if response and "Data" in response and "__count" not in response: + response["__count"] = len(response["Data"]) + elif not response: + return {"__count": 0, "Data": []} + return response + + async def get_closed_positions(self, top: int | None = None, skip: int = 0) -> dict: + if top is None: + top = self.api_limits["top_closed_positions"] + req = pf.closedpositions.ClosedPositionsMe( + params={ + "$top": top, "$skip": skip, + "AccountKey": self.account_key, + "FieldGroups": "ClosedPosition,ClosedPositionDetails,DisplayAndFormat,ExchangeInfo", + } + ) + response = await self.api_client.request(req) + return response if response else {"__count": 0, "Data": []} + + async def get_single_position(self, position_id: str) -> dict: + req = pf.positions.SinglePosition( + PositionId=position_id, + params={ + "ClientKey": self.client_key, + "AccountKey": self.account_key, + "FieldGroups": "PositionBase,PositionView,DisplayAndFormat,Costs,ExchangeInfo", + }, + ) + return await self.api_client.request(req) + + async def find_position_by_order_id_with_retry(self, order_id: str) -> dict: + """ + Finds position with exponential backoff (non-blocking). + Attempts order cancellation if not found after all retries. + """ + delays = [0.3, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0] # ~10.8s total max + max_attempts = min(len(delays), self.max_retries) + + for attempt in range(max_attempts): + await asyncio.sleep(delays[attempt]) + positions = await self.get_open_positions() + for p in positions.get("Data", []): + if p.get("PositionBase", {}).get("SourceOrderId") == order_id: + logger.info("Position found for order %s after %d attempt(s).", order_id, attempt + 1) + return p + logger.debug("Position not found for order %s (attempt %d/%d)", order_id, attempt + 1, max_attempts) + + # All retries exhausted — attempt cancellation + logger.critical("Position not found for order %s after %d retries. Cancelling.", order_id, max_attempts) + cancelled = await self.order_service.cancel_order(order_id) + raise PositionNotFoundException( + f"Position not found after {max_attempts} retries for order {order_id}", + order_id=order_id, + cancellation_attempted=True, + cancellation_succeeded=cancelled, + ) + + async def get_spending_power(self) -> float: + req = pf.balances.AccountBalances(params={"ClientKey": self.client_key}) + resp = await self.api_client.request(req) + if not resp or "SpendingPower" not in resp: + raise SaxoApiError("Invalid balance response, missing SpendingPower.") + power = resp["SpendingPower"] + logger.info("Spending power: %s", power) + return float(power) + + +# ────────────────────────────────────────────── +# Async TradingOrchestrator +# ────────────────────────────────────────────── + +class AsyncTradingOrchestrator: + """ + Orchestrates the full trade-execution flow asynchronously. + Key improvement: find_turbos and get_spending_power run in parallel. + """ + + def __init__( + self, + instrument_service: AsyncInstrumentService, + order_service: AsyncOrderService, + position_service: AsyncPositionService, + config_manager: ConfigurationManager, + db_order_manager: AsyncDbOrderManager, + db_position_manager: AsyncDbPositionManager, + ): + self.instrument_service = instrument_service + self.order_service = order_service + self.position_service = position_service + self.config = config_manager + self.db_order_manager = db_order_manager + self.db_position_manager = db_position_manager + self.buying_power_config = self.config.get_config_value("trade.config.buying_power", {}) + self.safety_margins = self.buying_power_config.get("safety_margins", {"bid_calculation": 1}) + self.reserve_cash_percent = self.buying_power_config.get("reserve_cash_percent", 0) + self.timezone = self.config.get_config_value("trade.config.general.timezone", "Europe/Paris") + self.time_of_day_config = self.config.get_config_value("trade.config.position_sizing.time_of_day_scaling", {"enabled": False}) + self.confidence_config = self.config.get_config_value("trade.config.position_sizing.confidence_scaling", {"enabled": False}) + + # ── Position sizing helpers (same logic, kept sync as they're pure computation) ── + + def _get_time_of_day_scale(self) -> float: + if not self.time_of_day_config.get("enabled", False): + return 100.0 + current_time = datetime.now(pytz.timezone(self.timezone)) + current_minutes = current_time.hour * 60 + current_time.minute + for period in self.time_of_day_config.get("periods", []): + start_m = period["start_hour"] * 60 + period.get("start_minute", 0) + end_m = period["end_hour"] * 60 + period.get("end_minute", 0) + if start_m <= current_minutes < end_m: + return period.get("scale_percent", 100) + return 100.0 + + def _get_confidence_scale(self, confidence) -> float: + if not self.confidence_config.get("enabled", False): + return 100.0 + if confidence is None: + confidence = self.confidence_config.get("default_confidence", 1.0) + for rule in self.confidence_config.get("scaling_rules", []): + if rule["min_confidence"] <= confidence < rule["max_confidence"]: + return rule.get("scale_percent", 100) + return 100.0 + + def _calculate_position_scale(self, confidence=None) -> float: + tod = self._get_time_of_day_scale() / 100.0 + conf = self._get_confidence_scale(confidence) / 100.0 + combined = tod * conf + logger.info("Position scale: tod=%.2f × conf=%.2f = %.2f", tod, conf, combined) + return combined + + def _calculate_bid_amount(self, turbo_info: dict, spending_power: float, position_scale: float = 1.0) -> int: + ask_price = turbo_info["selected_instrument"].get("latest_ask") + if ask_price is None: + ask_price = turbo_info["selected_instrument"].get("quote", {}).get("Ask") + if not ask_price or ask_price <= 0: + raise ValueError(f"Invalid ask price: {ask_price}") + + reserve = self.reserve_cash_percent + max_pct = self.buying_power_config.get("max_account_funds_to_use_percentage", 100) + effective_pct = max(0, max_pct - reserve) + available = spending_power * (effective_pct / 100.0) * position_scale + + safety = self.safety_margins.get("bid_calculation", 1) + required = ask_price * (1 + safety) + if available < required: + pre = 0 + else: + pre = (available / ask_price) - safety + + amount = int(math.floor(pre)) + if amount <= 0: + raise InsufficientFundsException( + f"Insufficient funds @ {ask_price}", + available_funds=available, + required_price=ask_price, + calculated_amount=amount, + ) + logger.info("Calculated bid amount: %d (available=%.2f, ask=%.4f, scale=%.2f)", amount, available, ask_price, position_scale) + return amount + + async def execute_trade_signal(self, exchange_id: str, underlying_uics: str, keywords: str, confidence: float = None) -> dict: + """ + Full async trade workflow. + Key optimisation: find_turbos and get_spending_power run **in parallel**. + """ + logger.info("--- Executing trade signal: %s on %s (conf=%s) ---", keywords, underlying_uics, confidence) + timestamps = {"start": time.time()} + validated_order = None + confirmed_position = None + turbo_info = None + + try: + # 0. Position scale (instant) + position_scale = self._calculate_position_scale(confidence) + timestamps["scale_calculated"] = time.time() + + # 1+2. PARALLEL: find turbo + get spending power + turbo_task = asyncio.create_task( + self.instrument_service.find_turbos(exchange_id, underlying_uics, keywords) + ) + balance_task = asyncio.create_task( + self.position_service.get_spending_power() + ) + turbo_info, spending_power = await asyncio.gather(turbo_task, balance_task) + timestamps["turbo_found"] = time.time() + timestamps["spending_power_fetched"] = time.time() + + # 3. Calculate amount + amount = self._calculate_bid_amount(turbo_info, spending_power, position_scale) + timestamps["amount_calculated"] = time.time() + + # 4. Place order + validated_order = await self.order_service.place_market_order( + uic=turbo_info["selected_instrument"]["uic"], + asset_type=turbo_info["selected_instrument"]["asset_type"], + amount=amount, + buy_sell="Buy", + ) + order_id = validated_order["OrderId"] + timestamps["order_placed"] = time.time() + + # 5. Confirm position (exponential backoff) + confirmed_position = await self.position_service.find_position_by_order_id_with_retry(order_id) + timestamps["position_confirmed"] = time.time() + + # 6. Persist to PostgreSQL + now_utc = datetime.now(pytz.utc) + order_data = { + "action": keywords, "buy_sell": "Buy", "order_id": order_id, + "order_amount": amount, "order_type": "Market", "order_kind": "main", + "order_submit_time": now_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), + "related_order_id": [], "position_id": confirmed_position.get("PositionId"), + "instrument_name": turbo_info["selected_instrument"]["description"], + "instrument_symbol": turbo_info["selected_instrument"]["symbol"], + "instrument_uic": turbo_info["selected_instrument"]["uic"], + "instrument_price": turbo_info["selected_instrument"].get("latest_ask"), + "instrument_currency": turbo_info["selected_instrument"]["currency"], + "order_cost": turbo_info["selected_instrument"].get("commissions", {}).get("CostBuy"), + } + pos_base = confirmed_position.get("PositionBase", {}) + pos_disp = confirmed_position.get("DisplayAndFormat", {}) + position_data = { + "action": keywords, "position_id": confirmed_position.get("PositionId"), + "position_amount": pos_base.get("Amount"), + "position_open_price": pos_base.get("OpenPrice"), + "position_total_open_price": (pos_base.get("Amount", 0) * pos_base.get("OpenPrice", 0)), + "position_status": pos_base.get("Status", "Open"), "position_kind": "main", + "execution_time_open": pos_base.get("ExecutionTimeOpen"), + "order_id": pos_base.get("SourceOrderId"), + "related_order_id": pos_base.get("RelatedOpenOrders", []), + "instrument_name": pos_disp.get("Description"), + "instrument_symbol": pos_disp.get("Symbol"), + "instrument_uic": pos_base.get("Uic"), + "instrument_currency": pos_disp.get("Currency"), + } + + try: + await asyncio.gather( + self.db_order_manager.insert_turbo_order_data(order_data), + self.db_position_manager.insert_turbo_open_position_data(position_data), + ) + except Exception as db_err: + logger.critical("CRITICAL DB ERROR after execution! Order=%s. %s", order_id, db_err, exc_info=True) + raise DatabaseOperationException( + f"Failed to persist trade {order_id}", operation="insert_trade_data", entity_id=order_id + ) from db_err + + timestamps["db_persisted"] = time.time() + timing = self._build_timing_summary(timestamps) + logger.info("Trade execution complete. Timing: %s", timing) + self._log_execution_timing(keywords, timestamps, confidence, position_scale) + + return { + "order_details": order_data, + "position_details": position_data, + "selected_turbo_info": turbo_info, + "execution_timing": timing, + "position_scale": position_scale, + "confidence": confidence, + "message": f"Successfully executed trade for {keywords}.", + } + + except PositionNotFoundException: + raise + except Exception as e: + if validated_order and not confirmed_position: + oid = validated_order.get("OrderId") + if oid: + logger.warning("Cancelling orphan order %s due to failure: %s", oid, e) + await self.order_service.cancel_order(oid) + raise + + @staticmethod + def _build_timing_summary(timestamps: dict) -> dict: + summary = {} + start = timestamps.get("start") + if not start: + return summary + steps = [ + ("scale_calculated", "Scale Calc"), + ("turbo_found", "Find Turbo"), + ("spending_power_fetched", "Get Balance"), + ("amount_calculated", "Calc Amount"), + ("order_placed", "Place Order"), + ("position_confirmed", "Confirm Pos"), + ("db_persisted", "DB Persist"), + ] + prev = start + for key, label in steps: + ts = timestamps.get(key) + if ts: + summary[label] = f"{(ts - prev) * 1000:.0f}ms" + prev = ts + total = timestamps.get("db_persisted", timestamps.get("position_confirmed", start)) + summary["TOTAL"] = f"{(total - start) * 1000:.0f}ms" + return summary + + def _log_execution_timing(self, action, timestamps, confidence, position_scale): + try: + tz = pytz.timezone(self.timezone) + now = datetime.now(tz) + log_path = self.config.get_config_value("logging.persistant.log_path", ".") + os.makedirs(log_path, exist_ok=True) + data = { + "action": action, "confidence": confidence, + "position_scale": position_scale, + "timestamp": now.strftime("%Y-%m-%d %H:%M:%S"), + "steps": {k: round((v - timestamps["start"]) * 1000) for k, v in timestamps.items() if k != "start"}, + } + path = os.path.join(log_path, f"execution_timing_{now.strftime('%Y-%m-%d')}.jsonl") + with open(path, "a") as f: + f.write(json.dumps(data) + "\n") + except Exception as e: + logger.error("Failed to log execution timing: %s", e) + + +# ────────────────────────────────────────────── +# Async PerformanceMonitor +# ────────────────────────────────────────────── + +class AsyncPerformanceMonitor: + """ + Monitors open positions, checks SL/TP/trailing stop, syncs DB. + Designed for the Position Monitor service (separate queue). + """ + + def __init__( + self, + position_service: AsyncPositionService, + order_service: AsyncOrderService, + config_manager: ConfigurationManager, + db_position_manager: AsyncDbPositionManager, + trading_rule, # TradingRule instance (sync is fine — it's pure computation) + send_telegram_fn, # async callable(message: str) -> None + ): + self.position_service = position_service + self.order_service = order_service + self.config = config_manager + self.db_position_manager = db_position_manager + self.trading_rule = trading_rule + self.send_telegram = send_telegram_fn + self.perf_config = self.config.get_config_value("trade.config.position_management", {}) + self.thresholds = self.perf_config.get("performance_thresholds", {"stoploss_percent": -15, "max_profit_percent": 60}) + self.trailing_stop_config = self.thresholds.get("trailing_stop", {"enabled": False, "activation_percent": 5, "drawdown_percent": 8}) + self.timezone = self.config.get_config_value("trade.config.general.timezone", "Europe/Paris") + self.logging_config = self.config.get_logging_config() + try: + day_cfg = self.trading_rule.get_rule_config("day_trading") + self.percent_profit_wanted = day_cfg.get("percent_profit_wanted_per_days", 1.0) + except Exception: + self.percent_profit_wanted = 1.0 + + async def check_positions_from_stream(self, streamed_positions: dict[str, dict]) -> dict: + """ + Check performance using **pre-fetched** position data from the + WebSocket stream instead of polling the REST API. + + Parameters + ---------- + streamed_positions : dict[str, dict] + Mapping of ``{PositionId: merged_position_dict}`` as maintained + by :class:`SaxoStreamClient`. + + Returns the same structure as :meth:`check_all_positions_performance`. + """ + return await self._evaluate_positions(streamed_positions) + + async def check_all_positions_performance(self) -> dict: + """Check all open positions, close if thresholds hit, update max perf.""" + logger.info("--- Checking performance of open positions ---") + db_positions = await self.db_position_manager.get_open_positions_ids_actions() + if not db_positions: + logger.info("No open positions to check.") + return {"closed_positions_processed": [], "db_updates": [], "errors": 0} + + try: + api_resp = await self.position_service.get_open_positions() + api_dict = {p["PositionId"]: p for p in api_resp.get("Data", [])} + except Exception as e: + logger.error("Failed to get API positions: %s", e) + return {"closed_positions_processed": [], "db_updates": [], "errors": 1} + + return await self._evaluate_positions(api_dict) + + async def _evaluate_positions(self, api_dict: dict[str, dict]) -> dict: + """ + Shared evaluation logic used by both REST-polled and stream-fed paths. + + Parameters + ---------- + api_dict : dict[str, dict] + ``{PositionId: position_dict}`` — from either API response or stream cache. + """ + db_positions = await self.db_position_manager.get_open_positions_ids_actions() + if not db_positions: + logger.info("No open positions to check.") + return {"closed_positions_processed": [], "db_updates": [], "errors": 0} + + positions_to_close = [] + db_updates = [] + errors = 0 + + for db_pos in db_positions: + pid = db_pos["position_id"] + if pid not in api_dict: + continue + + api_pos = api_dict[pid] + open_price = api_pos.get("PositionBase", {}).get("OpenPrice") + current_bid = api_pos.get("PositionView", {}).get("Bid") + if not open_price or not current_bid or open_price == 0: + continue + + perf_pct = round(((current_bid * 100) / open_price) - 100, 2) + logger.info("Pos %s: Open=%.4f, Bid=%.4f, Perf=%.2f%%", pid, open_price, current_bid, perf_pct) + + self._log_performance_detail(pid, api_pos, perf_pct) + + max_perf = await self.db_position_manager.get_max_position_percent(pid) + if perf_pct > max_perf: + db_updates.append((pid, {"position_max_performance_percent": perf_pct})) + + close_reason = None + + # Stop-loss + if perf_pct <= self.thresholds["stoploss_percent"]: + close_reason = f"Stoploss ({self.thresholds['stoploss_percent']}%) hit at {perf_pct}%" + try: + self.trading_rule.record_loss() + except Exception: + pass + + # Take-profit + elif perf_pct >= self.thresholds["max_profit_percent"]: + close_reason = f"Takeprofit ({self.thresholds['max_profit_percent']}%) hit at {perf_pct}%" + + # Trailing stop + if not close_reason and self.trailing_stop_config.get("enabled", False): + act_pct = self.trailing_stop_config.get("activation_percent", 5) + dd_pct = self.trailing_stop_config.get("drawdown_percent", 8) + if max_perf >= act_pct: + drawdown = max_perf - perf_pct + if drawdown >= dd_pct: + close_reason = f"Trailing Stop: peak {max_perf:.1f}%, now {perf_pct:.1f}% (dd {drawdown:.1f}% >= {dd_pct}%)" + + # Daily profit target + if not close_reason: + try: + today_pct = await self.db_position_manager.get_percent_of_the_day() + factor = (1 + today_pct / 100.0) * (1 + perf_pct / 100.0) - 1 + potential = round(factor * 100, 2) + if potential >= self.percent_profit_wanted: + close_reason = f"Daily profit target ({self.percent_profit_wanted}%) potentially met ({potential}%)" + except Exception as e: + logger.error("Daily profit check error for %s: %s", pid, e) + + if close_reason: + positions_to_close.append({"position_id": pid, "api_details": api_pos, "reason": close_reason}) + + # Execute closures concurrently (bounded) + processed = [] + sem = asyncio.Semaphore(3) # Max 3 concurrent closures + + async def _close_one(pos_info): + nonlocal errors + async with sem: + pid = pos_info["position_id"] + api_pos = pos_info["api_details"] + reason = pos_info["reason"] + + if not api_pos.get("PositionBase", {}).get("CanBeClosed", False): + processed.append({"id": pid, "status": "Skipped (Cannot Be Closed)"}) + return + + try: + direction = direction_from_amount(api_pos["PositionBase"]["Amount"]) + sell_dir = direction_invert(direction) + await self.order_service.place_market_order( + uic=api_pos["PositionBase"]["Uic"], + asset_type=api_pos["PositionBase"]["AssetType"], + amount=api_pos["PositionBase"]["Amount"], + buy_sell=sell_dir, + ) + ok = await self._fetch_and_update_closed_position(pid, f"Performance ({reason})") + processed.append({"id": pid, "status": "Closed" if ok else "Closed (DB Update Failed)"}) + if not ok: + errors += 1 + except Exception as e: + logger.error("Failed to close %s: %s", pid, e, exc_info=True) + await self.send_telegram(f"ERROR closing {pid}: {e}") + errors += 1 + processed.append({"id": pid, "status": f"Failed: {e}"}) + + await asyncio.gather(*[_close_one(p) for p in positions_to_close]) + + # Apply max-perf DB updates + for pid, data in db_updates: + try: + await self.db_position_manager.update_turbo_position_data(pid, data) + except Exception as e: + logger.error("Failed max-perf update for %s: %s", pid, e) + errors += 1 + + logger.debug("Perf check done. Closed=%d, MaxPerf updates=%d, Errors=%d", len(processed), len(db_updates), errors) + return {"closed_positions_processed": processed, "db_updates": db_updates, "errors": errors} + + async def sync_db_positions_with_api(self) -> dict: + """Compare DB open positions vs API, return updates for positions closed externally.""" + logger.debug("--- Syncing DB positions with API ---") + db_open_ids = await self.db_position_manager.get_open_positions_ids() + + try: + api_resp = await self.position_service.get_open_positions() + api_open_ids = {p["PositionId"] for p in api_resp.get("Data", [])} + except Exception as e: + logger.error("Failed to get API open positions: %s", e) + return {"updates_for_db": []} + + if not db_open_ids: + return {"updates_for_db": []} + + potentially_closed = [pid for pid in db_open_ids if pid not in api_open_ids] + if not potentially_closed: + logger.debug("All DB-open positions are still open on API.") + return {"updates_for_db": []} + + try: + closed_resp = await self.position_service.get_closed_positions(top=len(potentially_closed) + 50) + closed_map = { + p["ClosedPosition"]["OpeningPositionId"]: p + for p in closed_resp.get("Data", []) + if p and "ClosedPosition" in p and "OpeningPositionId" in p["ClosedPosition"] + } + except Exception as e: + logger.error("Failed to get API closed positions: %s", e) + return {"updates_for_db": []} + + updates = [] + for pid in potentially_closed: + if pid in closed_map: + cp = closed_map[pid]["ClosedPosition"] + dp = closed_map[pid].get("DisplayAndFormat", {}) + close_price = cp.get("ClosingPrice") + open_price = cp.get("OpenPrice") + amount = cp.get("Amount") + perf = round(((close_price * 100) / open_price) - 100, 2) if open_price and close_price else None + total_close = close_price * amount if close_price and amount else None + + update = { + "position_close_price": close_price, + "position_profit_loss": cp.get("ProfitLossOnTrade"), + "position_total_close_price": total_close, + "position_status": "Closed", + "position_total_performance_percent": perf, + "position_close_reason": "SaxoAPI", + "execution_time_close": cp.get("ExecutionTimeClose"), + } + updates.append((pid, update)) + await self.send_telegram(f"SYNC CLOSE: {pid} ({dp.get('Description','N/A')})\nPerf: {perf}%") + else: + logger.warning("ANOMALY: %s open in DB, not in API open or closed.", pid) + + return {"updates_for_db": updates} + + async def close_managed_positions_by_criteria(self, action_filter: str | None = None, exclude_position_id: str | None = None) -> dict: + """Close open positions, optionally filtered by action, with concurrency.""" + logger.info("--- Closing positions (filter=%s, exclude=%s) ---", action_filter, exclude_position_id) + + db_positions = await self.db_position_manager.get_open_positions_ids_actions() + if not db_positions: + return {"closed_initiated_count": 0, "errors_count": 0} + + try: + api_resp = await self.position_service.get_open_positions() + api_dict = {p["PositionId"]: p for p in api_resp.get("Data", [])} + except Exception as e: + logger.error("Failed to get API positions for closure: %s", e) + raise + + closed_count = 0 + error_count = 0 + sem = asyncio.Semaphore(3) + + async def _close_one(db_pos): + nonlocal closed_count, error_count + async with sem: + pid = db_pos["position_id"] + action = db_pos.get("action") + + if action_filter and action != action_filter: + return + if exclude_position_id and str(pid) == str(exclude_position_id): + return + if pid not in api_dict: + return + + api_pos = api_dict[pid] + pos_base = api_pos.get("PositionBase", {}) + if not pos_base.get("CanBeClosed", False): + return + + try: + direction = direction_from_amount(pos_base["Amount"]) + sell_dir = direction_invert(direction) + await self.order_service.place_market_order( + uic=pos_base["Uic"], + asset_type=pos_base["AssetType"], + amount=pos_base["Amount"], + buy_sell=sell_dir, + ) + closed_count += 1 + reason = f"Explicit Close ({action_filter or 'All'})" + await self._fetch_and_update_closed_position(pid, reason) + except Exception as e: + logger.error("Failed closing %s: %s", pid, e) + await self.send_telegram(f"ERROR closing {pid}: {e}") + error_count += 1 + + await asyncio.gather(*[_close_one(p) for p in db_positions]) + logger.info("Closure done. Initiated=%d, Errors=%d", closed_count, error_count) + return {"closed_initiated_count": closed_count, "errors_count": error_count} + + async def _fetch_and_update_closed_position(self, position_id: str, reason: str) -> bool: + """Fetch closed position from API after brief delay and update DB.""" + await asyncio.sleep(1.5) # Non-blocking wait for API to reflect closure + + try: + closed = await self.position_service.get_closed_positions(top=50) + for item in closed.get("Data", []): + cp = item.get("ClosedPosition", {}) + if cp.get("OpeningPositionId") == position_id: + dp = item.get("DisplayAndFormat", {}) + close_price = cp.get("ClosingPrice") + open_price = cp.get("OpenPrice") + amount = cp.get("Amount") + perf = round(((close_price * 100) / open_price) - 100, 2) if open_price and close_price and open_price != 0 else None + total_close = close_price * amount if close_price and amount else None + + update = { + "position_close_price": close_price, + "position_profit_loss": cp.get("ProfitLossOnTrade"), + "position_total_close_price": total_close, + "position_status": "Closed", + "position_total_performance_percent": perf, + "position_close_reason": reason, + "execution_time_close": cp.get("ExecutionTimeClose"), + } + await self.db_position_manager.update_turbo_position_data(position_id, update) + + if perf is not None and perf < 0: + try: + self.trading_rule.record_loss() + except Exception: + pass + + max_perf = await self.db_position_manager.get_max_position_percent(position_id) + today_pct = await self.db_position_manager.get_percent_of_the_day() + + msg = f"""--- CLOSED POSITION --- +Instrument: {dp.get('Description', 'N/A')} +Open: {open_price} → Close: {close_price} +Amount: {amount} | P/L: {cp.get('ProfitLossOnTrade')} +Perf: {perf}% | Max during trade: {max_perf}% +Reason: {reason} +Today realized: {today_pct}%""" + await self.send_telegram(msg) + return True + + logger.warning("Closed position %s not found in API.", position_id) + return False + + except Exception as e: + logger.error("Error updating closed position %s: %s", position_id, e, exc_info=True) + return False + + def _log_performance_detail(self, position_id, api_pos, perf_pct): + try: + tz = pytz.timezone(self.timezone) + now = datetime.now(tz) + pos_base = api_pos.get("PositionBase", {}) + pos_view = api_pos.get("PositionView", {}) + data = { + "position_id": position_id, + "performance": perf_pct, + "open_price": pos_base.get("OpenPrice"), + "bid": pos_view.get("Bid"), + "time": now.strftime("%Y-%m-%d %H:%M:%S"), + "current_hour": now.hour, + "current_minute": now.minute, + } + log_path = self.logging_config.get("persistant", {}).get("log_path", ".") + os.makedirs(log_path, exist_ok=True) + path = os.path.join(log_path, f"performance_{now.strftime('%Y-%m-%d')}.jsonl") + with open(path, "a") as f: + f.write(json.dumps(data) + "\n") + except Exception as e: + logger.error("Failed to write perf log: %s", e) diff --git a/src/trade/rules.py b/src/trade/rules.py index 639f5f6..d3fbfcc 100644 --- a/src/trade/rules.py +++ b/src/trade/rules.py @@ -15,7 +15,20 @@ def __init__(self, config_manager, db_position_manager): self.signal_validation_config = self.get_rule_config("signal_validation") self.market_hours_config = self.get_rule_config("market_hours") self.timezone = self.config_manager.get_config_value("trade.config.general.timezone", "Europe/Paris") + # Risk management config (optional rule) + self.risk_management_config = self.get_rule_config_safe("risk_management") + self.cooldown_after_loss_minutes = self.risk_management_config.get("cooldown_after_loss_minutes", 0) if self.risk_management_config else 0 + self.max_trades_per_day = self.risk_management_config.get("max_trades_per_day", 0) if self.risk_management_config else 0 + # Confidence scaling config (optional) + self.confidence_config = self.config_manager.get_config_value("trade.config.position_sizing.confidence_scaling", {}) + self.min_confidence_threshold = self.confidence_config.get("min_confidence_threshold", 0.0) + # Track last loss timestamp for cooldown + self._last_loss_timestamp = None logging.info(f"Trading rules using timezone: {self.timezone}") + if self.cooldown_after_loss_minutes > 0: + logging.info(f"Cooldown after loss: {self.cooldown_after_loss_minutes} minutes") + if self.max_trades_per_day > 0: + logging.info(f"Max trades per day: {self.max_trades_per_day}") def get_rule_config(self, rule_type): """ @@ -27,6 +40,16 @@ def get_rule_config(self, rule_type): return rule.get("rule_config", {}) raise TradingRuleViolation(f"Rule with type '{rule_type}' not found in the configuration.") + def get_rule_config_safe(self, rule_type): + """ + Retrieves the rule_config for a given rule_type, returning None if not found. + """ + trade_rules = self.config_manager.get_config_value("trade.rules", []) + for rule in trade_rules: + if rule.get("rule_type") == rule_type: + return rule.get("rule_config", {}) + return None + def check_signal_timestamp(self, signal_action, signal_timestamp): # Parse the signal_timestamp string into a datetime object signal_time = datetime.strptime(signal_timestamp, "%Y-%m-%dT%H:%M:%SZ") @@ -120,3 +143,59 @@ def check_if_open_position_is_same_signal(action, db_position_manager): f" with the same action {action}.") logging.info(message) raise TradingRuleViolation(message) + + def check_cooldown_after_loss(self): + """ + Checks if we are still in a cooldown period after the last losing trade. + Prevents entering trades too quickly after a loss. + """ + if self.cooldown_after_loss_minutes <= 0: + return # Cooldown disabled + + if self._last_loss_timestamp is None: + return # No loss recorded yet + + current_time = datetime.now(pytz.utc) + cooldown_end = self._last_loss_timestamp + timedelta(minutes=self.cooldown_after_loss_minutes) + + if current_time < cooldown_end: + remaining = (cooldown_end - current_time).total_seconds() + message = (f"Breaking trading rule: Cooldown active after last loss. " + f"Remaining: {remaining:.0f}s (cooldown: {self.cooldown_after_loss_minutes}min)") + logging.info(message) + raise TradingRuleViolation(message) + + def record_loss(self): + """Records the timestamp of a losing trade for cooldown tracking.""" + self._last_loss_timestamp = datetime.now(pytz.utc) + logging.info(f"Loss recorded at {self._last_loss_timestamp}. Cooldown of {self.cooldown_after_loss_minutes} minutes activated.") + + def check_max_trades_per_day(self): + """ + Checks if the maximum number of trades per day has been reached. + """ + if self.max_trades_per_day <= 0: + return # Limit disabled + + if self.db_position_manager is None: + return # No DB manager available + + today_trades = self.db_position_manager.get_today_trade_count() + if today_trades >= self.max_trades_per_day: + message = (f"Breaking trading rule: Maximum trades per day reached " + f"({today_trades}/{self.max_trades_per_day}).") + logging.info(message) + raise TradingRuleViolation(message) + + def check_confidence_threshold(self, confidence): + """ + Checks if the signal confidence meets the minimum threshold. + """ + if not self.confidence_config.get("enabled", False): + return # Confidence checking disabled + + if confidence is not None and confidence < self.min_confidence_threshold: + message = (f"Breaking trading rule: Signal confidence ({confidence:.2f}) is below " + f"minimum threshold ({self.min_confidence_threshold:.2f}).") + logging.info(message) + raise TradingRuleViolation(message) diff --git a/src/trader/__init__.py b/src/trader/__init__.py new file mode 100644 index 0000000..712239d --- /dev/null +++ b/src/trader/__init__.py @@ -0,0 +1,480 @@ +""" +Async Trader Service — consumes from ``trading-signals`` queue via aio-pika. + +Replaces the synchronous ``src/main.py`` Trader for signal processing. +Position monitoring is now handled by the separate Position Monitor service. +""" + +import asyncio +import json +import logging +import os +import sys +import traceback +from datetime import date, timedelta + +import aio_pika +import jsonschema + +# --- Configuration & Logging --- +from src.configuration import ConfigurationManager +from src.logging_helper import setup_logging + +# --- Async services --- +from src.trade.async_services import ( + AsyncSaxoApiClient, + AsyncInstrumentService, + AsyncOrderService, + AsyncPositionService, + AsyncTradingOrchestrator, + AsyncPerformanceMonitor, +) +from src.database.postgres import ( + PostgresConnectionManager, + AsyncDbOrderManager, + AsyncDbPositionManager, + AsyncDbTradePerformanceManager, + init_schema, +) +from src.mq_telegram.async_tools import AsyncTelegramSender +from src.saxo_authen import SaxoAuth +from src.trade.rules import TradingRule +from src.schema import SchemaLoader +from src.message_helper import ( + TelegramMessageComposer, + build_daily_trading_report_message, +) +from src.trade.exceptions import ( + TradingRuleViolation, + NoMarketAvailableException, + NoTurbosAvailableException, + PositionNotFoundException, + InsufficientFundsException, + ApiRequestException, + TokenAuthenticationException, + DatabaseOperationException, + PositionCloseException, + SaxoApiError, + OrderPlacementError, + ConfigurationError, +) + +logger = logging.getLogger(__name__) + +# --- Global version --- +APP_VERSION = "unknown" + + +def get_version() -> str: + try: + vf = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "VERSION") + with open(vf, "r") as f: + return f.read().strip() + except Exception: + return "unknown" + + +# ───────────────────────────────────────────────────── +# Signal handler (long / short) +# ───────────────────────────────────────────────────── + +async def handle_trading_signal( + data: dict, + trading_orchestrator: AsyncTradingOrchestrator, + performance_monitor: AsyncPerformanceMonitor, + trading_rule: TradingRule, + db_position_manager: AsyncDbPositionManager, + trade_turbo_exchange_id: str, + telegram: AsyncTelegramSender, +): + """Process a long/short signal end-to-end.""" + action = data["action"] + indice = data["indice"] + confidence = data.get("confidence") + composer = TelegramMessageComposer(data) + + try: + # 1. Rule checks (sync — pure computation + quick DB reads via asyncio.to_thread) + trading_rule.check_signal_timestamp(action, data.get("alert_timestamp")) + trading_rule.check_market_hours(data.get("signal_timestamp")) + indice_id = trading_rule.get_allowed_indice_id(indice) + + # Async-aware position duplicate check + open_positions = await db_position_manager.get_open_positions_ids_actions() + for pos in open_positions: + if pos.get("action") == action: + raise TradingRuleViolation(f"Duplicate signal: {action} position already open") + + # Async profit check + await _async_check_profit_per_day(trading_rule, db_position_manager) + + trading_rule.check_cooldown_after_loss() + trading_rule.check_max_trades_per_day() + if confidence is not None: + trading_rule.check_confidence_threshold(confidence) + + reserve_cash_pct = trading_orchestrator.reserve_cash_percent + + # 2. Execute based on mode + if reserve_cash_pct > 0: + # Decoupled: new trade first, then close old positions + result = await trading_orchestrator.execute_trade_signal( + exchange_id=trade_turbo_exchange_id, + underlying_uics=indice_id, + keywords=action, + confidence=confidence, + ) + close_result = await performance_monitor.close_managed_positions_by_criteria( + action_filter=None, + exclude_position_id=result["position_details"]["position_id"], + ) + composer.add_text_section( + "Post-Trade Closure", + f"Closed existing positions. Initiated: {close_result['closed_initiated_count']}, Errors: {close_result['errors_count']}", + ) + else: + # Classic: close old first, then open new + close_result = await performance_monitor.close_managed_positions_by_criteria(action_filter=None) + composer.add_text_section( + "Pre-Trade Closure", + f"Closed existing positions. Initiated: {close_result['closed_initiated_count']}, Errors: {close_result['errors_count']}", + ) + result = await trading_orchestrator.execute_trade_signal( + exchange_id=trade_turbo_exchange_id, + underlying_uics=indice_id, + keywords=action, + confidence=confidence, + ) + + # 3. Compose success message + composer.add_turbo_search_result(founded_turbo=result["selected_turbo_info"]) + composer.add_position_result(buy_details=result) + if "execution_timing" in result: + composer.add_text_section("Execution Timing", result["execution_timing"]) + if result.get("position_scale") is not None: + scale_info = f"Scale: {result['position_scale']}%" + if confidence is not None: + scale_info += f" (confidence: {confidence})" + composer.add_text_section("Position Sizing", scale_info) + + await telegram.send(composer.get_message()) + logger.info("Trade %s executed — Order %s, Position %s", + action, result["order_details"]["order_id"], result["position_details"]["position_id"]) + + except TradingRuleViolation as trv: + logger.warning("Rule violation: %s", trv) + # Don't spam telegram for rule violations + except (NoMarketAvailableException, NoTurbosAvailableException, InsufficientFundsException) as e: + logger.warning("Trade setup issue (%s): %s", type(e).__name__, e) + composer.add_generic_error(type(e).__name__, e) + await telegram.send(composer.get_message()) + except OrderPlacementError as e: + logger.error("Order rejected: %s", e) + composer.add_position_result(error=e) + await telegram.send(composer.get_message()) + except PositionNotFoundException as e: + logger.critical("Position not found after order: %s", e) + composer.add_generic_error("PositionNotFoundException", e, is_critical=True) + await telegram.send(composer.get_message()) + raise # Bubble up for service-level handling + except DatabaseOperationException as e: + logger.critical("DB error during trade: %s", e) + composer.add_generic_error("DatabaseOperationException", e, is_critical=True) + await telegram.send(composer.get_message()) + raise + + +async def _async_check_profit_per_day(trading_rule: TradingRule, db_pm: AsyncDbPositionManager): + """Async version of TradingRule.check_profit_per_day using async DB.""" + try: + day_config = trading_rule.get_rule_config("day_trading") + threshold = day_config.get("dont_enter_trade_if_day_profit_is_more_than") + if threshold is None: + return + today_pct = await db_pm.get_percent_of_the_day() + if today_pct >= threshold: + raise TradingRuleViolation( + f"Daily profit {today_pct}% >= limit {threshold}%" + ) + except TradingRuleViolation: + raise + except Exception as e: + logger.error("Error checking daily profit: %s", e) + + +# ───────────────────────────────────────────────────── +# Close handler (close-long / close-short / close-position) +# ───────────────────────────────────────────────────── + +async def handle_close_signal( + data: dict, + performance_monitor: AsyncPerformanceMonitor, + telegram: AsyncTelegramSender, +): + action = data["action"] + action_filter = None + if action == "close-long": + action_filter = "long" + elif action == "close-short": + action_filter = "short" + + result = await performance_monitor.close_managed_positions_by_criteria(action_filter=action_filter) + closed = result["closed_initiated_count"] + errors = result["errors_count"] + logger.info("Close action '%s': closed=%d, errors=%d", action, closed, errors) + if closed > 0: + await telegram.send(f"{action.upper()}: Closed {closed} position(s). Errors: {errors}") + + +# ───────────────────────────────────────────────────── +# Daily stats handler +# ───────────────────────────────────────────────────── + +async def handle_daily_stats( + data: dict, + db_position_manager: AsyncDbPositionManager, + db_perf_manager: AsyncDbTradePerformanceManager, + telegram: AsyncTelegramSender, +): + days = 7 + report_date = date.today() + history_start = date(report_date.year - 4, 1, 1) + + closed_trades, daily_profit_history, real_daily, best_daily, max_daily = await asyncio.gather( + db_position_manager.get_closed_trade_history(start_date=history_start, end_date=report_date), + db_position_manager.get_daily_profit_history(end_date=report_date), + db_position_manager.get_percent_of_last_n_days(days), + db_position_manager.get_best_percent_of_last_n_days(days), + db_position_manager.get_theoretical_percent_of_last_n_days_on_max(days), + ) + + message = build_daily_trading_report_message( + report_date=report_date, + closed_trades=closed_trades, + daily_profit_history=daily_profit_history, + daily_real=real_daily, + daily_best=best_daily, + daily_max=max_daily, + ) + await telegram.send(message) + await db_perf_manager.create_last_day_trade_performance_data() + logger.info("Daily stats sent.") + + +# ───────────────────────────────────────────────────── +# Message dispatcher +# ───────────────────────────────────────────────────── + +SIGNAL_ACTIONS = {"long", "short"} +CLOSE_ACTIONS = {"close-long", "close-short", "close-position"} +OPS_ACTIONS = {"check_positions_on_saxo_api", "daily_stats"} + + +async def dispatch_message( + message: aio_pika.IncomingMessage, + # injected dependencies + trading_orchestrator: AsyncTradingOrchestrator, + performance_monitor: AsyncPerformanceMonitor, + trading_rule: TradingRule, + db_position_manager: AsyncDbPositionManager, + db_perf_manager: AsyncDbTradePerformanceManager, + trade_turbo_exchange_id: str, + telegram: AsyncTelegramSender, +): + async with message.process(requeue=False): + try: + body = json.loads(message.body.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: + logger.error("Cannot decode message: %s", e) + await telegram.send(f"ERROR: Cannot decode MQ message: {e}") + return + + # Validate schema + try: + jsonschema.validate(instance=body, schema=SchemaLoader.get_trading_action_schema()) + except jsonschema.exceptions.ValidationError as e: + logger.error("Schema validation failed: %s", e.message) + await telegram.send(f"SCHEMA ERROR: {e.message}") + return + + action = body.get("action") + signal_id = body.get("signal_id", "N/A") + logger.info("Dispatching action=%s signal_id=%s", action, signal_id) + + try: + if action in SIGNAL_ACTIONS: + await handle_trading_signal( + body, trading_orchestrator, performance_monitor, + trading_rule, db_position_manager, + trade_turbo_exchange_id, telegram, + ) + elif action in CLOSE_ACTIONS: + await handle_close_signal(body, performance_monitor, telegram) + elif action == "daily_stats": + await handle_daily_stats(body, db_position_manager, db_perf_manager, telegram) + elif action == "check_positions_on_saxo_api": + # Position checks are handled by position_monitor service + # If message arrives here by mistake, just log and skip + logger.warning("check_positions_on_saxo_api should go to trading-ops queue. Skipping.") + else: + logger.error("Unknown action: %s", action) + await telegram.send(f"ERROR: Unknown action '{action}'") + + except (PositionNotFoundException, DatabaseOperationException) as critical: + logger.critical("CRITICAL error processing %s: %s", action, critical, exc_info=True) + await telegram.send(f"CRITICAL ERROR ({type(critical).__name__}): {critical}") + # For critical errors, we may want to restart + raise + except (TokenAuthenticationException, ConfigurationError) as fatal: + logger.critical("FATAL: %s", fatal, exc_info=True) + await telegram.send(f"FATAL ({type(fatal).__name__}): {fatal}") + raise + except Exception as e: + logger.error("Error processing action %s: %s", action, e, exc_info=True) + await telegram.send(f"ERROR processing {action}: {type(e).__name__}: {e}") + + +# ───────────────────────────────────────────────────── +# Account info helper (async-compatible via to_thread) +# ───────────────────────────────────────────────────── + +async def get_account_info_async(api_client: AsyncSaxoApiClient): + """Fetch AccountKey/ClientKey using the async API client.""" + import src.saxo_openapi.endpoints.portfolio as pf + from collections import namedtuple + + req = pf.accounts.AccountsMe() + rv = await api_client.request(req) + t = namedtuple("AcctInfo", "ClientId ClientKey AccountId AccountKey") + return t( + ClientId=rv["Data"][0]["ClientId"], + ClientKey=rv["Data"][0]["ClientKey"], + AccountId=rv["Data"][0]["AccountId"], + AccountKey=rv["Data"][0]["AccountKey"], + ) + + +# ───────────────────────────────────────────────────── +# Main entry point +# ───────────────────────────────────────────────────── + +async def main(): + global APP_VERSION + APP_VERSION = get_version() + + # 1. Configuration + config_path = os.getenv("WATA_CONFIG_PATH") + if not config_path: + print("FATAL: WATA_CONFIG_PATH not set", file=sys.stderr) + sys.exit(10) + config_manager = ConfigurationManager(config_path) + setup_logging(config_manager, "wata-trader-async") + logger.info("--- Starting WATA Async Trader v%s ---", APP_VERSION) + + # 2. Telegram sender (connect early for startup notifications) + telegram = AsyncTelegramSender(config_manager) + await telegram.connect() + + pg = None + api_client = None + + try: + # 3. PostgreSQL + logger.info("Connecting to PostgreSQL...") + pg = PostgresConnectionManager.from_config(config_manager) + await pg.connect() + await init_schema(pg) + + db_order_manager = AsyncDbOrderManager(pg) + db_position_manager = AsyncDbPositionManager(pg) + db_perf_manager = AsyncDbTradePerformanceManager(pg) + + # 4. Trading rules (sync TradingRule — uses async DB wrapper below) + trading_rule = TradingRule(config_manager, None) # db_position_manager passed separately + trade_turbo_exchange_id = config_manager.get_config_value("trade.config.turbo_preference.exchange_id") + + # 5. Saxo auth + API client + logger.info("Initialising Saxo API client...") + saxo_auth = SaxoAuth(config_manager) + api_client = AsyncSaxoApiClient(config_manager, saxo_auth) + await api_client.ensure_ready() + + # 6. Fetch account info + acct = await get_account_info_async(api_client) + account_key = acct.AccountKey + client_key = acct.ClientKey + logger.info("Account: %s, Client: %s", account_key, client_key) + + # 7. Async services + instrument_service = AsyncInstrumentService(api_client, config_manager, account_key) + order_service = AsyncOrderService(api_client, account_key, client_key) + position_service = AsyncPositionService(api_client, order_service, config_manager, account_key, client_key) + trading_orchestrator = AsyncTradingOrchestrator( + instrument_service, order_service, position_service, + config_manager, db_order_manager, db_position_manager, + ) + performance_monitor = AsyncPerformanceMonitor( + position_service, order_service, config_manager, + db_position_manager, trading_rule, telegram.send, + ) + + # 8. RabbitMQ consumer (aio-pika) + logger.info("Connecting to RabbitMQ...") + rmq_config = config_manager.get_rabbitmq_config() + rmq_url = f"amqp://{rmq_config['authentication']['username']}:{rmq_config['authentication']['password']}@{rmq_config['hostname']}/" + connection = await aio_pika.connect_robust(rmq_url) + channel = await connection.channel() + await channel.set_qos(prefetch_count=1) + + # Declare both queues — this service consumes from trading-signals + signals_queue = await channel.declare_queue("trading-signals", durable=True) + await channel.declare_queue("trading-ops", durable=True) + + async def on_message(msg: aio_pika.IncomingMessage): + await dispatch_message( + msg, + trading_orchestrator=trading_orchestrator, + performance_monitor=performance_monitor, + trading_rule=trading_rule, + db_position_manager=db_position_manager, + db_perf_manager=db_perf_manager, + trade_turbo_exchange_id=trade_turbo_exchange_id, + telegram=telegram, + ) + + await signals_queue.consume(on_message) + + startup_msg = f"WATA Async Trader v{APP_VERSION} is running (trading-signals queue)." + await telegram.send(startup_msg) + logger.info("Trader startup complete. Consuming from trading-signals...") + + # Keep running + try: + await asyncio.Future() # Run forever + except asyncio.CancelledError: + pass + + except Exception as e: + logger.critical("Unhandled startup error: %s", e, exc_info=True) + try: + await telegram.send(f"CRITICAL STARTUP FAILURE: {e}") + except Exception: + pass + sys.exit(1) + + finally: + logger.info("--- Shutting down WATA Async Trader ---") + if api_client is not None: + await api_client.close() + if pg is not None: + await pg.close() + await telegram.close() + + +if __name__ == "__main__": + try: + import uvloop + uvloop.install() + except ImportError: + pass + asyncio.run(main()) diff --git a/src/web_server/__init__.py b/src/web_server/__init__.py index e18ea49..53adf4d 100644 --- a/src/web_server/__init__.py +++ b/src/web_server/__init__.py @@ -63,7 +63,7 @@ async def verify_token(token: str): return token -def send_message_to_trading(action, indice, signal_timestamp, alert_timestamp): +def send_message_to_trading(action, indice, signal_timestamp, alert_timestamp, confidence=None): try: # Retrieve RabbitMQ credentials from the configuration rabbitmq_config = config_manager.get_rabbitmq_config() @@ -79,7 +79,7 @@ def send_message_to_trading(action, indice, signal_timestamp, alert_timestamp): ) ) channel = connection.channel() - channel.queue_declare(queue="trading-action") + channel.queue_declare(queue="trading-signals", durable=True) # Get the current time in UTC now_utc = datetime.now(pytz.utc) @@ -87,18 +87,20 @@ def send_message_to_trading(action, indice, signal_timestamp, alert_timestamp): # Generate a unique identifier for this signal signal_id = str(uuid.uuid4()) - message = json.dumps( - { + msg_payload = { "signal_id": signal_id, "action": action, "indice": indice, "signal_timestamp": signal_timestamp, "alert_timestamp": alert_timestamp, "mqsend_timestamp": now_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), - } - ) - channel.basic_publish(exchange="", routing_key="trading-action", body=message) - logging.info(f"Send message to channel trading-action, message {message}") + } + if confidence is not None: + msg_payload["confidence"] = confidence + + message = json.dumps(msg_payload) + channel.basic_publish(exchange="", routing_key="trading-signals", body=message) + logging.info(f"Send message to channel trading-signals, message {message}") return signal_id except pika.exceptions.AMQPConnectionError as e: logging.error(f"Failed to connect to RabbitMQ: {e}") @@ -139,6 +141,7 @@ async def webhook(request: Request): data["indice"], data["signal_timestamp"], data["alert_timestamp"], + confidence=data.get("confidence"), ) logging.info(f"Received data from {request.client.host} : {data}") return JSONResponse(content={"status": "success", "signal_id": signal_id}, status_code=200) diff --git a/tests/test_async_postgres_temporal_fields.py b/tests/test_async_postgres_temporal_fields.py new file mode 100644 index 0000000..3f4d0c9 --- /dev/null +++ b/tests/test_async_postgres_temporal_fields.py @@ -0,0 +1,103 @@ +import asyncio +import sys +import types +from datetime import date, datetime, timezone + +import pytest + + +asyncpg_stub = types.ModuleType("asyncpg") +asyncpg_stub.Pool = object +asyncpg_stub.Record = dict +asyncpg_stub.create_pool = None +sys.modules.setdefault("asyncpg", asyncpg_stub) + +from src.database.postgres import ( + AsyncDbOrderManager, + AsyncDbPositionManager, + AsyncDbTradePerformanceManager, +) + + +class FakeConnMgr: + def __init__(self): + self.calls = [] + + async def execute(self, query, *args): + self.calls.append((query, args)) + return "OK" + + +def test_async_order_manager_normalizes_iso_datetime_strings(): + conn_mgr = FakeConnMgr() + manager = AsyncDbOrderManager(conn_mgr) + + asyncio.run(manager.insert_turbo_order_data({ + "action": "short", + "buy_sell": "Buy", + "order_id": "5397119987", + "order_amount": 6, + "order_type": "Market", + "order_kind": "main", + "order_submit_time": "2026-04-27T19:42:58Z", + "related_order_id": [], + "position_id": "pos_123", + "instrument_name": "MiniFuture", + "instrument_symbol": "MF-123", + "instrument_uic": 55341056, + "instrument_price": 4.255, + "instrument_currency": "EUR", + "order_cost": 0.25, + })) + + _, args = conn_mgr.calls[0] + assert args[6] == datetime(2026, 4, 27, 19, 42, 58, tzinfo=timezone.utc) + + +def test_async_position_manager_normalizes_iso_datetime_strings_for_insert_and_update(): + conn_mgr = FakeConnMgr() + manager = AsyncDbPositionManager(conn_mgr) + + asyncio.run(manager.insert_turbo_open_position_data({ + "action": "short", + "position_id": "pos_123", + "position_amount": 6, + "position_open_price": 4.255, + "position_total_open_price": 25.53, + "position_status": "Open", + "position_kind": "main", + "execution_time_open": "2026-04-27T19:42:58Z", + "order_id": "5397119987", + "related_order_id": [], + "instrument_name": "MiniFuture", + "instrument_symbol": "MF-123", + "instrument_uic": 55341056, + "instrument_currency": "EUR", + })) + + _, insert_args = conn_mgr.calls[0] + assert insert_args[7] == datetime(2026, 4, 27, 19, 42, 58, tzinfo=timezone.utc) + + asyncio.run(manager.update_turbo_position_data("pos_123", { + "position_status": "Closed", + "execution_time_close": "2026-04-27T20:01:10Z", + })) + + _, update_args = conn_mgr.calls[1] + assert update_args[1] == datetime(2026, 4, 27, 20, 1, 10, tzinfo=timezone.utc) + + +def test_async_trade_performance_manager_normalizes_date_strings(): + conn_mgr = FakeConnMgr() + manager = AsyncDbTradePerformanceManager(conn_mgr) + + asyncio.run(manager.insert_trade_performance_data({ + "date_day": "2026-04-27T19:42:58Z", + "perf_day_real": 1.5, + "money_made_real": 12.0, + "trade_number_real": 3, + "max_perf_day_simulated": 2.1, + })) + + _, args = conn_mgr.calls[0] + assert args[0] == date(2026, 4, 27) \ No newline at end of file diff --git a/tests/test_async_services.py b/tests/test_async_services.py new file mode 100644 index 0000000..b27e3c4 --- /dev/null +++ b/tests/test_async_services.py @@ -0,0 +1,230 @@ +import asyncio +import sys +import types + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +from src.configuration import ConfigurationManager + +asyncpg_stub = types.ModuleType("asyncpg") +asyncpg_stub.Pool = type("Pool", (), {}) +asyncpg_stub.Record = dict +asyncpg_stub.create_pool = AsyncMock() +sys.modules.setdefault("asyncpg", asyncpg_stub) + +from src.trade.async_services import AsyncInstrumentService + + +@pytest.fixture +def mock_config_manager(): + manager = MagicMock(spec=ConfigurationManager) + + def get_config_value(key, default=None): + configs = { + "trade.config.general.api_limits": {"top_instruments": 200}, + "trade.config.turbo_preference.price_range": {"min": 4, "max": 15}, + "trade.config.general.retry_config": {"max_retries": 3, "retry_sleep_seconds": 1}, + "trade.config.turbo_cache": {"enabled": False, "ttl_seconds": 30}, + } + return configs.get(key, default) + + manager.get_config_value.side_effect = get_config_value + return manager + + +@pytest.fixture +def mock_api_client(): + client = MagicMock() + client.request = AsyncMock() + return client + + +@pytest.fixture +def instrument_service(mock_api_client, mock_config_manager): + return AsyncInstrumentService(mock_api_client, mock_config_manager, "account_key") + + +def test_find_turbos_uses_two_stage_infoprices_funnel(instrument_service, mock_api_client): + mock_api_client.request.side_effect = [ + { + "Data": [ + { + "Identifier": 1, + "Description": "TURBO LONG DAX 15000 CITI", + "AssetType": "WarrantKnockOut", + } + ] + }, + { + "Data": [ + { + "Uic": 101, + "Identifier": 1, + "AssetType": "WarrantKnockOut", + "Quote": { + "Bid": 10.0, + "Ask": 10.1, + "PriceTypeAsk": "Tradable", + "PriceTypeBid": "Tradable", + "MarketState": "Open", + }, + } + ] + }, + { + "Data": [ + { + "Uic": 101, + "DisplayAndFormat": { + "Description": "Final TURBO LONG DAX 15000 CITI", + "Symbol": "DAXL", + "Currency": "EUR", + "OrderDecimals": 2, + }, + "Commissions": {"CostBuy": 0.25}, + "InstrumentPriceDetails": {"LotSize": 1}, + } + ] + }, + ] + + result = asyncio.run(instrument_service.find_turbos("exchange1", "underlying1", "long")) + + assert result["selected_instrument"]["uic"] == 101 + assert result["selected_instrument"]["latest_ask"] == 10.1 + assert result["selected_instrument"]["description"] == "Final TURBO LONG DAX 15000 CITI" + assert result["selected_instrument"]["commissions"]["CostBuy"] == 0.25 + assert result["selected_instrument"]["subscription_context_id"] is None + assert result["selected_instrument"]["subscription_reference_id"] is None + assert mock_api_client.request.await_count == 3 + + initial_search_req = mock_api_client.request.await_args_list[0].args[0] + stage_one_req = mock_api_client.request.await_args_list[1].args[0] + stage_two_req = mock_api_client.request.await_args_list[2].args[0] + + assert initial_search_req.params["$top"] == 60 + assert stage_one_req.params["FieldGroups"] == "Quote" + assert stage_one_req.params["$top"] == 1 + assert stage_two_req.params["FieldGroups"] == "Commissions,DisplayAndFormat,InstrumentPriceDetails" + assert stage_two_req.params["$top"] == 1 + + +def test_find_turbos_short_circuits_retry_when_one_in_range_bid_exists(instrument_service, mock_api_client): + mock_api_client.request.side_effect = [ + { + "Data": [ + { + "Identifier": 1, + "Description": "TURBO LONG DAX 15000 CITI", + "AssetType": "WarrantKnockOut", + }, + { + "Identifier": 2, + "Description": "TURBO LONG DAX 14900 CITI", + "AssetType": "WarrantKnockOut", + }, + { + "Identifier": 3, + "Description": "TURBO LONG DAX 14800 CITI", + "AssetType": "WarrantKnockOut", + }, + ] + }, + { + "Data": [ + { + "Uic": 101, + "Identifier": 1, + "AssetType": "WarrantKnockOut", + "Quote": { + "Bid": 10.0, + "Ask": 10.1, + "PriceTypeAsk": "Tradable", + "PriceTypeBid": "Tradable", + "MarketState": "Open", + }, + }, + { + "Uic": 102, + "Identifier": 2, + "AssetType": "WarrantKnockOut", + "Quote": { + "Ask": 9.8, + "PriceTypeAsk": "Tradable", + "PriceTypeBid": "Tradable", + "MarketState": "Open", + }, + }, + { + "Uic": 103, + "Identifier": 3, + "AssetType": "WarrantKnockOut", + "Quote": { + "Ask": 9.6, + "PriceTypeAsk": "Tradable", + "PriceTypeBid": "Tradable", + "MarketState": "Open", + }, + }, + ] + }, + { + "Data": [ + { + "Uic": 101, + "DisplayAndFormat": { + "Description": "Final TURBO LONG DAX 15000 CITI", + "Symbol": "DAXL", + "Currency": "EUR", + "OrderDecimals": 2, + }, + "Commissions": {"CostBuy": 0.25}, + } + ] + }, + ] + + with patch("src.trade.async_services.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + result = asyncio.run(instrument_service.find_turbos("exchange1", "underlying1", "long")) + + assert result["selected_instrument"]["uic"] == 101 + assert mock_api_client.request.await_count == 3 + mock_sleep.assert_not_awaited() + + +def test_find_turbos_handles_empty_stage_two_details(instrument_service, mock_api_client): + mock_api_client.request.side_effect = [ + { + "Data": [ + { + "Identifier": 1, + "Description": "TURBO LONG DAX 15000 CITI", + "AssetType": "WarrantKnockOut", + } + ] + }, + { + "Data": [ + { + "Uic": 101, + "Identifier": 1, + "AssetType": "WarrantKnockOut", + "Quote": { + "Bid": 10.0, + "Ask": 10.1, + "PriceTypeAsk": "Tradable", + "PriceTypeBid": "Tradable", + "MarketState": "Open", + }, + } + ] + }, + {"Data": []}, + ] + + result = asyncio.run(instrument_service.find_turbos("exchange1", "underlying1", "long")) + + assert result["selected_instrument"]["description"] == "TURBO LONG DAX 15000 CITI" + assert result["selected_instrument"]["latest_bid"] == 10.0 + assert result["selected_instrument"]["parsed_data"]["price"] == "15000" \ No newline at end of file diff --git a/tests/test_message_helper.py b/tests/test_message_helper.py new file mode 100644 index 0000000..ef9725c --- /dev/null +++ b/tests/test_message_helper.py @@ -0,0 +1,232 @@ +from datetime import date + +from src.message_helper import ( + TelegramMessageComposer, + build_daily_trading_report_message, + calculate_winning_streak, + merge_daily_performance_series, +) + + +def test_add_text_section_accepts_execution_timing_dict(): + composer = TelegramMessageComposer({ + "action": "long", + "signal_id": "signal-123", + "signal_timestamp": "2026-04-30T18:46:44Z", + }) + + composer.add_text_section("Execution Timing", { + "Scale Calc": "0ms", + "Find Turbo": "458ms", + "TOTAL": "1088ms", + }) + + message = composer.get_message() + + assert "--- EXECUTION TIMING ---" in message + assert "```json" in message + assert '"TOTAL": "1088ms"' in message + + +def test_calculate_winning_streak_tracks_current_and_last_win_day(): + streak = calculate_winning_streak([ + {"day_date": "2026/06/01", "sum_profit": 1.44}, + {"day_date": "2026/06/02", "sum_profit": 1.20}, + {"day_date": "2026/06/03", "sum_profit": 1.95}, + {"day_date": "2026/06/04", "sum_profit": -2.44}, + ]) + + assert streak["current_streak"] == 0 + assert streak["best_streak"] == 3 + assert streak["last_win_date"] == date(2026, 6, 3) + + +def test_merge_daily_performance_series_aligns_real_best_and_max_rows(): + merged = merge_daily_performance_series( + { + "2026/06/04": -2.45, + "2026/06/03": 1.95, + "2026/06/02": 1.20, + }, + { + "2026/06/04": 0.72, + "2026/06/03": 1.95, + "2026/06/02": 1.20, + }, + { + "2026/06/04": 4.23, + "2026/06/03": 5.10, + "2026/06/02": 2.07, + }, + report_date=date(2026, 6, 4), + days=3, + ) + + assert merged[0]["day_date"] == date(2026, 6, 4) + assert merged[0]["real"] == -2.45 + assert merged[0]["best"] == 0.72 + assert merged[0]["max"] == 4.23 + assert merged[2]["day_date"] == date(2026, 6, 2) + + +def test_build_daily_trading_report_message_formats_modern_mobile_report(): + message = build_daily_trading_report_message( + report_date=date(2026, 6, 4), + closed_trades=[ + { + "action": "long", + "position_id": "y2022", + "performance_percent": 0.80, + "max_performance_percent": 1.10, + "profit_loss": 2.89, + "execution_time_close": "2022-08-10T10:00:00Z", + }, + { + "action": "long", + "position_id": "y2023", + "performance_percent": 1.10, + "max_performance_percent": 1.40, + "profit_loss": 4.56, + "execution_time_close": "2023-10-10T10:00:00Z", + }, + { + "action": "short", + "position_id": "y2024", + "performance_percent": 0.95, + "max_performance_percent": 1.20, + "profit_loss": 3.78, + "execution_time_close": "2024-12-10T10:00:00Z", + }, + { + "action": "long", + "position_id": "y2025", + "performance_percent": 1.05, + "max_performance_percent": 1.30, + "profit_loss": 4.12, + "execution_time_close": "2025-11-10T10:00:00Z", + }, + { + "action": "long", + "position_id": "w1", + "performance_percent": 1.12, + "max_performance_percent": 1.12, + "profit_loss": 5.50, + "execution_time_close": "2026-05-29T10:00:00Z", + }, + { + "action": "long", + "position_id": "w2", + "performance_percent": 1.44, + "max_performance_percent": 1.44, + "profit_loss": 6.20, + "execution_time_close": "2026-06-01T10:00:00Z", + }, + { + "action": "short", + "position_id": "w3", + "performance_percent": 1.20, + "max_performance_percent": 2.07, + "profit_loss": 4.80, + "execution_time_close": "2026-06-02T10:00:00Z", + }, + { + "action": "long", + "position_id": "w4", + "performance_percent": 1.95, + "max_performance_percent": 5.10, + "profit_loss": 8.00, + "execution_time_close": "2026-06-03T10:00:00Z", + }, + { + "action": "long", + "position_id": "d1", + "performance_percent": 0.72, + "max_performance_percent": 4.23, + "profit_loss": 0.72, + "execution_time_close": "2026-06-04T10:00:00Z", + }, + { + "action": "long", + "position_id": "d2", + "performance_percent": -0.34, + "max_performance_percent": 0.50, + "profit_loss": -0.34, + "execution_time_close": "2026-06-04T10:05:00Z", + }, + { + "action": "short", + "position_id": "d3", + "performance_percent": -2.82, + "max_performance_percent": 1.30, + "profit_loss": -2.82, + "execution_time_close": "2026-06-04T10:10:00Z", + }, + ], + daily_profit_history=[ + {"day_date": "2026/06/01", "sum_profit": 6.20}, + {"day_date": "2026/06/02", "sum_profit": 4.80}, + {"day_date": "2026/06/03", "sum_profit": 8.00}, + {"day_date": "2026/06/04", "sum_profit": -2.44}, + ], + daily_real={ + "2026/06/04": -2.45, + "2026/06/03": 1.95, + "2026/06/02": 1.20, + "2026/06/01": 1.44, + "2026/05/31": 0.00, + "2026/05/30": 0.00, + "2026/05/29": 1.12, + }, + daily_best={ + "2026/06/04": 0.72, + "2026/06/03": 1.95, + "2026/06/02": 1.20, + "2026/06/01": 1.44, + "2026/05/31": 0.00, + "2026/05/30": 0.00, + "2026/05/29": 1.12, + }, + daily_max={ + "2026/06/04": 4.23, + "2026/06/03": 5.10, + "2026/06/02": 2.07, + "2026/06/01": 1.34, + "2026/05/31": 0.00, + "2026/05/30": 0.00, + "2026/05/29": 0.57, + }, + ) + + assert "📊 Trading Report | 2026/06/04" in message + assert "🔥 Winning Streak: 0 Days (Last win: 2026/06/03)" in message + assert "🎯 Win Rate: 33.33% (1W | 2L | 0BE)" in message + assert "🟢 LONGS (2 trades | 1W - 1L)" in message + assert "🔴 SHORTS (1 trade | 0W - 1L)" in message + assert "--- 🗓 CUMULATIVE P/L ---" in message + assert "⏱ 7 Days:" in message + assert "--- 📊 LAST 7 DAYS (Real | Best | Max) ---" in message + assert "06/04:" in message + assert "--- 📅 LAST 10 WEEKS ---" in message + assert "--- 📅 LAST 12 MONTHS ---" in message + assert "June:" in message + assert "May:" in message + assert "--- 📅 LAST 5 YEARS ---" in message + assert "2026:" in message + assert "2025:" in message + assert "2024:" in message + assert "2023:" in message + assert "2022:" in message + + +def test_build_daily_trading_report_message_omits_monthly_and_yearly_sections_without_data(): + message = build_daily_trading_report_message( + report_date=date(2026, 6, 4), + closed_trades=[], + daily_profit_history=[], + daily_real={}, + daily_best={}, + daily_max={}, + ) + + assert "--- 📅 LAST 12 MONTHS ---" not in message + assert "--- 📅 LAST 5 YEARS ---" not in message \ No newline at end of file diff --git a/tests/test_saxo_api_action.py b/tests/test_saxo_api_action.py deleted file mode 100644 index 679f9c5..0000000 --- a/tests/test_saxo_api_action.py +++ /dev/null @@ -1,580 +0,0 @@ -import pytest -from unittest.mock import patch, MagicMock, call -import src.trade.api_actions as api_actions -from src.trade.api_actions import ( - TradingOrchestrator, - InstrumentService, - OrderService, - PositionService, - SaxoApiClient, - parse_saxo_turbo_description, - SaxoApiError, - InsufficientFundsException, - OrderPlacementError, - TokenAuthenticationException, - NoTurbosAvailableException, - NoMarketAvailableException, - PositionNotFoundException, - ApiRequestException, - PerformanceMonitor -) -from src.configuration import ConfigurationManager -from src.database import DbOrderManager, DbPositionManager -from src.trade.rules import TradingRule -from src.saxo_authen import SaxoAuth -from src.saxo_openapi.exceptions import OpenAPIError as SaxoOpenApiLibError -import requests -import json - -# region Fixtures - -@pytest.fixture -def mock_config_manager(): - """A mock for ConfigurationManager with a flexible side_effect.""" - manager = MagicMock(spec=ConfigurationManager) - - def get_config_value(key, default=None): - configs = { - "saxo_auth.env": "simulation", - "trade.config.general.api_limits": {"top_instruments": 200, "top_positions": 200, "top_closed_positions": 500}, - "trade.config.turbo_preference.price_range": {"min": 4, "max": 15}, - "trade.config.general.retry_config": {"max_retries": 3, "retry_sleep_seconds": 1}, - "trade.config.general.websocket": {"refresh_rate_ms": 10000}, - "trade.config.buying_power": {"safety_margins": {"bid_calculation": 1}, "max_account_funds_to_use_percentage": 100}, - "trade.config.position_management": {"performance_thresholds": {"stoploss_percent": -20, "max_profit_percent": 60}}, - "trade.config.general": {"timezone": "Europe/Paris"}, - "logging.persistant": {"log_path": "/tmp/logs"} - } - return configs.get(key, default) - - manager.get_config_value.side_effect = get_config_value - manager.get_logging_config.return_value = {"persistant": {"log_path": "/tmp/logs"}} - return manager - -@pytest.fixture -def mock_saxo_auth(): - """A mock for SaxoAuth.""" - auth = MagicMock(spec=SaxoAuth) - auth.get_token.return_value = "test_token" - return auth - -@pytest.fixture -def mock_db_order_manager(): - """A mock for DbOrderManager.""" - return MagicMock(spec=DbOrderManager) - -@pytest.fixture -def mock_db_position_manager(): - """A mock for DbPositionManager.""" - return MagicMock(spec=DbPositionManager) - -@pytest.fixture -def mock_trading_rule(): - """A mock for TradingRule.""" - rule = MagicMock(spec=TradingRule) - rule.get_rule_config.return_value = {"percent_profit_wanted_per_days": 1.0} - return rule - -@pytest.fixture -def mock_api_client(): - """A mock for SaxoApiClient that bypasses its internal SaxoOpenApiLib.""" - client = MagicMock(spec=SaxoApiClient) - return client - -# endregion - -# region Test Utility Functions - -def test_parse_saxo_turbo_description_valid(): - description = "TURBO LONG DAX 12345.67 CITI" - expected = { - "name": "TURBO", "kind": "LONG", "buysell": "DAX", - "price": "12345.67", "from": "CITI" - } - assert parse_saxo_turbo_description(description) == expected - -def test_parse_saxo_turbo_description_invalid(): - description = "This is not a valid turbo description" - assert parse_saxo_turbo_description(description) is None - -# endregion - -# region Test SaxoApiClient - -@patch('src.trade.api_actions.SaxoOpenApiLib') -def test_saxo_api_client_init_and_token_refresh(mock_saxo_lib, mock_config_manager): - """Test that the client initializes and refreshes the token correctly.""" - mock_auth = MagicMock(spec=SaxoAuth) - # This simulates the token changing on the third call to get_token - mock_auth.get_token.side_effect = ["token1", "token1", "token2"] - - # Initialization of the client calls _ensure_valid_token_and_api_instance once - client = SaxoApiClient(mock_config_manager, mock_auth) - mock_saxo_lib.assert_called_once_with(access_token="token1", environment="simulation", request_params={"timeout": 30}) - - # Calling it again with the same token should not trigger a refresh - client._ensure_valid_token_and_api_instance() - mock_saxo_lib.assert_called_once() - - # Calling it again after the token has "changed" should trigger a refresh - client._ensure_valid_token_and_api_instance() - mock_saxo_lib.assert_called_with(access_token="token2", environment="simulation", request_params={"timeout": 30}) - assert mock_saxo_lib.call_count == 2 - -@patch('src.trade.api_actions.SaxoOpenApiLib') -def test_saxo_api_client_request_success(mock_saxo_lib, mock_config_manager, mock_saxo_auth): - mock_api_instance = mock_saxo_lib.return_value - mock_api_instance.request.return_value = {"status": "success"} - - client = SaxoApiClient(mock_config_manager, mock_saxo_auth) - - response = client.request("some_endpoint_request_obj") - - assert response == {"status": "success"} - mock_api_instance.request.assert_called_once_with("some_endpoint_request_obj") - -@patch('src.trade.api_actions.SaxoOpenApiLib') -@pytest.mark.parametrize("status_code, error_code, error_content_str, expected_exception, is_order_endpoint", [ - (400, "InsufficientFunds", '{"Message": "Not enough money"}', InsufficientFundsException, False), - (400, "SomeError", '{"Message": "Bad request"}', OrderPlacementError, True), - (401, "AuthError", '{"Message": "Unauthorized"}', TokenAuthenticationException, False), - (429, "RateLimit", '{"Message": "Too many requests"}', SaxoApiError, False), - (500, "ServerError", 'Internal Server Error', SaxoApiError, False), -]) -def test_saxo_api_client_request_saxo_error_mapping(mock_saxo_lib, mock_config_manager, mock_saxo_auth, status_code, error_code, error_content_str, expected_exception, is_order_endpoint): - """Test that SaxoOpenApiLibError is correctly mapped to custom exceptions.""" - try: - content_json = json.loads(error_content_str) - content_json['ErrorCode'] = error_code - final_content = json.dumps(content_json) - except json.JSONDecodeError: - final_content = error_content_str - - mock_api_instance = mock_saxo_lib.return_value - mock_api_instance.request.side_effect = SaxoOpenApiLibError(code=status_code, content=final_content, reason="Some Reason") - - client = SaxoApiClient(mock_config_manager, mock_saxo_auth) - - mock_endpoint = MagicMock() - mock_endpoint.path = "/trade/v2/orders" if is_order_endpoint else "/some/other/endpoint" - - with pytest.raises(expected_exception): - client.request(mock_endpoint) - -@patch('src.trade.api_actions.SaxoOpenApiLib') -def test_saxo_api_client_request_connection_error(mock_saxo_lib, mock_config_manager, mock_saxo_auth): - mock_api_instance = mock_saxo_lib.return_value - mock_api_instance.request.side_effect = requests.RequestException("Connection failed") - client = SaxoApiClient(mock_config_manager, mock_saxo_auth) - with pytest.raises(ApiRequestException, match="Underlying request failed: Connection failed"): - client.request("some_endpoint") - -@patch('src.trade.api_actions.SaxoOpenApiLib') -def test_saxo_api_client_request_non_string_error_content(mock_saxo_lib, mock_config_manager, mock_saxo_auth): - """Test error handling when error content is not a string.""" - mock_api_instance = mock_saxo_lib.return_value - # Simulate error content being a dictionary instead of a string - mock_api_instance.request.side_effect = SaxoOpenApiLibError(code=500, content={"error": "detail"}, reason="Server Error") - client = SaxoApiClient(mock_config_manager, mock_saxo_auth) - with pytest.raises(SaxoApiError) as excinfo: - client.request("some_endpoint") - assert str({"error": "detail"}) in str(excinfo.value) - -@patch('src.trade.api_actions.SaxoOpenApiLib') -def test_saxo_api_client_request_invalid_json_error(mock_saxo_lib, mock_config_manager, mock_saxo_auth): - """Test error handling with invalid JSON content.""" - mock_api_instance = mock_saxo_lib.return_value - mock_api_instance.request.side_effect = SaxoOpenApiLibError(code=500, content="Not a valid JSON", reason="Server Error") - client = SaxoApiClient(mock_config_manager, mock_saxo_auth) - with pytest.raises(SaxoApiError, match="Not a valid JSON"): - client.request("some_endpoint") - -@patch('src.trade.api_actions.SaxoOpenApiLib') -def test_saxo_api_client_request_token_auth_exception_reraised(mock_saxo_lib, mock_config_manager, mock_saxo_auth): - """Test that TokenAuthenticationException is re-raised.""" - mock_saxo_auth.get_token.side_effect = TokenAuthenticationException("Token machine broke") - with pytest.raises(TokenAuthenticationException): - SaxoApiClient(mock_config_manager, mock_saxo_auth) - -@patch('src.trade.api_actions.SaxoOpenApiLib') -def test_saxo_api_client_unexpected_exception(mock_saxo_lib, mock_config_manager, mock_saxo_auth): - """Test wrapping of unexpected exceptions.""" - mock_api_instance = mock_saxo_lib.return_value - mock_api_instance.request.side_effect = Exception("Something totally unexpected") - client = SaxoApiClient(mock_config_manager, mock_saxo_auth) - with pytest.raises(ApiRequestException, match="Unexpected wrapper error: Something totally unexpected"): - client.request("some_endpoint") - -# endregion - -# region Test InstrumentService - -class TestInstrumentService: - - @pytest.fixture - def instrument_service(self, mock_api_client, mock_config_manager): - return InstrumentService(mock_api_client, mock_config_manager, "account_key") - - @patch('src.trade.api_actions.tr.infoprices.InfoPrices') - def test_get_infoprices_for_asset_type_success(self, mock_infoprices_req, instrument_service, mock_api_client): - mock_api_client.request.return_value = {"Data": ["price_info"]} - result = instrument_service._get_infoprices_for_asset_type("123,456", "Exchange1", "AssetType1") - assert result == {"Data": ["price_info"]} - mock_api_client.request.assert_called_once() - - @patch('time.sleep', return_value=None) - @patch('src.trade.api_actions.rd.instruments.Instruments') - @patch('src.trade.api_actions.tr.infoprices.InfoPrices') - @patch('src.trade.api_actions.tr.prices.CreatePriceSubscription') - def test_find_turbos_happy_path(self, mock_price_sub_req, mock_infoprices_req, mock_instruments_req, mock_sleep, instrument_service, mock_api_client): - mock_api_client.request.side_effect = [ - {"Data": [{"Identifier": 1, "Description": "TURBO LONG DAX 15000 CITI", "AssetType": "WarrantKnockOut"}]}, - {"Data": [{"Uic": 101, "Identifier": 1, "AssetType": "WarrantKnockOut", "Quote": {"Bid": 10, "Ask": 10.1, "PriceTypeAsk": "Tradable", "PriceTypeBid": "Tradable", "MarketState": "Open"}}]}, - {"Snapshot": {"Uic": 101, "DisplayAndFormat": {"Description": "Final TURBO LONG DAX 15000 CITI"}, "Quote": {"Ask": 10.05, "Bid": 9.95}}} - ] - result = instrument_service.find_turbos("exchange1", "underlying1", "long") - assert result['selected_instrument']['uic'] == 101 - assert result['selected_instrument']['latest_ask'] == 10.05 - assert mock_api_client.request.call_count == 3 - - def test_find_turbos_no_initial_instruments(self, instrument_service, mock_api_client): - mock_api_client.request.return_value = {"Data": []} - with pytest.raises(NoTurbosAvailableException): - instrument_service.find_turbos("e1", "u1", "long") - - @patch('src.trade.api_actions.parse_saxo_turbo_description') - def test_find_turbos_sorting_error(self, mock_parse_description, instrument_service, mock_api_client): - # Simulate data that will cause a sorting error - mock_api_client.request.return_value = {"Data": [{"Identifier": 1, "Description": "A valid description"}]} - # Mock the parsing result to have a non-numeric price - mock_parse_description.return_value = {"price": "not_a_number"} - with pytest.raises(ValueError, match="Could not sort instruments by parsed price"): - instrument_service.find_turbos("e1", "u1", "long") - - @patch('time.sleep', return_value=None) - def test_find_turbos_price_subscription_fails(self, mock_sleep, instrument_service, mock_api_client): - # Happy path until the last step (price subscription) - mock_api_client.request.side_effect = [ - {"Data": [{"Identifier": 1, "Description": "TURBO LONG DAX 15000 CITI", "AssetType": "WarrantKnockOut"}]}, - {"Data": [{"Uic": 101, "Identifier": 1, "AssetType": "WarrantKnockOut", "Quote": {"Bid": 10, "Ask": 10.1, "PriceTypeAsk": "Tradable", "PriceTypeBid": "Tradable", "MarketState": "Open"}}]}, - # This time, the subscription fails - ApiRequestException("Subscription failed") - ] - - result = instrument_service.find_turbos("e1", "u1", "long") - # Should fall back to using the InfoPrice data - assert result is not None - assert result['selected_instrument']['uic'] == 101 - # latest_ask should be from the InfoPrice response, not the (failed) subscription - assert result['selected_instrument']['latest_ask'] == 10.1 - assert result['selected_instrument']['subscription_context_id'] is None # Should be None on failure - - @patch('time.sleep', return_value=None) - def test_find_turbos_no_infoprice_data(self, mock_sleep, instrument_service, mock_api_client): - # First call to get instruments succeeds - mock_api_client.request.side_effect = [ - {"Data": [{"Identifier": 1, "Description": "TURBO LONG DAX 15000 CITI", "AssetType": "WarrantKnockOut"}]}, - # Subsequent calls to get infoprices fail - None, None, None - ] - with pytest.raises(NoMarketAvailableException, match="Failed to obtain valid InfoPrice data"): - instrument_service.find_turbos("e1", "u1", "long") - - @patch('time.sleep', return_value=None) - def test_find_turbos_no_quote_in_infoprice_data(self, mock_sleep, instrument_service, mock_api_client): - # The bid check loop should exit gracefully if no items have a "Quote" field - mock_api_client.request.side_effect = [ - {"Data": [{"Identifier": 1, "Description": "TURBO LONG DAX 15000 CITI", "AssetType": "WarrantKnockOut"}]}, - # InfoPrice data is missing the "Quote" field - {"Data": [{"Uic": 101, "Identifier": 1, "AssetType": "WarrantKnockOut"}]}, - ] - with pytest.raises(NoMarketAvailableException, match="No instruments with Bid data available after retries and final filtering."): - instrument_service.find_turbos("e1", "u1", "long") - -# endregion - -# region Test OrderService - -class TestOrderService: - @pytest.fixture - def order_service(self, mock_api_client): - return OrderService(mock_api_client, "account_key", "client_key") - - @patch('src.trade.api_actions.tr.orders.Order') - def test_place_market_order_success(self, mock_order_req, order_service, mock_api_client): - mock_api_client.request.return_value = {"OrderId": "12345"} - result = order_service.place_market_order(uic=1, asset_type="FxSpot", amount=100, buy_sell="Buy") - assert result == {"OrderId": "12345"} - mock_api_client.request.assert_called_once() - - def test_place_market_order_api_error(self, order_service, mock_api_client): - mock_api_client.request.side_effect = OrderPlacementError("API rejected order") - with pytest.raises(OrderPlacementError): - order_service.place_market_order(uic=1, asset_type="FxSpot", amount=100, buy_sell="Buy") - - @patch('src.trade.api_actions.tr.orders.CancelOrders') - def test_cancel_order_unexpected_exception(self, mock_cancel_req, order_service, mock_api_client): - mock_api_client.request.side_effect = Exception("Unexpected error") - result = order_service.cancel_order("123") - assert result is False - -# endregion - -# region Test PositionService - -class TestPositionService: - @pytest.fixture - def order_service(self, mock_api_client): - return OrderService(mock_api_client, "account_key", "client_key") - - @pytest.fixture - def position_service(self, mock_api_client, order_service, mock_config_manager): - return PositionService(mock_api_client, order_service, mock_config_manager, "account_key", "client_key") - - @patch('src.trade.api_actions.pf.positions.PositionsMe') - def test_get_open_positions_success(self, mock_positions_req, position_service, mock_api_client): - mock_api_client.request.return_value = {"Data": [{"PositionId": "pos1"}]} - result = position_service.get_open_positions() - assert result["__count"] == 1 - assert result["Data"][0]["PositionId"] == "pos1" - - @patch('src.trade.api_actions.pf.closedpositions.ClosedPositionsMe') - def test_get_closed_positions_success(self, mock_closed_positions_req, position_service, mock_api_client): - mock_api_client.request.return_value = {"Data": [{"PositionId": "pos1"}]} - result = position_service.get_closed_positions() - assert result["Data"][0]["PositionId"] == "pos1" - - @patch('src.trade.api_actions.pf.positions.SinglePosition') - def test_get_single_position_success(self, mock_single_position_req, position_service, mock_api_client): - mock_api_client.request.return_value = {"PositionId": "pos1"} - result = position_service.get_single_position("pos1") - assert result["PositionId"] == "pos1" - - @patch.object(PositionService, 'get_open_positions') - def test_find_position_by_order_id_with_retry_found_first_try(self, mock_get_open_positions, position_service): - mock_get_open_positions.return_value = {"Data": [{"PositionBase": {"SourceOrderId": "order1"}, "PositionId": "pos1"}]} - result = position_service.find_position_by_order_id_with_retry("order1") - assert result["PositionId"] == "pos1" - mock_get_open_positions.assert_called_once() - - @patch('time.sleep', return_value=None) - @patch.object(PositionService, 'get_open_positions') - @patch.object(OrderService, 'cancel_order') - def test_find_position_by_order_id_with_retry_not_found_and_cancel_success(self, mock_cancel_order, mock_get_open_positions, mock_sleep, position_service): - mock_get_open_positions.return_value = {"Data": []} - mock_cancel_order.return_value = True - - with pytest.raises(PositionNotFoundException) as excinfo: - position_service.find_position_by_order_id_with_retry("order1") - - assert "Successfully cancelled" in str(excinfo.value) - assert excinfo.value.cancellation_succeeded is True - assert mock_get_open_positions.call_count == 5 - mock_cancel_order.assert_called_once_with("order1") - - @patch('time.sleep', return_value=None) - @patch.object(PositionService, 'get_open_positions') - @patch.object(OrderService, 'cancel_order') - def test_find_position_by_order_id_with_retry_not_found_and_cancel_fail(self, mock_cancel_order, mock_get_open_positions, mock_sleep, position_service): - mock_get_open_positions.return_value = {"Data": []} - mock_cancel_order.return_value = False - - with pytest.raises(PositionNotFoundException) as excinfo: - position_service.find_position_by_order_id_with_retry("order1") - - assert "Failed to cancel" in str(excinfo.value) - assert excinfo.value.cancellation_succeeded is False - - @patch('src.trade.api_actions.pf.balances.AccountBalances') - def test_get_spending_power_invalid_value(self, mock_balances_req, position_service, mock_api_client): - mock_api_client.request.return_value = {"SpendingPower": "not a number"} - with pytest.raises(SaxoApiError, match="Invalid SpendingPower value received"): - position_service.get_spending_power() - -# endregion - -# region Test TradingOrchestrator - -class TestTradingOrchestrator: - - @pytest.fixture - def trading_orchestrator(self, mock_config_manager, mock_db_order_manager, mock_db_position_manager): - instrument_service = MagicMock(spec=InstrumentService) - order_service = MagicMock(spec=OrderService) - position_service = MagicMock(spec=PositionService) - - return TradingOrchestrator( - instrument_service, - order_service, - position_service, - mock_config_manager, - mock_db_order_manager, - mock_db_position_manager - ) - - def test_calculate_bid_amount_success(self, trading_orchestrator): - turbo_info = {"selected_instrument": {"latest_ask": 10, "decimals": 2}} - amount = trading_orchestrator._calculate_bid_amount(turbo_info, 1000) - assert amount == 99 - - def test_execute_trade_signal_happy_path(self, trading_orchestrator, mock_db_order_manager, mock_db_position_manager): - trading_orchestrator.instrument_service.find_turbos.return_value = { - "selected_instrument": {"uic": 123, "asset_type": "TypeA", "latest_ask": 10, "decimals": 2, "description": "Desc", "symbol": "Sym", "currency": "EUR", "commissions": {}} - } - trading_orchestrator.position_service.get_spending_power.return_value = 1000 - trading_orchestrator.order_service.place_market_order.return_value = {"OrderId": "order1"} - trading_orchestrator.position_service.find_position_by_order_id_with_retry.return_value = { - "PositionId": "pos1", "PositionBase": {}, "DisplayAndFormat": {} - } - - result = trading_orchestrator.execute_trade_signal("e1", "u1", "long") - - assert result is not None - mock_db_order_manager.insert_turbo_order_data.assert_called_once() - mock_db_position_manager.insert_turbo_open_position_data.assert_called_once() - - def test_calculate_bid_amount_invalid_ask_price(self, trading_orchestrator): - turbo_info = {"selected_instrument": {"latest_ask": None, "decimals": 2}} - with pytest.raises(ValueError, match="Invalid ask price for bid calculation"): - trading_orchestrator._calculate_bid_amount(turbo_info, 1000) - - def test_execute_trade_signal_db_error(self, trading_orchestrator, mock_db_order_manager): - trading_orchestrator.instrument_service.find_turbos.return_value = { - "selected_instrument": {"uic": 123, "asset_type": "TypeA", "latest_ask": 10, "decimals": 2, "description": "Desc", "symbol": "Sym", "currency": "EUR", "commissions": {}} - } - trading_orchestrator.position_service.get_spending_power.return_value = 1000 - trading_orchestrator.order_service.place_market_order.return_value = {"OrderId": "order1"} - trading_orchestrator.position_service.find_position_by_order_id_with_retry.return_value = { - "PositionId": "pos1", "PositionBase": {}, "DisplayAndFormat": {} - } - mock_db_order_manager.insert_turbo_order_data.side_effect = Exception("DB Error") - - from src.trade.exceptions import DatabaseOperationException - with pytest.raises(DatabaseOperationException): - trading_orchestrator.execute_trade_signal("e1", "u1", "long") - -# endregion - -# region Test PerformanceMonitor - -class TestPerformanceMonitor: - - @pytest.fixture - def performance_monitor(self, mock_config_manager, mock_db_position_manager, mock_trading_rule): - position_service = MagicMock(spec=PositionService) - order_service = MagicMock(spec=OrderService) - rabbit_connection = MagicMock() - - return PerformanceMonitor( - position_service, - order_service, - mock_config_manager, - mock_db_position_manager, - mock_trading_rule, - rabbit_connection - ) - - @patch('time.sleep', return_value=None) - @patch('src.trade.api_actions.send_message_to_mq_for_telegram') - def test_fetch_and_update_closed_position_in_db_success(self, mock_send_message, mock_sleep, performance_monitor, mock_db_position_manager): - performance_monitor.position_service.get_closed_positions.return_value = { - "Data": [{"ClosedPosition": {"OpeningPositionId": "pos1", "ClosingPrice": 120, "OpenPrice": 100, "Amount": 10}, "DisplayAndFormat": {}}] - } - result = performance_monitor._fetch_and_update_closed_position_in_db("pos1", "Test Close") - assert result is True - mock_db_position_manager.update_turbo_position_data.assert_called_once() - mock_send_message.assert_called_once() - - @patch.object(PerformanceMonitor, '_log_performance_detail') - @patch.object(PerformanceMonitor, '_fetch_and_update_closed_position_in_db') - def test_check_all_positions_performance_triggers_stoploss(self, mock_update_db, mock_log_perf, performance_monitor): - performance_monitor.db_position_manager.get_open_positions_ids_actions.return_value = [{"position_id": "pos1"}] - performance_monitor.position_service.get_open_positions.return_value = { - "Data": [{"PositionId": "pos1", "PositionBase": {"OpenPrice": 100, "Amount": 10, "CanBeClosed": True, "Uic": 1, "AssetType": "T"}, "PositionView": {"Bid": 79}}] - } - performance_monitor.db_position_manager.get_max_position_percent.return_value = -10.0 - mock_update_db.return_value = True - result = performance_monitor.check_all_positions_performance() - performance_monitor.order_service.place_market_order.assert_called_once() - mock_update_db.assert_called_once() - - def test_check_all_positions_performance_no_positions(self, performance_monitor): - performance_monitor.db_position_manager.get_open_positions_ids_actions.return_value = [] - result = performance_monitor.check_all_positions_performance() - assert result == {"closed_positions_processed": [], "db_updates": [], "errors": 0} - - @patch('src.trade.api_actions.send_message_to_mq_for_telegram') - def test_close_managed_positions_by_criteria(self, mock_send_message, performance_monitor): - performance_monitor.db_position_manager.get_open_positions_ids_actions.return_value = [ - {"position_id": "pos1", "action": "long"}, - {"position_id": "pos2", "action": "short"}, - ] - performance_monitor.position_service.get_open_positions.return_value = { - "Data": [ - {"PositionId": "pos1", "PositionBase": {"Amount": 10, "CanBeClosed": True, "Uic": 1, "AssetType": "T"}}, - {"PositionId": "pos2", "PositionBase": {"Amount": -10, "CanBeClosed": True, "Uic": 2, "AssetType": "T"}}, - ] - } - performance_monitor.order_service.place_market_order.return_value = {"OrderId": "close_order"} - with patch.object(performance_monitor, '_fetch_and_update_closed_position_in_db', return_value=True): - result = performance_monitor.close_managed_positions_by_criteria(action_filter="long") - - assert result["closed_initiated_count"] == 1 - assert performance_monitor.order_service.place_market_order.call_count == 1 - - @patch('src.trade.api_actions.send_message_to_mq_for_telegram') - def test_sync_db_positions_with_api_success(self, mock_send_message, performance_monitor): - performance_monitor.db_position_manager.get_open_positions_ids.return_value = ["pos1_closed", "pos2_open"] - performance_monitor.position_service.get_open_positions.return_value = {"Data": [{"PositionId": "pos2_open"}]} - performance_monitor.position_service.get_closed_positions.return_value = { - "Data": [{"ClosedPosition": {"OpeningPositionId": "pos1_closed"}, "DisplayAndFormat": {}}] - } - result = performance_monitor.sync_db_positions_with_api() - assert len(result["updates_for_db"]) == 1 - assert result["updates_for_db"][0][0] == "pos1_closed" - - @patch('os.path.exists', return_value=True) - @patch('builtins.open', new_callable=MagicMock) - def test_log_performance_detail(self, mock_open, mock_path_exists, performance_monitor): - api_pos = { - "PositionBase": {"ExecutionTimeOpen": "2023-01-01T12:00:00Z"}, - "PositionView": {} - } - performance_monitor._log_performance_detail("pos1", api_pos, 1.23) - mock_open.assert_called_once() - handle = mock_open.return_value.__enter__() - handle.write.assert_called_once() - written_content = handle.write.call_args[0][0] - import json - log_data = json.loads(written_content) - assert log_data["position_id"] == "pos1" - assert log_data["performance"] == 1.23 - - @patch('time.sleep', return_value=None) - def test_fetch_and_update_closed_position_in_db_not_found(self, mock_sleep, performance_monitor): - performance_monitor.position_service.get_closed_positions.return_value = {"Data": []} - result = performance_monitor._fetch_and_update_closed_position_in_db("pos1", "Test Close") - assert result is False - - def test_check_all_positions_performance_api_fail(self, performance_monitor): - performance_monitor.db_position_manager.get_open_positions_ids_actions.return_value = [{"position_id": "pos1"}] - performance_monitor.position_service.get_open_positions.side_effect = ApiRequestException("API Error") - result = performance_monitor.check_all_positions_performance() - assert result["errors"] == 1 - - def test_close_managed_positions_no_filter(self, performance_monitor): - performance_monitor.db_position_manager.get_open_positions_ids_actions.return_value = [ - {"position_id": "pos1", "action": "long"}, - ] - performance_monitor.position_service.get_open_positions.return_value = { - "Data": [ - {"PositionId": "pos1", "PositionBase": {"Amount": 10, "CanBeClosed": True, "Uic": 1, "AssetType": "T"}}, - ] - } - performance_monitor.order_service.place_market_order.return_value = {"OrderId": "close_order"} - with patch.object(performance_monitor, '_fetch_and_update_closed_position_in_db', return_value=True): - result = performance_monitor.close_managed_positions_by_criteria() - - assert result["closed_initiated_count"] == 1 - -# endregion \ No newline at end of file